<!-- Generated by fyi::use_fyi_md() on 2026-01-14 --> <!-- Regenerate with: fyi::use_fyi_md("diffuseR") -->

fyi: diffuseR

Exported Functions (diffuseR::)

Function	Arguments
apply_interleaved_rotary_emb	x, freqs
apply_split_rotary_emb	x, freqs
auto_devices	model, strategy
bpe_tokenizer	tokenizer_path
clear_vram	verbose
CLIPTokenizer	prompt, merges, vocab_file, pad_token
configure_vae_for_profile	vae, profile
ddim_scheduler_create	num_train_timesteps, num_inference_steps, eta, beta_schedule, beta_start, beta_end, rescale_betas_zero_snr, dtype, device
ddim_scheduler_step	model_output, timestep, sample, schedule, eta, use_clipped_model_output, thresholding, generator, variance_noise, clip_sample, set_alpha_to_one, prediction_type, dtype, device
decode_bpe	tokenizer, ids, skip_special_tokens
diagonal_gaussian_distribution	parameters
dit_offloaded_forward	hidden_states, layers, device, ...
download_component	model_name, component, device, overwrite, show_progress
download_model	model_name, devices, unet_dtype_str, overwrite, show_progress, download_models
encode_bpe	tokenizer, text, add_special_tokens, max_length, padding, truncation, return_tensors
encode_text_ltx2	prompt, backend, model_path, tokenizer_path, text_encoder, embeddings_file, api_url, max_sequence_length, caption_channels, device, dtype
encode_with_gemma3	prompts, model, tokenizer, max_sequence_length, scale_factor, device, dtype, verbose
filename_from_prompt	prompt, datetime
flowmatch_calculate_shift	seq_len, base_seq_len, max_seq_len, base_shift, max_shift
flowmatch_scale_noise	sample, timestep, noise, schedule
flowmatch_scheduler_create	num_train_timesteps, shift, use_dynamic_shifting, base_shift, max_shift, base_seq_len, max_seq_len, invert_sigmas, shift_terminal, time_shift_type
flowmatch_scheduler_step	model_output, timestep, sample, schedule, generator
flowmatch_set_timesteps	schedule, num_inference_steps, device, mu, sigmas, timesteps
gemma3_config_ltx2
gemma3_text_model	config
gemma3_tokenizer	tokenizer_path
img2img	input_image, prompt, negative_prompt, img_dim, model_name, pipeline, devices, unet_dtype_str, download_models, scheduler, num_inference_steps, strength, guidance_scale, seed, save_file, filename, metadata_path, use_native_decoder, use_native_text_encoder, use_native_unet, ...
is_blackwell_gpu
latents_to_video	latents, vae, file, fps, ...
load_decoder_weights	native_decoder, torchscript_path, verbose
load_gemma3_text_encoder	model_path, device, dtype, verbose
load_ltx2_connectors	weights_path, config_path, device, dtype, verbose
load_ltx2_transformer	weights_dir, config_path, device, dtype, verbose
load_ltx2_vae	weights_path, config_path, device, dtype, verbose
load_model_component	component, model_name, device, unet_dtype_str, download, use_native
load_pipeline	model_name, m2d, i2i, unet_dtype_str, use_native_decoder, use_native_text_encoder, use_native_unet, ...
load_text_encoder_weights	native_encoder, torchscript_path, verbose
load_text_encoder2_weights	native_encoder, torchscript_path, verbose
load_to_gpu	module, device
load_unet_sdxl_weights	native_unet, torchscript_path, verbose
load_unet_weights	native_unet, torchscript_path, verbose
ltx_video_downsampler3d	in_channels, out_channels, stride, spatial_padding_mode
ltx_video_upsampler3d	in_channels, stride, residual, upscale_factor, spatial_padding_mode
ltx2_memory_profile	vram_gb, model
ltx2_text_connectors	caption_channels, text_proj_in_factor, video_connector_num_attention_heads, video_connector_attention_head_dim, video_connector_num_layers, video_connector_num_learnable_registers, audio_connector_num_attention_heads, audio_connector_attention_head_dim, audio_connector_num_layers, audio_connector_num_learnable_registers, connector_rope_base_seq_len, rope_theta, rope_double_precision, causal_temporal_positioning, rope_type
ltx2_video_causal_conv3d	in_channels, out_channels, kernel_size, stride, dilation, groups, spatial_padding_mode
ltx2_video_decoder3d	in_channels, out_channels, block_out_channels, spatio_temporal_scaling, layers_per_block, patch_size, patch_size_t, resnet_norm_eps, is_causal, inject_noise, timestep_conditioning, upsample_residual, upsample_factor, spatial_padding_mode
ltx2_video_down_block3d	in_channels, out_channels, num_layers, dropout, resnet_eps, resnet_act_fn, spatio_temporal_scale, downsample_type, spatial_padding_mode
ltx2_video_encoder3d	in_channels, out_channels, block_out_channels, spatio_temporal_scaling, layers_per_block, downsample_type, patch_size, patch_size_t, resnet_norm_eps, is_causal, spatial_padding_mode
ltx2_video_mid_block3d	in_channels, num_layers, dropout, resnet_eps, resnet_act_fn, inject_noise, timestep_conditioning, spatial_padding_mode
ltx2_video_resnet_block3d	in_channels, out_channels, dropout, eps, non_linearity, inject_noise, timestep_conditioning, spatial_padding_mode
ltx2_video_transformer_3d_model	in_channels, out_channels, patch_size, patch_size_t, num_attention_heads, attention_head_dim, cross_attention_dim, vae_scale_factors, pos_embed_max_pos, base_height, base_width, audio_in_channels, audio_out_channels, audio_patch_size, audio_patch_size_t, audio_num_attention_heads, audio_attention_head_dim, audio_cross_attention_dim, audio_scale_factor, audio_pos_embed_max_pos, audio_sampling_rate, audio_hop_length, num_layers, activation_fn, qk_norm, norm_elementwise_affine, norm_eps, caption_channels, attention_bias, attention_out_bias, rope_theta, rope_double_precision, causal_offset, timestep_scale_multiplier, cross_attn_timestep_scale_multiplier, rope_type
ltx2_video_up_block3d	in_channels, out_channels, num_layers, dropout, resnet_eps, resnet_act_fn, spatio_temporal_scale, inject_noise, timestep_conditioning, upsample_residual, upscale_factor, spatial_padding_mode
ltx2_video_vae	in_channels, out_channels, latent_channels, block_out_channels, decoder_block_out_channels, layers_per_block, decoder_layers_per_block, spatio_temporal_scaling, decoder_spatio_temporal_scaling, decoder_inject_noise, downsample_type, upsample_residual, upsample_factor, timestep_conditioning, patch_size, patch_size_t, resnet_norm_eps, scaling_factor, encoder_causal, decoder_causal, encoder_spatial_padding_mode, decoder_spatial_padding_mode
models2devices	model_name, devices, unet_dtype_str, download_models
offload_to_cpu	module, gc
pack_text_embeds	text_hidden_states, sequence_lengths, padding_side, scale_factor, eps, device
per_channel_rms_norm	channel_dim, eps
post_quant_conv	x, dtype, device
preprocess_image	input, device, width, height
quant_conv	x, dtype, device
rope_embedder_create	dim, patch_size, patch_size_t, base_num_frames, base_height, base_width, scale_factors, theta, causal_offset, double_precision, rope_type, num_attention_heads
rope_forward	embedder, coords, device
rope_prepare_video_coords	embedder, batch_size, num_frames, height, width, device, fps
save_frames	video, dir, prefix, format, verbose
save_image	img, save_to, normalize
save_video	video, file, fps, format, backend, quality, verbose
scheduler_add_noise	original_latents, noise, timestep, scheduler_obj
sdxl_memory_profile	vram_gb
sequential_cfg_forward	model, latents, timestep, prompt_embeds, negative_prompt_embeds, guidance_scale, ...
text_encoder_native	vocab_size, context_length, embed_dim, num_layers, num_heads, mlp_dim, apply_final_ln
text_encoder2_native	vocab_size, context_length, embed_dim, num_layers, num_heads, mlp_dim
tokenize_gemma3	tokenizer, text, max_length, padding, return_tensors
txt2img	prompt, model_name, ...
txt2img_sd21	prompt, negative_prompt, img_dim, pipeline, devices, unet_dtype_str, download_models, scheduler, timesteps, initial_latents, num_inference_steps, guidance_scale, seed, save_file, filename, metadata_path, use_native_decoder, use_native_text_encoder, use_native_unet, ...
txt2img_sdxl	prompt, negative_prompt, img_dim, pipeline, devices, memory_profile, unet_dtype_str, download_models, scheduler, timesteps, initial_latents, num_inference_steps, guidance_scale, seed, save_file, filename, metadata_path, use_native_decoder, use_native_text_encoder, use_native_unet, verbose, ...
txt2vid_ltx2	prompt, negative_prompt, width, height, num_frames, fps, num_inference_steps, guidance_scale, memory_profile, text_backend, text_api_url, vae, dit, connectors, seed, output_file, output_format, return_latents, verbose
unet_native	in_channels, out_channels, block_out_channels, layers_per_block, cross_attention_dim, attention_head_dim
unet_native_from_torchscript	torchscript_path, verbose
unet_sdxl_native	in_channels, out_channels, block_out_channels, layers_per_block, transformer_layers_per_block, cross_attention_dim, attention_head_dim, addition_embed_dim, addition_time_embed_dim
unet_sdxl_native_from_torchscript	torchscript_path, verbose
vae_decoder_native	latent_channels, out_channels
validate_resolution	height, width, num_frames, profile
vocab_size	tokenizer
vram_report	label

Internal Functions (diffuseR:::)

Function	Arguments
.build_fallback_devices	model, strategy
.detect_vram	use_free
.ffmpeg_available
.flowmatch_index_for_timestep	timestep, schedule
.flowmatch_init_step_index	timestep, schedule
.flowmatch_stretch_shift_to_terminal	t, shift_terminal
.flowmatch_time_shift	mu, sigma, t, shift_type
`%
apply_bpe_merges	tokens, merge_priority, vocab
apply_interleaved_rotary_emb_list	x, freqs
apply_rotary_pos_emb	q, k, cos, sin
BasicTransformerBlock	dim, n_heads, d_head, context_dim
CLIPAttention	embed_dim, num_heads
CLIPMLP	in_dim, hidden_dim, gelu_type
CLIPTransformerBlock	embed_dim, num_heads, mlp_dim, gelu_type
create_sliding_window_mask	seq_len, window_size, device
detect_text_encoder_architecture	torchscript_path
detect_unet_architecture	torchscript_path
detect_unet_sdxl_architecture	torchscript_path
Downsample2D	channels
encode_single	tokenizer, text, add_special_tokens
feed_forward	dim, dim_out, mult, dropout, activation_fn, inner_dim, bias
FeedForward	dim, mult
GEGLU	dim_in, dim_out
gelu_activation	dim_in, dim_out, approximate, bias
gemma3_attention	config, layer_idx
gemma3_decoder_layer	config, layer_idx
gemma3_mlp	config
gemma3_rms_norm	dim, eps
gemma3_rotary_embedding	dim, max_position_embeddings, base, scaling_factor
get_bos_id	tokenizer
get_component_file_path	component, model_dir, device, unet_dtype_str
get_eos_id	tokenizer
get_pad_id	tokenizer
get_required_components	model_name
get_timestep_embedding	timesteps, embedding_dim, flip_sin_to_cos, downscale_freq_shift, scale, max_period
greedy_tokenize	text, vocab, byte_fallback, unk_token
group_norm_32	channels
load_gemma3_weights	model, weights, verbose
load_ltx2_connector_weights	connectors, weights, verbose
load_ltx2_transformer_sharded	transformer, weights_dir, index_path, verbose
load_ltx2_transformer_weights	transformer, weights, verbose
load_ltx2_vae_weights	vae, weights, verbose
load_text_encoders	model_name, device, download
ltx2_ada_layer_norm_single	embedding_dim, num_mod_params, use_additional_conditions
ltx2_attention	query_dim, heads, kv_heads, dim_head, dropout, bias, cross_attention_dim, out_bias, qk_norm, norm_eps, norm_elementwise_affine, rope_type
ltx2_audio_video_rotary_pos_embed	dim, patch_size, patch_size_t, base_num_frames, base_height, base_width, sampling_rate, hop_length, scale_factors, theta, causal_offset, modality, double_precision, rope_type, num_attention_heads
ltx2_connector_transformer_1d	num_attention_heads, attention_head_dim, num_layers, num_learnable_registers, rope_base_seq_len, rope_theta, rope_double_precision, eps, causal_temporal_positioning, rope_type
ltx2_rotary_pos_embed_1d	dim, base_seq_len, theta, double_precision, rope_type, num_attention_heads
ltx2_transformer_block_1d	dim, num_attention_heads, attention_head_dim, activation_fn, eps, rope_type
ltx2_video_transformer_block	dim, num_attention_heads, attention_head_dim, cross_attention_dim, audio_dim, audio_num_attention_heads, audio_attention_head_dim, audio_cross_attention_dim, qk_norm, activation_fn, attention_bias, attention_out_bias, eps, elementwise_affine, rope_type
pixart_alpha_combined_timestep_size_embeddings	embedding_dim, size_emb_dim, use_additional_conditions
pixart_alpha_text_projection	in_features, hidden_size, out_features, act_fn
print.bpe_tokenizer	x, ...
quick_gelu	x
repeat_kv	hidden_states, n_rep
rescale_zero_terminal_snr	betas
rms_norm	dim, eps, elementwise_affine
rotate_half	x
save_video_av	video, file, fps, verbose
save_video_ffmpeg	video, file, fps, format, quality, verbose
setup_dtype	devices, unet_dtype_str
SpatialTransformer	in_channels, n_heads, d_head, depth, context_dim
standardize_devices	devices, required_components
timestep_embedding	timesteps, dim, flip_sin_to_cos, downscale_freq_shift
timestep_embedding_module	in_channels, time_embed_dim, act_fn, out_dim
timesteps_module	num_channels, flip_sin_to_cos, downscale_freq_shift, scale
token_get_pairs	symbols
token_merge_pair_once	symbols, bigram
UNetCrossAttention	query_dim, context_dim, heads, dim_head
UNetResBlock	in_channels, out_channels, time_embed_dim
Upsample2D	channels
VAEAttentionBlock	channels
VAEMidBlock	channels
VAEResnetBlock	in_channels, out_channels
VAEUpBlock	in_channels, out_channels, num_resnets, add_upsample

Options

No options found in diffuseR.

Documentation: diffuseR

apply_bpe_merges

Apply BPE merge rules

Description

Apply BPE merge rules

Usage

apply_bpe_merges(tokens, merge_priority, vocab)

apply_interleaved_rotary_emb_list

Apply interleaved rotary embedding

Description

Apply interleaved rotary embedding

Usage

apply_interleaved_rotary_emb_list(x, freqs)

apply_interleaved_rotary_emb

Apply interleaved rotary embeddings

Description

Applies rotary position embeddings to query or key tensors using the interleaved format where real and imaginary components alternate.

Usage

apply_interleaved_rotary_emb(x, freqs)

Arguments

• x: torch tensor. Query or key tensor of shape (B, S, C).
• freqs: List. Contains cos_freqs and sin_freqs from rope_forward().

Value

torch tensor. Rotated tensor with same shape as input.

apply_rotary_pos_emb

Apply rotary position embeddings

Description

Apply rotary position embeddings

Usage

apply_rotary_pos_emb(q, k, cos, sin)

Arguments

• q: Query tensor [batch, heads, seq, head_dim]
• k: Key tensor [batch, heads, seq, head_dim]
• cos: Cosine embeddings [batch, seq, head_dim]
• sin: Sine embeddings [batch, seq, head_dim]

apply_split_rotary_emb

Apply split rotary embeddings

Description

Applies rotary position embeddings to query or key tensors using the split format where first half is real and second half is imaginary.

Usage

apply_split_rotary_emb(x, freqs)

Arguments

• x: torch tensor. Query or key tensor.
• freqs: List. Contains cos_freqs and sin_freqs from rope_forward().

Value

torch tensor. Rotated tensor with same shape as input.

auto_devices

Auto-Configure Device Assignment

Description

Automatically determines optimal device configuration for diffusion model components based on available VRAM and GPU architecture. Uses gpuctl for detection if available, otherwise falls back to sensible defaults.

Usage

auto_devices(model = "sdxl", strategy = "auto")

Arguments

• model: Character. Model type: "sd21" or "sdxl".
• strategy: Character. Memory strategy: "auto" (default), "full_gpu", "unet_gpu", or "cpu_only". See Details.

Details

Strategies:

• "auto": Detect VRAM and choose best strategy (requires gpuctl)
• "full_gpu": All components on CUDA (16GB+ VRAM for SDXL)
• "unet_gpu": Only unet on CUDA, rest on CPU (8GB+ VRAM)
• "cpu_only": All components on CPU

If gpuctl is not installed, "auto" falls back to "unet_gpu" which works on most modern GPUs (8GB+ VRAM).

On Blackwell GPUs (RTX 50xx), "unet_gpu" is forced due to TorchScript compatibility issues, regardless of available VRAM.

Value

A named list of device assignments suitable for models2devices().

Examples

# Auto-detect best configuration
devices <- auto_devices("sdxl")

# Use with models2devices
m2d <- models2devices("sdxl", devices = auto_devices("sdxl"))

# Force CPU-only
devices <- auto_devices("sdxl", strategy = "cpu_only")

BasicTransformerBlock

Basic Transformer Block

Description

Basic Transformer Block

Usage

BasicTransformerBlock(dim, n_heads, d_head, context_dim = NULL)

bpe_tokenizer

BPE Tokenizer

Description

Native R implementation of Byte-Pair Encoding tokenizer. Loads from HuggingFace tokenizer.json format.

Usage

bpe_tokenizer(tokenizer_path)

Arguments

• tokenizer_path: Path to tokenizer.json or directory containing it.

Value

A bpe_tokenizer object.

clear_vram

Clear VRAM Cache

Description

Forces garbage collection and clears CUDA memory cache.

Usage

clear_vram(verbose = FALSE)

Arguments

• verbose: Logical. Print memory status before/after.

Value

Invisibly returns NULL.

Examples

clear_vram()

CLIPAttention

CLIP Attention Block

Description

Multi-head self-attention with separate Q/K/V projections (HuggingFace style)

Usage

CLIPAttention(embed_dim, num_heads)

Arguments

• embed_dim: Embedding dimension
• num_heads: Number of attention heads

CLIPMLP

CLIP MLP Block

Description

Feed-forward network with configurable activation

Usage

CLIPMLP(in_dim, hidden_dim, gelu_type = "tanh")

Arguments

• in_dim: Input dimension
• hidden_dim: Hidden dimension
• gelu_type: GELU variant: "tanh" (tanh approximation), "quick" (QuickGELU), "exact" (standard GELU)

CLIPTokenizer

Tokenize a prompt

Description

Tokenize a prompt

Usage

CLIPTokenizer(
  prompt,
  merges = system.file("tokenizer/merges.txt", package = "diffuseR"),
  vocab_file = system.file("tokenizer/vocab.json", package = "diffuseR"),
  pad_token = 0L
)

Arguments

• prompt: A character string prompt describing the image to generate.
• merges: Path to the merges file (BPE merges).
• vocab_file: Path to the vocabulary file (token->id mapping).
• pad_token: The token ID used for padding (default is 0).

Value

A 2D torch tensor of shape c(1, 77) containing the token IDs.

CLIPTransformerBlock

CLIP Transformer Block

Description

Pre-norm transformer block with attention and MLP (HuggingFace style)

Usage

CLIPTransformerBlock(embed_dim, num_heads, mlp_dim, gelu_type = "tanh")

Arguments

• embed_dim: Embedding dimension
• num_heads: Number of attention heads
• mlp_dim: MLP hidden dimension
• gelu_type: GELU variant: "tanh", "quick", or "exact"

configure_vae_for_profile

Configure VAE for Memory Profile

Description

Sets VAE tiling parameters based on memory profile.

Usage

configure_vae_for_profile(vae, profile)

Arguments

• vae: The LTX2 VAE module.
• profile: Memory profile from ltx2_memory_profile().

Value

The VAE (modified in place).

Examples

profile <- ltx2_memory_profile(vram_gb = 8)
vae <- load_ltx2_vae(...)
configure_vae_for_profile(vae, profile)

create_sliding_window_mask

Create sliding window causal attention mask

Description

Create sliding window causal attention mask

Usage

create_sliding_window_mask(seq_len, window_size, device = "cpu")

ddim_scheduler_create

Create a DDIM Scheduler

Description

Creates a Denoising Diffusion Implicit Models (DDIM) scheduler for use with diffusion models. DDIM schedulers provide a deterministic sampling process that offers faster inference compared to DDPM while maintaining high quality outputs.

Usage

ddim_scheduler_create(
  num_train_timesteps = 1000,
  num_inference_steps = 50,
  eta = 0,
  beta_schedule = c("linear", "scaled_linear", "cosine"),
  beta_start = 0.00085,
  beta_end = 0.012,
  rescale_betas_zero_snr = FALSE,
  dtype = torch::torch_float32(),
  device = c(torch::torch_device("cpu"), torch::torch_device("cuda"))
)

Arguments

• num_train_timesteps: Integer. The number of diffusion steps used to train the model. Default: 1000
• num_inference_steps: Integer. The number of diffusion steps used for inference. Fewer steps typically means faster inference at the cost of sample quality. Default: 50
• eta: Numeric. Controls the amount of stochasticity. When eta=0, the sampling process is deterministic. When eta=1, the sampling process is equivalent to DDPM. Default: 0
• beta_schedule: Character. The beta schedule to use. Options are: - "linear": Linear beta schedule from beta_start to beta_end
• "scaled_linear": Scaled linear schedule, generally gives better results
• "cosine": Cosine schedule that approaches zero smoothly Default: "linear"
• beta_start: Numeric. The starting value for the beta schedule. Default: 0.00085
• beta_end: Numeric. The final value for the beta schedule. Default: 0.012
• rescale_betas_zero_snr: Logical. If TRUE, rescales the beta values
• dtype: The data type to use for computations. Default is torch_float32(). Options are torch_float16() and torch_float32().
• device: The device to use for computations. Options are torch_device("cpu"), torch_device("cuda").

Details

DDIM (Denoising Diffusion Implicit Models) was introduced by Song et al. (2020) as an extension to DDPM (Denoising Diffusion Probabilistic Models). It offers a deterministic sampling process and allows for controlling the number of inference steps independently from the training process.

The scheduler contains the noise schedule and methods for computing alpha, beta, and other parameters used in the diffusion process.

Value

A DDIM scheduler object that can be used with diffusion models to generate samples.

References

Song, J., Meng, C., & Ermon, S. (2020). "Denoising Diffusion Implicit Models." https://arxiv.org/abs/2010.02502

Examples

# Create a DDIM scheduler with custom parameters
scheduler <- ddim_scheduler_create(
  num_train_timesteps = 1000,
  num_inference_steps = 30,
  eta = 0.5,
  beta_schedule = "scaled_linear"
)

ddim_scheduler_step

Perform a DDIM scheduler step

Description

Performs a single denoising step using the DDIM (Denoising Diffusion Implicit Models) algorithm. This function takes the output from a diffusion model at a specific timestep and computes the previous (less noisy) sample in the diffusion process.

Usage

ddim_scheduler_step(
  model_output,
  timestep,
  sample,
  schedule,
  eta = 0,
  use_clipped_model_output = FALSE,
  thresholding = FALSE,
  generator = NULL,
  variance_noise = NULL,
  clip_sample = FALSE,
  set_alpha_to_one = FALSE,
  prediction_type = c("epsilon", "sample", "v_prediction"),
  dtype = torch::torch_float32(),
  device = "cpu"
)

Arguments

• model_output: Numeric array. The output from the diffusion model, typically representing predicted noise or the denoised sample depending on prediction_type.
• timestep: Integer. The current timestep in the diffusion process.
• sample: Numeric array. The current noisy sample at timestep t.
• schedule: List. The DDIM scheduler object containing the necessary parameters created from ddim_scheduler_create()
• eta: Numeric. Controls the stochasticity of the process. When eta=0, DDIM is deterministic. When eta=1, it's equivalent to DDPM. Default: 0
• use_clipped_model_output: Logical. Whether to clip the model output before computing the sample update. Can improve stability. Default: FALSE
• thresholding: Logical. Whether to apply thresholding to the output. Default: FALSE
• generator: An optional random number generator for reproducibility. Default: NULL
• variance_noise: Optional pre-generated noise for the variance when eta > 0. If NULL and eta > 0, noise will be generated. Default: NULL
• clip_sample: Logical. Whether to clip the sample. Default: FALSE
• set_alpha_to_one: Logical. Whether to override the final alpha value to 1. Used for numerical stability in the final step. Default: FALSE
• prediction_type: Character. The type of prediction the model outputs. Options are:
• "epsilon": The model predicts the noise
• "sample": The model predicts the denoised sample directly
• "v_prediction": The model predicts the velocity vector (v) Default: "epsilon"
• dtype: The data type to use for computations. Default is torch_float32().
• device: The device to use for computations. Options are "cpu" and "cuda".

Details

The DDIM step function implements the core sampling algorithm of DDIM described in Song et al. 2020. It computes the previous sample x_t-1 given the current sample x_t and the model output.

The algorithm differs from DDPM by using a non-Markovian diffusion process that allows for deterministic sampling and fewer inference steps without sacrificing quality.

When using prediction_type="epsilon" (most common), the model predicts the noise that was added to create the current noisy sample. For prediction_type="sample", the model predicts the clean sample directly. The v_prediction option implements the v-parameterization from Salimans & Ho (2022).

Value

A list containing:

• prev_sample: The less noisy sample at timestep t-1
• pred_original_sample: The predicted denoised sample

References

Song, J., Meng, C., & Ermon, S. (2020). "Denoising Diffusion Implicit Models." https://arxiv.org/abs/2010.02502

Salimans, T., & Ho, J. (2022). "Progressive Distillation for Fast Sampling of Diffusion Models." https://arxiv.org/abs/2202.00512

Examples

# Perform a denoising step
result <- ddim_scheduler_step(
  model_output = model_output,
  timestep = timestep,
  sample = sample,
  eta = 0,  # Deterministic sampling
  prediction_type = "epsilon")

decode_bpe

Decode token IDs to text

Description

Decode token IDs to text

Usage

decode_bpe(tokenizer, ids, skip_special_tokens = TRUE)

Arguments

• tokenizer: A bpe_tokenizer object.
• ids: Integer vector or matrix of token IDs.
• skip_special_tokens: Logical. Skip special tokens in output.

Value

Character string or vector.

detect_text_encoder_architecture

Detect text encoder architecture from TorchScript file

Description

Detect text encoder architecture from TorchScript file

Usage

detect_text_encoder_architecture(torchscript_path)

Arguments

• torchscript_path: Path to TorchScript encoder .pt file

Value

List with vocab_size, context_length, embed_dim, num_layers, num_heads, mlp_dim

detect_unet_architecture

Detect UNet architecture from TorchScript file

Description

Detect UNet architecture from TorchScript file

Usage

detect_unet_architecture(torchscript_path)

Arguments

• torchscript_path: Path to TorchScript UNet .pt file

Value

List with architecture parameters

detect_unet_sdxl_architecture

Detect SDXL UNet architecture from TorchScript file

Description

Detect SDXL UNet architecture from TorchScript file

Usage

detect_unet_sdxl_architecture(torchscript_path)

Arguments

• torchscript_path: Path to TorchScript SDXL UNet .pt file

Value

List with architecture parameters

diagonal_gaussian_distribution

Diagonal Gaussian Distribution

Description

Represents a diagonal Gaussian distribution for VAE latents.

Usage

diagonal_gaussian_distribution(parameters)

Arguments

• parameters: Tensor of concatenated mean and log variance.

dit_offloaded_forward

DiT Layer-by-Layer Forward Pass

Description

Runs transformer layers one at a time, moving each to GPU before computation and back to CPU after. For extreme memory-constrained scenarios.

Usage

dit_offloaded_forward(hidden_states, layers, device = "cuda", ...)

Arguments

• hidden_states: Input tensor.
• layers: List of transformer layers (on CPU).
• device: Target device for computation.
• ...: Additional arguments passed to each layer.

Value

Output tensor (on CPU).

Examples

# During low-VRAM inference
output <- dit_offloaded_forward(
  hidden_states,
  model$transformer_blocks,
  device = "cuda",
  encoder_hidden_states = text_embeds
)

dot-build_fallback_devices

Build fallback device configuration

Description

Build fallback device configuration

Usage

.build_fallback_devices(model, strategy)

Arguments

• model: Character. Model type.
• strategy: Character. Memory strategy.

Value

Named list of device assignments.

dot-detect_vram

Detect Available VRAM

Description

Uses gpuctl if available.

Usage

.detect_vram(use_free = FALSE)

Arguments

• use_free: Logical. If TRUE, return free VRAM. If FALSE, return total.

Value

Numeric. VRAM in GB, or 0 if no GPU detected.

dot-ffmpeg_available

Check if FFmpeg is Available

Description

Check if FFmpeg is Available

Usage

.ffmpeg_available()

Value

Logical. TRUE if ffmpeg is in PATH.

download_component

Download TorchScript model component for Stable Diffusion

Description

Downloads the required model file (e.g., UNet, decoder, text encoder, and tokenizer) for a given Stable Diffusion model into a local user-specific data directory. The files will be stored in a persistent path returned by [tools::R_user_dir()], typically: -

• Each model is stored in its own subdirectory for better organization. If the files already exist, they will not be downloaded again unless overwrite = TRUE.

Usage

download_component(
  model_name = "sd21",
  component,
  device = "cpu",
  overwrite = FALSE,
  show_progress = TRUE
)

Arguments

• model_name: Character string, the name of the model to download (e.g., "sd21").
• component: Character string, the specific model component to download (e.g., "unet", "decoder", "text_encoder").
• device: Character string, the device type for which the model is intended (e.g., "cpu" or "cuda").
• overwrite: Logical; if TRUE, re-downloads the model files even if they already exist.
• show_progress: Logical; if TRUE (default), displays a progress bar during download.

Value

The local file path to the specific model directory (as a string).

Examples

model_dir <- download_model("sd21")

download_model

Download TorchScript model files for Stable Diffusion

Description

Downloads the required model files (e.g., UNet, decoder, text encoder, and tokenizer) for a given Stable Diffusion model into a local user-specific data directory. The files will be stored in a persistent path returned by [tools::R_user_dir()], typically: -

• Each model is stored in its own subdirectory for better organization. If the files already exist, they will not be downloaded again unless overwrite = TRUE.

Usage

download_model(
  model_name = "sd21",
  devices = list(unet = "cpu", decoder = "cpu", text_encoder = "cpu"),
  unet_dtype_str = NULL,
  overwrite = FALSE,
  show_progress = TRUE,
  download_models = FALSE
)

Arguments

• model_name: Name of the model (e.g., "sd21" for stable-diffusion-2-1)
• devices: Either a single device string or a named list with elements 'unet', 'decoder', 'text_encoder'; optionally 'encoder'
• unet_dtype_str: Optional: "float16" or "float32" (only applies if unet uses CUDA)
• overwrite: If TRUE, overwrite existing model files
• show_progress: Show download progress messages
• download_models: If TRUE, download the model files from Hugging Face

Value

A list with model_dir and model_files

Examples

model_dir <- download_model("sd21")

Downsample2D

Downsample Block

Description

Downsample Block

Usage

Downsample2D(channels)

encode_bpe

Encode text to token IDs

Description

Encode text to token IDs

Usage

encode_bpe(
  tokenizer,
  text,
  add_special_tokens = TRUE,
  max_length = NULL,
  padding = "none",
  truncation = FALSE,
  return_tensors = "list"
)

Arguments

• tokenizer: A bpe_tokenizer object.
• text: Character string or vector to encode.
• add_special_tokens: Logical. Add BOS/EOS tokens.
• max_length: Integer. Maximum sequence length (NULL for no limit).
• padding: Character. Padding strategy: "none", "max_length", or "longest".
• truncation: Logical. Truncate to max_length.
• return_tensors: Character. Return type: "list" or "pt" (torch tensors).

Value

List with input_ids and attention_mask.

encode_single

Encode a single text string

Description

Encode a single text string

Usage

encode_single(tokenizer, text, add_special_tokens = TRUE)

encode_text_ltx2

Encode Text for LTX2

Description

Encodes text prompts for LTX2 video generation. Supports multiple backends: - "gemma3": Native R torch Gemma3 text encoder - "precomputed": Load pre-computed embeddings from file - "api": Call an HTTP API for text encoding - "random": Generate random embeddings (for testing only)

Usage

encode_text_ltx2(
  prompt,
  backend = "random",
  model_path = NULL,
  tokenizer_path = NULL,
  text_encoder = NULL,
  embeddings_file = NULL,
  api_url = NULL,
  max_sequence_length = 1024L,
  caption_channels = 3840L,
  device = "cpu",
  dtype = torch::torch_float32()
)

Arguments

• prompt: Character vector of prompts.
• backend: Character. Backend to use ("gemma3", "precomputed", "api", "random").
• model_path: Character. Path to Gemma3 model directory (for "gemma3" backend).
• tokenizer_path: Character. Path to tokenizer (for "gemma3" backend, defaults to model_path).
• text_encoder: Pre-loaded Gemma3 text encoder module (for "gemma3" backend).
• embeddings_file: Character. Path to pre-computed embeddings (for "precomputed" backend).
• api_url: Character. URL of text encoding API (for "api" backend).
• max_sequence_length: Integer. Maximum sequence length (default 1024).
• caption_channels: Integer. Caption embedding dimension (default 3840).
• device: Character. Device for tensors.
• dtype: torch_dtype. Data type for tensors.

Value

List with prompt_embeds and prompt_attention_mask tensors.

encode_with_gemma3

Encode text with Gemma3 for LTX-2

Description

Full pipeline for encoding text prompts using Gemma3 text encoder. Returns packed embeddings ready for LTX-2 connectors.

Usage

encode_with_gemma3(
  prompts,
  model = NULL,
  tokenizer = NULL,
  max_sequence_length = 1024L,
  scale_factor = 8,
  device = "cuda",
  dtype = "float16",
  verbose = TRUE
)

Arguments

• prompts: Character vector of prompts.
• model: Gemma3 text model (or path to load from).
• tokenizer: Gemma3 tokenizer (or path to load from).
• max_sequence_length: Integer. Maximum sequence length.
• scale_factor: Numeric. Scale factor for packing (default 8).
• device: Character. Device for computation.
• dtype: Character. Data type.
• verbose: Logical. Print progress.

Value

List with prompt_embeds and prompt_attention_mask.

feed_forward

FeedForward module

Description

FeedForward module

Usage

feed_forward(
  dim,
  dim_out = NULL,
  mult = 4L,
  dropout = 0,
  activation_fn = "gelu-approximate",
  inner_dim = NULL,
  bias = TRUE
)

FeedForward

FeedForward Network

Description

FeedForward Network

Usage

FeedForward(dim, mult = 4L)

filename_from_prompt

Generate a filename from a prompt

Description

This function generates a filename from a prompt by removing all non-alphanumeric characters and replacing them with underscores. The filename is limited to 50 characters. If datetime is set to TRUE, the current date and time are prepended to the filename.

Usage

filename_from_prompt(prompt, datetime = TRUE)

Arguments

• prompt: A character string representing the prompt.
• datetime: Logical indicating whether to prepend the current date and time to the filename. Default is TRUE.

Value

A character string representing the generated filename.

Examples

filename_from_prompt("A beautiful sunset over the mountains")
filename_from_prompt("A beautiful sunset over the mountains", datetime = FALSE)

flowmatch_calculate_shift

Calculate shift for dynamic shifting

Description

Computes the shift parameter (mu) based on sequence length for resolution-dependent timestep shifting.

Usage

flowmatch_calculate_shift(
  seq_len,
  base_seq_len = 256L,
  max_seq_len = 4096L,
  base_shift = 0.5,
  max_shift = 1.15
)

Arguments

• seq_len: Integer. The sequence length (num_patches).
• base_seq_len: Integer. Base sequence length. Default: 256
• max_seq_len: Integer. Maximum sequence length. Default: 4096
• base_shift: Numeric. Base shift value. Default: 0.5
• max_shift: Numeric. Maximum shift value. Default: 1.15

Value

Numeric. The computed shift value (mu).

flowmatch_scale_noise

Scale noise for flow matching forward process

Description

Applies the forward process in flow-matching: interpolates between the clean sample and noise.

Usage

flowmatch_scale_noise(sample, timestep, noise, schedule)

Arguments

• sample: torch tensor. The clean sample.
• timestep: torch tensor. The current timestep.
• noise: torch tensor. The noise tensor.
• schedule: List. The FlowMatch scheduler object.

Value

torch tensor. The noisy sample at timestep t.

flowmatch_scheduler_create

Create a FlowMatch Euler Discrete Scheduler

Description

Creates a FlowMatch scheduler for use with flow-matching diffusion models like LTX-2. FlowMatch schedulers use Euler integration for sampling, which is simpler and often faster than DDIM-style schedulers.

Usage

flowmatch_scheduler_create(
  num_train_timesteps = 1000L,
  shift = 1,
  use_dynamic_shifting = FALSE,
  base_shift = 0.5,
  max_shift = 1.15,
  base_seq_len = 256L,
  max_seq_len = 4096L,
  invert_sigmas = FALSE,
  shift_terminal = NULL,
  time_shift_type = c("exponential", "linear")
)

Arguments

• num_train_timesteps: Integer. The number of diffusion steps used to train the model. Default: 1000
• shift: Numeric. The shift value for the timestep schedule. Default: 1.0
• use_dynamic_shifting: Logical. Whether to apply timestep shifting on-the-fly based on the image/video resolution. Default: FALSE
• base_shift: Numeric. Value to stabilize generation. Increasing reduces variation. Default: 0.5
• max_shift: Numeric. Maximum shift allowed. Increasing encourages more variation. Default: 1.15
• base_seq_len: Integer. Base sequence length for dynamic shifting. Default: 256
• max_seq_len: Integer. Maximum sequence length for dynamic shifting. Default: 4096
• invert_sigmas: Logical. Whether to invert the sigmas (used by some models like Mochi). Default: FALSE
• shift_terminal: Numeric or NULL. End value of shifted schedule. Default: NULL
• time_shift_type: Character. Type of dynamic shifting: "exponential" or "linear". Default: "exponential"

Details

FlowMatch (Flow Matching) is a framework for training continuous normalizing flows by regressing onto target probability paths. The Euler discrete scheduler implements simple Euler integration for sampling from trained flow models.

The core update rule is: prev_sample = sample + dt * model_output where dt = sigma_next - sigma_current.

Value

A FlowMatch scheduler object (list) containing:

• sigmas: The noise schedule
• timesteps: The timestep schedule
• num_train_timesteps: Training timesteps
• config: All configuration parameters

References

Lipman, Y., Chen, R. T. Q., Ben-Hamu, H., Nickel, M., & Le, M. (2022). "Flow Matching for Generative Modeling." https://arxiv.org/abs/2210.02747

Examples

# Create a FlowMatch scheduler
scheduler <- flowmatch_scheduler_create(
  num_train_timesteps = 1000,
  shift = 1.0
)

# Set timesteps for inference
scheduler <- flowmatch_set_timesteps(scheduler, num_inference_steps = 8)

flowmatch_scheduler_step

Perform a FlowMatch scheduler step

Description

Performs a single denoising step using Euler integration. This is the core sampling function for FlowMatch models.

Usage

flowmatch_scheduler_step(
  model_output,
  timestep,
  sample,
  schedule,
  generator = NULL
)

Arguments

• model_output: torch tensor. The output from the diffusion model (velocity prediction).
• timestep: Numeric. The current timestep.
• sample: torch tensor. The current noisy sample.
• schedule: List. The FlowMatch scheduler object.
• generator: torch generator or NULL. Random generator for reproducibility.

Details

The FlowMatch Euler step is remarkably simple: prev_sample = sample + dt * model_output where dt = sigma_next - sigma_current.

This implements the Euler method for solving the probability flow ODE in continuous normalizing flows.

Value

A list containing:

• prev_sample: The denoised sample at the previous timestep
• schedule: The updated scheduler with incremented step_index

flowmatch_set_timesteps

Set timesteps for inference

Description

Configures the scheduler timesteps for a specific number of inference steps. This must be called before using the scheduler for denoising.

Usage

flowmatch_set_timesteps(
  schedule,
  num_inference_steps = 50L,
  device = "cpu",
  mu = NULL,
  sigmas = NULL,
  timesteps = NULL
)

Arguments

• schedule: List. The FlowMatch scheduler object.
• num_inference_steps: Integer. Number of denoising steps. Default: 50
• device: Character or torch device. Device for tensors. Default: "cpu"
• mu: Numeric or NULL. Shift parameter for dynamic shifting. Required if use_dynamic_shifting is TRUE. Default: NULL
• sigmas: Numeric vector or NULL. Custom sigma values. Default: NULL
• timesteps: Numeric vector or NULL

Value

Updated scheduler with configured timesteps and sigmas.

GEGLU

GEGLU Feedforward

Description

GEGLU Feedforward

Usage

GEGLU(dim_in, dim_out)

gelu_activation

GELU activation with optional approximation

Description

GELU activation with optional approximation

Usage

gelu_activation(dim_in, dim_out, approximate = "none", bias = TRUE)

gemma3_attention

Gemma3 Attention

Description

Multi-head attention with Grouped Query Attention (GQA) and optional sliding window attention.

Usage

gemma3_attention(config, layer_idx = 0L)

Arguments

• config: Model configuration.
• layer_idx: Integer. Layer index for layer-specific settings.

gemma3_config_ltx2

Create Gemma3 configuration for LTX-2

Description

Returns the default configuration used by LTX-2's text encoder.

Usage

gemma3_config_ltx2()

Value

List with model configuration parameters.

gemma3_decoder_layer

Gemma3 Decoder Layer

Description

Single transformer block with pre-norm attention and MLP.

Usage

gemma3_decoder_layer(config, layer_idx = 0L)

Arguments

• config: Model configuration.
• layer_idx: Integer. Layer index.

gemma3_mlp

Gemma3 MLP

Description

Feed-forward network with gated linear units and GELU activation.

Usage

gemma3_mlp(config)

Arguments

• config: List with hidden_size and intermediate_size.

gemma3_rms_norm

Gemma3 RMS Normalization

Description

RMSNorm with optional addition of 1 to weights (Gemma-style).

Usage

gemma3_rms_norm(dim, eps = 1e-06)

Arguments

• dim: Integer. Hidden dimension.
• eps: Numeric. Epsilon for numerical stability.

gemma3_rotary_embedding

Gemma3 Rotary Position Embeddings

Description

Standard RoPE with optional scaling factor for extended context.

Usage

gemma3_rotary_embedding(
  dim,
  max_position_embeddings = 8192L,
  base = 10000,
  scaling_factor = 1
)

Arguments

• dim: Integer. Head dimension.
• max_position_embeddings: Integer. Maximum sequence length.
• base: Numeric. RoPE base frequency.
• scaling_factor: Numeric. Optional scaling factor for extended context.

gemma3_text_model

Gemma3 Text Model

Description

Full Gemma3 text encoder model.

Usage

gemma3_text_model(config)

Arguments

• config: Model configuration list.

gemma3_tokenizer

Gemma3 Tokenizer

Description

Native R tokenizer for Gemma3 using BPE. Loads from HuggingFace tokenizer.json format.

Usage

gemma3_tokenizer(tokenizer_path)

Arguments

• tokenizer_path: Character. Path to tokenizer directory or tokenizer.json file.

Value

A gemma3_tokenizer object (extends bpe_tokenizer).

get_bos_id

Get BOS token ID

Description

Get BOS token ID

Usage

get_bos_id(tokenizer)

get_eos_id

Get EOS token ID

Description

Get EOS token ID

Usage

get_eos_id(tokenizer)

get_pad_id

Get padding token ID

Description

Get padding token ID

Usage

get_pad_id(tokenizer)

get_required_components

Get required components for each model type

Description

This function returns a list of required components for each supported model type.

Usage

get_required_components(model_name)

Arguments

• model_name: A character string representing the name of the model.

Value

A character vector of required components for the specified model.

get_timestep_embedding

Get timestep embedding (sinusoidal)

Description

Get timestep embedding (sinusoidal)

Usage

get_timestep_embedding(
  timesteps,
  embedding_dim,
  flip_sin_to_cos = FALSE,
  downscale_freq_shift = 1,
  scale = 1,
  max_period = 10000
)

greedy_tokenize

Greedy longest match tokenization

Description

Greedy longest match tokenization

Usage

greedy_tokenize(text, vocab, byte_fallback = FALSE, unk_token = NULL)

group_norm_32

Group Normalization (32 groups)

Description

Group Normalization (32 groups)

Usage

group_norm_32(channels)

img2img

Image-to-Image Generation with Stable Diffusion

Description

This function generates an image based on an input image and a text prompt using the Stable Diffusion model. It allows for various configurations such as model name, device, scheduler, and more.

Usage

img2img(
  input_image,
  prompt,
  negative_prompt = NULL,
  img_dim = 512,
  model_name = c("sd21", "sdxl"),
  pipeline = NULL,
  devices = "auto",
  unet_dtype_str = "float16",
  download_models = FALSE,
  scheduler = "ddim",
  num_inference_steps = 50,
  strength = 0.8,
  guidance_scale = 7.5,
  seed = NULL,
  save_file = TRUE,
  filename = NULL,
  metadata_path = NULL,
  use_native_decoder = FALSE,
  use_native_text_encoder = FALSE,
  use_native_unet = FALSE,
  ...
)

Arguments

• input_image: Path to the input image or a tensor representing the image.
• prompt: Text prompt to guide the image generation.
• negative_prompt: Optional negative prompt to guide the image generation.`
• img_dim: Dimension of the output image (default: 512).
• model_name: Name of the Stable Diffusion model to use (default: "sd21").
• pipeline: Optional pre-loaded pipeline. If NULL, it will be loaded based on model_name.
• devices: A named list of devices for each model component (e.g., list(unet = "cuda", decoder = "cpu", text_encoder = "cpu", encoder = "cpu")).
• unet_dtype_str: Optional A character for dtype of the unet component (typically "torch_float16" for cuda and "torch_float32" for cpu).
• download_models: Logical indicating whether to download models if not found (default: FALSE).
• scheduler: Scheduler to use for the diffusion process (default: "ddim").
• num_inference_steps: Number of diffusion steps (default: 50).
• strength: Strength of the image-to-image transformation (default: 0.8).
• guidance_scale: Scale for classifier-free guidance (default: 7.5).
• seed: Random seed for reproducibility (default: NULL).
• save_file: Logical indicating whether to save the generated image.
• filename: Optional filename for saving the image. If NULL, a default name is generated.
• metadata_path: Path to save metadata (default: NULL).
• use_native_decoder: Logical; if TRUE, uses native R torch decoder instead of TorchScript. Native decoder has better GPU compatibility (especially Blackwell).
• use_native_text_encoder: Logical; if TRUE, uses native R torch text encoder instead of TorchScript. Native text encoder has better GPU compatibility (especially Blackwell).
• use_native_unet: Logical; if TRUE, uses native R torch UNet instead of TorchScript. Native UNet has better GPU compatibility (especially Blackwell).
• ...: Additional arguments for future use.

Value

An image array and metadata

is_blackwell_gpu

Check if GPU is Blackwell Architecture

Description

Blackwell GPUs (RTX 50xx) may need special handling.

Usage

is_blackwell_gpu()

Value

Logical. TRUE if Blackwell GPU detected.

Examples

if (is_blackwell_gpu()) {
  message("Using Blackwell-compatible settings")
}

latents_to_video

Create Video from Latents (Helper)

Description

Convenience function to decode latents and save video in one step.

Usage

latents_to_video(latents, vae, file, fps = 24, ...)

Arguments

• latents: Tensor of latents from generation.
• vae: VAE decoder module.
• file: Output file path.
• fps: Frames per second.
• ...: Additional arguments to save_video.

Value

Invisibly returns the output file path.

load_decoder_weights

Load weights from TorchScript decoder into native decoder

Description

Load weights from TorchScript decoder into native decoder

Usage

load_decoder_weights(native_decoder, torchscript_path, verbose = TRUE)

Arguments

• native_decoder: Native VAE decoder module
• torchscript_path: Path to TorchScript decoder .pt file
• verbose: Print loading progress

Value

The native decoder with loaded weights (invisibly)

load_gemma3_text_encoder

Load Gemma3 Text Model from safetensors

Description

Loads pre-trained Gemma3 weights from HuggingFace safetensors files.

Usage

load_gemma3_text_encoder(
  model_path,
  device = "cpu",
  dtype = "float16",
  verbose = TRUE
)

Arguments

• model_path: Character. Path to directory containing model files.
• device: Character. Device to load model to.
• dtype: Character. Data type ("float32", "float16", "bfloat16").
• verbose: Logical. Print loading progress.

Value

Initialized gemma3_text_model with loaded weights.

load_gemma3_weights

Load weights into Gemma3 model

Description

Load weights into Gemma3 model

Usage

load_gemma3_weights(model, weights, verbose = TRUE)

load_ltx2_connector_weights

Load weights into LTX2 connectors module

Description

Load weights into LTX2 connectors module

Usage

load_ltx2_connector_weights(connectors, weights, verbose = TRUE)

Arguments

• connectors: LTX2 connectors module
• weights: Named list of weight tensors
• verbose: Print progress

load_ltx2_connectors

Load LTX2 Text Connectors from safetensors

Description

Load pre-trained LTX2 connector weights from HuggingFace safetensors file.

Usage

load_ltx2_connectors(
  weights_path,
  config_path = NULL,
  device = "cpu",
  dtype = "float32",
  verbose = TRUE
)

Arguments

• weights_path: Character. Path to safetensors file.
• config_path: Character. Optional path to config.json.
• device: Character. Device to load weights to. Default: "cpu"
• dtype: Character. Data type ("float32", "float16"). Default: "float32"
• verbose: Logical. Print loading progress. Default: TRUE

Value

Initialized ltx2_text_connectors module

load_ltx2_transformer_sharded

Load sharded transformer weights

Description

Load sharded transformer weights

Usage

load_ltx2_transformer_sharded(
  transformer,
  weights_dir,
  index_path,
  verbose = TRUE
)

Arguments

• transformer: LTX2 transformer module
• weights_dir: Directory containing sharded safetensors
• index_path: Path to index.json
• verbose: Print progress

load_ltx2_transformer_weights

Load weights into LTX2 transformer module

Description

Load weights into LTX2 transformer module

Usage

load_ltx2_transformer_weights(transformer, weights, verbose = TRUE)

Arguments

• transformer: LTX2 transformer module
• weights: Named list of weight tensors
• verbose: Print progress

load_ltx2_transformer

Load LTX2 DiT Transformer from safetensors

Description

Load pre-trained LTX2 transformer weights from HuggingFace safetensors files. Supports both single file and sharded multi-file loading.

Usage

load_ltx2_transformer(
  weights_dir,
  config_path = NULL,
  device = "cpu",
  dtype = "float16",
  verbose = TRUE
)

Arguments

• weights_dir: Character. Directory containing safetensors files.
• config_path: Character. Optional path to config.json. If NULL, uses default config.
• device: Character. Device to load weights to. Default: "cpu"
• dtype: Character. Data type ("float32", "float16", "bfloat16"). Default: "float16"
• verbose: Logical. Print loading progress. Default: TRUE

Value

Initialized ltx2_video_transformer3d module

load_ltx2_vae_weights

Load weights into LTX2 VAE module

Description

Maps HuggingFace safetensors parameter names to R module parameters.

Usage

load_ltx2_vae_weights(vae, weights, verbose = TRUE)

Arguments

• vae: LTX2 VAE module
• weights: Named list of weight tensors from safetensors
• verbose: Logical. Print loading progress

Value

The VAE with loaded weights (invisibly)

load_ltx2_vae

Load LTX2 Video VAE from safetensors

Description

Load pre-trained LTX2 VAE weights from a HuggingFace safetensors file.

Usage

load_ltx2_vae(
  weights_path,
  config_path = NULL,
  device = "cpu",
  dtype = "float32",
  verbose = TRUE
)

Arguments

• weights_path: Character. Path to safetensors file or directory containing weights.
• config_path: Character. Optional path to config.json. If NULL and weights_path is a directory, looks for config.json in that directory. Otherwise uses default config.
• device: Character. Device to load weights to. Default: "cpu"
• dtype: Character or torch dtype. Data type. Default: "float32"
• verbose: Logical. Print loading progress. Default: TRUE

Value

Initialized ltx2_video_vae module

load_model_component

Load a specific component of a diffusion model

Description

Loads a TorchScript model component (UNet, decoder, or text encoder) from the local model directory, downloading it first if necessary.

Usage

load_model_component(
  component,
  model_name = "sd21",
  device = "cpu",
  unet_dtype_str = NULL,
  download = TRUE,
  use_native = FALSE
)

Arguments

• component: Character string, the component to load: "unet", "decoder", or "text_encoder".
• model_name: Character string, the name of the model to use.
• device: Character string, the torch device to load the model onto ("cpu" or "cuda").
• unet_dtype_str: Optional; the data type for the UNet model. If NULL, defaults to float32 for CPU and float16 for CUDA.
• download: Logical; if TRUE (default), downloads the model if it doesn't exist locally.
• use_native: Logical; if TRUE, uses native R torch modules instead of TorchScript. Supported for unet, decoder, text_encoder, and text_encoder2. Native modules have better GPU compatibility (especially on Blackwell/RTX 50xx).

Value

A torch model object.

Examples

unet <- load_model_component("unet", "sd21", "cpu")

load_pipeline

Load a diffusion model pipeline

Description

This function loads a diffusion model pipeline consisting of a UNet, VAE decoder, and text encoder. It initializes the models and sets up the environment for inference.

Usage

load_pipeline(
  model_name,
  m2d,
  i2i = FALSE,
  unet_dtype_str,
  use_native_decoder = FALSE,
  use_native_text_encoder = FALSE,
  use_native_unet = FALSE,
  ...
)

Arguments

• model_name: The name of the model to load.
• m2d: A list containing model-to-device mappings and configurations.
• i2i: Logical indicating whether to load the encoder for img2img().
• unet_dtype_str: A string representing the data type for the UNet model (e.g., "float32", "float16").
• use_native_decoder: Logical; if TRUE, uses native R torch decoder instead of TorchScript. Native decoder has better GPU compatibility (especially Blackwell).
• use_native_text_encoder: Logical; if TRUE, uses native R torch text encoder instead of TorchScript. Native text encoder has better GPU compatibility (especially Blackwell).
• use_native_unet: Logical; if TRUE, uses native R torch UNet instead of TorchScript. Native UNet has better GPU compatibility (especially Blackwell).
• ...: Additional arguments passed to the model loading functions.

Value

An environment containing the loaded models and configuration.

Examples

pipeline <- load_pipeline("my_model", device = "cuda")

load_text_encoder_weights

Load weights from TorchScript text encoder into native encoder

Description

Load weights from TorchScript text encoder into native encoder

Usage

load_text_encoder_weights(native_encoder, torchscript_path, verbose = TRUE)

Arguments

• native_encoder: Native text encoder module
• torchscript_path: Path to TorchScript encoder .pt file
• verbose: Print loading progress

Value

The native encoder with loaded weights (invisibly)

load_text_encoder2_weights

Load weights from TorchScript text encoder 2 into native encoder

Description

Load weights from TorchScript text encoder 2 into native encoder

Usage

load_text_encoder2_weights(native_encoder, torchscript_path, verbose = TRUE)

Arguments

• native_encoder: Native text encoder 2 module
• torchscript_path: Path to TorchScript encoder .pt file
• verbose: Print loading progress

Value

The native encoder with loaded weights (invisibly)

load_to_gpu

Load Module to GPU

Description

Moves a torch module and all its parameters to CUDA.

Usage

load_to_gpu(module, device = "cuda")

Arguments

• module: A torch nn_module.
• device: Character. Target device (default "cuda").

Value

The module (modified in place).

Examples

load_to_gpu(model)
output <- model(x)
offload_to_cpu(model)

load_unet_sdxl_weights

Load weights from TorchScript SDXL UNet into native SDXL UNet

Description

Load weights from TorchScript SDXL UNet into native SDXL UNet

Usage

load_unet_sdxl_weights(native_unet, torchscript_path, verbose = TRUE)

Arguments

• native_unet: Native SDXL UNet module
• torchscript_path: Path to TorchScript SDXL UNet .pt file
• verbose: Print loading progress

Value

The native UNet with loaded weights (invisibly)

load_unet_weights

Load weights from TorchScript UNet into native UNet

Description

Load weights from TorchScript UNet into native UNet

Usage

load_unet_weights(native_unet, torchscript_path, verbose = TRUE)

Arguments

• native_unet: Native UNet module
• torchscript_path: Path to TorchScript UNet .pt file
• verbose: Print loading progress

Value

The native UNet with loaded weights (invisibly)

ltx_video_downsampler3d

LTX Video Downsampler 3D

Description

Spatiotemporal downsampling with strided pixel unshuffle + convolution.

Usage

ltx_video_downsampler3d(
  in_channels,
  out_channels,
  stride = 1L,
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels.
• out_channels: Integer. Output channels.
• stride: Integer or vector of 3. Downsampling stride.
• spatial_padding_mode: Character. Padding mode.

ltx_video_upsampler3d

LTX Video Upsampler 3D

Description

Spatiotemporal upsampling with pixel shuffle + optional residual.

Usage

ltx_video_upsampler3d(
  in_channels,
  stride = 1L,
  residual = FALSE,
  upscale_factor = 1L,
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels.
• stride: Integer or vector of 3. Upsampling stride.
• residual: Logical. Whether to use residual connection.
• upscale_factor: Integer. Channel upscale factor.
• spatial_padding_mode: Character. Padding mode.

ltx2_ada_layer_norm_single

LTX2 AdaLayerNorm Single

Description

LTX2 AdaLayerNorm Single

Usage

ltx2_ada_layer_norm_single(
  embedding_dim,
  num_mod_params = 6L,
  use_additional_conditions = FALSE
)

ltx2_attention

LTX2 Attention module

Description

LTX2 Attention module

Usage

ltx2_attention(
  query_dim,
  heads = 8L,
  kv_heads = 8L,
  dim_head = 64L,
  dropout = 0,
  bias = TRUE,
  cross_attention_dim = NULL,
  out_bias = TRUE,
  qk_norm = "rms_norm_across_heads",
  norm_eps = 1e-06,
  norm_elementwise_affine = TRUE,
  rope_type = "interleaved"
)

ltx2_audio_video_rotary_pos_embed

LTX2 Audio-Video Rotary Positional Embeddings

Description

LTX2 Audio-Video Rotary Positional Embeddings

Usage

ltx2_audio_video_rotary_pos_embed(
  dim,
  patch_size = 1L,
  patch_size_t = 1L,
  base_num_frames = 20L,
  base_height = 2048L,
  base_width = 2048L,
  sampling_rate = 16000L,
  hop_length = 160L,
  scale_factors = c(8L, 32L, 32L),
  theta = 10000,
  causal_offset = 1L,
  modality = "video",
  double_precision = TRUE,
  rope_type = "interleaved",
  num_attention_heads = 32L
)

ltx2_connector_transformer_1d

1D Connector Transformer for LTX2

Description

1D Connector Transformer for LTX2

Usage

ltx2_connector_transformer_1d(
  num_attention_heads = 30L,
  attention_head_dim = 128L,
  num_layers = 2L,
  num_learnable_registers = 128L,
  rope_base_seq_len = 4096L,
  rope_theta = 10000,
  rope_double_precision = TRUE,
  eps = 1e-06,
  causal_temporal_positioning = FALSE,
  rope_type = "interleaved"
)

ltx2_memory_profile

Get LTX-2 Memory Profile

Description

Determines optimal memory configuration based on available VRAM.

Usage

ltx2_memory_profile(vram_gb = NULL, model = "ltx2-2b")

Arguments

• vram_gb: Numeric. Available VRAM in GB, or NULL for auto-detection.
• model: Character. Model variant: "ltx2-2b" (default) or "ltx2-2b-distilled".

Details

Memory profiles for LTX-2:

• high: 16GB+ - DiT and VAE on GPU, text via API
• medium: 12GB - DiT on GPU, VAE tiled, text via API
• low: 8GB - DiT offloaded layer-by-layer, VAE tiled small, text via API
• very_low: 6GB - Maximum offloading, minimum tiles
• cpu_only: All on CPU

Value

A list with memory profile settings.

Examples

# Auto-detect profile
profile <- ltx2_memory_profile()

# Specific VRAM
profile <- ltx2_memory_profile(vram_gb = 8)

ltx2_rotary_pos_embed_1d

1D Rotary Position Embeddings for LTX2 Text Connectors

Description

1D Rotary Position Embeddings for LTX2 Text Connectors

Usage

ltx2_rotary_pos_embed_1d(
  dim,
  base_seq_len = 4096L,
  theta = 10000,
  double_precision = TRUE,
  rope_type = "interleaved",
  num_attention_heads = 32L
)

ltx2_text_connectors

LTX2 Text Connectors

Description

Transforms packed text encoder hidden states for video and audio streams.

Usage

ltx2_text_connectors(
  caption_channels = 3840L,
  text_proj_in_factor = 1L,
  video_connector_num_attention_heads = 30L,
  video_connector_attention_head_dim = 128L,
  video_connector_num_layers = 2L,
  video_connector_num_learnable_registers = NULL,
  audio_connector_num_attention_heads = 16L,
  audio_connector_attention_head_dim = 128L,
  audio_connector_num_layers = 2L,
  audio_connector_num_learnable_registers = NULL,
  connector_rope_base_seq_len = 4096L,
  rope_theta = 10000,
  rope_double_precision = TRUE,
  causal_temporal_positioning = FALSE,
  rope_type = "interleaved"
)

Arguments

• caption_channels: Integer. Dimension of caption embeddings (default 3840).
• text_proj_in_factor: Integer. Factor for input projection (default 1).
• video_connector_num_attention_heads: Integer. Number of attention heads for video connector.
• video_connector_attention_head_dim: Integer. Attention head dimension for video.
• video_connector_num_layers: Integer. Number of transformer layers for video.
• video_connector_num_learnable_registers: Integer. Number of learnable registers for video.
• audio_connector_num_attention_heads: Integer. Number of attention heads for audio connector.
• audio_connector_attention_head_dim: Integer. Attention head dimension for audio.
• audio_connector_num_layers: Integer. Number of transformer layers for audio.
• audio_connector_num_learnable_registers: Integer. Number of learnable registers for audio.
• connector_rope_base_seq_len: Integer. Base sequence length for RoPE.
• rope_theta: Numeric. RoPE theta parameter.
• rope_double_precision: Logical. Use double precision for RoPE.
• causal_temporal_positioning: Logical. Use causal temporal positioning.
• rope_type: Character. RoPE type ("interleaved" or "split").

Value

nn_module for text connectors.

ltx2_transformer_block_1d

1D Transformer Block for LTX2 Text Connectors

Description

1D Transformer Block for LTX2 Text Connectors

Usage

ltx2_transformer_block_1d(
  dim,
  num_attention_heads,
  attention_head_dim,
  activation_fn = "gelu-approximate",
  eps = 1e-06,
  rope_type = "interleaved"
)

ltx2_video_causal_conv3d

LTX2 Video Causal 3D Convolution

Description

3D convolution with runtime-selectable causal or non-causal padding. Causal mode pads temporally by repeating first frame. Non-causal mode pads temporally by repeating first and last frames.

Usage

ltx2_video_causal_conv3d(
  in_channels,
  out_channels,
  kernel_size = 3L,
  stride = 1L,
  dilation = 1L,
  groups = 1L,
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels.
• out_channels: Integer. Output channels.
• kernel_size: Integer or vector of 3. Convolution kernel size.
• stride: Integer or vector of 3. Stride.
• dilation: Integer or vector of 3. Dilation.
• groups: Integer. Convolution groups. Default: 1
• spatial_padding_mode: Character. Padding mode for spatial dims. Default: "zeros"

ltx2_video_decoder3d

LTX2 Video Decoder

Description

Decodes latent representations back to video frames.

Usage

ltx2_video_decoder3d(
  in_channels = 128L,
  out_channels = 3L,
  block_out_channels = c(256L, 512L, 1024L),
  spatio_temporal_scaling = c(TRUE, TRUE, TRUE),
  layers_per_block = c(5L, 5L, 5L, 5L),
  patch_size = 4L,
  patch_size_t = 1L,
  resnet_norm_eps = 1e-06,
  is_causal = FALSE,
  inject_noise = c(FALSE, FALSE, FALSE, FALSE),
  timestep_conditioning = FALSE,
  upsample_residual = c(TRUE, TRUE, TRUE),
  upsample_factor = c(2L, 2L, 2L),
  spatial_padding_mode = "reflect"
)

Arguments

• in_channels: Integer. Latent channels.
• out_channels: Integer. Output channels (typically 3 for RGB).
• block_out_channels: Integer vector. Output channels per block.
• spatio_temporal_scaling: Logical vector. Whether each block upscales.
• layers_per_block: Integer vector. Number of layers per block.
• patch_size: Integer. Spatial patch size.
• patch_size_t: Integer. Temporal patch size.
• resnet_norm_eps: Numeric. Epsilon for normalization.
• is_causal: Logical. Whether to use causal convolutions.
• inject_noise: Logical vector. Whether to inject noise per block.
• timestep_conditioning: Logical. Whether to use timestep conditioning.
• upsample_residual: Logical vector. Whether upsamplers use residual.
• upsample_factor: Integer vector. Channel upscale factors.
• spatial_padding_mode: Character. Padding mode.

ltx2_video_down_block3d

LTX2 Video Down Block 3D

Description

Encoder down block with multiple ResNet layers and optional downsampling.

Usage

ltx2_video_down_block3d(
  in_channels,
  out_channels = NULL,
  num_layers = 1L,
  dropout = 0,
  resnet_eps = 1e-06,
  resnet_act_fn = "swish",
  spatio_temporal_scale = TRUE,
  downsample_type = "conv",
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels.
• out_channels: Integer or NULL. Output channels.
• num_layers: Integer. Number of ResNet layers.
• dropout: Numeric. Dropout rate.
• resnet_eps: Numeric. Epsilon for normalization.
• resnet_act_fn: Character. Activation function.
• spatio_temporal_scale: Logical. Whether to use downsampling.
• downsample_type: Character. Type of downsampling.
• spatial_padding_mode: Character. Padding mode.

ltx2_video_encoder3d

LTX2 Video Encoder

Description

Encodes video frames into latent space with 3D causal convolutions.

Usage

ltx2_video_encoder3d(
  in_channels = 3L,
  out_channels = 128L,
  block_out_channels = c(256L, 512L, 1024L, 2048L),
  spatio_temporal_scaling = c(TRUE, TRUE, TRUE, TRUE),
  layers_per_block = c(4L, 6L, 6L, 2L, 2L),
  downsample_type = c("spatial", "temporal", "spatiotemporal", "spatiotemporal"),
  patch_size = 4L,
  patch_size_t = 1L,
  resnet_norm_eps = 1e-06,
  is_causal = TRUE,
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels (typically 3 for RGB).
• out_channels: Integer. Latent channels.
• block_out_channels: Integer vector. Output channels per block.
• spatio_temporal_scaling: Logical vector. Whether each block downscales.
• layers_per_block: Integer vector. Number of layers per block.
• downsample_type: Character vector. Type of downsampling per block.
• patch_size: Integer. Spatial patch size.
• patch_size_t: Integer. Temporal patch size.
• resnet_norm_eps: Numeric. Epsilon for normalization.
• is_causal: Logical. Whether to use causal convolutions.
• spatial_padding_mode: Character. Padding mode.

ltx2_video_mid_block3d

LTX2 Video Mid Block 3D

Description

Middle block with ResNet layers and optional timestep conditioning.

Usage

ltx2_video_mid_block3d(
  in_channels,
  num_layers = 1L,
  dropout = 0,
  resnet_eps = 1e-06,
  resnet_act_fn = "swish",
  inject_noise = FALSE,
  timestep_conditioning = FALSE,
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels.
• num_layers: Integer. Number of ResNet layers.
• dropout: Numeric. Dropout rate.
• resnet_eps: Numeric. Epsilon for normalization.
• resnet_act_fn: Character. Activation function.
• inject_noise: Logical. Whether to inject noise.
• timestep_conditioning: Logical. Whether to use timestep conditioning.
• spatial_padding_mode: Character. Padding mode.

ltx2_video_resnet_block3d

LTX2 Video ResNet Block 3D

Description

3D ResNet block with per-channel RMS normalization and optional noise injection and timestep conditioning.

Usage

ltx2_video_resnet_block3d(
  in_channels,
  out_channels = NULL,
  dropout = 0,
  eps = 1e-06,
  non_linearity = "silu",
  inject_noise = FALSE,
  timestep_conditioning = FALSE,
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels.
• out_channels: Integer or NULL. Output channels (defaults to in_channels).
• dropout: Numeric. Dropout rate. Default: 0.0
• eps: Numeric. Epsilon for normalization. Default: 1e-6
• non_linearity: Character. Activation function. Default: "silu"
• inject_noise: Logical. Whether to inject noise. Default: FALSE
• timestep_conditioning: Logical. Whether to use timestep conditioning. Default: FALSE
• spatial_padding_mode: Character. Padding mode. Default: "zeros"

ltx2_video_transformer_3d_model

LTX2 Video Transformer 3D Model (Audio-Video)

Description

Full audio-video transformer matching HuggingFace diffusers implementation.

Usage

ltx2_video_transformer_3d_model(
  in_channels,
  out_channels,
  patch_size,
  patch_size_t,
  num_attention_heads,
  attention_head_dim,
  cross_attention_dim,
  vae_scale_factors,
  32L,
  32L),
  pos_embed_max_pos,
  base_height,
  base_width,
  audio_in_channels,
  audio_out_channels,
  audio_patch_size,
  audio_patch_size_t,
  audio_num_attention_heads,
  audio_attention_head_dim,
  audio_cross_attention_dim,
  audio_scale_factor,
  audio_pos_embed_max_pos,
  audio_sampling_rate,
  audio_hop_length,
  num_layers,
  activation_fn,
  qk_norm,
  norm_elementwise_affine,
  norm_eps,
  caption_channels,
  attention_bias,
  attention_out_bias,
  rope_theta,
  rope_double_precision
)

Arguments

• in_channels: Integer. Video input channels (default: 128).
• out_channels: Integer. Video output channels (default: 128).
• patch_size: Integer. Spatial patch size (default: 1).
• patch_size_t: Integer. Temporal patch size (default: 1).
• num_attention_heads: Integer. Video attention heads (default: 32).
• attention_head_dim: Integer. Video attention head dimension (default: 128).
• cross_attention_dim: Integer. Video cross-attention dimension (default: 4096).
• vae_scale_factors: Integer vector. VAE scale factors (default: c(8, 32, 32)).
• pos_embed_max_pos: Integer. Max position for RoPE (default: 20).
• base_height: Integer. Base height for RoPE (default: 2048).
• base_width: Integer. Base width for RoPE (default: 2048).
• audio_in_channels: Integer. Audio input channels (default: 128).
• audio_out_channels: Integer. Audio output channels (default: 128).
• audio_patch_size: Integer. Audio patch size (default: 1).
• audio_patch_size_t: Integer. Audio temporal patch size (default: 1).
• audio_num_attention_heads: Integer. Audio attention heads (default: 32).
• audio_attention_head_dim: Integer. Audio head dimension (default: 64).
• audio_cross_attention_dim: Integer. Audio cross-attention dim (default: 2048).
• audio_scale_factor: Integer. Audio scale factor (default: 4).
• audio_pos_embed_max_pos: Integer. Audio max position (default: 20).
• audio_sampling_rate: Integer. Audio sampling rate (default: 16000).
• audio_hop_length: Integer. Audio hop length (default: 160).
• num_layers: Integer. Number of transformer layers (default: 48).
• activation_fn: Character. Activation function (default: "gelu-approximate").
• qk_norm: Character. QK normalization type (default: "rms_norm_across_heads").
• norm_elementwise_affine: Logical. Use elementwise affine in norms (default: FALSE).
• norm_eps: Numeric. Epsilon for normalization (default: 1e-6).
• caption_channels: Integer. Caption embedding channels (default: 3840).
• attention_bias: Logical. Use bias in attention (default: TRUE).
• attention_out_bias: Logical. Use bias in attention output (default: TRUE).
• rope_theta: Numeric. Theta for RoPE (default: 10000).
• rope_double_precision: Logical. Use double precision for RoPE (default: TRUE).
• causal_offset: Integer. Causal offset for RoPE (default: 1).
• timestep_scale_multiplier: Numeric. Timestep scale (default: 1000).
• cross_attn_timestep_scale_multiplier: Numeric. Cross-attn timestep scale (default: 1000).
• rope_type: Character. RoPE type: "interleaved" or "split" (default: "interleaved").

Value

An nn_module representing the LTX2 video transformer.

ltx2_video_transformer_block

LTX2 Video Transformer Block (Audio-Video)

Description

LTX2 Video Transformer Block (Audio-Video)

Usage

ltx2_video_transformer_block(
  dim,
  num_attention_heads,
  attention_head_dim,
  cross_attention_dim,
  audio_dim,
  audio_num_attention_heads,
  audio_attention_head_dim,
  audio_cross_attention_dim,
  qk_norm = "rms_norm_across_heads",
  activation_fn = "gelu-approximate",
  attention_bias = TRUE,
  attention_out_bias = TRUE,
  eps = 1e-06,
  elementwise_affine = FALSE,
  rope_type = "interleaved"
)

ltx2_video_up_block3d

LTX2 Video Up Block 3D

Description

Decoder up block with upsampling and ResNet layers.

Usage

ltx2_video_up_block3d(
  in_channels,
  out_channels = NULL,
  num_layers = 1L,
  dropout = 0,
  resnet_eps = 1e-06,
  resnet_act_fn = "swish",
  spatio_temporal_scale = TRUE,
  inject_noise = FALSE,
  timestep_conditioning = FALSE,
  upsample_residual = FALSE,
  upscale_factor = 1L,
  spatial_padding_mode = "zeros"
)

Arguments

• in_channels: Integer. Input channels.
• out_channels: Integer or NULL. Output channels.
• num_layers: Integer. Number of ResNet layers.
• dropout: Numeric. Dropout rate.
• resnet_eps: Numeric. Epsilon for normalization.
• resnet_act_fn: Character. Activation function.
• spatio_temporal_scale: Logical. Whether to use upsampling.
• inject_noise: Logical. Whether to inject noise.
• timestep_conditioning: Logical. Whether to use timestep conditioning.
• upsample_residual: Logical. Whether upsampler uses residual.
• upscale_factor: Integer. Channel upscale factor.
• spatial_padding_mode: Character. Padding mode.

ltx2_video_vae

LTX2 Video VAE

Description

Full VAE with encoder and decoder, supporting tiled encoding/decoding for GPU-poor memory management.

Usage

ltx2_video_vae(
  in_channels,
  out_channels,
  latent_channels,
  block_out_channels,
  512L,
  1024L,
  2048L),
  decoder_block_out_channels,
  512L,
  1024L),
  layers_per_block,
  6L,
  6L,
  2L,
  2L),
  decoder_layers_per_block,
  5L,
  5L,
  5L),
  spatio_temporal_scaling,
  TRUE,
  TRUE,
  TRUE),
  decoder_spatio_temporal_scaling,
  TRUE,
  TRUE),
  decoder_inject_noise,
  FALSE,
  FALSE,
  FALSE),
  downsample_type,
  "temporal",
  "spatiotemporal",
  "spatiotemporal"),
  upsample_residual,
  TRUE,
  TRUE),
  upsample_factor,
  2L,
  2L),
  timestep_conditioning,
  patch_size,
  patch_size_t,
  resnet_norm_eps
)

Arguments

• in_channels: Integer. Input channels.
• out_channels: Integer. Output channels.
• latent_channels: Integer. Latent space channels.
• block_out_channels: Integer vector. Encoder block channels.
• decoder_block_out_channels: Integer vector. Decoder block channels.
• layers_per_block: Integer vector. Encoder layers per block.
• decoder_layers_per_block: Integer vector. Decoder layers per block.
• spatio_temporal_scaling: Logical vector. Encoder scaling.
• decoder_spatio_temporal_scaling: Logical vector. Decoder scaling.
• decoder_inject_noise: Logical vector. Noise injection per decoder block.
• downsample_type: Character vector. Downsampling types.
• upsample_residual: Logical vector. Upsampler residual flags.
• upsample_factor: Integer vector. Upsampler factors.
• timestep_conditioning: Logical. Whether to use timestep conditioning.
• patch_size: Integer. Spatial patch size.
• patch_size_t: Integer. Temporal patch size.
• resnet_norm_eps: Numeric. Normalization epsilon.
• scaling_factor: Numeric. Latent scaling factor.
• encoder_causal: Logical. Encoder causality.
• decoder_causal: Logical. Decoder causality.
• encoder_spatial_padding_mode: Character. Encoder padding mode.
• decoder_spatial_padding_mode: Character. Decoder padding mode.

models2devices

models2devices

Description

This function sets up the model directory, device configuration, and data types for diffusion models. It checks the validity of the model name and devices, detects model type, and downloads the model if necessary.

Usage

models2devices(
  model_name,
  devices = "cpu",
  unet_dtype_str = NULL,
  download_models = FALSE
)

Arguments

• model_name: A character string representing the name of the model to be used.
• devices: A character string or a named list specifying the devices for different components of the model.
• unet_dtype_str: A character string specifying the data type for the UNet model.
• download_models: Logical indicating whether to download models if they are not found.

Value

A list containing the model directory, model files, device configuration, UNet data type, and CPU/CUDA devices.

offload_to_cpu

Offload Module to CPU

Description

Moves a torch module and all its parameters to CPU.

Usage

offload_to_cpu(module, gc = TRUE)

Arguments

• module: A torch nn_module.
• gc: Logical. Run garbage collection after offload.

Value

The module (modified in place).

Examples

model$to(device = "cuda")
output <- model(x)
offload_to_cpu(model)

pack_text_embeds

Pack Text Embeddings (Gemma-style)

Description

Normalizes and packs text encoder hidden states from multiple layers. This is used when working with raw Gemma outputs.

Usage

pack_text_embeds(
  text_hidden_states,
  sequence_lengths,
  padding_side = "left",
  scale_factor = 8,
  eps = 1e-06,
  device = "cpu"
)

Arguments

• text_hidden_states: Tensor of shape [batch, seq_len, hidden_dim, num_layers].
• sequence_lengths: Integer vector of valid sequence lengths per batch item.
• padding_side: Character. "left" or "right".
• scale_factor: Numeric. Scale factor for normalization (default 8).
• eps: Numeric. Epsilon for numerical stability.
• device: Character. Device for tensors.

Value

Tensor of shape [batch, seq_len, hidden_dim * num_layers].

per_channel_rms_norm

Per-channel RMS normalization

Description

Normalizes tensor by root-mean-square along the channel dimension: y = x / sqrt(mean(x^2, dim=channel_dim, keepdim=TRUE) + eps)

Usage

per_channel_rms_norm(channel_dim = 2L, eps = 1e-08)

Arguments

• channel_dim: Integer. Dimension for RMS computation (1-indexed). Default: 2 (channels)
• eps: Numeric. Small constant for numerical stability. Default: 1e-8

pixart_alpha_combined_timestep_size_embeddings

PixArt Alpha Combined Timestep Size Embeddings

Description

PixArt Alpha Combined Timestep Size Embeddings

Usage

pixart_alpha_combined_timestep_size_embeddings(
  embedding_dim,
  size_emb_dim,
  use_additional_conditions = FALSE
)

pixart_alpha_text_projection

PixArt Alpha Text Projection

Description

PixArt Alpha Text Projection

Usage

pixart_alpha_text_projection(
  in_features,
  hidden_size,
  out_features = NULL,
  act_fn = "gelu_tanh"
)

post_quant_conv

Post Quant Conv

Description

This function applies a quantized convolution operation to an input tensor. It is typically used in the context of image post processing, particularly in generative models like Stable Diffusion XL.

Usage

post_quant_conv(x, dtype, device)

Arguments

• x: Input tensor to be processed.
• dtype: Data type for the tensor (e.g., "torch_float16" or "torch_float32").
• device: Device on which the tensor is located (e.g., "cpu" or "cuda").

Value

Processed tensor after applying the quantized convolution.

preprocess_image

Preprocess image for Stable Diffusion

Description

Preprocess image for Stable Diffusion

Usage

preprocess_image(input, device = "cpu", width = 512, height = 512)

Arguments

• input: File path to .jpg or .png, or a 3D array
• device: Target device for torch ("cpu" or "cuda")
• width: Desired width of the output image
• height: Desired height of the output image

Value

Torch tensor of shape c(1, 3, 512, 512), scaled to c(-1, 1)

print.bpe_tokenizer

Print BPE Tokenizer

Description

Print BPE Tokenizer

Usage

print.bpe_tokenizer(x, ...)

Arguments

• x: A bpe_tokenizer object.
• ...: Additional arguments (ignored).

quant_conv

Quant Conv

Description

This function applies a quantized convolution operation to an input tensor. It is typically used in the context of image processing, particularly in generative models like Stable Diffusion.

Usage

quant_conv(x, dtype, device)

Arguments

• x: Input tensor to be processed.
• dtype: Data type for the tensor (e.g., "torch_float16" or "torch_float32").
• device: Device on which the tensor is located (e.g., "cpu" or "cuda").

Value

Processed tensor after applying the quantized convolution.

quick_gelu

QuickGELU activation

Description

GELU approximation used by OpenAI CLIP: x * sigmoid(1.702 * x)

Usage

quick_gelu(x)

Arguments

• x: Input tensor

rms_norm

RMS Normalization

Description

RMS Normalization

Usage

rms_norm(dim, eps = 1e-06, elementwise_affine = TRUE)

rope_embedder_create

Create RoPE position embedder for video

Description

Creates a RoPE embedder configured for video generation, handling spatiotemporal coordinates (frames, height, width).

Usage

rope_embedder_create(
  dim,
  patch_size = 1L,
  patch_size_t = 1L,
  base_num_frames = 20L,
  base_height = 2048L,
  base_width = 2048L,
  scale_factors = c(8, 32, 32),
  theta = 10000,
  causal_offset = 1L,
  double_precision = TRUE,
  rope_type = c("interleaved", "split"),
  num_attention_heads = 32L
)

Arguments

• dim: Integer. Dimension for RoPE (typically attention head dim * num_heads).
• patch_size: Integer. Spatial patch size. Default: 1
• patch_size_t: Integer. Temporal patch size. Default: 1
• base_num_frames: Integer. Base number of frames for normalization. Default: 20
• base_height: Integer. Base height for normalization. Default: 2048
• base_width: Integer. Base width for normalization. Default: 2048
• scale_factors: Numeric vector of length 3. VAE scale factors for (temporal, height, width). Default: c(8, 32, 32)
• theta: Numeric. Base frequency for RoPE. Default: 10000.0
• causal_offset: Integer. Offset for causal VAE modeling. Default: 1
• double_precision: Logical. Whether to use float64 for frequency computation. Default: TRUE
• rope_type: Character. Type of RoPE: "interleaved" or "split". Default: "interleaved"
• num_attention_heads: Integer. Number of attention heads (for split RoPE). Default: 32

Value

A list containing RoPE configuration and methods.

rope_forward

Compute RoPE frequencies from coordinates

Description

Converts spatiotemporal coordinates to (cos, sin) frequency tensors for applying rotary embeddings.

Usage

rope_forward(embedder, coords, device = NULL)

Arguments

• embedder: List. RoPE embedder configuration.
• coords: torch tensor. Coordinate tensor from rope_prepare_video_coords().
• device: Character or torch device. Device for output tensors.

Value

A list with:

• cos_freqs: Cosine frequencies tensor
• sin_freqs: Sine frequencies tensor

rope_prepare_video_coords

Prepare video coordinates for RoPE

Description

Creates per-dimension patch boundaries for video coordinates in pixel space. Returns tensor of shape (batch_size, 3, num_patches, 2) where dimension 1 represents (frame, height, width) and dimension 3 represents (start, end).

Usage

rope_prepare_video_coords(
  embedder,
  batch_size,
  num_frames,
  height,
  width,
  device = "cpu",
  fps = 24
)

Arguments

• embedder: List. RoPE embedder configuration.
• batch_size: Integer. Batch size.
• num_frames: Integer. Number of latent frames.
• height: Integer. Latent height.
• width: Integer. Latent width.
• device: Character or torch device. Device for tensors.
• fps: Numeric. Video frames per second. Default: 24.0

Value

torch tensor of shape (batch_size, 3, num_patches, 2).

rotate_half

Rotate half of the hidden dims

Description

Rotate half of the hidden dims

Usage

rotate_half(x)

save_frames

Save Video Frames as Individual Images

Description

Save Video Frames as Individual Images

Usage

save_frames(video, dir, prefix = "frame_", format = "png", verbose = TRUE)

Arguments

• video: Array of video frames [T, H, W, C].
• dir: Directory to save frames in.
• prefix: Character. Filename prefix (default "frame_").
• format: Character. Image format: "png" or "jpg".
• verbose: Logical.

Value

Invisibly returns vector of saved file paths.

save_image

Save and Display an Image from a Torch Tensor

Description

Converts a Torch tensor to a normalized RGB image array, saves it as a PNG file, and optionally displays it in the RStudio Viewer pane using grid::grid.raster().

Usage

save_image(img, save_to = "output.png", normalize = TRUE)

Arguments

• img: A numeric with shape [3, H, W].
• save_to: File path for the PNG image (default is "output.png").
• normalize: Logical; whether to normalize pixel values to [0, 1]. Default is TRUE.

Value

Invisibly returns the saved file path.

Examples

save_image(output_tensor, "sample.png")

save_video_av

Save Video using av Package

Description

Save Video using av Package

Usage

save_video_av(video, file, fps = 24, verbose = TRUE)

Arguments

• video: Array of video frames [T, H, W, C].
• file: Output file path.
• fps: Frames per second.
• verbose: Logical.

save_video_ffmpeg

Save Video using FFmpeg

Description

Save Video using FFmpeg

Usage

save_video_ffmpeg(
  video,
  file,
  fps = 24,
  format = "mp4",
  quality = 85,
  verbose = TRUE
)

Arguments

• video: Array of video frames [T, H, W, C].
• file: Output file path.
• fps: Frames per second.
• format: Output format.
• quality: Quality level 1-100.
• verbose: Logical.

save_video

Save Video to File

Description

Saves a video array to a file in various formats.

Usage

save_video(
  video,
  file,
  fps = 24,
  format = NULL,
  backend = "auto",
  quality = 85,
  verbose = TRUE
)

Arguments

• video: Array of video frames with shape [T, H, W, C] where C is 3 (RGB). Values should be in [0, 1] range.
• file: Character. Output file path. Extension determines format.
• fps: Numeric. Frames per second (default 24).
• format: Character. Output format: "mp4", "gif", "webm", or "frames". If NULL, inferred from file extension.
• backend: Character. Backend to use: "ffmpeg", "av", or "auto".
• quality: Integer. Quality level 1-100 (for lossy formats).
• verbose: Logical. Print progress messages.

Value

Invisibly returns the output file path.

Examples

# Save as MP4
save_video(video_array, "output.mp4", fps = 24)

# Save as GIF
save_video(video_array, "output.gif", fps = 10)

# Save as individual frames
save_video(video_array, "frames/", format = "frames")

scheduler_add_noise

Add noise to latents using DDIM scheduler

Description

This function adds noise to the original latents according to the DDIM scheduler's diffusion process. It computes the noisy latents based on the original latents, noise, and the current timestep.

Usage

scheduler_add_noise(original_latents, noise, timestep, scheduler_obj)

Arguments

• original_latents: A torch tensor representing the original latents.
• noise: A torch tensor representing the noise to be added.
• timestep: An integer representing the current timestep in the diffusion process.
• scheduler_obj: A list containing the DDIM scheduler parameters, including alphas_cumprod and timesteps. The alphas_cumprod represents how much of the original signal remains at each timestep of the diffusion process.

Details

The noise is added according to the standard diffusion forward process formula: noised_latents = sqrt(alpha_cumprod) * original_latents + sqrt(1-alpha_cumprod) * noise

Where alpha_cumprod is the cumulative product of (1-beta) values up to the specified timestep, with beta being the noise schedule.

Value

A torch tensor containing the noised latents, which represents the original latents with the appropriate amount of noise added for the given timestep.

Examples

# Assuming we have latents, noise, and a scheduler
noised_latents <- scheduler_add_noise(
  original_latents = latents,
  noise = torch::torch_randn_like(latents),
  timestep = scheduler$timesteps[1],
  scheduler_obj = scheduler
)

sdxl_memory_profile

Get SDXL Memory Profile

Description

Determines optimal memory configuration for SDXL image generation based on available VRAM.

Usage

sdxl_memory_profile(vram_gb = NULL)

Arguments

• vram_gb: Numeric. Available VRAM in GB, or NULL for auto-detection.

Details

Memory profiles for SDXL:

• full_gpu: 16GB+ - All components on CUDA
• balanced: 10-12GB - UNet + decoder on CUDA, text encoders on CPU
• unet_gpu: 6-10GB - Only UNet on CUDA, everything else CPU
• cpu_only: <6GB - All on CPU

Each profile also specifies:

• cfg_mode: "batched" or "sequential" (sequential halves peak memory)
• cleanup: "none", "phase", or "step" (when to clear VRAM)
• dtype: "float16" or "float32"
• max_resolution: maximum image dimension

Value

A list with memory profile settings.

Examples

# Auto-detect profile
profile <- sdxl_memory_profile()

# Specific VRAM
profile <- sdxl_memory_profile(vram_gb = 8)

sequential_cfg_forward

Sequential CFG Forward Pass

Description

Runs unconditional and conditional forward passes separately to halve peak activation memory. For GPU-poor scenarios.

Usage

sequential_cfg_forward(
  model,
  latents,
  timestep,
  prompt_embeds,
  negative_prompt_embeds,
  guidance_scale,
  ...
)

Arguments

• model: The DiT model.
• latents: Current latent tensor.
• timestep: Current timestep tensor.
• prompt_embeds: Conditional prompt embeddings.
• negative_prompt_embeds: Unconditional prompt embeddings.
• guidance_scale: CFG scale.
• ...: Additional arguments to model forward pass.

Value

The CFG-combined noise prediction.

Examples

noise_pred <- sequential_cfg_forward(
  model, latents, timestep,
  prompt_embeds, negative_prompt_embeds,
  guidance_scale = 4.0
)

setup_dtype

Set up dtype based on device configuration

Description

Set up dtype based on device configuration

Usage

setup_dtype(devices, unet_dtype_str)

Arguments

• devices: A character string or a named list specifying the devices for model components.
• unet_dtype_str: A character string specifying the data type for the UNet model.

Value

A torch dtype object based on the main computation device.

SpatialTransformer

Spatial Transformer (Attention Block)

Description

Spatial Transformer (Attention Block)

Usage

SpatialTransformer(in_channels, n_heads, d_head, depth = 1L, context_dim = NULL)

standardize_devices

Standardize devices configuration

Description

This function standardizes the device configuration for model components. It checks if the devices parameter is a single string or a named list, and fills in missing components with reasonable defaults.

Usage

standardize_devices(devices, required_components)

Arguments

• devices: A character string or a named list specifying the devices for model components.
• required_components: A character vector of required components for the model.

Value

A named list of devices for each required component.

text_encoder_native

Native CLIP Text Encoder

Description

Native R torch implementation of CLIP text encoder. Replaces TorchScript for better GPU compatibility.

Usage

text_encoder_native(
  vocab_size = 49408,
  context_length = 77,
  embed_dim = 768,
  num_layers = 12,
  num_heads = 12,
  mlp_dim = 3072,
  apply_final_ln = TRUE
)

Arguments

• vocab_size: Vocabulary size (default 49408)
• context_length: Maximum sequence length (default 77)
• embed_dim: Embedding dimension
• num_layers: Number of transformer layers
• num_heads: Number of attention heads
• mlp_dim: MLP hidden dimension
• apply_final_ln: Whether to apply final layer norm (default TRUE). Set to FALSE to match TorchScript exports that don't include final LN.

Value

An nn_module representing the text encoder

text_encoder2_native

Native CLIP Text Encoder 2 (OpenCLIP ViT-bigG for SDXL)

Description

Native R torch implementation of OpenCLIP text encoder used in SDXL. Returns both hidden states and pooled output.

Usage

text_encoder2_native(
  vocab_size = 49408,
  context_length = 77,
  embed_dim = 1280,
  num_layers = 32,
  num_heads = 20,
  mlp_dim = 5120
)

Arguments

• vocab_size: Vocabulary size (default 49408)
• context_length: Maximum sequence length (default 77)
• embed_dim: Embedding dimension (default 1280)
• num_layers: Number of transformer layers (default 32)
• num_heads: Number of attention heads (default 20)
• mlp_dim: MLP hidden dimension (default 5120)

Value

An nn_module representing the text encoder

timestep_embedding_module

Timestep embedding MLP

Description

Timestep embedding MLP

Usage

timestep_embedding_module(
  in_channels,
  time_embed_dim,
  act_fn = "silu",
  out_dim = NULL
)

timestep_embedding

Sinusoidal Timestep Embedding

Description

Sinusoidal Timestep Embedding

Usage

timestep_embedding(
  timesteps,
  dim,
  flip_sin_to_cos = TRUE,
  downscale_freq_shift = 0L
)

Arguments

• timesteps: Tensor of timesteps (batch_size,)
• dim: Embedding dimension
• flip_sin_to_cos: If TRUE, output [cos, sin] instead of [sin, cos]. SDXL uses TRUE (default), SD21 uses FALSE.
• downscale_freq_shift: Frequency shift parameter. SDXL uses 0 (default), SD21 uses 1. With 0: exponent = log(10000) / half_dim. With 1: exponent = log(10000) / (half_dim - 1).

Value

Tensor (batch_size, dim)

timesteps_module

Timesteps module

Description

Timesteps module

Usage

timesteps_module(
  num_channels,
  flip_sin_to_cos = TRUE,
  downscale_freq_shift = 0,
  scale = 1L
)

tokenize_gemma3

Tokenize text for Gemma3

Description

Tokenize text for Gemma3

Usage

tokenize_gemma3(
  tokenizer,
  text,
  max_length = 1024L,
  padding = "max_length",
  return_tensors = "pt"
)

Arguments

• tokenizer: Gemma3 tokenizer object.
• text: Character vector of prompts.
• max_length: Integer. Maximum sequence length.
• padding: Character. Padding strategy ("left", "right", "max_length", "none").
• return_tensors: Character. Return type ("list" or "pt" for torch tensors).

Value

List with input_ids and attention_mask.

txt2img_sd21

Generate an image from a text prompt using a diffusion pipeline

Description

This function generates an image based on a text prompt using the Stable Diffusion model. It allows for various configurations such as model name, device, scheduler, and more.

Usage

txt2img_sd21(
  prompt,
  negative_prompt = NULL,
  img_dim = 768,
  pipeline = NULL,
  devices = "auto",
  unet_dtype_str = NULL,
  download_models = FALSE,
  scheduler = "ddim",
  timesteps = NULL,
  initial_latents = NULL,
  num_inference_steps = 50,
  guidance_scale = 7.5,
  seed = NULL,
  save_file = TRUE,
  filename = NULL,
  metadata_path = NULL,
  use_native_decoder = FALSE,
  use_native_text_encoder = FALSE,
  use_native_unet = FALSE,
  ...
)

Arguments

• prompt: A character string prompt describing the image to generate.
• negative_prompt: Optional negative prompt to guide the generation.
• img_dim: Dimension of the output image (e.g., 512 for 512x512).
• pipeline: Optional A pre-loaded diffusion pipeline. If NULL, it will be loaded based on the model name and devices.
• devices: A named list of devices for each model component (e.g., list(unet = "cuda", decoder = "cpu", text_encoder = "cpu")).
• unet_dtype_str: Optional A character for dtype of the unet component (typically "float16" for cuda and "float32" for cpu; float32 is available for cuda).
• download_models: Logical indicating whether to download the model files if they are not found.
• scheduler: Scheduler to use (e.g., "ddim", "euler").
• timesteps: Optional A vector of timesteps to use.
• initial_latents: Optional initial latents for the diffusion process.
• num_inference_steps: Number of inference steps to run.
• guidance_scale: Scale for classifier-free guidance (typically 7.5).
• seed: Optional seed for reproducibility.
• save_file: Logical indicating whether to save the generated image.
• filename: Optional filename for saving the image. If NULL, a default name is generated.
• metadata_path: Optional file path to save metadata.
• use_native_decoder: Logical; if TRUE, uses native R torch decoder instead of TorchScript. Native decoder has better GPU compatibility (especially Blackwell).
• use_native_text_encoder: Logical; if TRUE, uses native R torch text encoder instead of TorchScript. Native text encoder has better GPU compatibility (especially Blackwell).
• use_native_unet: Logical; if TRUE, uses native R torch UNet instead of TorchScript. Native UNet has better GPU compatibility (especially Blackwell).
• ...: Additional parameters passed to the diffusion process.

Value

An image array and metadata

Examples

img <- txt2img("a cat wearing sunglasses in space", device = "cuda")

txt2img_sdxl

Generate an image from a text prompt using SDXL

Description

Generate an image from a text prompt using SDXL

Usage

txt2img_sdxl(
  prompt,
  negative_prompt,
  img_dim,
  pipeline,
  devices,
  memory_profile,
  unet_dtype_str,
  download_models,
  scheduler,
  timesteps,
  initial_latents,
  num_inference_steps,
  guidance_scale,
  seed,
  save_file,
  filename,
  metadata_path,
  use_native_decoder,
  use_native_text_encoder,
  use_native_unet,
  verbose
)

Arguments

• prompt: A character string prompt describing the image to generate.
• negative_prompt: Optional negative prompt to guide the generation.
• img_dim: Dimension of the output image (e.g., 512 for 512x512).
• pipeline: Optional A pre-loaded diffusion pipeline. If NULL, it will be loaded based on the model name and devices.
• devices: A named list of devices for each model component (e.g., list(unet = "cuda", decoder = "cpu", text_encoder = "cpu")), or "auto" to use auto_devices(), or NULL to use memory_profile devices.
• memory_profile: Character or list. Memory profile for GPU-poor optimization: "auto" for auto-detection, or a profile name ("full_gpu", "balanced", "unet_gpu", "cpu_only"), or a list from sdxl_memory_profile(). When specified, overrides devices parameter.
• unet_dtype_str: Optional A character for dtype of the unet component (typically "float16" for cuda and "float32" for cpu; float32 is available for cuda).
• download_models: Logical indicating whether to download the model files if they are not found.
• scheduler: Scheduler to use (e.g., "ddim", "euler").
• timesteps: Optional A vector of timesteps to use.
• initial_latents: Optional initial latents for the diffusion process.
• num_inference_steps: Number of inference steps to run.
• guidance_scale: Scale for classifier-free guidance (typically 7.5).
• seed: Optional seed for reproducibility.
• save_file: Logical indicating whether to save the generated image.
• filename: Optional filename for saving the image. If NULL, a default name is generated.
• metadata_path: Optional file path to save metadata.
• use_native_decoder: Logical; if TRUE, uses native R torch decoder instead of TorchScript. Native decoder has better GPU compatibility (especially Blackwell).
• use_native_text_encoder: Logical; if TRUE, uses native R torch text encoder instead of TorchScript. Native text encoder has better GPU compatibility (especially Blackwell).
• use_native_unet: Logical; if TRUE, uses native R torch UNet instead of TorchScript. Native UNet has better GPU compatibility (especially Blackwell).
• verbose: Logical. Print progress and memory status messages.
• ...: Additional parameters passed to the diffusion process.

Value

An image array and metadata

Examples

# Basic usage with auto-detection
img <- txt2img_sdxl("a cat wearing sunglasses in space")

# GPU-poor mode (8GB VRAM)
img <- txt2img_sdxl("a sunset over mountains", memory_profile = "unet_gpu")

# Explicit memory profile
profile <- sdxl_memory_profile(vram_gb = 8)
img <- txt2img_sdxl("a forest path", memory_profile = profile)

txt2img

Generate an image from a text prompt using a diffusion pipeline

Description

Generate an image from a text prompt using a diffusion pipeline

Usage

txt2img(prompt, model_name = c("sd21", "sdxl"), ...)

Arguments

• prompt: A character string prompt describing the image to generate.
• model_name: Name of the model to use (e.g., "sd21").
• ...: Additional parameters passed to the diffusion process.

Value

A tensor or image object, depending on implementation.

Examples

img <- txt2img("a cat wearing sunglasses in space", device = "cuda")

txt2vid_ltx2

Generate Video from Text Prompt using LTX-2

Description

Generates video using the LTX-2 diffusion transformer model.

Usage

txt2vid_ltx2(
  prompt,
  negative_prompt = NULL,
  width = 768L,
  height = 512L,
  num_frames = 121L,
  fps = 24,
  num_inference_steps = 8L,
  guidance_scale = 4,
  memory_profile = "auto",
  text_backend = "api",
  text_api_url = NULL,
  vae = NULL,
  dit = NULL,
  connectors = NULL,
  seed = NULL,
  output_file = NULL,
  output_format = "mp4",
  return_latents = FALSE,
  verbose = TRUE
)

Arguments

• prompt: Character. Text prompt describing the video to generate.
• negative_prompt: Character. Optional negative prompt.
• width: Integer. Video width in pixels (default 768).
• height: Integer. Video height in pixels (default 512).
• num_frames: Integer. Number of frames to generate (default 121).
• fps: Numeric. Frames per second (default 24).
• num_inference_steps: Integer. Number of denoising steps (default 8 for distilled).
• guidance_scale: Numeric. CFG scale (default 4.0).
• memory_profile: Character or list. Memory profile: "auto" for auto-detection, or a profile from ltx2_memory_profile().
• text_backend: Character. Text encoding backend: "api", "precomputed", or "random".
• text_api_url: Character. URL for text encoding API (if backend = "api").
• vae: Optional. Pre-loaded VAE module.
• dit: Optional. Pre-loaded DiT transformer module.
• connectors: Optional. Pre-loaded text connectors module.
• seed: Integer. Random seed for reproducibility.
• output_file: Character. Path to save output video (NULL for no save).
• output_format: Character. Output format: "mp4", "gif", or "frames".
• return_latents: Logical. If TRUE, also return final latents.
• verbose: Logical. Print progress messages.

Value

A list with:

• video: Array of video frames [frames, height, width, channels]
• latents: (if return_latents=TRUE) Final latent tensor
• metadata: Generation metadata

Examples

# Basic usage
result <- txt2vid_ltx2("A cat walking on a beach at sunset")

# With specific settings
result <- txt2vid_ltx2(
  prompt = "A timelapse of clouds moving over mountains",
  width = 512,
  height = 320,
  num_frames = 61,
  num_inference_steps = 8,
  seed = 42,
  output_file = "clouds.mp4"
)

unet_native_from_torchscript

Create native UNet from TorchScript

Description

Detects architecture and loads weights from a TorchScript UNet file.

Usage

unet_native_from_torchscript(torchscript_path, verbose = TRUE)

Arguments

• torchscript_path: Path to TorchScript UNet .pt file
• verbose: Print loading progress

Value

A native UNet module with loaded weights

unet_native

Native UNet for Stable Diffusion

Description

Native R torch implementation of UNet2DConditionModel. Replaces TorchScript for better GPU compatibility.

Usage

unet_native(
  in_channels = 4L,
  out_channels = 4L,
  block_out_channels = c(320L, 640L, 1280L, 1280L),
  layers_per_block = 2L,
  cross_attention_dim = 1024L,
  attention_head_dim = 64L
)

Arguments

• in_channels: Input channels (default 4 for latent space)
• out_channels: Output channels (default 4)
• block_out_channels: Channel multipliers per block
• layers_per_block: Number of ResBlocks per down/up block
• cross_attention_dim: Context dimension from text encoder
• attention_head_dim: Dimension per attention head

Value

An nn_module representing the UNet

unet_sdxl_native_from_torchscript

Create native SDXL UNet from TorchScript

Description

Create native SDXL UNet from TorchScript

Usage

unet_sdxl_native_from_torchscript(torchscript_path, verbose = TRUE)

Arguments

• torchscript_path: Path to TorchScript SDXL UNet .pt file
• verbose: Print loading progress

Value

A native SDXL UNet module with loaded weights

unet_sdxl_native

Native SDXL UNet

Description

Native R torch implementation of SDXL UNet2DConditionModel. SDXL has a different architecture from SD21: - 3 down/up blocks (not 4) - Variable transformer depth per block - Additional conditioning via add_embedding

Usage

unet_sdxl_native(
  in_channels = 4L,
  out_channels = 4L,
  block_out_channels = c(320L, 640L, 1280L),
  layers_per_block = 2L,
  transformer_layers_per_block = c(0L, 2L, 10L),
  cross_attention_dim = 2048L,
  attention_head_dim = 64L,
  addition_embed_dim = 1280L,
  addition_time_embed_dim = 256L
)

Arguments

• in_channels: Input channels (default 4 for latent space)
• out_channels: Output channels (default 4)
• block_out_channels: Channel multipliers per block
• layers_per_block: Number of ResBlocks per down/up block
• transformer_layers_per_block: Transformer depth per block
• cross_attention_dim: Context dimension from text encoder
• attention_head_dim: Dimension per attention head
• addition_embed_dim: Dimension for additional embeddings
• addition_time_embed_dim: Dimension for time embedding projection

Value

An nn_module representing the SDXL UNet

UNetCrossAttention

Cross-Attention for UNet

Description

Cross-Attention for UNet

Usage

UNetCrossAttention(query_dim, context_dim = NULL, heads = 8L, dim_head = 64L)

UNetResBlock

ResNet Block for UNet

Description

ResNet Block for UNet

Usage

UNetResBlock(in_channels, out_channels, time_embed_dim)

Upsample2D

Upsample Block

Description

Upsample Block

Usage

Upsample2D(channels)

vae_decoder_native

Native VAE Decoder

Description

Native R torch implementation of the SDXL VAE decoder. Replaces TorchScript decoder for better GPU compatibility.

Usage

vae_decoder_native(latent_channels = 4, out_channels = 3)

Arguments

• latent_channels: Number of latent channels (default 4)
• out_channels: Number of output channels (default 3 for RGB)

Value

An nn_module representing the VAE decoder

Examples

decoder <- vae_decoder_native()
load_decoder_weights(decoder, "path/to/decoder.pt")
latents <- torch::torch_randn(c(1, 4, 64, 64))
image <- decoder(latents)

VAEAttentionBlock

VAE Attention Block

Description

Self-attention for VAE mid block

Usage

VAEAttentionBlock(channels)

Arguments

• channels: Number of channels

VAEMidBlock

VAE Mid Block

Description

VAE Mid Block

Usage

VAEMidBlock(channels)

Arguments

• channels: Number of channels

VAEResnetBlock

VAE ResNet Block

Description

VAE ResNet Block

Usage

VAEResnetBlock(in_channels, out_channels)

Arguments

• in_channels: Input channels
• out_channels: Output channels

VAEUpBlock

VAE Up Block

Description

VAE Up Block

Usage

VAEUpBlock(in_channels, out_channels, num_resnets = 3, add_upsample = TRUE)

Arguments

• in_channels: Input channels
• out_channels: Output channels
• num_resnets: Number of resnet blocks (default 3)
• add_upsample: Whether to add upsampler

validate_resolution

Validate Resolution Against Profile

Description

Checks if requested resolution fits within memory profile limits.

Usage

validate_resolution(height, width, num_frames, profile)

Arguments

• height: Integer. Requested height.
• width: Integer. Requested width.
• num_frames: Integer. Requested number of frames.
• profile: Memory profile from ltx2_memory_profile().

Value

List with adjusted height, width, num_frames and warning if adjusted.

Examples

profile <- ltx2_memory_profile(vram_gb = 8)
validated <- validate_resolution(720, 1280, 60, profile)

vocab_size

Get vocabulary size

Description

Get vocabulary size

Usage

vocab_size(tokenizer)

Arguments

• tokenizer: A bpe_tokenizer object.

Value

Integer vocabulary size.

vram_report

Report VRAM Usage

Description

Prints current VRAM usage using gpuctl.

Usage

vram_report(label = "")

Arguments

• label: Character. Label for the report.

Value

Invisibly returns a list with used and free VRAM in GB.

Examples

vram_report("After model load")