o jCÄiO(ã@szdZddlZddlZddlZddlmZej dd¡Zej dd¡Z dZ dZ dZ d Z d d „Zee ƒZGd d „d ejƒZdS)uÒDinoVideoModel: predict future VAE latent tokens from a single DINO-encoded frame. Uses a 3-step diffusion process: during training, a random noise level is sampled and the 3D conv blocks predict the noise residual. During inference, 3 reverse diffusion steps denoise from pure Gaussian noise to predicted VAE tokens. Architecture: 1. Frozen DINO ViT-S/16 extracts (B, D, 16, 16) patch features from first frame 2. Temporal expand + temporal embeddings → (B, D, N_FRAMES, 16, 16) 3. Concatenate with noised VAE tokens z_t → (B, D + vae_embed_dim, N_FRAMES, 16, 16) 4. Add timestep embedding (broadcast over spatial/temporal dims) 5. 3D conv blocks predict noise ε → (B, vae_embed_dim, N_FRAMES, 16, 16) éNÚ DINO_REPO_DIRz/data/cameron/dinov3ÚDINO_WEIGHTS_PATHzM/data/cameron/dinov3/weights/dinov3_vits16plus_pretrain_lvd1689m-4057cbaa.pthééécs‡fdd„tˆƒDƒS)uICosine schedule for alpha_bar: alpha_bar[t] = cos²(Ï€/2 · (t+1)/(T+1)).cs0g|]}t tjd|dˆd¡d‘qS)éé)ÚmathÚcosÚpi)Ú.0Út©Zn_steps©ú#s0z._cosine_alpha_bar_schedule..)ÚrangerrrrÚ_cosine_alpha_bar_schedule!srcsneZdZdZeeef‡fdd„ Z‡fdd„Zdd„Z dd „Z d d „Z d d „Z ddd„Z e ¡dd„ƒZ‡ZS)ÚDinoVideoModelzWPredict future VAE latent tokens via 3-step diffusion from a single DINO-encoded frame.c sŒtƒ ¡||_||_||_t|_t|ƒ}| dt j |t j d¡t d|›ddd„|Dƒ›ƒt dƒt j jtdd td |_|j ¡D]}d |_qD|j ¡|jj|_t d |j›d ƒ|j}t t  d||dd¡d¡|_t |||¡|_||}t tj||dddt ¡tj||dddt ¡tj|ddddt ¡tjd|dd¡|_t d|›d|›d|›d|›ƒt d|›d|›dƒdS)NÚ alpha_bar)ÚdtypezDiffusion schedule (z steps): alpha_bar = cSsg|]}|d›‘qS)z.4fr)r Úarrrr7sz+DinoVideoModel.__init__..zLoading DINOv3 model...Zdinov3_vits16plusÚlocal)ÚsourceÚweightsFz"Frozen DINOv3 backbone (embed_dim=ú)rg{®Gáz”?r)Ú kernel_sizeÚpaddingé)rz3D conv blocks: z -> z -> 256 -> z Output: (B, z, u, 16, 16) — noise prediction) ÚsuperÚ__init__Ún_framesÚ vae_embed_dimÚn_diffusion_stepsÚDINO_PATCH_SIZEÚ patch_sizerÚregister_bufferÚtorchÚtensorÚfloat32ÚprintÚhubÚloadrrÚdinoÚ parametersÚ requires_gradÚevalÚ embed_dimÚnnÚ ParameterÚrandnÚtemporal_embedÚ EmbeddingÚtimestep_embedÚ SequentialÚConv3dÚGELUÚ conv3d_blocks)Úselfr!r"r#rÚparamÚDÚin_ch©Ú __class__rrr ,sF ü  ù zDinoVideoModel.__init__cs(tƒ |¡t|dƒr|j |¡|_|S)Nr-)rÚtoÚhasattrr-)r<Údevicer@rrrB]s  zDinoVideoModel.toc Cs|jd}|j |¡\}\}}|jjD]}|jjr!|jj||dnd}|||ƒ}q|jjr[|j |dd…d|jjd…f¡}|j |dd…|jjdd…f¡} t j || gdd}n|j |¡}|dd…|jjdd…f} |   ||||j ¡} |   dddd¡ ¡} | S)zÂExtract patch features from frozen DINO backbone. Args: x: (B, 3, 256, 256) ImageNet-normalized input Returns: patch_features: (B, D, H_p, W_p) r)ÚHÚWNr©Údimrr)Úshaper-Zprepare_tokens_with_masksÚblocksZ rope_embedZuntie_cls_and_patch_normsZcls_normZn_storage_tokensÚnormr'ÚcatÚreshaper1ÚpermuteÚ contiguous) r<ÚxÚBZx_tokensÚH_pZW_pÚblkZ rope_sincosZ x_norm_clsZx_norm_patchesZ patch_tokensZpatch_featuresrrrÚ_extract_dino_featurescs   $$ z%DinoVideoModel._extract_dino_featurescCsZt ¡ | |¡}Wdƒn1swY| d¡ dd|jdd¡}||j}|S)uãExtract DINO features and expand temporally. Args: x: (B, 3, 256, 256) ImageNet-normalized input Returns: cond: (B, D, N_FRAMES, 16, 16) — DINO features + temporal embeddings Nréÿÿÿÿ)r'Úno_gradrTÚ unsqueezeÚexpandr!r5)r<rPZfeatsÚcondrrrÚ_get_dino_cond}s ÿ zDinoVideoModel._get_dino_condcCsR| |¡}tj||gdd}| |¡}||dd…dd…dddf}| |¡}|S)aˆPredict noise given conditioning image, noised tokens, and timestep. Args: x: (B, 3, 256, 256) ImageNet-normalized input frame z_t: (B, vae_embed_dim, N_FRAMES, 16, 16) noised VAE tokens t: (B,) integer timestep indices in [0, n_diffusion_steps) Returns: eps_pred: (B, vae_embed_dim, N_FRAMES, 16, 16) predicted noise rrGN)rZr'rLr7r;)r<rPÚz_tr rYÚhÚt_embZeps_predrrrÚ predict_noises   zDinoVideoModel.predict_noisecCsR|j|}|dd…ddddf}t |¡}t |¡|t d|¡|}||fS)aForward diffusion: add noise to clean tokens. Args: z_0: (B, C, T, H, W) clean VAE tokens t: (B,) integer timestep indices Returns: z_t: (B, C, T, H, W) noised tokens eps: (B, C, T, H, W) the noise that was added Nçð?)rr'Ú randn_likeÚsqrt)r<Úz_0r Ú alpha_bar_tÚepsr[rrrÚforward_diffusion¥s   z DinoVideoModel.forward_diffusionNc CsV|jd}|j}|durtjd|j|f|d}| ||¡\}}| |||¡}|||fS)u Training forward pass: sample noise level, predict clean xâ‚€. Args: x: (B, 3, 256, 256) ImageNet-normalized input z_0: (B, vae_embed_dim, N_FRAMES, 16, 16) clean target VAE tokens t: (B,) optional timestep indices. If None, sampled randomly. Returns: x0_pred: (B, vae_embed_dim, N_FRAMES, 16, 16) predicted clean tokens z_0: (B, vae_embed_dim, N_FRAMES, 16, 16) actual clean tokens (target) t: (B,) sampled timestep indices rN©rD)rIrDr'Úrandintr#rer^) r<rPrbr rQrDr[rdÚx0_predrrrÚforward·s  zDinoVideoModel.forwardcCs,|jd}|j}tj||j|jdd|d}tt|jƒƒD]w}tj |f||tj d}|  |||¡}|j |}|dkr‘|t  |¡|t  d|¡} |j |d} || } d| } t  | ¡| d|} t  | ¡d| d|}| |||}d| d|| }t  |¡}||t |¡}q|}q|S)u¸3-step reverse diffusion (xâ‚€-prediction): denoise from pure noise to predicted VAE tokens. At each step the network directly predicts xâ‚€. We derive ε from xâ‚€_pred to compute the DDPM posterior for stepping to the next noise level. Args: x: (B, 3, 256, 256) ImageNet-normalized input Returns: z_0_pred: (B, vae_embed_dim, N_FRAMES, 16, 16) predicted clean VAE tokens rrrf)rDrr_r)rIrDr'r4r"r!Úreversedrr#ÚfullÚlongr^rr rar`)r<rPrQrDÚzÚstepr rhrcZ eps_derivedÚalpha_bar_prevÚalpha_tZbeta_tZcoef_z0Zcoef_ztÚmuZ posterior_varÚsigmarrrÚsampleÓs(    zDinoVideoModel.sample)NN)Ú__name__Ú __module__Ú __qualname__Ú__doc__ÚN_FRAMESÚ VAE_EMBED_DIMÚN_DIFFUSION_STEPSr rBrTrZr^rerir'rVrsÚ __classcell__rrr@rr)sÿ 1 r)rwÚosr r'Útorch.nnr2ÚenvironÚgetrrr$rxryrzrZ ALPHA_BARÚModulerrrrrÚs" þ