o 9iJ4 @s>dZddlZddlmZddlZddlmZddlmZm Z   d5de deej d ej d e d e d ej f d dZ  d6ddZddZdefddZGdddejjZddZGdddejjZd7dd Zd!d"Zd#d$Zd%d&Zd'd(ZGd)d*d*ejZGd+d,d,ejZd-d.Z d/d0Z!d1d2Z"Gd3d4d4ejZ#dS)8a partially adopted from https://github.com/openai/improved-diffusion/blob/main/improved_diffusion/gaussian_diffusion.py and https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py and https://github.com/openai/guided-diffusion/blob/0ba878e517b276c45d1195eb29f6f5f72659a05b/guided_diffusion/nn.py thanks! N)Optional) rearrangerepeatTFmerge_strategy mix_factorimage_only_indicator apply_sigmoidis_attnreturncCs|dks|dkr |}n;|dkr0t|tjdd|jdt|d}|r*t|d}nt|d}n|d krD|}|r>t|d}nt|d}nt|rMt|S|S) Nfixedlearnedlearned_with_imagesdevice ... -> ... 1b t -> (b t) 1 1zb t -> b 1 t 1 1fixed_with_images)torchwhereboolonesrrNotImplementedErrorsigmoid)rrrrr alpharK/data/cameron/vidgen/generative-models/sgm/modules/diffusionmodules/util.py get_alphas$    r-C6?{Gz?cCs0|dkrtj|d|d|tjdd}|S)Nlinearg?)dtype)rlinspacefloat64numpy)schedule n_timestep linear_start linear_endbetasrrrmake_beta_schedule2sr+cCs4|j^}}|d|}|j|gdt|dRS)N)rr)shapegatherreshapelen)atx_shapeb_outrrrextract_into_tensorBs  r7inputsc s|rCfddD}fddD}fddD}fddD}t|t|t|}tj|t|t|||g|RS|diS)a Evaluate a function without caching intermediate activations, allowing for reduced memory at the expense of extra compute in the backward pass. This differs from the original checkpoint function borrowed from https://github.com/openai/guided-diffusion/blob/0ba878e517b276c45d1195eb29f6f5f72659a05b/guided_diffusion/nn.py in that it also works with non-tensor inputs :param func: the function to evaluate. :param inputs: the argument dictionary to pass to `func`. :param params: a sequence of parameters `func` depends on but does not explicitly take as arguments. :param flag: if False, disable gradient checkpointing. cs g|] }t|tjr|qSr isinstancerTensor.0keyr8rr Us z$mixed_checkpoint..cs$g|]}t|tjr|qSrr9r<r?rrr@V cs g|] }t|tjs|qSrr9r<r?rrr@Ys cs$g|]}t|tjs|qSrr9r<r?rrr@\rANr)tupleMixedCheckpointFunctionapplyr0) funcr8paramsflag tensor_keys tensor_inputsnon_tensor_keysnon_tensor_inputsargsrr?rmixed_checkpointHs,     rMc@$eZdZeddZeddZdS)rCcGs||_|||_tttd|_t||kr!t||ks#Jddt|t |d|jD|_ ddt|t ||j|jD|_ ||_ t ||jd|_ t|j di|j |j }Wd|S1sqwY|S)Nenabledr! cache_enabledcSi|]\}}||qSrrr=r>valrrr s z3MixedCheckpointFunction.forward..cSrRrrrSrrrrUsr) end_tensorsend_non_tensorsris_autocast_enabledget_autocast_gpu_dtypeis_autocast_cache_enabledgpu_autocast_kwargsr0ziplist input_tensorsinput_non_tensors run_function input_paramsno_grad)ctxr`length_tensorslength_non_tensorsrHrJrLoutput_tensorsrrrforwardms8    zMixedCheckpointFunction.forwardc sfddjD_t8tjjjdijfddjD}jdi|j}Wdn1s;wYWdn1sJwYtj j |t j j |dd}`` ~d|djdjj|jdS) Ncs"i|] }|j|dqST)r^detachrequires_grad_r<rcrrrUsz4MixedCheckpointFunction.backward..cs$i|]}|j|j|qSr)r^view_asr<rkrrrUsT allow_unused)NNNNNNr)r^r enable_gradcudaampautocastr[r`r_autogradgradr]valuesrarVrWrc output_gradsshallow_copiesrf input_gradsrrkrbackwards8 "    z MixedCheckpointFunction.backwardN__name__ __module__ __qualname__ staticmethodrgr{rrrrrCls  'rCcCs4|rt|t|}tj|t|g|RS||S)a Evaluate a function without caching intermediate activations, allowing for reduced memory at the expense of extra compute in the backward pass. :param func: the function to evaluate. :param inputs: the argument sequence to pass to `func`. :param params: a sequence of parameters `func` depends on but does not explicitly take as arguments. :param flag: if False, disable gradient checkpointing. )rBCheckpointFunctionrDr0)rEr8rFrGrLrrr checkpoints rc@rN)rcGs~||_t|d||_t||d|_tttd|_t |j|j}Wd|S1s8wY|S)NrO) r`r]r^rarrXrYrZr[rb)rcr`lengthrLrfrrrrgs  zCheckpointFunction.forwardc Gsdd|jD|_t0tjjjdi|jdd|jD}|j|}Wdn1s1wYWdn1s@wYtjj ||j|j |dd}|`|` ~d|S)NcSsg|] }|dqSrh)rirjr=xrrrr@sz/CheckpointFunction.backward..cSsg|]}||qSr)rlrrrrr@sTrm)NNr) r^rrprqrrrsr[r`rtrurarwrrrr{s""  zCheckpointFunction.backwardNr|rrrrrs   r'cCs|sV|d}tt| tjd|tjd|j|jd}|dddf|d}tj t |t |gdd}|drTtj |t |ddddfgdd}|St |d |d }|S) aX Create sinusoidal timestep embeddings. :param timesteps: a 1-D Tensor of N indices, one per batch element. These may be fractional. :param dim: the dimension of the output. :param max_period: controls the minimum frequency of the embeddings. :return: an [N x dim] Tensor of positional embeddings. r"r)startendr!rNr,dimrzb -> b d)d)rexpmathlogarangefloat32torfloatcatcossin zeros_liker) timestepsr max_period repeat_onlyhalffreqsrL embeddingrrrtimestep_embeddings(  rcCs|D]}|q|S)z< Zero out the parameters of a module and return it. ) parametersrizero_)moduleprrr zero_modules rcCs |D] }||q|S)z9 Scale the parameters of a module and return it. )rrimul_)rscalerrrr scale_modules rcCs|jttdt|jdS)z6 Take the mean over all non-batch dimensions. rr)meanr]ranger0r-)tensorrrr mean_flatsrcCs td|S)z Make a standard normalization layer. :param channels: number of input channels. :return: an nn.Module for normalization. ) GroupNorm32)channelsrrr normalization!s rc@seZdZddZdS)SiLUcCs|t|Sro)rrselfrrrrrg,sz SiLU.forwardN)r}r~rrgrrrrr+s rcseZdZfddZZS)rcst||jSro)superrgrtyper!r __class__rrrg1szGroupNorm32.forward)r}r~rrg __classcell__rrrrr0srcOV|dkr tj|i|S|dkrtj|i|S|dkr$tj|i|Std|)z4 Create a 1D, 2D, or 3D convolution module. rr"unsupported dimensions: )nnConv1dConv2dConv3d ValueErrordimsrLkwargsrrrconv_nd5rcOstj|i|S)z! Create a linear module. )rLinear)rLrrrrr Bsr cOr)z8 Create a 1D, 2D, or 3D average pooling module. rr"rr)r AvgPool1d AvgPool2d AvgPool3drrrrr avg_pool_ndIrrc sxeZdZgdZ  ddededeffdd Zd ejd ejfd d Z ddejdejd e ejd ejfddZ Z S) AlphaBlender)r r r r rrrrearrange_patterncst||_||_||jvsJd|j|jdkr)|dt|gdS|jdks3|jdkrC|dtj t|gdSt d|j)Nzmerge_strategy needs to be in r rr r zunknown merge strategy ) r__init__rr strategiesregister_bufferrr;register_parameterr Parameterr)rrrrrrrrYs      zAlphaBlender.__init__rr cCs|jdkr |j}|S|jdkrt|j}|S|jdkrC|dus$Jdt|tjdd|jdtt|jd}t||j }|St )Nr r r zneed image_only_indicator ...rrr) rrrrrrrrrrr)rrrrrrrss    zAlphaBlender.get_alphaN x_spatial x_temporalcCs2||}||j|d||j|}|S)Ng?)rrr!)rrrrrrrrrrgs zAlphaBlender.forward)r rro) r}r~rrrstrrrr;rrrgrrrrrrVs,r)TF)rr)rF)$__doc__rtypingrrtorch.nnreinopsrrrr;rrr+r7dictrMrtFunctionrCrrrrrrrModuler GroupNormrrr rrrrrrsP   ! $L $