\[fryddlmZddlZddlZddlZddlmZddl mcm Z GddejZ dZdZdZd'dZd Zd Zd(d Zd ZGd dej,j.ZGddej2ZGddej2ZGddej8eZGddej2ZGddej2ZGddeZ Gddej2Z!dZ"Gddej2Z#Gd d!ej2Z$Gd"d#ej2Z% d)d$Z& d*d%Z' d+d&Z(y),)abstractmethodNceZdZfdZxZS) GroupNorm32cpt||jj|jSN)superforwardfloattypedtype)selfx __class__s 3/home/cameronsmith/repos/minimal-diffusion/unets.pyr zGroupNorm32.forward s'wqwwy)..qww77)__name__ __module__ __qualname__r __classcell__rs@rrr s 88rrc|dk(rtj|i|S|dk(rtj|i|S|dk(rtj|i|St d|)z4 Create a 1D, 2D, or 3D convolution module. unsupported dimensions: )nnConv1dConv2dConv3d ValueErrordimsargskwargss rconv_ndr%sh qyyy$)&)) yy$)&)) yy$)&)) /v6 77rc,tj|i|S)z! Create a linear module. )rLinear)r#r$s rlinearr(s 99d %f %%rc|dk(rtj|i|S|dk(rtj|i|S|dk(rtj|i|St d|)z8 Create a 1D, 2D, or 3D average pooling module. rrrr)r AvgPool1d AvgPool2d AvgPool3dr r!s r avg_pool_ndr-$sh qy||T,V,, ||T,V,, ||T,V,, /v6 77rct||D]8\}}|jj|j|d|z :y)a# Update target parameters to be closer to those of source parameters using an exponential moving average. :param target_params: the target parameter sequence. :param source_params: the source parameter sequence. :param rate: the EMA rate (closer to 1 means slower). r)alphaN)zipdetachmul_add_) target_params source_paramsratetargsrcs r update_emar91sF 6; c 4 %%cT%:;rcl|jD] }|jj"|S)z< Zero out the parameters of a module and return it. ) parametersr1zero_)moduleps r zero_moduler?>s3      Mrctd|S)z Make a standard normalization layer. :param channels: number of input channels. :return: an nn.Module for normalization. )r)channelss r normalizationrCGs r8 $$rc |dz}tjtj| tjd|tj z|z j |j}|dddfj|dz}tjtj|tj|gd}|dzr5tj|tj|ddddfgd}|S) aY Create sinusoidal timestep embeddings. :param timesteps: a 1-D Tensor of N indices, one per batch element. These may be fractional. :param dim: the dimension of the output. :param max_period: controls the minimum frequency of the embeddings. :return: an [N x dim] Tensor of positional embeddings. rr)startendr )deviceNdimr) thexpmathlogarangefloat32torGr catcossin zeros_like) timestepsrJ max_periodhalffreqsr# embeddings rtimestep_embeddingr[Qs !8D FF *  t2:: NNQUU b   b!  QW  # # %d 3Dt bffTl3N  ? kk&&     0 0 0  '     k))!Y G  ? ?s#&CC C8CCC(N)rrr staticmethodr rzrrr^r^xs(**rr^c >eZdZdZ ddedededeffd ZdZxZS) AttentionPool2dzS Adapted from CLIP: https://github.com/openai/CLIP/blob/main/clip/model.py spacial_dim embed_dimnum_heads_channels output_dimc8t|tjt j ||dzdz|dzz |_td|d|zd|_td||xs|d|_ ||z|_ t|j|_ y)Nrrg?r) r__init__r ParameterrKrandnpositional_embeddingr%qkv_projc_proj num_heads QKVAttention attention)r rrrrrs rrzAttentionPool2d.__init__s $&LL HHY q 01 4 5 S8H H% ! 9a)mQ? aJ,C)QG "&88%dnn5rc||j^}}}|j||d}tj|j dd|gd}||j dddddfj |jz}|j|}|j|}|j|}|dddddfS)NrHT)rJkeepdimrIr) shapereshaperKrRmeanrrQr rrr)r rbc_spatials rr zAttentionPool2d.forwards''1x IIaB  FFAFFr4F0!4" = ))$1*588A A MM!  NN1  KKNAqzrr)rrr__doc__intrr rrs@rr~r~s= 666 6  6 rr~c eZdZdZedZy) TimestepBlockzT Any module where forward() takes timestep embeddings as a second argument. cy)zJ Apply the module to `x` given `emb` timestep embeddings. Nr|r rembs rr zTimestepBlock.forwardsrN)rrrrrr r|rrrrs  rrceZdZdZdZy)TimestepEmbedSequentialzt A sequential module that passes timestep embeddings to the children that support it as an extra input. cX|D]$}t|tr |||}||}&|Sr) isinstancer)r rrlayers rr zTimestepEmbedSequential.forwards6 E%/!SM!H   rN)rrrrr r|rrrrs  rrc*eZdZdZdfd ZdZxZS)UpsampleaA An upsampling layer with an optional convolution. :param channels: channels in the inputs and outputs. :param use_conv: a bool determining if a convolution is applied. :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then upsampling occurs in the inner-two dimensions. ct|||_|xs||_||_||_|r*t ||j|jdd|_yy)Nrrpadding)rrrB out_channelsuse_convr"r%conv)r rBrr"rrs rrzUpsample.__init__sW   (4H   dmmT5F5FSTUDI rc|jd|jk(sJ|jdk(rItj||jd|jddz|jddzfd}ntj|dd}|jd|jd cxk(rdk(rnntj |d }|j r|j|}|S) Nrrrnearest)mode) scale_factorrrH)rrrr)rrBr"F interpolatepadrr)r routs rr zUpsample.forwardswwqzT]]** 99>--AGGAJ Q Q?iC-- BC 772;!''"+ * *%%\*C ==))C.C rrNrrrrrr rrs@rrrsV rrc*eZdZdZdfd ZdZxZS) DownsampleaD A downsampling layer with an optional convolution. :param channels: channels in the inputs and outputs. :param use_conv: a bool determining if a convolution is applied. :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then downsampling occurs in the inner-two dimensions. c*t|||_|xs||_||_||_|dk7rdnd}|r+t ||j|jd|d|_y|j|jk(sJt||||_y)Nrr)rrrr)strider) kernel_sizer) rrrBrrr"r%opr-)r rBrr"rrrs rrzDownsample.__init__s   (4H   aiY dmmT%6%6&RSDG==D$5$55 5!$F6JDGrc`|jd|jk(sJ|j|S)Nr)rrBrr rs rr zDownsample.forward s'wwqzT]]**wwqzrrrrs@rrrs Krrc>eZdZdZ dfd ZdZdZxZS)ResBlocka A residual block that can optionally change the number of channels. :param channels: the number of input channels. :param emb_channels: the number of timestep embedding channels. :param dropout: the rate of dropout. :param out_channels: if specified, the number of out channels. :param use_conv: if True and out_channels is specified, use a spatial convolution instead of a smaller 1x1 convolution to change the channels in the skip connection. :param dims: determines if the signal is 1D, 2D, or 3D. :param use_checkpoint: if True, use gradient checkpointing on this module. :param up: if True, use this block for upsampling. :param down: if True, use this block for downsampling. c t |||_||_||_|xs||_||_||_||_tjt|tjt|||j dd|_| xs| |_| r%t!|d||_t!|d||_nG| r%t'|d||_t'|d||_n tj(x|_|_tjtjt+||rd|j zn |j |_tjt|j tjtj.|t1t||j |j dd|_|j |k(rtj(|_y|r t|||j dd|_yt|||j d|_y)NrrrFr)r>)rrrB emb_channelsdropoutrruse_checkpointuse_scale_shift_normr SequentialrCSiLUr% in_layersupdownrh_updx_updrIdentityr( emb_layersDropoutr? out_layersskip_connection) r rBrrrrrr"rupdownrs rrzResBlock.__init__!s   ( (4H  ,$8! ( # GGI D(D$5$5q! D  jD !(E48DJ!(E48DJ #HeT:DJ#HeT:DJ&(kkm 3DJ-- GGI )=D%%%4CTCT  -- $++ , GGI JJ ! d//1B1BAqQ       (#%;;=D #*h 1 11a$D $+44;L;La#PD rcft|j||f|j|jS)a Apply the block to a Tensor, conditioned on a timestep embedding. :param x: an [N x C x ...] Tensor of features. :param emb: an [N x emb_channels] Tensor of timestep embeddings. :return: an [N x C x ...] Tensor of outputs. )re_forwardr;rrs rr zResBlock.forwardas/ MMAs8T__%68K8K  rc|jrQ|jdd|jd}}||}|j|}|j|}||}n|j|}|j |j |j }t|jt|jkr1|d}t|jt|jkr1|jrS|jd|jdd}}tj|dd\} } ||d| zz| z}||}n||z}|j|}|j||zS)NrH).NrrrrI)rrrrrr r r`rrrrKchunkr) r rrin_restin_convhemb_outout_normout_restscaleshifts rrzResBlock._forwardlsD ;;#~~cr2DNN24FWG A 1 A 1 A Aq!A//#&++AGG4'-- 3qww</i(G'-- 3qww</  $ $!%!3T__QR5HhH88GQA6LE5 q5y)E1A AG A"A##A&**r)NFFrFFFrrrrrr rrrs@rrrs0 ("   >Q@  +rrc8eZdZdZ dfd ZdZdZxZS)AttentionBlocka An attention block that allows spatial positions to attend to each other. Originally ported from here, but adapted to the N-d case. https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/models/unet.py#L66. ct|||_|dk(r||_n||zdk(s Jd|d|||z|_||_t ||_td||dzd|_|rt|j|_ nt|j|_ ttd||d|_ y)NrHrzq,k,v channels z' is not divisible by num_head_channels rr)rrrBrrrCnormr%qkvrrQKVAttentionLegacyr?proj_out)r rBrnum_head_channelsruse_new_attention_orderrs rrzAttentionBlock.__init__s    "&DN,,1 f  *QRcQde f&)::DN,!(+ 1h1 a8 ")$..9DN0?DN#GAx1$EF rcPt|j|f|jdS)NT)rerr;rs rr zAttentionBlock.forwards $--!t/@$GGrc|j^}}}|j||d}|j|j|}|j |}|j |}||zj||g|S)NrH)rrrrrr)r rrrspatialrrs rrzAttentionBlock._forwardsq1w IIaB hhtyy|$ NN3  MM! Aq!.g..r)rrHFFrrs@rrrs' % G:H/rrc|dj^}}}ttj|}d|z|dzz|z}|xjt j |gz c_y)a( A counter for the `thop` package to count the operations in an attention operation. Meant to be used like: macs, params = thop.profile( model, inputs=(inputs, timestamps), custom_ops={QKVAttention: QKVAttention.count_flops}, ) rrN)rrnpprod total_opsrK DoubleTensor)model_xyrrr num_spatial matmul_opss rcount_flops_attnrs]qTZZNAq7bggg&'KQ+*+a/J OOr |44Orc8eZdZdZfdZdZedZxZS)rzh A module which performs QKV attention. Matches legacy QKVAttention + input/ouput heads shaping c0t|||_yrrrn_headsr rrs rrzQKVAttentionLegacy.__init__  rcD|j\}}}|d|jzzdk(sJ|d|jzz}|j||jz|dz|j|d\}}}dt j t j |z } t jd|| z|| z} t j| jdj| j} t jd| |} | j|d|S)z Apply QKV attention. :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs. :return: an [N x (H * C) x T] tensor after attention. rrrrI bct,bcs->btsrH bts,bcs->bct) rrrsplitrMsqrtrKeinsumsoftmaxr r r r rbswidthrnchqkvrweightas rr zQKVAttentionLegacy.forwards  IIE6DLL()Q.. q4<<' (++b4<</a@FFrqFQ1aDIIdiim,, AIq5y FLLN388F IInfa 0yyR((rct|||Srrrrrs r count_flopszQKVAttentionLegacy.count_flopsr1--r rrrrrr r{rrrs@rrrs&)$..rrc8eZdZdZfdZdZedZxZS)rzP A module which performs QKV attention and splits in a different order. c0t|||_yrrrs rrzQKVAttention.__init__rrc |j\}}}|d|jzzdk(sJ|d|jzz}|jdd\}}}dtjtj|z } t j d|| zj||jz|||| zj||jz||} t j| jdj| j} t j d| |j||jz||} | j|d|S)z Apply QKV attention. :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs. :return: an [N x (H * C) x T] tensor after attention. rrrrIrrHr) rrrrMrrKrviewrr r r rrs rr zQKVAttention.forwards$  IIE6DLL()Q.. q4<<' ())A1)%1aDIIdiim,,  Y  R$,,.F ; Y  R$,,.F ;  FLLN388F IInfaiiT\\8I2v.V WyyR((rct|||Srr r s rrzQKVAttention.count_flopsrrrrs@rrrs&)(..rrcHeZdZdZ dfd ZddZxZS) UNetModela The full UNet model with attention and timestep embedding. :param in_channels: channels in the input Tensor. :param emb_dim: base dimension of timestep embedding. :param model_channels: base channel count for the model. :param out_channels: channels in the output Tensor. :param num_res_blocks: number of residual blocks per downsample. :param attention_resolutions: a collection of downsample rates at which attention will take place. May be a set, list, or tuple. For example, if this contains 4, then at 4x downsampling, attention will be used. :param dropout: the dropout probability. :param channel_mult: channel multiplier for each level of the UNet. :param conv_resample: if True, use learned convolutions for upsampling and downsampling. :param dims: determines if the signal is 1D, 2D, or 3D. :param num_classes: if specified (as an int), then this model will be class-conditional with `num_classes` classes. :param use_checkpoint: use gradient checkpointing to reduce memory usage. :param num_heads: the number of attention heads in each attention layer. :param num_heads_channels: if specified, ignore num_heads and instead use a fixed channel width per attention head. :param num_heads_upsample: works with num_heads to set a different number of heads for upsampling. Deprecated. :param use_scale_shift_norm: use a FiLM-like conditioning mechanism. :param resblock_updown: use residual blocks for up/downsampling. :param use_new_attention_order: use a different attention pattern for potentially increased efficiency. ct!||dk(r|}||_||_||_||_||_||_||_| |_ | |_ | |_ | |_ |rtjntj|_||_||_||_||z}t)j*t-||t)j.t-|||_|jt)j2| ||_t7| d|zx}}t)j8t;t=| ||ddg|_||_ |g}d}tC| D]4\}}tE|D]}tG|||t7||z| | |g}t7||z}||vr|jItK|| ||||j>jIt;||xj@|z c_ |jI||tM| dz k7s|}|j>jIt;|rtG||||| | |d ntO|| | | |}|jI||d z}|xj@|z c_ 7t;tG|||| | | tK|| |||tG|||| | | |_(|xj@|z c_ t)j8g|_)tUtC| dddD]\}}tE|dzD]}|jW} tG|| z||t7||z| | |g}t7||z}||vr|jItK|| ||||r?||k(r:|}|jI|rtG||||| | |d ntY|| | | |d z}|jRjIt;||xj@|z c_ t)j*t[|t)j.t]t=| ||dd|_/y)NrHrrrr)rr"rr)rrrrT)rr"rrr)r"rr)r"rr)rr"rrr)0rr image_size in_channelsmodel_channelsrnum_res_blocksattention_resolutionsr channel_mult conv_resample num_classesrrKfloat16rPr rrnum_heads_upsamplerrr(r time_embed Embedding label_embr ModuleListrr% input_blocks _feature_size enumeraterangerappendrr`r middle_block output_blocksripoprrCr?r)"r rrrrrrtime_emb_factorrrrr"r ruse_fp16rrr"rresblock_updownrtime_embed_dimrinput_chinput_block_chansdslevelmult_layersout_chiichrs" rrzUNetModel.__init__(s.   #!* $&,(,%:" (*&,#+RZZ "!2"4'/9-- >> 2 GGI >> 2     '\\+~FDNLOn<==XMM $WT;Aq%Q R S  D $\22 )KE4>* -&%()>%?!'5-A ./..MM&+9&/.?4K !!(()@&)IJ""b("!((,3 -4L)A--!!((++!*#)/!%+91E!% ( Dv$!((,a""b("e2 )h4 -%9   -#"3(?   -%9  ! 2 b ]]2. , 782>+ )KE4>A-.* )'++-S&%($)>%?!'5-A $./..MM&+9&8.?4K Q.0FMM+!*#)/!%+91E# &b-dQWX 1HB""))*A6*JK""b("U* )+ )Z== "  GGI h aK L rc|du|jduk(sJdg}|jt||j}|j3|j|jdfk(sJ||j |z}|j |j}|jD]}|||}|j||j||}|jD]2}tj||jgd}|||}4|j |j}|j|S)a Apply the model to an input batch. :param x: an [N x C x ...] Tensor of inputs. :param timesteps: a 1-D batch of timesteps. :param y: an [N] Tensor of labels, if class-conditional. :return: an [N x C x ...] Tensor of outputs. Nzr`sQ 8",,8 8& 8 ;%*$*--*>bii@  BII   bmm] ryyD:o+}o+d-/RYY-/`5(..@.299.Dz z~ ,b ,b -r