
    [Qf!5                     8   d dl Z d dlmZ d dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlZd ZddZd Z G d dej                        Zd	 Z G d
 dej                        Z G d dej                        Z G d dej                        Z G d dej                        Zy)    N)
functional)	rearrangerepeatc                     t        | d      S )Nz... c x y -> ... (x y) c)r   xs    4/home/cameronsmith/repos/onestep_diffusion/models.py<lambda>r
      s    i"<=     c           	      f    t        | d|"t        | j                  d      dz              S |      S )Nz... (x y) c -> ... c x y      ?r   )r   intsize)srcr   s     r	   r
   r
      s2    i,FdedmSXXVX\\^M_I`u stu r   c           	      6   d }g }t        t        |       dz
        D ]S  }|j                  t        j                  | |   | |dz                   |j                  t        j
                                U t        j                  |d d  }|j                  |       |S )Nc                     t        |       t        j                  k(  r;t        | d      r.t        j                  j                  | j                  ddd       y y y )Nweightg        relufan_in)anonlinearitymode)typennLinearhasattrinitkaiming_normal_r   )ms    r	   init_weights_normalz%make_net.<locals>.init_weights_normal   sF    7biiq(#''CfS['\ $  r      )rangelenappendr   r   ReLU
Sequentialapply)dimsr!   layersinets        r	   make_netr.      s    ] F3t9Q; !biiQQqS	23bggi ! --
%CII!"Jr   c                   6     e Zd Zd fd	ZddZd ZddZ xZS )MyDDPMc                    t         t        |           d||f}|| _        || _        || _        || _        t        j                  |||      j                         | _
        d| j                  z
  | _        t        j                  t        t        | j                              D cg c]'  }t        j                  | j                  d |dz          ) c}      j                         | _        y c c}w Nr"   )superr0   __init__n_stepsdevice	image_chwnetworktorchlinspacecudabetasalphastensorr$   r%   prod
alpha_bars)
selfr8   r5   min_betamax_betar6   imslr7   r,   	__class__s
            r	   r4   zMyDDPM.__init__   s    fd$&dD/	"^^Hh@EEG
$**n,,QVWZ[_[f[fWgQh'iA

4;;vA3F(G'ijooq'is   ",C)c                 
   |j                   \  }}}}| j                  |   }|1t        j                  ||||      j	                  | j
                        }d|z
  j                         j                  |ddd      |z  }	||	z   }
|
|	fS r2   )shaper@   r9   randntor6   sqrtreshape)rA   x0tetanchwa_bareta_noisys              r	   forwardzMyDDPM.forward)   s    XX
1a";++aAq),,T[[9C %i''1a3c9T	Tzr   c                 &    | j                  ||      S N)r8   )rA   r   rM   s      r	   backwardzMyDDPM.backward6   s     ||Aq!!r   c           
      >   |x}}	t        j                  d| j                  |      j                  t         j                        }
g }g }t        j                         5  t        j                  ||||	      j                         }t        t        t        t        | j                              ddd               d   \  }}t        j                  |d      |z  j                         j                         }| j                  ||      }||z
  cddd       S # 1 sw Y   yxY w)zjGiven a DDPM model, a number of samples to be generated and a device, returns some newly generated samplesr   Nr#   r"   )npr:   r5   astypeuintr9   no_gradrH   r;   list	enumerater$   oneslongrY   )rA   	n_samplesr6   frames_per_gifgif_namerP   rD   	just_lastrQ   rR   
frame_idxsframesxsr   idxrM   time_tensor	noise_ests                     r	   generate_new_imageszMyDDPM.generate_new_images;   s    ![[DLL.AHHQ
]]_ 	Iq!Q/446A)Dt||)<$=dd$CDEaHFC ::i3a7==?DDFK}}Q4IY;	 	 	s   B.DD)   g-C6?g{Gz?N   rX   )   Nd   zsampling.gifr"   ro   F)__name__
__module____qualname__r4   rV   rY   rm   __classcell__rE   s   @r	   r0   r0      s    	r"
r   r0   c           	         t        j                  | |      }t        j                  t        |      D cg c]  }ddd|z  |z  z  z   c}      }|j	                  d|f      }t        j
                  |       j	                  | df      }t        j                  ||d d d d df   z        |d d d d df<   t        j                  ||d d d d df   z        |d d dd df<   |S c c}w )Nr"   i'     )r9   zerosr>   r$   rK   arangesincos)rO   d	embeddingjwkrM   s         r	   sinusoidal_embeddingr   L   s    Aq!I	%(CQq6a!eai00C	DB	QF	BQA'AyyR#A#Y/Ia!e		!b3Q3i-0Ia1f Ds   C"c                   &     e Zd Zd fd	Zd Z xZS )
MySceneRepc                 \   t         t        |           ||f}t        j                  ||      | _        t        ||      | j
                  j                  _        | j
                  j                  d       |dddg}t        j                  t        d      D cg c]u  }t        j                  t        d      D cg c]N  }t        ||t        |dkD        z      |d   d|z  z  |d   d|z  z  f||t        |dkD        z      ||dz            P c} w c}}      | _        t        j                  t        j                          g|dd	 D 	cg c]  }	t        j"                  |	|	d	dd       c}	z         | _        t        j                  ||d d d
   D 
cg c]  }
|
dz  	 c}
z   D 
cg c]S  }
t        j                  t        j&                  ||
      t        j(                         t        j&                  |
|
            U c}
      | _        t        j                  t        d	      D cg c]"  }t-        j.                  |d
   d	|d
   dz        $ c}      j1                         | _        t        j                  d|d
         | _        t        j                  |d   dz  |d   dz  z  |d
         | _        |d   g|dd  z   }t        j                  dD cg c]  }t        j                  t        d      D cg c][  }t        d||t        |dk(        z      z  |d   d|z  z  |d   d|z  z  f||t        |dk(        z      dz  ||   |dk  rdndz        ] c}  c}}      | _        t        j"                  |d   |ddd      | _        y c c}w c c}}w c c}	w c c}
w c c}
w c c}w c c}w c c}}w )NF@            r   rx   r"      r#      )rx   r"   r   )r3   r   r4   r   	Embedding
time_embedr   r   datarequires_grad_
ModuleListr$   r(   MyBlockr   conv_blocks_downIdentityConv2d
down_convsr   SiLUtime_embeddingsattn_modules
CrossAttn_r;   
self_attnsscene_tokensspatial_embconv_blocks_upconv_out)rA   r5   time_emb_dimrD   in_chresr*   r,   r   dimr}   _dims_rE   s                r	   r4   zMySceneRep.__init__X   s   j$(*$K ,,w=&:7L&Q#&&u- bS! "
 1X/
 	 MM#Ah( T!C!H*-A1c!fadlKTRSTWXYZ[X[T\R\M]_cdefgdg_hi( / !  --aefghiaj8kZ]3QPQST9U8k(kl  "}}  ~B  QU  VZ  XZ  VZ  Q[  C\  LM  DE  FG  DG  C\  ~\  .]xybmmBIIlTU<VXZX_X_Xaceclclmnpqcr.s  .]   ^ --bghibj(k]^)@)@b!DQSHVWK)X(klqqsLLDH5<<QA(EtBxP AwiQR  mm
 -
 	 MM#Ah( QuQs1a4y[11CFAqDL#a&!Q$,OQVWXY\]^`a]aYbWbQcdeQeglmngovwxyvyqr  @A  hB  C( -  		$q'5!Q:/(/ 9l C\  .] )l(-sK   !N	
7AN
N	
 N
N
"AN"'N!N(
/A N#N(
N	
#N(
c                 v   | j                  |      }t        |      }|g}t        | j                  | j                  | j
                        D ]A  \  }}}|j                   | ||d          ||      j                  |ddd      z                C t         | j                  d   |d                | j
                  d   |      z   | j                  j                  d    z   }t        j                  || j                  j                  d    j                  |dd      fd      }| j                  D ]  }	 |	||      } t!        |d d d | j                  j                  j#                  d       f   |d   j#                  d      dz        }|}
t        | j$                  |d d d   | j
                  dd        D ]e  \  }}}t        j                  |t'        j(                  |
|j*                  dd  d	      fd
      } || ||      j                  |ddd      z         }
g | j-                  |
      j/                         S )Nr#   r"   r   r   rx   r   r   bilinear)r   r   )r   r%   zipr   r   r   r&   rK   ch_secr   r   r9   catr   expandr   ch_fstr   r   FinterpolaterG   r   tanh)rA   r   rM   rO   xs_down
conv_block	down_convtime_emb	mid_feats	self_attn	curr_featx_prevcat_feats                r	   rV   zMySceneRep.forward   s1   OOAF /243H3HY]YmYm/n 	a+J	8NNJy'=@S@STUVXYZ[\@]']^`	a -4??1-gbk:;>Ud>R>RST>UVW>XX[_[k[k[r[rsw[xx	IIy):):)A)A$)G)N)NqQSTV)WXYZ[	TIi	)6T)T9Q'I):):)A)A)F)Fq)I(I'I%IJGTVKL\L\]_L`bcLcd	 	*-d.A.A'$B$-PTPdPdefegPh*i 	K&Jvhyy&!--	&,,rsBSYc*d!eklmH"8HQK,?,?"Qq,I#IJI	K }}Y',,..r   )  rq   ro   r"   rr   rs   rt   r4   rV   ru   rv   s   @r	   r   r   W   s    %;N/r   r   c                   &     e Zd Zd fd	Zd Z xZS )MySceneRep_c                    t         t        |           d}t        j                  ||      | _        t        ||      | j
                  j                  _        | j
                  j                  d       t        j                  t        d      D cg c]  }t        j                  d|r|nd       c}      | _        t        j                  d|      | _        t        j                  t        d      D cg c]  }t#        j$                  |d|d	z         c}      j'                         | _        t        j                  d|      | _        t        j                  t        d	      D cg c]#  }t        j,                  |d
k(  rdn||dd	d      % c}      | _        t        j                  d|      | _        t3        g d      | _        t3        g d      | _        y c c}w c c}w c c}w )Nr   F   rq   r   	   )r   fdim_outr   rx   r   r"   r   )paddingi   )r   r   r       )r   r   r   r"   )r3   r   r4   r   r   r   r   r   r   r   r   r$   r   	time_linsconv_modulesPixelNeRFEncoderconv_encr   r   r;   r   r   r   r   r   r.   
comb_featsimg_dec)rA   r5   r   fdimr,   r   rE   s         r	   r4   zMySceneRep_.__init__   sa   j$(* ,,w=&:7L&Q#&&u- QVWXQY'ZA		#adQ(G'Z[$55CN--Z_`aZb(cUV)@)@aa)P(cdiikLLD1--dijkdl(m_`11$tAaXY)Z(mn<<6"?3- ([(c(ms   !G2!G$(G"c                 P   | j                  |      j                  d      } | j                  d   |      d   j                  dd|j	                  d      |j	                  d            }| j                  t        j                  ||fd            }t        j                  |d      }t        j                  t        | j                  j                        d    |j                  dd        }t        j                  t        ||z    | j                  d   |      d   z         | j                  j                  d    j                  t!        |      dd      fd      }t#        | j$                        D ].  \  }	}
 |
||       | j                  |	dz      |      d d d f   z   }0 t        |d d d | j                  j                  j	                  d       f   |j	                  d            }t        t        j                  t        j                  ||j                  dd        |gd            }t        | j'                  |      |j	                  d            }t        | j)                  t        t        j                  ||j                  dd                    |j	                  d            }|S )Nr"   r   ).NNr#   r   r   )scale_factor)r   squeezer   r   r   r   r9   r   r   r   r   r   r   rG   r   r   r%   r`   r   r   r   )rA   r   rM   t_embtime_emb_hires
conv_featsfeats2pos_emb_lowfeats3r,   attn
low_and_hifeats4decs                 r	   rV   zMySceneRep_.forward   sB   "**1-(t~~a(/>EEbAFFSUJWXW]W]^`Wab]]599a-?#BC

 zr:mmF4+;+;+B+B$CD$I&,,WYWZJ[\		6&"45FT^^A5Fu5Mm5\"\],,33D9@@Q2NPPQS0qFAd4v;NObt~~^_`a^aObchOijklpjpOq;q&qva B$"3"3":":"?"?"B!B BBCFKKPROT EIIq}}VJ<L<LRS<Q'RS]&^_`ab

3JOOB4GHT\\&vaggbcl)K"LMaffUWjY
r   r   rq   r   rv   s   @r	   r   r      s    .*r   r   c                   (     e Zd Zd fd	ZddZ xZS )r   c
                 ,   t         t        |           t        j                  |      | _        |	| _        t        j                  |||||      | _        t        j                  |||||      | _	        |t        j                         n|| _        || _        y rX   )r3   r   r4   r   	LayerNormlnlast_actr   conv1conv2r   
activation	normalize)rA   rG   in_cout_ckernel_sizestrider   r   r   r   rE   s             r	   r4   zMyBlock.__init__   sr    gt%',,u%YYtUKI
YYue[&'J
'1'9"'')z"r   c                     | j                   r| j                  |      n|}| j                  |      }| j                  |      }| j	                  |      }| j
                  r| j                  |      }|S rX   )r   r   r   r   r   r   )rA   r   r   outs       r	   rV   zMyBlock.forward   sX     NNdggajjjoooc"jjo== 4#
r   )r   r"   r"   NTT)Tr   rv   s   @r	   r   r      s    #r   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )MyUNetc                    t         t        |           t        j                  ||      | _        t        ||      | j
                  j                  _        | j
                  j                  d       | j                  |d      | _        t        j                  t        ddd      t        ddd      t        ddd            | _        t        j                  ddddd      | _        | j                  |d      | _        t        j                  t        ddd	      t        d
d	d	      t        d
d	d	            | _        t        j                  d	d	ddd      | _        | j                  |d	      | _        t        j                  t        dd	d      t        ddd      t        ddd            | _        t        j                  t        j                  dddd      t        j,                         t        j                  ddddd            | _        | j                  |d      | _        t        j                  t        ddd	      t        dd	d	      t        dd	d            | _        t        j                  t        j4                  ddddd      t        j,                         t        j4                  dddd            | _        | j                  |d      | _        t        j                  t        ddd      t        ddd	      t        dd	d	            | _        t        j4                  d	d	ddd      | _        | j                  |d      | _        t        j                  t        ddd	      t        d
d	d      t        ddd            | _         t        j4                  ddddd      | _!        | j                  |d	      | _"        t        j                  t        dd	d      t        ddd      t        dddd            | _#        t        j                  ddddd      | _$        y )NFr"   )r"   ro   ro   
   )r   ro   ro   r   rx   )r      r      )r   r   r   )r      r   (   )r   r   r   )r   r   r   )r   r   r   P   )r   r   r   )r   r   r   )r   ro   ro   )r   r   )%r3   r   r4   r   r   r   r   r   r   r   _make_tete1r(   r   b1r   down1te2b2down2te3b3r   down3te_midb_midConvTranspose2dup1te4b4up2te5b5up3te_outb_outr   )rA   r5   r   rE   s      r	   r4   zMyUNet.__init__   sM   fd$& ,,w=&:7L&Q#&&u- ==q1--KB'L"b)L"b)

 YYr2q!Q/
==r2--L"b)L"b)L"b)

 YYr2q!Q/
==r2--JB'JB'JB'

 ]]IIb"a#GGIIIb"aA&

 mmL"5]]JB'JB'JB'

 ==r2q!Q/GGIr2q!,
 ==r2--JB'JB'JB'
 %%b"aA6==r2--L"b)L"b)L"b)
 %%b"aA6mmL"5]]L"b)L"b)L"bE:

 		"aAq1r   c           	         | j                  |      }t        |      }| j                  || j                  |      j	                  |ddd      z         }| j                  | j                  |      | j                  |      j	                  |ddd      z         }| j                  | j                  |      | j                  |      j	                  |ddd      z         }| j                  | j                  |      | j                  |      j	                  |ddd      z         }t        j                  || j!                  |      fd      }| j#                  || j%                  |      j	                  |ddd      z         }t        j                  || j'                  |      fd      }	| j)                  |	| j+                  |      j	                  |ddd      z         }	t        j                  || j-                  |	      fd      }
| j/                  |
| j1                  |      j	                  |ddd      z         }
| j3                  |
      }
|
S )Nr#   r"   r   )r   r%   r   r   rK   r   r   r   r   r   r   r   r   r   r9   r   r   r   r   r   r  r   r  r  r  r   )rA   r   rM   rO   out1out2out3out_midout4out5r   s              r	   rV   zMyUNet.forward$  s   OOAFwwq488A;..q"a;;<wwtzz$'$((1+*=*=aQ*JJKwwtzz$'$((1+*=*=aQ*JJK**TZZ-A0F0Fq"aQR0SSTyy$ 12:wwtdhhqk11!RA>>?yy$/Q7wwtdhhqk11!RA>>?iitxx~.A6jjt{{1~55aQBBCmmC 
r   c                     t        j                  t        j                  ||      t        j                         t        j                  ||            S rX   )r   r(   r   r   )rA   dim_indim_outs      r	   r   zMyUNet._make_te;  s8    }}IIfg&GGIIIgw'
 	
r   r   )rr   rs   rt   r4   rV   r   ru   rv   s   @r	   r   r      s    K2Z.
r   r   rX   )r9   torch.nnr   numpyr[   r   r   r   einopsr   r   r   r   r   r.   Moduler0   r   r   r   r   r    r   r	   <module>r     s        $ $  >u-RYY -^	=/ =/~,")) ,\bii $j
RYY j
r   