
    fE                         d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ d Zd	Z G d
 d      Z	 	 	 	 	 	 	 	 	 	 	 	 ddZy)    N)time)tqdm)EasyDict)
DataLoader)DistributedSampler)	rearrangec                     | d   S )N).NNNN xs    7/home/cameronsmith/repos/minimal-diffusion/diffusion.py<lambda>r      s    45     gh㈵>c            	       V    e Zd ZdZd
dZddZd Zdi dddddddf	dZdi dddddfd	Zy)GaussianDiffusionzGaussian diffusion process with 1) Cosine schedule for beta values (https://arxiv.org/abs/2102.09672)
    2) L_simple training objective from https://arxiv.org/abs/2006.11239.
    c                      | _         | _         fd _         j                   j                   j                    j                         _        d  _         fd _        d  _        y )Nc                     t        j                  | j                  z  dz   dz  t         j                  z  dz        dz  S )NgMb?gT㥛 ?   )mathcos	timestepspi)tselfs    r   r   z,GaussianDiffusion.__init__.<locals>.<lambda>    s8    dhhDNN 2U :eCdggMPQQRVWW r   c                 &    | j                  dd      S )N   )clampr   s    r   r   z,GaussianDiffusion.__init__.<locals>.<lambda>&   s    !''"a. r   c           	          j                  dt        t        |j                  |   j	                               z   z  | t        d|j                  |   z
  j	                               |z  z
  z        S Nr   )clamp_x0small_constunsqueeze3x	alpha_barsqrt)xtepsr   scalarsr   s       r   r   z,GaussianDiffusion.__init__.<locals>.<lambda>'   s    MM1K8I8I!8L8Q8Q8S,T TUY[^iklov  pA  pA  BC  pD  lD  kJ  kJ  kL  _M  PS  _S  ZS  T  U r   c                    t        |j                  |   j                         |j                  |   z  t        d|j                  |   z
  |j
                  |   j                         z  z   z        |z  t        |j
                  |   |j                  |   z
  t        d|j                  |   z
  |j
                  |   j                         z  z   z        | z  z   S r    )r#   r$   r%   betar"   alpha)r&   x0r   r(   s       r   r   z,GaussianDiffusion.__init__.<locals>.<lambda>+   sG   {W5F5Fq5I5N5N5PSZS_S_`aSb5bgrtux  yJ  yJ  KL  yM  uM  QX  Q^  Q^  _`  Qa  Qf  Qf  Qh  th  hh  5i  (j  mo  (oGMM!,w/@/@/CCUVY`YjYjklYmUmqxq~q~  @A  rB  rG  rG  rI  UI  II  J  K  NP  P(P r   )r   devicealpha_bar_schedulerget_all_scalarsr(   r!   get_x0_from_xt_epsget_pred_mean_from_x0_xt)r   r   r-   s   `  r   __init__zGaussianDiffusion.__init__   sf    "W 	  ++$$dnndkk
 1#
P 	%r   Nc                    i }|qt        j                  t        j                  t	        |      D cg c]#  }t        d ||dz          ||      z  z
  d      % c}            j                  |      |d<   n||d<   t        j                  |d         |d<   d|d   z
  |d<   t        j                  |d   d      |d	<   |d   dd d|d	   dd
 z
  z  d|d	   dd z
  z  |d<   t        j                  |d   dd |d   g      |d<   t        j                  |d         |d<   t        t        |j                         D cg c]  \  }}||j                         f c}}            S c c}w c c}}w )zv
        Using alpha_bar_scheduler, get values of all scalars, such as beta, beta_hat, alpha, alpha_hat, etc.
        Nr   g+?r*   beta_logr+   r   )dimr$   r   
beta_tildebeta_tilde_log)torch
from_numpynparrayrangemintologcumprodcatr   dictitemsfloat)	r   r.   r   r-   betasall_scalarsr   kvs	            r   r/   z!GaussianDiffusion.get_all_scalars/   s    ="'"2"2 "'y!1
 	  3AE :=PQR=S SS!
# b  #(K"'))K,?"@J ;v#66G#(==W1E1#MK #;{+CR002;{+AB//1 	L!
 %*II&q+[-FG%
L! ).		+l2K(L$%;;L;L;NO!Qq!'')nOPQQ32 Ps   (E$<E)c                 $   t        j                  |      }t        | j                  j                  |   j                               |z  t        d| j                  j                  |   z
  j                               |z  z   }|j                         |fS )zSingle step of the forward process, where we add noise in the image.
        Note that we will use this paritcular realization of noise vector (eps) in training.
        r   )r8   
randn_liker#   r(   r$   r%   rD   )r   r,   r   r'   r&   s        r   sample_from_forward_processz-GaussianDiffusion.sample_from_forward_processR   s     r" ..q16689B>1t||55a88>>@ACGH 	
 xxz3r   FTc                    d}	 |}|xs | j                   }t        j                  d| j                   dz
  |dt              }| j                  d   |   }d|t
        j                  j                  j                  |ddgd      d d z  z
  }| j                  | j                  || j                  |      }g }|xr d	|
v }t        d
|       |j                         D ci c]  \  }}|dd  g  }}}t        t        j                  |      d d d   |d d d         D ]  \  }}|r| j!                  t#        |
d	   d      |      \  }}t#        |d      |d<   t#        |d      |d<   | j!                  t#        |
d   d      |      \  }}t#        |d      |d<   t#        |d      |d<   t        j$                  |gt'        t)        |j+                               d         z  | j                        }t        j$                  |gt'        t)        |j+                               d         z  | j                        }|j-                  ||z  |      \  }}|j/                         D ci c]=  }|j1                  dd      |v s|j1                  dd      ||j1                  dd         ? }}|j3                  |j                         D ci c]D  \  }}||r'||   ||   z
  j5                         j7                         nt        j8                  |      F c}}       ||fD  cg c]6  } | j                         D ci c]  \  }}|d|vrt#        |d      n| c}}8 c}}} \  }}|j                         D ci c]-  \  }}|| j;                  ||j1                  dd         |||      / }!}}|!j                         D ci c]-  \  }}|| j=                  ||j1                  dd         |||      / }"}}|dk(  r|"}n|"j/                         D ci c]g  }|j1                  dd      t?        |d   |dz
           jA                         |!|   z  dt?        |d   |dz
           z
  jA                         ||   z  z   i }}||fD  cg c]6  } | j                         D ci c]  \  }}|d|vrt#        |d      n| c}}8 c}}} \  }}|"j                         D ]  \  }}||dd     j3                  |      g   |j                         D ci c]  \  }}|dd  | c}}|j                         D ci c](  \  }}t'        |      s|t        jB                  |d      * c}}|d   j/                         D #ci c]/  }d|z   t        jB                  |D #cg c]
  }#|#|   d     c}#      1 c}#}fS c c}}w c c}w c c}}w c c}}w c c}}} w c c}}w c c}}w c c}w c c}}w c c}}} w c c}}w c c}}w c c}#w c c}#}w )NTr   r   numendpointdtyper$         ?valuer   scene_graphzusing teacher forcing :   b o c -> b 1 c o 1b 1 c o 1 -> b o c noised_scene_grapheps_scene_graphcamerasnoised_cameraseps_camerasr-   noisedr'   rgb   r   epsloss_)"r   r:   linspaceintr(   r8   nn
functionalpadr/   r.   r-   printrC   ziparangerK   r   tensorlenlistvaluesdenoise_scene_graphkeysreplaceappendsquaremean
zeros_liker0   r1   r#   r%   stack)$r   modelxTr   model_kwargsddimrT   rZ   conditioning
use_directmodel_inputteacher_forcingfinalnew_timestepsr$   	new_betasr(   eps_loss_allrG   rH   	intermedsir   xt_scene_graphrY   
xt_camerasr\   	current_tcurrent_sub_t	model_outscene_graph_latentpred_epsilondpred_x0	pred_meanr   s$                                       r    diff_sample_from_reverse_processz2GaussianDiffusion.diff_sample_from_reverse_processb   s    	  /	Q(:	TX`cdLL-m<	)ehh&9&9&=&=i!QWZ&=&[\_]_&``a	&&(@(@)T[[Zcd 'HM[,H'8 (-{{}5!1QR585	5		),TrT2M$B$4GH 	FDAq262R2RS\]hiv]w  yM  TN  PQ  3R/.7G\.]*++4_EZ+['(*.*J*J9U`ajUk  mA  LB  DE  +F'
K*3J?T*U&''0=R'Sm$ aS3tELLN/CA/F+G%GPTP[P[\I!LL!s43G3J/K)KTXT_T_`M,1,E,EelFZ\e,f)I)didndndp  \_`tut}t}  G  HM  uN  R[  u[AIIhu5i		(SX@Y6ZZ  \L  \  CO  CU  CU  CW  !X{~{|}~oE!HYq\$9#A#A#C#H#H#Jchcscstucv!v  !X  Y D  EQ  ~R   S   Sxyijipipir sbebcdeUV9Q/C#D\]!] s   SE, {G  {M  {M  {O  Psvstuvq00%		%8Q2RTUWdfmnn  PG  P CJ  CP  CP  CR  S{~{|}~488%		%PX@Y:Z\]_lnuvv  SI  SAvyu '^^-/ 52{7;CWXehiXiCj7k7p7p7ru|}~u7{7;+?PQ@Q+RSSYY[^jkl^mm8n n / / @E  FR  S   T   Tyzjkjqjqjs tcfcdefVW9Q/D#E]^!^ t   TE, (E!)AabE*:*A*A!*D)EE9	F< %*KKM2SQq12q2R[RaRaRc3n31QgjklgmAekk!A6F4F3nYefgYhYmYmYopTUAekk*N1AaDJ*NOOp  sEF 	F? 6  \ !X s   S P S/ !u   T 33n*Nps   V)V/6(V/>A	V4W 
/V:W 
-2W52WA,W:W
W/W
W%)W+=W+/W6
W1	W6
:W 
W
1W6
c
                    d}	 |j                          |}
|xs | j                  }t        j                  d| j                  dz
  |dt              }| j
                  d   |   }d|t        j                  j                  j                  |ddgd      d d	 z  z
  }| j                  | j                  || j                  |      }t        d
       d\  }}t        |
j                               ddgz   D ci c]  }|dd  g 
 }}t!        t        j"                  |      d d d	   |d d d	         D ]  \  }}t        j$                         5  t        j&                  |gt)        t        |
j+                               d         z  | j                        }t        j&                  |gt)        t        |
j+                               d         z  | j                        }|	 }|dkD  xr | xs |	} ||
|z  |fd|dkD  xr | |dkD  xr | xr dd|ddi}|
j                         D ci c]=  }|j-                  dd      |v s|j-                  dd      ||j-                  dd         ? }}d}|s|
|fD cg c]6  }|j/                         D ci c]  \  }}|d|vrt1        |d      n| c}}8 c}}}\  }
}|j/                         D ci c]-  \  }}|| j3                  |
|j-                  dd         |||      / }}}|j/                         D ci c]-  \  }}|| j5                  |
|j-                  dd         |||      / }}}nm|
j                         D ci c]!  }|d|vrt1        ||dd     d      n||dd     # }
}|
j/                         D ci c]  \  }}|j-                  dd      | }}}|dk(  r|}
n|r|s|j                         D ci c]g  }|j-                  dd      t7        |d   |dz
           j9                         |   z  dt7        |d   |dz
           z
  j9                         ||   z  z   i }
}np|j/                         D ci c]U  \  }}|j-                  dd      |t7        |j:                  |   j9                               t        j<                  |      z  z   W }
}}|rP|
j/                         D ci c]4  \  }}|d|vrt1        |j?                         d      n|j?                         6 }
}}ng|
|fD cg c]R  }|j/                         D ci c]4  \  }}|d|vrt1        |j?                         d      n|j?                         6 c}}T c}}}\  }
}|t        j"                  |      t        j@                  t        j                  dt)        t        j"                  |            dz
  tC        d|                  jE                  t                 v r;t        |       |j/                         D ]  \  }}||dd     jG                  |      g  d d d         |
j/                         D ci c]  \  }}|dd  | c}}|j/                         D ci c](  \  }}t)        |      s|t        jH                  |d      * c}}fS c c}w c c}w c c}}w c c}}}w c c}}w c c}}w c c}w c c}}w c c}w c c}}w c c}}w c c}}w c c}}}w # 1 sw Y   xY wc c}}w c c}}w )NFr   r   TrM   r$   rQ   rR   r   zstarting diff)NNr[   rX   rU   r]   i  i  i  )use_skipsample_rand_globalclip_global_latentr}   r^   r'   r_   rV   rW   
   r`   r   )%evalr   r:   rb   rc   r(   r8   rd   re   rf   r/   r.   r-   rg   rl   ro   rh   ri   no_gradrj   rk   rm   rp   rC   r   r0   r1   r#   r%   r6   rJ   detachroundr=   astyperq   ru   )r   rv   rw   r   rx   ry   rT   rZ   rz   r{   r~   r   r$   r   r(   sgcamrG   r   r   r   r   r   vanilla
direct_rgbr   r   r   rH   r   r   s                                  r   sample_from_reverse_processz-GaussianDiffusion.sample_from_reverse_process   sB    	 	

 /	Q(:	TX`cdLL-m<	)ehh&9&9&=&=i!QWZ&=&[\_]_&``a	&&(@(@)T[[Zcd 	o3%)%**,%79IJ^8_%_`1QR58`	`		),TrT2M$B$4GH D	NDAq CN!LL!s43G3J/K)KTXT_T_`	 %aS3tELLN7KA7N3O-OX\XcXc d &0 cE1'k@j
 "%,"6	 qDefgjejezszoz/0u/JW/JqMYqjoq	 inhrhrht   `cdxy  yB  yB  CK  LQ  yR  V_  y_		(5 9)AIIhW\D]:^ ^   `   ` !
 " GL  MY  FZ  ([  ([  ABqrqxqxqz({jmjklmPU]^P^9Q7K+Lde)e({  ([&E, CO  CU  CU  CW  X{~{|}~q!8!8%		%PX@Y:Z\]_lnu!vv  XG  X KR  KX  KX  KZ  ![  DG  DE  FG4#@#@%		RWX`HaBbdegtv}#~!~  ![I  ![ @E  @J  @J  @L  Mz{1QV^_Q_Yy12'78LMenopqrqsoteuu  ME  M IN V18E!:1!< VI V 6%EJ &/^^%5!7 ! "#5!:{7S^K_`mpq`qKr?s?x?x?z  ~E  FG  ~H  @H#${7;3GXYHY3Z'[#[!a!a!cfrstfu!u@v "v !7 !7 >G__=N!P69a "#5!:AWM_M_`mMnMsMsMu@v#..q1A2 =2 "2 !P !P   BG  BM  BM  BO  &Pz}z{|}aW\deWe	!((*=R(Sklksksku&u  &Pu  &P \a  bn  [o  *p  *p  VW  GH  GN  GN  GP  +Q  @C  @A  BC1\aij\jYqxxzBW-Xpqpxpxpz+z  +Q  *p(eL 		),RXXbkk!SS\I]E^abEbdghjktdu6v-w-~-~  @C  .D  E  E*%(0M!)AabE2B2I2I!2L1MMGCN CND	NR $);;=1CAa!"a1QZQ`Q`Qb2m#!Afijkfl1U[[15E3E2mmmU a0  ` )|  ([ X ![ M !W!7!P &P +Q  *p{CN CNP 22ms   2ZC[&$Z>(Z&[&4Z*Z$)Z*/[&
2Z1
<[&2Z7
[&&Z= [&[
2![&A,[?[&A[
/[&9[
 [&[$9[[#C[&[3([9<[9[&$Z**/[&[[&&[0	)i  zcuda:0)N)	__name__
__module____qualname____doc__r2   r/   rK   r   r   r
   r   r   r   r      sg    
&!RF" $(buRWaesw  DI  VZ  koDFN $(buRWaesw  DIlnr   r   c                 d   g g g df\  }}}}|d   j                  d      }d}t        t        j                  | |
j                  |z  z              5 }|| k  r9|j                         D ci c]S  \  }}d|z   t        j                  |g|j                  dd  j                         j                  |
j                        U }}}d}|j                  |||d|i|
j                  ||||		      \  }}|
j                  rQ|g}|j                  t        j                   |      j#                         j%                         j'                                |t)        t+        |j-                               d         |z  z  }|j/                  d       || k  r9ddd       fS c c}}w # 1 sw Y   fS xY w)
a  use this function to sample any number of images from a given
        diffusion model and diffusion process.

    Args:
        N : Number of images
        model : Diffusion model
        diffusion : Diffusion process
        xT : Starting instantiation of noise vector.
        sampling_steps : Number of sampling steps.
        batch_size : Batch-size for sampling.
        num_channels : Number of channels in the image.
        image_size : Image size (assuming square images).
        num_classes : Number of classes in the dataset (needed for class-conditioned models)
        args : All args from the argparser.

    Returns: Numpy array with N images and corresponding labels.
    r   r_   r   r   )totalnoised_Ny)rT   rZ   rz   r{   )sizer   r   ceil
batch_sizerC   r8   randnshaperD   r>   r-   r   ry   
class_condrq   rA   r   cpunumpyrk   rl   rm   updater:   concatenate	transposer   uint8)Nrv   	diffusionxT_sampling_stepsr   num_channels
image_sizen_framesnum_classesargsrT   rZ   rz   r{   samplesintermed_sampleslabelsnum_samplesnum_processespbarrG   rH   rw   r   
gen_imagesgen_intermedslabels_lists                               r   sample_N_imagesr     s   D 68R]2Gv{5zr"JM	DIIa4??]#BCD	E Ao knjsjsjuvcfcdef)A+u{{:DDKKMQQRVR]R]^^vBv A'0'L'LeUWYgjmopiqsws|s|  JU  ^e  s  LV'L  (W$J}  !ceii4;;=AACIIKL 3tBIIK034}DDKKKN' Ao* m### w	* m##s    F#+AFCF#F##F/)N   @          r   NNFNNF)cv2copyr   argparser   r:   r   r   easydictr   wandbr8   torch.distributeddistributeddisttorch.utils.datar   torch.utils.data.distributedr   einopsr   r#   r"   r   r   r
   r   r   <module>r      sv    
            ' ;  6}n }nD 		>\r   