
    Ӥf                     R   d Z ddlZddlZddlmZ ddlmZmZ ddlmZ	 ddl
ZddlZddlZddlZddlZddlmZ ddlmZ ddlmZ ddlZddlmZ dd	lmZmZmZmZmZmZ ddl Z dd
l!m"Z" ddl#m$Z$m%Z% d Z&ddZ'd Z(d Z)d Z*d Z+d Z,d Z-d Z.d Z/d Z0 G d dejb                        Z2y)z$Code for pixelnerf and alternatives.    N)nn)	rearrangerepeat)
functional)deepcopy)cm)tqdm)CallableListOptionalTuple	GeneratorDict)defaultdict)
CrossAttn_PositionalEncodingNoFreqFactorc                     t        | d      S )Nz... c x y -> ... (x y) c)r   xs    5/home/cameronsmith/repos/inference_as_optim/models.py<lambda>r      s    i"<=     c           	      f    t        | d|"t        | j                  d      dz              S |      S )Nz... (x y) c -> ... c x yg      ?r   )r   intsize)srcr   s     r   r   r      s2    i,FdedmSXXVX\\^M_I`u stu r   c                 d    t        j                  | t        j                  | ddgf         fd      S )N.r   )torchcat	ones_liker   s    r   r   r      s)    eii5??1S!W:#> ?C r   c                 .    | dd df   d| ddd f   z   z  S )N.r   gh㈵> r   s    r   r   r      s%    aCRCj$qRSz/2 r   c                 4    t        j                  | |dd      S )NbilinearTmodealign_corners)Finterpolater   ys     r   r   r      s    azM r   c                 @    t        j                  | |dz  dz
  dd      S )N      r&   Tr'   )r*   grid_sampler,   s     r   r   r      s    a!A:DQ r   c                     t        | j                        dk(  rt        | |      S t        | j                  dd      |j                  dd            j	                  d| j                  d d       S )N   r   r0   r/   )lenshape
grid_samp_flatten	unflattenr,   s     r   r   r      so    3qww<?
1Q 
199UVWX>Z[ZcZcdefgZh@i@s@stuvwv}v}  A  @A  wB  AC r   c                 B    t        t        j                  d||             S )Nb...cij,b...ckj->b...cki)unhomr    einsum)crdsKs     r   r   r      s    5.HD!QR r   c           	      d    t        t        j                  d|t        |             dd df   |      S )Nr:   .   )projectr    r<   hom)r=   posesr>   s      r   r   r      s6    %,,7QRWX[\`Xa*bcfgihigici*jlm!o r   c                 h   t         j                  d|d| f   j                  t              j	                  ddd      }t        j                  t        j                  |d      j                               j                         }|t        j                  |dz
  | dz
  g      z  }|j                         S )Nr   r0   r/   r   )axis)npmgridastypefloat	transposer    
from_numpyflipcopytensorcuda)hwuvs      r   	make_xpixrS      s    	xxAq1u$$U+55aA>",1134::<"	ELL!A#qs$$"	r   c           	      6   d }g }t        t        |       dz
        D ]S  }|j                  t        j                  | |   | |dz                   |j                  t        j
                                U t        j                  |d d  }|j                  |       |S )Nc                     t        |       t        j                  k(  r;t        | d      r.t        j                  j                  | j                  ddd       y y y )Nweightg        relufan_in)anonlinearityr(   )typer   Linearhasattrinitkaiming_normal_rV   )ms    r   init_weights_normalz%make_net.<locals>.init_weights_normal$   sF    7biiq(#''CfS['\ $  r   r0   r   )ranger4   appendr   r\   ReLU
Sequentialapply)dimsra   layersinets        r   make_netrk   #   s    ] F3t9Q; !biiQQqS	23bggi ! --
%CII!"Jr   c                   D     e Zd Z fdZd Zi dfdZi fdZi fdZ xZS )SceneLearnerc                    t         |           || _        d}t        j                  d|      | _        t        |ddg      | _        t        |dz  dddg      | _        t        j                  j                  dd| j                  j                         | _        | j                  j                  j                  | _        t        j"                         x| j                   d<   | j                  j                  _        t%        j&                         | _        |j*                  }g d	d |j,                   | _        t        j.                  | j,                  D cg c]  }t        j0                  d
||dz  z          c}      j3                         | _        | j4                  D ].  }t        j6                  j9                  |j:                  dd       0 d}t=        d|      | _        t        j                  |dz  dz   |      | _         t        j.                  tC        d      D cg c]  }tE        |dd       c}      j3                         | _#        t        |ddg      | _        t        |ddg      | _$        y c c}w c c}w )N@   i   r0   r/      zintel-isl/MiDaSMiDaS_small)
pretrained)r0   r3         i  r   g{Gz?)meanstd   r3   r@   )%super__init__argsr   r\   fmap_downprojrk   	depth_estcorr_weighter_perpointr    hubloadscratch_netmidasscratchoutput_conv	midas_outIdentityconv_modulesPixelNeRFEncoderimg_encfdimspatial_dims
ModuleList	EmbeddingrO   latentsr^   normal_rV   r   posenc	pix_embedrb   r   decoderrgb_est)selfrz   r   sr   n_freq_	__class__s          r   ry   zSceneLearner.__init__1   s   	 YYs40!41+.&.Qs2a/@&A#YY^^$5}UYU^U^UjUjQj^k
zz))559;Fq$**,,8#446YY&'9(9(9:}}TM^M^%_bll4ad&C%_`eegJArwwqxxaTJJ 4Qv>6!8A:d3}}eAh%Oja&<%OPUUW!41+.b, &` &Ps   #I2I7c                 ^   |d   j                   dd  }t        d       }t        j                         5  t	        t        j
                  |d   |d   z        j                  d            D ]r  \  }} | ||      }t        j                  |d   |d   gd      |d<   t        j                  |d   |d   gd	      |d<   t        j                  |d
   |d
   gd	      |d
<   t 	 d d d        |d   j                  d	|      t        |d   |d         t        |d
   d|d         dS # 1 sw Y   DxY w)Nrgbr   c                  H    t        j                  g       j                         S N)r    rN   rO   r$   r   r   r   z.SceneLearner.render_full_img.<locals>.<lambda>W   s    ell2&6&;&;&= r   r   r0   rt   )
render_pixdepthr   masksz b l1 l2 1 (x y) -> b l1 l2 1 x yr   )r   r   r   )r5   r   r    no_grad	enumeratearangechunkr!   r8   ch_fstr   )r   model_inputimsizeout_alljpixouts          r   render_full_imgzSceneLearner.render_full_imgU   s@   5!'',=>]]_ 	O"5<<q	&)0C#D#J#J2#NO O#;#6$yy'%.U)DRH!&GG,<S\+J2!N !&GG,<S\+J2!N 	O	O G$..r&9q	2gg./QTZ[\T]^ 		O 	Os   B,D##D,Nc                 X   |d   j                   dd  }t        |d         t        | j                        dz   }}|!t        j                  |d   |d   z        d d }t        | j                  dd      |   }t        | j                  | j                  |            d||      }d|vrt        | j                  | j                        D 	
cg c]>  \  }	}
t         |
|d	   j                  d
            j                  d
|	dz  d
f      |	      @ }}	}
|j                  | j!                  |d                ||d<   t        j"                  |d   D 
cg c],  }
t%        t'        |
|d    j)                  |d
d
d
                  . c}
d      }t        t        j"                  t+        |      D cg c]:  }t        j,                  t+        |      D cg c]  }t/        ||k         c}      < c}}      j1                         d|      }t3        t'        |j                  dd      |d   j)                  ||dz  z  d
d
d
            d||      }t        |d|j5                  d            }||z  }|}| j6                  D ]  } |||      } t3        |d      }t9        j:                  | j=                  |            j                  d
      }t9        j>                  | jA                  |            }||||jC                  ddddd      |j1                         dz  S c c}
}	w c c}
w c c}w c c}}w )Nr   r   r0   r   i  zxy c -> b xy l 1 c)lbr   idxr   r/   zx y -> b x y 1 1 1)r   )NNz!(b l1  l2) 1 1 xy -> b xy l1 l2 1)r   l1zb xy l c -> b xy l2 l c)l2zb xy l1 1 c -> b l1 xy cr@   r3   )r   r   r   r   )"r5   r4   r   r    randpermrS   r7   r   r   r   zipr   r   squeezer8   rc   r   stackch_sec	grid_sampexpandrb   rN   rI   rO   r   r   r   r*   softplusr|   sigmoidr   permute)r   r   r   r   r   r   n_latentxpixr=   r   latentr   r   ri   lowres_maskshires_masksfmap
cross_attnr   r   s                       r   forwardzSceneLearner.forwarde   s<   5!'',U+,S1B1B-CA-E(ENN6!9VAY;N,OPUQU,Vz &!))!A.z:dnnT[[%678LxZ[\ K'pstx  uF  uF  GK  GS  GS  qT  Udldeflvf[%7%?%?%CDNNrSTVWSWXZR[\]^_  UG  UNNDLLU);<>%,K	"++epqze{|[avitDz7H7H2bQS7T&UV|}~ ekkinowix*yde5<<eT\o8^q!t8^+_*yz  B  CW  Z[  \ )L,@,@1,Ed9oF\F\]^_gij_j]jkmnpqsFt"u  y\  _`  dl  m !:gll1oNK' ,,GJz'$/GG89 

4>>$/088<iiT*+''!Aa2#*	
 
 	
3 U } 9_*ys%   AL1L*!L&L! 	L&!L&c                    |d   j                   dd  }| j                  |d   j                  d            }| j                  t	        | j                  dd      d    j                  t        |d         dd      j                               }| j                  |j                  d|      d d dd ddd df   j                  dd            }| j                  D ]  } ||d d d f   |      } t        j                  t        ||d   dz        |d	      }t        j                  t        | j!                  t#        j$                  |t'        |      fd            |d               j                  d      }t        j(                  t        | j+                  t#        j$                  |t'        |      fd            |d               }	|||	d
z  S )Nr   r   r   r   r   r0   r/   r&   r(   )r   r   )r5   latent_codesr   r   rS   r7   r   r4   rO   r   r8   r   r*   r+   r   r   r|   r    r!   r   r   r   )
r   r   r   r   r   r=   r   r   r   r   s
             r   forward_autodecoder_globalz'SceneLearner.forward_autodecoder_global   s   5!'',((U);)C)CB)GH{{9f-55a:4@GGKX]L^H_`bcefkkmn ~~dnnQv6qAadd{CKKAaPQ,,TJz,qv:Nt/TT }}VDA6v:N

6$..D;Nr1R"STZ[\T]^_gghijiit||EIItF4L6I",MNvVWyYZ
 
 	
r   c           	         |d   j                   dd  }| j                  |d         }t        | j                  t	        |            |j                  d            }t        j                  ||d      }t        j                  t        | j                  t	        |            |d               j                  d      }|d|iz  S )Nr   r   r&   r   r   r0   r   )r5   r   r   r{   r   r   r*   r+   r   r|   r   )r   r   r   r   r   r   s         r   forward_unetzSceneLearner.forward_unet   s    5!'',||K./d((6tyy}E}}T&j9

6$..">vayIJRRSTUE
 
 	
r   )	__name__
__module____qualname__ry   r   r   r   r   __classcell__)r   s   @r   rm   rm   0   s/    -H  (*d *
X ;= 
* -/ 
r   rm   r   )3__doc__r    torchvisionr   einopsr   r   torch.nnr   r*   numpyrF   sysrandomtimeosrM   r   matplotlib.pyplotpyplotplt
matplotlibr   wandbr	   typingr
   r   r   r   r   r   r   collectionsr   attn_modulesr   r   r   r   rB   r;   interpr6   r   rA   warprS   rk   Modulerm   r$   r   r   <module>r      s    *   $ $         C C  # B >uC2MQ
 C	RoA
299 A
r   