
    )fT                        d dl mc mZ d dlZd dlZd dlmZ d dlZd dlZd dlZd dl	m
Z
mZmZ d dlmZ d Zd Z G d dej                         Zd dlmZ  G d d	ej                         Z G d
 dej                         Z G d de      Z G d de      Zd Z G d dej                         Zd Z G d dej                         Z G d dej                         Z G d dej                         Z G d de      Z G d de      Z G d dej                         Z G d  d!ej                         Z G d" d#ej                         Z  G d$ d%e       Z! G d& d'e       Z" G d( d)e       Z#y)*    N)	rearrangerepeatreduce)einsumc                 
    | d uS N )vals    ;/home/cameronsmith/repos/inference_as_optim/attn_modules.pyexistsr      s    d?    c                      t        |       r| S |S r   )r   )r
   ds     r   defaultr      s    +3$1$r   c                   <     e Zd ZdZ	 ddededdf fdZd	dZ xZS )
PositionalEncodingNoFreqFactorzPositionalEncoding module

    Maps v to positional encoding representation phi(v)

    Arguments:
        i_dim (int): input dimension for v
        N_freqs (int): #frequency to sample (default: 10)
    i_dimN_freqsreturnNc                     t         |           || _        |d|z  |z  z   | _        || _        d| j                  dz
  }}dt        j                  ||| j                        z  | _        y )N      )super__init__r   out_dimr   torchlinspace
freq_bands)selfr   r   ab	__class__s        r   r   z'PositionalEncodingNoFreqFactor.__init__   sb     	
Gu44$,,"1u~~aDLLAAr   c                     |r|gng }| j                   D ]4  }||z  }|t        j                  |      t        j                  |      gz  }6 t        j                  |d      S )Ndim)r   r   sincoscat)r   vstart_with_vpefreqfvs         r   forwardz&PositionalEncodingNoFreqFactor.forward+   s[     aSbOO 	1DB599R=%))B-00B	1 yy$$r   )
   )T)__name__
__module____qualname____doc__intr   r/   __classcell__r"   s   @r   r   r      s7     BB B 
	B%r   r   c                   (     e Zd Zd fd	ZddZ xZS )
CrossAttn_c           
      |   t         |           ||z  }|dz  | _        || _        || _        t        j                  ||d      | _        t        j                  ||dz  d      | _        t        j                  ||      | _	        t        j                  t        j                  |t        d|z              t        j                         t        j                  t        d|z        |            | _        t        j                  | j                  g      | _        t        j                  | j                  g      | _        y N      Fbiasr      )r   r   scaleheadschnnLinearto_qto_kvproj
Sequentialr5   GELUout	LayerNormln_1ln_2)r   rB   rA   dim_head	inner_dimr"   s        r   r   zCrossAttn_.__init__5   s    u$	%

IIb)%8	YYr9q=u=
IIi,	==IIb#ad)$GGIIIc!B$i$
 LL$''+	LL$''+	r   c                     t        |j                        dkD  rJ | |j                  dd      |j                  dd      ||||      j                  d|j                  d d       S | j	                  |      }| j	                  |      }| j
                  | j                  |      }	| j                  |      j                  dd      \  }
}t        fd|	|
|f      \  }	}
}t        d	|	|
      | j                  z  }|j                  |      }t        d
||      }|rt        |d      S t        |d      }| j                  |      |z   }| j                  | j!                  |            |z   }|r|t        |d      fS |S )N   r   r   r$   r%   c                      t        | d      S Nzb n (h d) -> (b h) n dhr   trW   s    r   <lambda>z$CrossAttn_.forward.<locals>.<lambda>V       	!-E K r   b i d, b j d -> b i jb i j, b j d -> b i dz(b h) n d -> b n h drV   (b h) n d -> b n (h d))lenshapeflatten	unflattenrL   rA   rE   rF   chunkmapr   r@   softmaxr   rG   rJ   rM   )r   xy	attn_maskreturn_headsreturn_attnsoftmax_axisx_lny_lnqkr*   simattnrJ   rW   s                  @r   r/   zCrossAttn_.forwardJ   s}   qww<>		!B		!B	,{[ghrrstuvu|u|  ~A  A  vB  C  Cyy|yy|JJIIdOzz$%%aR%01KaQRTUYW1a ,a3djj@{{|{,,dA6	#.D JJ5;iinq hhtyy~&,CNId#9A>?WTWWr   )   @   )NFFr$   r1   r2   r3   r   r/   r6   r7   s   @r   r9   r9   4   s    ,*Xr   r9   c                   :     e Zd Z	 	 d fd	ZddZd ZddZ xZS )LightFieldModelc           
         t         
|           || _        d| _        || _        || _        || _        || _        || _        d}| j                  r|dz  }| j                  r-|dz  }t        j                  d      j                         | _        | j                  s|dv r\|dk(  r,t        j                  | j                  dd|dd	
      | _        n4|dk(  r.d}	t        j!                  ddd|d|	|	      | _        n	|dk(  rt#        j$                  t#        j&                  d| j                  z   | j                        t        j)                  | j                  | j                  | j                        t        j)                  | j                  | j                  | j                        t        j)                  | j                  | j                  | j                        t#        j&                  | j                  d            | _        | j                  sz|dk(  r=t*        j-                  | j                  d| j                  | j                        | _        n8|dk(  r3t*        j1                  | j                  dd| j                  d      | _        t3        | j                         t3        t5        j6                  d | j                  j9                         D                     y )N   rQ   r   )r   r   r   rQ   hyperlow_rankrelu   Tlayernorm_na)	hidden_chnum_hidden_layersin_featuresout_featuresoutermost_linearnormsireng      >@rs   )r   hidden_featureshidden_layersr   r   hidden_omega_0first_omega_0concat)size_insize_outsize_hr{   )hyper_in_featureshyper_hidden_layershyper_hidden_featureshypo_moduler|   i   
leaky_relu)r   r   r   r   nonlinearityc              3   Z   K   | ]#  }t        j                  |j                         % y wr   )npprodra   ).0params     r   	<genexpr>z+LightFieldModel.__init__.<locals>.<genexpr>   s     MeRWWU[[)Ms   )+)r   r   
latent_dimnum_hidden_units_phi
fit_singleparameterizationconditioningdepthalphar   onescuda
backgroundcustom_layersFCBlockphiSirenrC   rH   rD   ResnetBlockFChyperlayersHyperNetwork	hyper_phiLowRankHyperNetworkprintr   sum
parameters)r   r   r   networkr   r   r   r   out_channelsomega_0r"   s             r   r   zLightFieldModel.__init__k   s   $$'!$ 0(

::AL::AL#jj6;;=DO??l.CC& (004;T;Thi=>\lp  xF 1 GG#(..1cab<H[_pw=D / F X%}}		!DOO+T-F-FG++D4M4MX\XqXq373L3L , N++D4M4MX\XqXq373L3L , N++D4M4MX\XqXq373L3L , N		$33Q7	DH W$!,!9!9DOONOPTP_P_FJhh ": "P z)!,!@!@SWSbSbUVWZMQXXNZ	 "A "\ 	dhhbffMtxx7J7J7LMMNr   c                       j                   r j                  }|S  j                  dv r j                         fd}|S  j                  dk(  r fd}S )Nrz   c                 *    j                  |       S )N)params)r   )rg   phi_weightsr   s    r   r[   z:LightFieldModel.get_light_field_function.<locals>.<lambda>   s    DHHQ{H; r   r   c                     | j                   d d \  }}j                  |dj                        j                  d|d      }j	                  t        j                  || fd            S )Nr   r   r$   r%   )ra   viewr   r   r   r   r)   )rg   r!   n_pixz_repr   zs       r   r   z5LightFieldModel.get_light_field_function.<locals>.phi   sY    772A;5q!T__5<<QqIxx		5!*" =>>r   )r   r   r   r   )r   r   r   r   s   `` @r   get_light_field_functionz(LightFieldModel.get_light_field_function   sa    ??((C 
 "77..+K;C 
 (*? 
r   c                     |d   }|d   j                  d      }|d   j                  d      }|d   j                         j                  d      }|||fS )Nquery	cam2worldr   )end_dim
intrinsicsuv)rb   float)r   input
query_dictposer   r   s         r   get_query_camzLightFieldModel.get_query_cam   si    7^
+&..q.9-55a5@
##%--a-8Z##r   c                 B   i }|d   }|d   j                   d d \  }}|d   j                   dd \  }	}
| j                  s?d|v r|d   }n| j                  |      }||d<   |j                  ||	z  | j                        }| j                  |      \  }}}| j                  dk(  rt        j                  |||      }n|d d d ddf   d d d d d f   }t        j                  |||      }t        j                  ||d	      \  }}t        j                  |d
      }t        j                  |d
      }t        j                  ||fd
      }||d<   ||d<   ||d<   ||d<   |j                  d       |j                  ||	z  |
d      |d<   | j!                  | j                  rd n      }||d<   |rt#        j"                         } ||d         }|r%t#        j"                         }|z
  }t%        |        |dd df   }| j&                  r |dddf   }|j                  ||	|
d      |d<   |j                  ||	|
d      }| j(                  rd|dd
d f   j                  ||	|
d      }dt        j*                  t        j,                  |             z
  }||z  d|z
  | j.                  z  z   }||d<   |rat        j0                         5  | j!                        }t2        j5                  |||      d   }|j                  ||	|
d      }||d<   d d d        ||d<   |S # 1 sw Y   xY w)Nr   r   r   r   rQ   r   pluckerd   )radiusr$   r%   intsec_1intsec_2ray_dir
ray_originTr~   coordslf_function.r?   r   r   rgb)ra   r   get_zr   r   r   r   geometryplucker_embeddingget_ray_directionsray_sphere_intersectF	normalizer   r)   requires_grad_r   timer   r   r   expabsr   enable_gradutillight_field_depth_map)r   r   r
   compute_depthtimingout_dictr   r!   n_ctxtn_qryr   r   
query_posequery_intrinsicsquery_uvlight_field_coordsr   r   r   r   r   t0lf_outt1total_nr   r   r   weights                                r   r/   zLightFieldModel.forward   s>   g$K%%bq)	6T{((1-ue|#JJJu%HSMq5y$//2A151C1CE1J.
$h  I-!)!;!;JRb!c#Arr1H-aqj9J11(JHXYG!)!>!>z7[^!>!_Hh{{84H{{84H!&Hh+?R!H#+HZ #+HZ ")HY%/H\"))$//44QuWeQG33DOODQRS"-		2Xh/0		2rBwW	8KS"1"Wo::3!8$E %

1eUA >HWhhq%*::38$))!UE1=EEIIe$4#455F3,!f*!??C &HW""$ *";;A>223EzS^_`gh

1eUA6$)!	* * *s   ALL)r   r}   Fr{   FFr   FFF)r1   r2   r3   r   r   r   r/   r6   r7   s   @r   rw   rw   j   s     GMLQ7Or$@r   rw   c                   (     e Zd Zd fd	ZddZ xZS )LFAutoDecoderc                     t        |   d||d| || _        t        j                  || j
                        | _        t        j                  j                  | j                  j                  dd       y )N)r   r   r   {Gz?)meanstdr	   )
r   r   num_instancesrC   	Embeddingr   latent_codesinitnormal_r   )r   r   r   r   kwargsr"   s        r   r   zLFAutoDecoder.__init__   s]    \JAQ\U[\*LLH
))00qdCr   c                 T    |d   d   j                         }| j                  |      }|S )Nr   instance_idx)longr   )r   r   r
   instance_idcsr   s        r   r   zLFAutoDecoder.get_z  s.    g~6;;=m,r   )r   Fr1   r2   r3   r   r   r6   r7   s   @r   r   r      s    Dr   r   c                   (     e Zd Zd fd	ZddZ xZS )	LFEncoderc                 n    t         |   ||d       || _        t        j	                  |      | _        y )Nr|   )r   )c_dim)r   r   r   conv_modulesResnet18encoder)r   r   r   r   r   r"   s        r   r   zLFEncoder.__init__
  s5    %5JO*#,,:,>r   c                     |d   d   j                   d   }t        j                  t        j                  |d   d               }| j	                  |      }|j                  d      j                  d|d      }|dz  }|S )Nr   r   r   contextr   r   )ra   r   lin2imgflatten_first_twor	  	unsqueezer   )r   r   r
   r   r   r   s         r   r   zLFEncoder.get_z  sv    gt$**1-ll411%	2B52IJKLLKKN!!!UA.	T	r   )r   r{   r  r  r7   s   @r   r  r  	  s    ?
r   r  c                     t        | t        j                        st        | t        j                        rkt        j                  j
                  j                  | j                         | j                  +t        j
                  j                  | j                  d       y y t        | t        j                        rUt        j
                  j                  | j                  d       t        j
                  j                  | j                  d       y y )Nr   g      ?)
isinstancerC   rD   Conv2dr   r   xavier_uniform_r   r>   	constant_rK   )ms    r   init_weightsr    s    !RYY:a#;%%ahh/66GGaffa( 	Ar||	$
!&&!$
!((C( 
%r   c                   $     e Zd Z fdZd Z xZS )simple_CrossAttentionc           	      |   t         |           t        j                  t	        d      D cg c]  }t        j
                  ||       c}      j                         | _        t        j
                  ||      j                         | _        t        j                  t	        d      D cg c]  }t        j
                  ||       c}      j                         | _	        t        j
                  ||      j                         | _
        t        j                  t        j
                  ||      t        j                         t        j
                  ||            | _        t        j                  |g      | _        t        j                  |g      | _        | j#                  t$               |dz  | _        y c c}w c c}w )Nr   rQ   r<   )r   r   rC   
ModuleListrangerD   r   kv_crossq_crosskqv_selfrG   rH   rI   	out_crossrK   rL   rM   applyr  r@   )r   slot_dim_r"   s      r   r   zsimple_CrossAttention.__init__$  s(   mm5QR8$TaRYYx%A$TUZZ\YYx1668mm5QR8$TaRYYx%A$TUZZ\))HX.335	IIh)GGIIIh)
 LL(,	LL(,	

< d#
 %U$Ts   F4(F9c                    | j                   D cg c]
  } ||       c}\  }}| j                  |      }t        j                  d||g      | j                  z  j                  d      }t        j                  d||g      }| j                  | j                  |            |z   }	| j                  | j                  |	      |	z         }	|	S c c}w )Nzbtc,blc->btlr   zbtc,btl->blc)
r  r  r   r   r@   rf   rL   rG   rM   r  )
r   rg   rh   fkeyr
   r   ARrJ   s
             r   r/   zsimple_CrossAttention.forward9  s    #}}-!1-Cll1o<<E{3DJJ>
G
G
J
,,~s1g
.IIdiil#A%IIdnnS)#-.
 .s   C	ru   r7   s   @r   r  r  "  s    $*	r   r  c                      t        | d||      S )Nz&b c (h p1) (w p2) -> b (h w) (p1 p2) c)p1p2rX   )rg   
block_sizes     r   unfold_non_overlappingr+  E  s    Q@ZT^__r   c                   *     e Zd Z	 	 d fd	Zd Z xZS )PatchedSelfAttentionc           
      <   t         |           || _        || _        || _        || _        || _        || _        || _        || _	        d| _
        |
| _        | j                  r|
dkD  sJ d       |	rAt        j                  |
d      | _        | j                  j                  dz  |
z  |
z   | _        nd | _        |
| _        | j                  rut#        j$                  t'        j(                  ddd| j                  | j
                  z  f            | _        t"        j,                  j/                  | j*                  d	d
d       | j                  dk(  r||k(  sJ d       | j                  }| j                  r|| j                   z  }t#        j0                  || j
                  | j                  z  dz  d      | _        t#        j0                  || j
                  | j                  z  d      | _        | j                  dkD  r<t#        j0                  | j
                  | j                  z  | j                        | _        || j                  }t#        j8                  t#        j0                  | j                  t;        d| j                  z              t#        j<                         t#        j0                  t;        d| j                  z        |            | _        t#        j@                  | j                  g      | _!        t#        j@                  | j                  g      | _"        y )Ntokenr   z6need to pass a coord_dim if coordinates are to be usedT	input_diminclude_inputr   c                     | S r   r	   rg   s    r   r[   z/PatchedSelfAttention.__init__.<locals>.<lambda>`       r   r   g        r}   fan_in)r    r   modeNIf not multi-head, number of channels needs to be equivalent to head dimensionFr=   r?   )#r   r   rB   ksnhhd
shift_equidownhalocoordinates	down_modeorig_coord_dimr   PositionalEncodingr,   num_encoding_functions	coord_dimrC   	Parameterr   zeros
down_tokenr   kaiming_normal_rD   rF   query_embed	multi_outrH   r5   rI   rJ   rK   rL   rM   )r   channelsneighborhood_size	num_headshead_dimshift_equivariantr=  r>  r?  pos_encodingrD  out_chargsr   in_chr"   s                  r   r   zPatchedSelfAttention.__init__J  sJ   #+		& 'q=Z"ZZ#66Z^6_DG!WW;;A=iG	QDN!DG&DN99 ll5;;1a7Q+RSDOGG##DOOsV^#_77a<X%x(xxT^^#EYYudggdggoa&7eD
99UDGGdgg,=EJ77Q;YYtwwtww@DN>WWF==IIdggs1TWW9~.GGIIIc!DGG)nf-
 LL$''+	LL$''+	r   c           
      ,	    |j                   \  }}}}t        | j                        } j                  |      }t        | j                        }	 j                  rL j
                  r|	|	j                  dd      z  }	 j                  |	      }
t        j                  |
|fd      } j                  t        j                  | j                   j                  dz  z    j                   j                        }t        |d j                  	      } j                  rt        j                  | j                   j                  dz  z    j                   j                        }t        |d j                  	      } j
                  r||j                  dd      z  } j                  |      }t        j                  ||fd      }n|} j!                  |      j#                  dd      \  }} j$                  r j&                  d
k(  r_t)        |dd      j+                  d      }t)        |	dd      }t-         j.                  d|t1        ||z   j                  dz  z              }n j&                  dk(  rW j3                  |      }t)        |dd      j+                  d      }t)        |dd      j+                  d      }t)        |	dd      }nH j&                  dk(  r9|dd dd d f   }|	ddd d f   } j3                  |      }n j3                  |      }t5         fd||f      \  }}}t7        d||      }|t9        j:                   j<                        z  j?                  d      }t7        d||      }t        |d j@                        } jC                  |      }||dd  j                  f   z   } jE                   jG                  |            |z   } j$                  rft        |jI                  d      d| j                  z  | j                  z        }t        |d| j                  z  | j                  z        }||fS t        |d|| j                  z  | j                  z   j                   j                        }||fS )NrS   T)r&   keepdimsr$   r%   r   kernel_sizestridepaddingzb (c k) i -> b i k ccr/  zb n k d -> b n dr   z!b hw k ch -> (b b1) (hw hw1) k ch)b1hw1averagesub.r   r   c                 4    t        | dj                        S Nzb n k (h d) -> (b h) n k drV   r   r:  rZ   r   s    r   r[   z.PatchedSelfAttention.forward.<locals>.<lambda>  s    	!-ITWW U r   zb m j d, b m k d -> b m j kzb m j k, b m k d -> b m j dz(b h) m j d -> b m j (h d)rV   b (h w) c -> b c h wrW   wz&b (h w) (p1 p2) c -> b c (h p1) (w p2))r!   rW   rf  r(  r)  )%ra   r+  r9  rL   r?  r<  r   r,   r   r)   r>  r   unfoldr   rB   rA  rF   rd   r=  r@  r   r  r   rG  r5   rI  re   r   r   sqrtr;  rf   r:  rJ  rJ   rM   squeeze)r   inpr   r!   r!  rW   rf  inp_unf_vaninp_unf
coords_unfcoords_unf_pekv_inpcoords_kv_inprp   r*   ro   rq   rr   rJ   s   `                  r   r/   zPatchedSelfAttention.forward  s*   YY
1a,S$'':))K(+FDGG<
joo"toDD
 GGJ/Mii 8bAG99 XXc499q=1HSWSZSZdhdmdmnFv'=IF !twwUV?Vaeahahrvr{r{ | )-9OSWSfSf g??!]%7%7B%7%NNM $ 6FM#:CFzz&!''r'2199~~($[2DfMWWXZ[
,>G4??,OTU[^_`ab_bcgcjcjlmcm_m[no9,$$[110&9CCBG$[2DfMWWXZ[
,>G5()#rr1*5#CAI.$$W-  )AUXY[\^_W`a1a2Aq9bggdgg&&//B/72D!< 9TWWEnnS!KXdggX..hhtyy~&,99CKKBK/1GAQUQXQXL^_cgcjcj^jlCv'=!tww,TUY]Y`Y`T`bF F{ C!IQSTX\X_X_S_efjnjqjqeq#ww4774CF{r   )
r?   r?   rt   TFNTTr   Nru   r7   s   @r   r-  r-  I  s    mrUY3,jEr   r-  c                   (     e Zd Zd fd	ZddZ xZS )GlobalAttentionc           
         t         |           ||z  }t        ||      }|dz  | _        || _        || _        t        j                  ||d      | _        t        j                  ||dz  d      | _	        t        j                  ||      | _
        t        j                  t        j                  |t        d|z              t        j                         t        j                  t        d|z        |            | _        t        j                  | j
                  g      | _        t        j                  | j
                  g      | _        y r;   )r   r   r   r@   rA   rB   rC   rD   rE   rF   rG   rH   r5   rI   rJ   rK   rL   rM   )r   rB   	query_dimcontext_dimrA   rN   rO   r"   s          r   r   zGlobalAttention.__init__  s    u$	k95%

IIi?	YY{IMF
IIi,	==IIb#ad)$GGIIIc!B$i$
 LL$''+	LL$''+	r   c                 F   |j                   \  }}}}t        |d      }| j                  |      }| j                  | j	                  |      }	t        ||      }| j                  |      j                  dd      \  }
}t        fd|	|
|f      \  }	}
}t        d|	|
      | j                  z  }|j                  d      }t        d||      }t        |d	      }| j                  |      |z   }| j                  | j                  |            |z   }t        |d
||      }|S )Nb ch h w -> b (h w) chr   r$   r%   c                      t        | d      S rU   rX   rY   s    r   r[   z)GlobalAttention.forward.<locals>.<lambda>  r\   r   r]   r^   r_   rV   rd  re  )ra   r   rL   rA   rE   r   rF   rd   re   r   r@   rf   rG   rJ   rM   )r   rg   r   r  r!   r!  heightwidthrm   ro   rp   r*   rq   rr   rJ   rW   s                  @r   r/   zGlobalAttention.forward  s   gg1fea12yy|JJIIdO'4(zz'"(((31KaQRTUYW1a,a3djj@ {{r{",dA65;iinq hhtyy~&,3vG
r   )Nrs   rt   )NNru   r7   s   @r   rr  rr    s    ,*r   rr  c                   $     e Zd Z fdZd Z xZS )CrossAttentionc                 r   t         |           || _        || _        || _        t        j                  || j                  | j                  z  dz  d      | _        t        j                  || j                  | j                  z  d      | _        | j                  dkD  r<t        j                  | j                  | j                  z  | j                        | _	        t        j                  t        j                  | j                  | j                        t        j                         t        j                  | j                  | j                              | _        t        j                  | j                  g      | _        t        j                  | j                  g      | _        | j!                  t"               y )Nr   Fr=   r   )r   r   rB   r:  r;  rC   rD   rF   q_embmh_projrH   rI   rJ   rK   rL   rM   r  r  )r   r   rM  rN  	kv_in_dimquery_in_dimr"   s         r   r   zCrossAttention.__init__  s   YYy$''DGG*;a*?eL
YY|TWWtww->UK
77Q;99TWWtww%6@DL==IIdggtww'GGIIIdggtww'
 LL$''+	LL$''+	

< r   c                 ^   t        d||      }|t        j                  | j                        z  j	                  d      }t        d||      }t        |d| j                        }| j                  |      }| j                  |      |z   }| j                  | j                  |      |z         }|S )Nzb n d, b n k d -> b n kr$   r%   zb n k, b n k d -> b n dr_   rV   )r   r   rh  r;  rf   r   r:  r  rL   rM   rJ   )r   rp   r*   ro   query_inrq   rr   rJ   s           r   	attentionzCrossAttention.attention  s    .15bggdgg&&//B/7.a85All3iinx'ii+,
r   )r1   r2   r3   r   r  r6   r7   s   @r   r|  r|    s    !.r   r|  c                   $     e Zd Z fdZd Z xZS )PatchedCrossAttentionc                 2    t        |   di | || _        y )Nr	   )r   r   r9  )r   nh_sizer   r"   s      r   r   zPatchedCrossAttention.__init__!  s    "6"r   c                     |j                   d   } j                  |      }t        |d j                        }| j                  kD  r?t        j                  | j                  d j                  dz        }t        |d|      }n*t        |d	      j                  d
      j                  d
      }t        j                  ||j                  d
      j                  d      dd      d d d d dd d f   }t        |d j                        } j                  |      j                  dd
      \  }}t         fd||f      \  }} j                  ||||      }	|	S )NrS   b n (nh hd) -> (b nh) n hdr:  r   r   rV  z*b ch (feat_h feat_w) -> b ch feat_h feat_w)feat_hz*b ch feat_h feat_w -> b (ch feat_h feat_w)r$   nearestF)r7  align_cornersr   z b (c k) n_query -> b n_query k crZ  r%   c                 4    t        | dj                        S ra  rb  rc  s    r   r[   z/PatchedCrossAttention.forward.<locals>.<lambda>7      Yq*F$''R r   )ra   r~  r   r:  r9  r   rg  r  grid_samplefliprB   rF   rd   re   r  )
r   kv_inr  query_coordsr  ro   neighborhoodsrp   r*   rJ   s
   `         r   r/   zPatchedCrossAttention.forward%  sA   RJJx a5$''BDGGHHU477VW<XEe%QZ`aEe%QR\\]_`jjE e\->->r-B-L-LQ-O+4EKKLaQRTU:W!-1SW[W^W^_zz-(..qb.91RUVXYTZ[1nnQ1h/
r   ru   r7   s   @r   r  r     s    r   r  c                       e Zd Zd Zy)GlobalCrossAttentionc                 *     j                  |      }t        |d j                        }t        |d      j                  d      } j	                  |      j                  dd      \  }}t         fd||f      \  }} j                  ||||      }|S )	Nr  r  z*b ch feat_h feat_w -> b (feat_h feat_w) chr   r   r$   r%   c                 4    t        | dj                        S ra  rb  rc  s    r   r[   z.GlobalCrossAttention.forward.<locals>.<lambda>E  r  r   )r~  r   r:  r  rF   rd   re   r  )r   r  r  r  ro   rp   r*   rJ   s   `       r   r/   zGlobalCrossAttention.forward>  s    JJx a5$''B%!MNXXYZ[zz% &&qb&11RUVXYTZ[1nnQ1h/
r   N)r1   r2   r3   r/   r	   r   r   r  r  =  s    
r   r  c                   *     e Zd Z	 	 d fd	Zd Z xZS )MultiScaleDecoderc
                 b   t         |           || _        || _        || _        || _        || _        || _        || _        || _	        |	rAt        j                  |d      | _        | j                  j                  dz  |z  |z   | _        nd | _        || _        | j                  dk(  r||k(  sJ d       | j                  rbt        j                  t        j                   | j                  | j                  z   | j                        t        j"                               | _        t        j&                         | _        t+        t-        | j                              D ]  }
| j                  }| j                  rX| xj(                  t/        | j                  | j                  | j
                  | j                  || j                        gz  c_        s| xj(                  t1        | j                  | j                  | j
                  | j                  |      gz  c_         | j3                  t4               y )	NTr0  r   c                     | S r   r	   r4  s    r   r[   z,MultiScaleDecoder.__init__.<locals>.<lambda>]  r5  r   r   r8  )r   rM  rN  r  r  r  )r   rM  rN  r  r  )r   r   rB   r:  cdr;  sir9  patchedconcat_query_coordsr   rB  r,   rC  rC   rH   rD   rI   	init_projr  r  r  r`   r  r  r  r  )r   rK  
scale_idcsrM  rD  rN  rL  r  r  rP  ir  r"   s               r   r   zMultiScaleDecoder.__init__L  s   ##6 #66Z^6_DGgg44Q6y@JDG!DGDG77a<X%x(xx##]]		$''DGG+TWW5	DN
 s477|$ 	A77L||)TWWZ^ZaZa48GG,`d`g`gi# 
 (DGGtwwY]Y`Y`3777W# 	 	

< r   c                    |j                   \  }}}g }|d   j                  d      j                  d      j                  d      }t        |d|      }| j                  r9t        j                  || j                  |      fd      }| j                  |      }| j                  D cg c]  }||   	 }	}t        |	d d d   | j                        D ]!  \  }
} ||
||      }|j                  |       # |S c c}w )Nr$   r   b n ch -> b (n n1) chn1r%   )r  r  r  )ra   ri  r  r   r  r   r)   r,   r  r  zipr  append)r   multi_scale_inpr  r!   nrB   scale_embeddingsr   idxscale_featsfeatsatts               r   r/   zMultiScaleDecoder.forwardz  s    %%1b#++B/77;EEaHu5!<##IIudggl&;<"EENN5)E7;ww?s+??k$B$/@ 	+JE3ee,OE##E*	+   @s   &C6)rs   r   rt   rQ   TFTru   r7   s   @r   r  r  K  s    PR\`,!\ r   r  c                   &     e Zd Zd fd	Zd Z xZS )MultiscaleImgEncoderc                    t         |           || _        || _        || _        t        j                  ||dd      | _        t        j                         | _	        t        t        t        j                  |      dz
              D ]c  }| j                  j                  t        |d|dk(  ddddd	
      t        |ddddd      t        |ddddd      t        |d ddddd	      g       e | j                  j                  t        |ddddd      t        |ddddd      t        |ddddd      t        |d ddddd	      t!        ||dd      t!        ||dd      t!        ||dd      t        |d ddddd	      t!        ||dd      t!        ||dd      t!        ||dd      t        |d ddddd	      t!        ||dd      t!        ||dd      t!        ||dd      t        |d ddddd	      g       | j#                  t$               y )Nr   r   )rY     r   rs   r?   FT)rK  r>  r?  rD  rM  rL  rO  rP  )rK  r>  r?  rD  rM  rL  )rK  r>  r?  rD  rM  rL  r=  rt   )rB   rt  rA   rN   )r   r   icr  mcrC   r  in_embedr  layersr  r5   r   log2extendr-  rr  r  r  )r   
sidelengthin_channelsrD  mid_chr  r"   s         r   r   zMultiscaleImgEncoder.__init__  sR   		+vq!Dmmos277:.234 	AKK$f1!Q$Z[gh|}7<4Q$f1%[\hi}~$f1%[\hi}~$f4U^_kl  AB  IM  N  	 	 &qeWXdeyz{ &qeWXdeyz{ &qeWXdeyz{ &tZ[gh|}  EI  Jvq2Nvq2Nvq2N &tZ[gh|}  EI  Jvq2Nvq2Nvq2N &tZ[gh|}  EI  Jvq2Nvq2Nvq2N &tZ[gh|}  EI  J!
 	& 	

< r   c           	         |}|}|j                   \  }}}t        |dt        t        j                  |                  }t        |dt        t        j                  |                  }| j                  |      }g }g }	t        | j                        D ]3  \  }
} |||      \  }}|j                  |       |	j                  |       5 ||	fS )Nzb (h w) ch -> b ch h wrV   )	ra   r   r5   r   rh  r  	enumerater  r  )r   r   r   rg   r[  r!   r  rB   multi_scale_featsmulti_scale_coordsr  layers               r   r/   zMultiscaleImgEncoder.forward  s    771ba1S_Ea1S_EMM!!$++. 	)HAuA;DAq$$Q'%%a(		) !"444r   )rQ   r   ry   ru   r7   s   @r   r  r    s    %!N5r   r  c                   (     e Zd Zd fd	ZddZ xZS )TransformerLightFieldc                     t         |           d| _        t        |d      | _        t        j                  t        j                         t        j                  |d            | _	        | j                  t               y )Nr   rt   )r  r  rQ   )r   r   
num_scalesr  img_encoderrC   rH   rI   rD   rgb_outr  r  r   	latent_chr"   s     r   r   zTransformerLightField.__init__  sY    /yRP}}GGIIIi#

 	

< r   c                 ^   i }|d   }|d   }|d   d d df   }|d   d d df   }	| j                  ||	dz  dz
        \  }
}| j                  |
||      \  }}t        j                  |d   |d   |d	         d d df   }| j	                  |||      }| j                  |      d d d f   }||d<   |S )
Nr   r  r   r   r   rt   r   r   r   )r  intermediate_encoder   r   
ray_decoder  )r   r   r
   r   r   r   r   r  r   r   ms_feats	ms_coordsintermed_featsintermed_coords
query_rays	ray_featss                   r   r/   zTransformerLightField.forward  s    g	"enQT"T]1a4 "..sBrEAI>)*.*B*B8YX_*`'//k0BE$KQVWcQdefgijfjk
OOJP	ll9%ag.r      r   ru   r7   s   @r   r  r    s    !r   r  c                   *     e Zd Z fdZd Zd Z xZS )GlobalTransLightFieldc           
      (   t         |           d}t        j                  t        j                  |dz   |      t        j
                         t        j                  ||      t        j
                         t        j                  |d            | _        y )Nr  r~   rQ   )r   r   rC   rH   rD   rI   r  )r   r   r  r"   s      r   r   zGlobalTransLightField.__init__  sc    	}}IIik9-GGIIIi+GGIIIi#
r   c                 
    ||fS r   r	   r   r  r   r  s       r   r  z)GlobalTransLightField.intermediate_encode      f}r   c                     |j                   d   }|d   j                  d      j                  d      j                  d      }t        |d|      }t	        j
                  ||fd      S )Nr   r$   r  r  r%   )ra   ri  r  r   r   r)   )r   r  r  r   r  r   s         r   r  z GlobalTransLightField.ray_decode  s`    Q"Ib!))"-77:1-!4yy!Zb11r   r1   r2   r3   r   r  r  r6   r7   s   @r   r  r    s    

2r   r  c                   ,     e Zd Zd fd	Zd Zd Z xZS )Multiscale2DLightFieldc                 8   t         |           t        |dg dddd      | _        t	        j
                  |dd      | _        t	        j                  t	        j                         t	        j
                  |d            | _	        | j                  t               y )NFr0            r~   T)rK  r  r  rD  r  rP  rQ   r=   )r   r   r  ray_decoderrC   rD   	coord_maprH   rI   r  r  r  r  s     r   r   zMultiscale2DLightField.__init__  s{    ,i[kwx BFTXZ 9ad;}}GGIIIi#
 	

< r   c                 
    ||fS r   r	   r  s       r   r  z*Multiscale2DLightField.intermediate_encode  r  r   c                 ,    | j                  ||      d   S )Nr$   )r  )r   r  r  r   s       r   r  z!Multiscale2DLightField.ray_decode  s    z2266r   r  r  r7   s   @r   r  r    s    !7r   r  c                   ,     e Zd Zd fd	Zd Zd Z xZS )MultiscalePointLightFieldc                    t         |           g d| _        t        |d| j                  dd      | _        t        j                  |dd      | _        t        j                  t        dd      g      | _
        t        j                  d	|      | _        t        j                  t        j                         t        j                  |d
            | _        | j                  t                y )Nr  Tr   )rK  r  r  rD  r  r   r=   r  )rt  r~   rQ   )r   r   r  r  
pc_decoderrC   rD   	depth_mapr  rr  r  r  rH   rI   r  r  r  r  s     r   r   z"MultiscalePointLightField.__init__  s    *+YY]YhYh67TS9ad;==C3/*
  1i0}}GGIIIi#
 	

< r   c                    t        |d   d      }| j                  ||      }g }|D ]a  }t        j                  | j	                  |            }|j                  t        j                  |||d   d d df   |d   d d df                c ||fS )Nr   rw  r   r   )r   r  r   squarer  r  r   world_from_xy_depth)	r   r  r   r  r  pc_feats	pc_coordsrg   r   s	            r   r  z-MultiscalePointLightField.intermediate_encode0  s     ,DE??5,7	 	ALL!23E,,\5'+BVWXZ[W[B\^efr^stuwxtx^yz	
 ""r   c                 b   g }t        | j                  ||      D ]  \  }}}t        j                  ||      \  }}	|j	                         ddddddf   }
t
        j                  |	|
      }t
        j                  ||
      }|j                   |||              | j                  ||      d   S )zfeats: list of 2D features.N	   r$   )r  r  r   plucker_to_pointargsortr   index_pointsr  )r   r  r  r   feats_concatdecoderfeatcoorddist_matlocal_coordsknn_idx
knn_coords	knn_featss                r   r  z$MultiscalePointLightField.ray_decode<  s     $'(8(8%$H 	@ GT5%-%>%>z6%R"Hl&&(Arr2G**<AJ))%9I	: >?	@ z2266r   r  r  r7   s   @r   r  r    s    !*
#7r   r  )$torch.nn.functionalrC   
functionalr   r   r   torch.nnnumpyr   einopsr   r   r   r   r   r   Moduler   r9   rw   r   r  r  r  r+  r-  rr  r|  r  r  r  r  r  r  r  r  r	   r   r   <module>r     sI           , , %%RYY %@ 1X 1XlNbii NbO  ) BII  F`{299 {|0bii 0f#RYY #LN :> @ 		 @ F:5299 :5zBII B21 2.72 7.07 5 07r   