
    gf                        d Z ddlZddlZddlmZ ddlZddlmZmZ ddlm	Z	 ddl
mZ ddlZddlZddlZddlZddlZddlmZ ddlmZ ddlZddlmZ dd	lmZmZmZmZmZmZ dd
l m!Z! ejD                  jG                  d       ddl$m%Z%m&Z& ddl'Z'd Z(ddZ)d Z*d Z+d Z,d Z-d Z.d Z/d Z0 G d dejb                        Z2y)z$Code for pixelnerf and alternatives.    N)nn)	rearrangerepeat)
functional)deepcopy)cm)tqdm)CallableListOptionalTuple	GeneratorDict)defaultdictz./third_party/co-tracker/)
Visualizerread_video_from_pathc                     t        | d      S )Nz... c x y -> ... (x y) c)r   xs    ,/home/cameronsmith/repos/flowmap++/models.py<lambda>r      s    i"<=     c           	      f    t        | d|"t        | j                  d      dz              S |      S )Nz... (x y) c -> ... c x y      ?r   )r   intsize)srcr   s     r   r   r      s2    i,FdedmSXXVX\\^M_I`u stu r   c                 d    t        j                  | t        j                  | ddgf         fd      S )N.r   )torchcat	ones_liker   s    r   r   r      s)    eii5??1S!W:#> ?C r   c                 .    | dd df   d| ddd f   z   z  S )N.r    h㈵> r   s    r   r   r      s%    aCRCj$qRSz/2 r   c                 @    t        j                  | |dz  dz
  dd      S )N      bilinearborder)modepadding_mode)Fgrid_sampler   ys     r   r   r      s    a!A:8T r   c                     t        | j                        dk(  rt        | |      S t        | j                  dd      |j                  dd            j	                  d| j                  d d       S )N   r   r)   r(   )lenshape
grid_samp_flatten	unflattenr0   s     r   r   r      so    3qww<?
1Q 
199UVWX>Z[ZcZcdefgZh@i@s@stuvwv}v}  A  @A  wB  AC r   c                 B    t        t        j                  d||             S )Nb...cij,b...ckj->b...cki)unhomr!   einsum)crdsKs     r   r   r      s    5.HD!QR r   c           	      d    t        t        j                  d|t        |             dd df   |      S )Nr:   .   )projectr!   r<   hom)r=   posesr>   s      r   r   r      s6    %,,7QRWX[\`Xa*bcfgihigici*jlm!o r   c           	      6   d }g }t        t        |       dz
        D ]S  }|j                  t        j                  | |   | |dz                   |j                  t        j
                                U t        j                  |d d  }|j                  |       |S )Nc                     t        |       t        j                  k(  r;t        | d      r.t        j                  j                  | j                  ddd       y y y )Nweightg        relufan_in)anonlinearityr,   )typer   Linearhasattrinitkaiming_normal_rF   )ms    r   init_weights_normalz%make_net.<locals>.init_weights_normal!   sF    7biiq(#''CfS['\ $  r   r)   r    )ranger4   appendr   rL   ReLU
Sequentialapply)dimsrQ   layersinets        r   make_netr[       s    ] F3t9Q; !biiQQqS	23bggi ! --
%CII!"Jr   c                   B     e Zd Z fdZi fdZi fdZi fdZd Z xZS )FlowMapc           	          t                    | _        ddlm} |j
                  j                          _        ddlm}  ||j
                  d       _	        t        j                  j                  d       ddlm}  |dd	d
d	ddd      j                          _        t#        j$                  d      }d|v r|d   n|} j                   j'                  |d        j                   j)                         D ]	  }d|_         ddlm}  |t0        j                  j3                  d            j                          _        d}	t7        j8                  d|	       _        t=        |	dd	g       _        t=        |	dz  ddd	g       _         t=        |	dd	g       _!        t=        |	dd	g       _"        dd l#}
t7        jH                  |
jK                  d      t7        jL                  d|	dd	             _'        t7        jL                  |	d	dd	       _(        t7        jL                  |	d	dd	       _)        t7        jL                  |	d	dd	       _*        t7        jL                  |	ddd	       _+        d _,        t=        |	dd jX                  g       _-         fd _.        d  _/        d\   _0         _1        d _2        |jf                  sti        d       d }t"        jj                  jm                  d!d"d#$       t"        jj                  j%                  |d%d#&      j                          _7         jn                  j)                         D ]	  }d|_         ti        d'       y y )(Nr   )Raft_Large_Weights)
raft_largeF)weightsprogressz	./gmflow/)GMFlow   r)      swinr3      )feature_channels
num_scalesupsample_factornum_headattention_typeffn_dim_expansionnum_transformer_layersz8./gmflow/gmflow-scale1-mixdata-train320x576-4c3a6e9a.pthmodel)strict)CoTrackerPredictorzB./third_party/co-tracker/checkpoints/cotracker_stride_4_wind_8.pth)
checkpointi   @   r(   )in_chr@   )padding    c                 4    t        | dj                        S )Nzb ... -> (b o) ...o)r   n_rig)r   selfs    r   r   z"FlowMap.__init__.<locals>.<lambda>W   s    va(<tzzJ r   c                     t        | d      S )Nzb t o ... -> (b o) t ...)r   r   s    r   r   z"FlowMap.__init__.<locals>.<lambda>X   s    &+E"F r   NNzloading depth estimatorzisl-org/ZoeDepthzintel-isl/MiDaSDPT_BEiT_L_384T)force_reloadZoeD_N)
pretrainedzdone loading depth estimator)8super__init__argstorchvision.models.optical_flowr_   DEFAULT
transformsraft_transformsr`   raft_syspathrS   gmflow.gmflowrc   cudagm_flowr!   loadload_state_dict
parametersrequires_gradcotracker.predictorrq   osjoin
co_trackerr   rL   fmap_downprojr[   	depth_estcorr_weighter_perpoint
static_estmask_scorerconv_modulesrU   PixelNeRFEncoderConv2d
resnet_enc
depth_conv
focal_convtemperature_convaffinities_convrz   rig_predictoro_repo_switchcorr_weightsdepthstep	load_saveprinthubhelpmodel_zoe_n)r{   r   r_   r`   rc   rr   ra   paramrq   fdimr   repo	__class__s   `           r   r   zFlowMap.__init__/   s   	 	G199DDF>(:(B(BUS
 	$(sqRS^_pv  KL  ef  h  m  m  oZZ Z[
)0J)>*W%J$$WU$;\\,,.KEe0CK 	;,  HL  :M  N  S  S  U YYs40!41+.&.Qs2a/@&A#"DA;/#T"QK0l&C&C!&C&LRYYWZ[_`ajkMlm		$q15		$q15 iiQq;YYtBq;
%tC4::&>?J
F &/$$*	~~+,%DIINN,.>TNR$yy~~dH~NSSUD))446Se8KS01 r   c                 t   | xj                   dz  c_         |d   j                  dd  }|d   j                  d d |d   j                  d      c\  }}}d}t        j                  |d   |d   z        d | }| j
                  j                  r1t        j                  d|d   |d   z  dz
  |      j                         }| j                  |       t        |d   |d         }	t        j                  |d   |	fd      }
t        j                  | j                  |
j                  dd      dz  dz         |d	
      j!                  d||f      |d<   t        j"                  | j%                  |d   j                  dd            j!                  d||f      dz         }|	|z   }| j
                  j&                  s| j)                  |d   j                  dd            j!                  d||f      j+                         j                  dd      j-                  d      }t        j.                  d      d    j1                         j3                  |      j5                  ||dd      |d<   dx|d   d<   |d   d<   ||d   z  |d   d<   ||d   d<   |d   d d d df   t7        |d         z   }t9        j:                  |d   |d   d       d   }|t7        |      z  }t        j<                  t        ||d         d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }t        j<                  |d   d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }| jE                  t        j                  |t7        |d         d d dd f   fd            j+                         jG                  d      }t        jH                  | jK                  |d   j                  dd            j!                  d||f      d      }|d   j?                  d      }tM        tO        |d d dgf   |d d dgf         d      }t        j                  |j                  dd      dd	
      j!                  d||f      }t        jP                  dt7        |      |jA                  d            }tM        |d      }| jS                  |d   j                  dd            j!                  d||f      jU                         dz  dz   }||z  tW        jX                  |j                  d            dz  z  j[                  d      jG                  d      }d  }d!g d"}}t5         |ddt]        |dz              d#|j                  d      ||dz
  $      j_                         }t        j                  |D  cg c]%  }  ||  | t]        |ta        |      z  dz              ' c} d      jb                  }!tM        |d d dd f   d%      |!j_                         d d d d d f   z   }"t        jd                  |"dk  |"dkD  z  |d d d d d |"j                  d      f   |"      }"t9        jf                  tM        tO        t        |d d dd f   |d         tM        |"d&|'            d(|"j                  d      )      tM        tO        t        ||d         tM        |"d&|'            d(|"j                  d      )      tM        tO        t        ||d         tM        |"d&|'            d(|"j                  d      )      tO        t        |d d dd f   d*      |"      jA                  d      z  jG                  d            d   }#ti        |dz
  dd      D ]:  } t        j                  |#d d d | f   |#d d | dz
  gf   |#d d | d f   z  fd      }#< t        j                  t        j.                  d+      j3                  |#      d,   jk                  |#j                  d      ddd      |#fd      }#t        j                  tl        j8                  jn                  jq                  |#d-d dd df   d./      |#d-d ddf   fd      }$||$j?                  d      z  j!                  d||j                  d      f      js                  d      }%t7        t        j                  t        |%d*      j                  dd      |d	
      j!                  d||f            }%t        j.                  d+      d0   jk                  |||%j                  d      dd      j3                  |%      }&tl        j8                  jn                  ju                  |%d-d d+f         |&d-d dd df<   |%d-d+d f   |&d-d ddf<   tw        t        jP                  d1|&d d d df   jy                         |&d d dd f   z  t{        |d d dd f               d-d df   |d   d d dd f         |d   d d dd f   z
  }'tM        t        j<                  t        ||d         j                  dd      |d   j                  dd      j?                  d      dz  dz
        jA                  d      j!                  d||f      d2      }(tM        t        j<                  t        |&j                  dd      |d         j                  dd      |d   j                  dd      j?                  d      dz  dz
  d34      jA                  d      j!                  d||f      d5d+6      })t5        |)d7|8      jy                         t5        |)d9|8      z  }*t        jP                  d:|*t{        |(            d-d df   }+t}        t        jP                  d;|d   d d df   |+            jG                  dd      },|i d<tM        |d=|j                  d      >      d?tM        |d=|j                  d      >      d@|dAt7        |      dBt7        |      dC|'dD|%j!                  d|      dE|&j!                  dF|      dG|dH|#dI|,d|d   dJdd.|d   z   z  dd.t7        |      z   z  z
  j                         j-                         dKz  dLt        ||d         dM|#d d dN|d   z  S c c} w )ONr)   rgbr   r(   i,  r   	zoe_depthr   r*   r,   fmapr    dimr@   
intrinsics.r   r(   .r)   r(   	org_ratio.r   r   .r)   r)   x_pixbwd_flow-C6?minpred_tracksb t c p 1 -> b t p c)rs   rs   zb t p c, b s c -> b t p szb t p s -> (b s) t p 1g      I@re   c                     t        j                  t        j                  t        d      D cg c]  }t        j                  | dz   |dz
  |      ! c}       j                  dd      S c c}w )Nr(   g{Gz?r)   )r!   stackmeshgridrR   linspacer7   )sen_s       r   r   z!FlowMap.forward.<locals>.<lambda>   sV    U^^ejklem=n`aennQsUSTUXSXYZ>[=n-o!p!x!xyz{|!} =ns   $A(iD  )r         ?g?zc p -> (b o) t p 1 c)ry   btzb t s 1 c -> (b s) t 1 1 cz(b s) t p 1 c -> b t (p s) 1 c)r   zb t c (p s) 1 -> (b s) t p c)prs   r3   r}   .r%   )epsNNNzbtpij,btpj->btpib t c p 1 -> b p t cr+   )r-   zb t (x y) p 1 -> b p t x yr   zb p t x y -> b p s t x yr   zb p t x y -> b p t s x yzbksnij,bksj->bksnibij,bksnj->bksni	rig_masksz(b o) t xy 1 -> b t o xy 1rx   rig_masks_unnormaffinity_emb	res_depthr   flow_from_pose
lie_perpixpose_perpixeye_surf	poses_alltrack_reprojs
zoe_d_loss      $@r   rC   	flow_inp_)@r   r5   r   r!   randpermr   overfitr   longget_flowch_fstr"   r.   interpolater   r7   r8   softplusr   use_gt_intrinsicsr   sigmoidmeaneyefloattor   ch_secgeometryget_world_raysr/   	unsqueezesqueezepermuter   clip	normalizer   r   	grid_sampr<   r   maxnpsqrtsoftmaxr   r   r4   Twhere
procrustesrR   expandkorniaconversionsrotation_matrix_to_quaternionsumquaternion_to_rotation_matrixrA   inverserB   r;   square)-r{   model_inputoutimsizer   r   n_trgtn_samprand_subset	depth_inprgbdr   r   focal
corresp_uvrdsr   corresp_surfcorresp_featr   r   aff_src_samp_locsaff_sourcesaffinity_emb_lowaff_sim_allpixaffinity_mask_unnormtemperatureaffinity_maskget_xpixn_total_samp	samp_amtsuniform_grid_pixrY   local_samp_gridssample_locsrC   	poses_lier   r   adj_opt_flow
eye_trackspose_trackspose_all_to_all_perpixtrack_surf_reprojsr   s-                                                r   forwardzFlowMap.forwardf   s   		1	5!'', '--bq1+e2D2I2I!2L1fnnVAYvay%89'6B99ENN1VAYvay=PQR=RSY,Z,_,_,ak 	k" ;{3F1I>	yy+e,Y7:mmDOODLL1<Mb<PQS<S,TU[aklvvwxz{  }C  zD  EFJJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	I% yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  CE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q/4[5M/MK%g./4K%g. "'*1SbS51&Z9P2QQ
%%k'&:;|;TUYZ[\]VE]"}}VHVAY%?#2#%F%N%NqQR%ST^TfTfghijTkTuTuvwTxyzTz{|T|}  F  F  GI  J  R  R  ST  UV  WX  Y  c  c  de  gh  io  pq  iq  fr  s }}[%83B3%?%G%G!%LZM_M_`abcMdMnMnopMqrsMstuMuv~~  @B  C  K  K  LM  NO  PQ  R  \  \  ]^  `a  bh  ij  bj  _k  l22599l6R]^dReKfghijikgkKl=mnp3qrzz|  B  B  GK  B  L {{D$8$8V9L9T9TUVWX9Y$Z$d$defhijpgq$rxy{ (6@@D	,q!u*=?PQRTUSVQV?W XYop==)=)=a)B7PZ[eefgijkqhrs&A6JZC[]h]p]pqr]st(8PQ++K,?,G,G!,LMWWXY[\]cZdeiikloopqq-k9277>CVCVWYCZ;[\];]^gglmgnssx|s} ~!$[Y!(1Qs<3C/D"EF\_j_o_opq_ruvy  AB  zB  C  H  H  J 99en%o`ahr!Cc)n9TWY8Y4Z&[%opqrtt 1!AB$ 78TUVfVkVkVmnrswxyz~n~Vkk;q=;q="ABRSTUVWkXcXhXhijXkWkSkBlmxy ##ix!"~fQi @;Wwz{A|}  ]  `k  `p  `p  qr  `s  ti|F1I >yUuxy?z{  }[  ^i  ^n  ^n  op  ^q  ri|F1I >yUuxy?z{  }[  ^i  ^n  ^n  op  ^q  rf1!AB$7;KHPPQRSTUYUY^bUYUc	  vax"%dAuyy%"1"+eAqQRseGnUZ[\]^]_[_U`F`9abc/dud		599Q<??51)<CCEJJqMRTUWXZ[\abcde IIv::XXY^_bcedecefhghfh_hYinrXstyz}  A  @A  A  BD  {D  uE  F  GI  J	#I$7$7$;;FFq!KL\L\]^L_I`aeefgh
AMM&B*?*G*G!*LVYcdnnoprstzq{|}
iil>299!F:??SUCVWYZ\]``akl!'!<!<!Z!Z[efijlkljlfl[m!nC2A2I!+CF!3C2I -?AcrcE@R@Z@Z@\]hijklkmim]n@nors{|}~  A  }A  tB  pC  !D  EH  IK  JK  IK  EK  !L0;L0I!AB$0ORR]^eRfghijikgkRlm Q]]6(6!9+E+M+MaPQ+RS^_lSmSuSuvwxySz  TE  TE  FH  TI  JK  TK  LM  TM  N  V  V  WY  Z  d  d  ef  hi  jp  gq  r36
ammF;3F3Fr"3MfUVi,X,`,`abcd,efqr  gA  gI  gI  JK  LM  gN  gX  gX  Y[  g\  ]^  g^  _`  g`-577>wr{99QPQRXzCZ\x{|~!'4NQW!X!`!`!bciju  wQ  TZ  d[  "["\\*>?UVYZdVefgjkmlmkmgmnell+=k,>WXYZ[X[>\]opqvvwxyz{  
	-0L{O_O_`aObc
y)=>Z]h]m]mno]pq
 <
 y)	

 F5M
 \
 --b8
 +//6:
 x
 
 M
 K0
 !T+k"::;AtF5M?Q<RRZZ\aacdgg
 F<q	:
 E"1I
  Z0!
 
 	
] &ps   ?*t5c                 
   | xj                   dz  c_         |d   j                  dd  }|d   j                  d d |d   j                  d      c\  }}}d}t        j                  |d   |d   z        d | }| j
                  j                  r1t        j                  d|d   |d   z  dz
  |      j                         }| j                  |       t        |d   |d         }	t        j                  |d   |	fd      }
t        j                  | j                  |
j                  dd      dz  dz         |d	
      j!                  d||f      |d<   t        j"                  | j%                  |d   j                  dd            j!                  d||f      dz         }||	z   }| j
                  j&                  s| j)                  |d   j                  dd            j!                  d||f      j+                         j                  dd      j-                  d      }t        j.                  d      d    j1                         j3                  |      j5                  ||dd      |d<   dx|d   d<   |d   d<   ||d   z  |d   d<   ||d   d<   |d   d d d df   t7        |d         z   }t9        j:                  |d   |d   d       d   }|t7        |      z  }t        j<                  t        ||d         d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }t        j<                  |d   d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }| jE                  t        j                  |t7        |d         d d dd f   fd            j+                         jG                  d      }t9        jH                  |d d dd |f   |d d d d |f   |d d d d |f         d   }|}tK        |dz
  dd      D ]:  }t        j                  |d d d |f   |d d |dz
  gf   |d d |d f   z  fd      }< t        j                  t        j.                  d      j3                  |      d   jM                  |j                  d      ddd      |fd      }tO        |d d dd f   |d d d df   jQ                         |d d dd f   z  |d   d d dd f         |d   d d dd f   z
  }tS        t        j<                  t        ||d         j                  dd      |d   j                  dd      j?                  d      dz  dz
        jA                  d      j!                  d||f      d      }t        jT                  dt5        |d|      jQ                         t5        |d |      z  tW        |            d!d df   }tY        t        jT                  d"|d   d d df   |            jG                  dd      }|t7        |      t7        |      ||d   |dd#|d   z   z  dd#t7        |      z   z  z
  j[                         j-                         d$z  t        ||d         ||d   d%	z  S )&Nr)   r   r   r(   i  r   r   r   r*   r   r   r    r   r@   r   r   r   r   r   r   r   r   r   r   r3   r}   r   r   zbsnij,bksj->bksnizb t x y -> b s t x yr   zb t x y -> b t s x y.r   r%   r   )	r   r   r   r   r   r   r   rC   r   ).r   r5   r   r!   r   r   r   r   r   r   r   r"   r.   r   r   r7   r8   r   r   r   r   r   r   r   r   r   r   r   r   r   r/   r   r   r   r   r   r  rR   r  warpr	  r   r<   rB   r;   r
  )r{   r  r  r  r   r   r  r  r  r  r  r   r   r  r  r  r   r  r  r   
adj_transfrC   rY   r&  r'  r*  r   s                              r   forward_staticzFlowMap.forward_static   s   		1	5!'', '--bq1+e2D2I2I!2L1fnnVAYvay%89'6B99ENN1VAYvay=PQR=RSY,Z,_,_,ak 	k" ;{3F1I>	yy+e,Y7:mmDOODLL1<Mb<PQS<S,TU[aklvvwxz{  }C  zD  EFJJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	I% yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  CE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q/4[5M/MK%g./4K%g. "'*1SbS51&Z9P2QQ
%%k'&:;|;TUYZ[\]VE]"}}VHVAY%?#2#%F%N%NqQR%ST^TfTfghijTkTuTuvwTxyzTz{|T|}  F  F  GI  J  R  R  ST  UV  WX  Y  c  c  de  gh  io  pq  iq  fr  s }}[%83B3%?%G%G!%LZM_M_`abcMdMnMnopMqrsMstuMuv~~  @B  C  K  K  LM  NO  PQ  R  \  \  ]^  `a  bh  ij  bj  _k  l22599l6R]^dReKfghijikgkKl=mnp3qrzz|  B  B  GK  B  L ((!AB{2B)C\RSTUVaRaEbdpqrst  vA  rA  eB  D  EF  G
vax"%dAuyy%"1"+eAqQRseGnUZ[\]^]_[_U`F`9abc/dud		599Q<??51)<CCEJJqMRTUWXZ[\abcde HQqrTN53B3<+?+?+A%!"++MkZfNghijkjlhlNmnoz  |C  pD  EF  GH  GI  EI  pJ  J Q]]6(6!9+E+M+MaPQ+RS^_lSmSuSuvwxySz  TE  TE  FH  TI  JK  TK  LM  TM  N  V  V  WY  Z  d  d  ef  hi  jp  gq  r  sI  K
"\\*=fUKadj>k>s>s>uv|  ~C  DZ  ]c  wd  ?d  eh  is  et  u  vy  z|  {|  z|  v|  }ell+=k,>WXYZ[X[>\]opqvvwxyz{y)5M)#K0)T+k"::;AtF5M?Q<RRZZ\aacdgg"<q	:$Z0

 

 
	
r   c                 (
   | xj                   dz  c_         |d   j                  dd  }|d   j                  d d |d   j                  d      c\  }}}| j                  |       t	        |d   |d         }t        j                  |d   |fd      }t        j                  | j                  |j                  dd      dz  dz         |d	      j                  d||f      |d
<   t        j                  | j                  |d
   j                  dd            j                  d||f      dz         }	|	|z   }
| j                  j                  s| j!                  |d
   j                  dd            j                  d||f      j#                         j                  dd      j%                  d      }t        j&                  d      d    j)                         j+                  |
      j-                  ||dd      |d<   dx|d   d<   |d   d<   ||d   z  |d   d<   ||d   d<   t/        t1        |d
   |d   j3                  d            d      }| j5                  t        j                  |d d dgf   j7                  d|dd      |fd            j#                         j9                  d      }t;        j<                  d      d d d   }t?        j@                  |d   |d   d       d   }|t1        |
|d   j3                  d            jC                  d      z  }t?        jD                  |d|d d f   |d d dgf   j7                  d|dd      d|d d f   |d|d d f         d   }t        jF                  d|jI                         tK        |d d dgf   j7                  d|dd                  dd df   }tM        ||d         j9                  dd      }t        jN                  |d   |d   d d dgf         }||d   z
  |j3                  d      z  jQ                         j%                         }t?        j@                  |d   |d   d       d   tS        |
      z  }tU        |d d dd f   |d d d df   jI                         |d d dd f   z  |d   d d dd f         |d   d d dd f   z
  }|tS        |	      tS        |
      ||dz  |d   dd|d   z   z  ddtS        |
      z   z  z
  jQ                         j%                         d z  t	        |d!      ||d"   d#	z  S )$Nr)   r   r   r(   r   r   r   r*   r   r   r    r   r@   r   r   r   r   r   r   r   r   r   r   i  
   .zbtij,btpj->btpipred_visibilityr   g      Y@r%   r   *   r   )	r   r   r   point_track_lossr   r   r   rC   r   )+r   r5   r   r   r   r!   r"   r.   r   r   r7   r8   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   aranger   r   r   r  r<   r	  rB   rA   minimumr
  r   r-  )r{   r  r  r  r   r   r  r  r  r   r   r  
track_featr   neighb_list	rds_trackeye_surf_trackrC   point_track_surf_reprojpoint_track_reprojvis_maskr4  r   r&  s                           r   forward_zFlowMap.forward_%  s&   		1	5!'', '--bq1+e2D2I2I!2L1f 	k" ;{3F1I>	yy+e,Y7:mmDOODLL1<Mb<PQS<S,TU[aklvvwxz{  }C  zD  EFJJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	I% yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  CE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q/4[5M/MK%g./4K%g. !;v+>M@Z@d@deg@h!i  kA  B
22599jA3>O>V>VWYZ`acdf>ghr=stv3wx  A  A  C  H  H  MQ  H  R ii&tt, ++K,F{S_G`aefghi	"Yu[5O5Y5YZ\5]%^%f%fgh%ii##N3{13D$E~VWYZX[V[G\GcGcdfgmnpqsGtux  zE  FG  vG  HH  JV  WZ  [f  gh  Wh  Ji  j  kl  m"',,/@QTUcdeghfidiUjUqUqrtu{|~  @B  VC  RD  #E  FI  JL  KL  JL  FL  #M$%<[=VW\\]^_`a==->!?L]@^_`bcad_d@ef0;}3MMQYQcQcdfQggpprwwy **;w+?L@YZ^_`abeklqerrHQqrTN53B3<+?+?+A%!"++MkZfNghijkjlhlNmnoz  |C  pD  EF  GH  GI  EI  pJ  Jy)5M)/3#K0T+k"::;AtF5M?Q<RRZZ\aacdgg"<3$Z0

 

 
	
r   c                 b    |d   j                   d d |d   j                  d      |d   j                   dd  c\  }}}d|vrt        d       t        j                         5  t        j                   j                  j                  t        j                  |d   j                  dd      d	z  d
            |d   j                   dd        }t        |j                  d|d   j                   d d             |d<   d d d        t        d        fdt        j                         5   j                  j                  rdnd _        d|vr j                  |d   d d dd f   j                  dd      j!                  t        j"                        |d   d d d df   j                  dd      j!                  t        j"                              \  }} ||      j                  d||dz
  f      |d<    j                  j$                  rg g }	}t'        t        j(                  d|dz
   j                        j+                               D ]  \  }
}t        d|        j-                  |d   d d d|d      \  }}|t        j.                  |d   j                   dd  d d d         dz
  d   j1                         z  }|j3                  |       |	j3                  |        j                  j                  s* j                  j                  r j4                  dz  dk(  st        d       t7        dd      j9                  |d   ||d|
z  |       	 t        j:                  |d      |d<   t        j:                  |	d      }	t        j:                  |d   t        j<                  |d         j?                  |dz
  ddd      f      |d<   t        j:                  |	t        j@                  |	      j?                  |dz
  dd      f      |d<   d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r(   r)   r   r   zdoing zoe depth	rgb_larger      r   )scale_factorzdone doing zoe depthc                    t        d| j                         	j                  j                  rwt	        |       dkD  ri| d t	        |       dz   |d t	        |       dz   }}| t	        |       dz  d  |t	        |       dz  d  }}t        j                   ||       ||      f      S 	j                  j                  r.	j                  j                  s	j                  | |d      d   }n6| dz  dz   dz  |dz  dz   dz  }} 	j                  | |dgdgdgd	
      d   d   }|t        j                  | j                  dd  d d d         dz
  j                  |       d d d d d f   z  }t        j                  |dd      S )Nz
doing raftr3   r(   rv   )num_flow_updatesr    r   rA  F)attn_splits_listcorr_radius_listprop_radius_listpred_bidir_flow
flow_predsr   r)   r*   T)r,   	antialias)r   r5   r   r   r4   r!   r"   r   r   tensorr   r.   r   )
r   r1   x_lefty_leftx_righty_rightr  r  raftr{   s
          r   rP  zFlowMap.get_flow.<locals>.rafti  s   ,qww' yy  SVAX !*3q619a
Qmv"#CFAIJ-#a&!)*yy$vf"5d776K!LMMyy  ):):JJqBJ7;rT"WcM1R47C-!LL1qcB4bdaev{L|  ~J  K  LN  Ou||AGGBCL2$679==a@aTAQRRC==VdKKr   r   r    zdoing video trackingre   T)	grid_sizegrid_query_framebackward_trackingr      z+writing point track video for visualizationz	./output/d   )save_dir	pad_valueztracks_%02d)videotracks
visibilityfilenamequery_framer   r2  )!r5   r   r   r!   no_gradr.   r   r   inferr7   r   r8   r   r   n_track_framesr   r   uint8point_track	enumerater   r   r   rK  r   rS   r   r   	visualizer"   r#   r  
zeros_like)r{   r  r   r   r  zoe_est
raft_inp_x
raft_inp_yr   visibilitiesrY   start_frame
pred_trackrZ  pred_track_normr  rP  s   `              @@r   r   zFlowMap.get_flow]  s   '.44Ra8U9K9P9PQR9ST_`eTfTlTlmompTq1fVk)#$ e--(8(8(>(>q}}[YdMeMmMmnopqMrsvMv  EH  @I  )J  KV  W\  K]  Kc  Kc  df  dg  Kh  i+1'2C2CAkRWFXF^F^_a`aFb2c+dK(e ()	L& ]]_ 	~%)YY%6%6AD, )-(<(<[=UVWXYXZVZ=[=c=cdefg=h=k=klqlwlw=x  {F  GR  {S  TU  VY  WY  VY  TY  {Z  {b  {b  cd  ef  {g  {j  {j  kp  kv  kv  {w  )x%
:*.z**F*P*PQRTUV\]^V^S_*`J' 99((-/K)25>>!F1HTM`M`3a3f3f3h)i 	U+4[A04K@XY[Z[@\hi  gr  FJ  1K-
:*4ell;{C[CaCabdbeCfgkikgkCl6mno6oq  6A  6F  6F  6H  +H#**?;L<O<OPZ<[  99,,TYY5F5F499UW<YZ?!"OP&sKUU]hit]u  I  V`  kx  yz  kz  HSU  T	U 05yyQ/GK.!&<!:L 05yy+m:TUZUdUdepq~e  VA  VH  VH  IJ  KL  IL  MO  PR  SU  VV  :W  0XK.3899l5K[K[\hKiKpKpqrstqtuwxzK{=|3}K 125	~ 	~5e e4	~ 	~s'   #BP*G$P%P%"C-P%P"%P.)	__name__
__module____qualname__r   r+  r/  r>  r   __classcell__)r   s   @r   r]   r]   .   s1    52n (* }
~ /1 >
@ )+ 7
p9~r   r]   )N)3__doc__r!   torchvisionr   r  einopsr   r   torchvision.opsopstorch.nnr   r.   numpyr   r   randomtimer   copyr   
matplotlibr   wandbr	   typingr
   r   r   r   r   r   collectionsr   r   rS   cotracker.utils.visualizerr   r   r   r   r   rB   r;   r6   r   rA   r-  r[   Moduler]   r&   r   r   <module>r     s    *    $  $        C C # + , G  >uC2T
 C	Roh~bii h~r   