
    53g                        d dl Z d dlZd dl mZ d dlZd dlmZmZ d dlmZ d dl	m
Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlmZ d dlmZmZmZmZmZmZ d d	lm Z  d dl!m"Z# ejH                  jK                  d
       d dl&m'Z'm(Z( d dl)Z)d Z*ddZ+d Z,d Z-d Z.d Z/d Z0d Z1d Z2 G d dejf                        Z4y)    N)nn)	rearrangerepeat)
functional)deepcopy)cm)tqdm)CallableListOptionalTuple	GeneratorDict)defaultdictz./third_party/co-tracker/)
Visualizerread_video_from_pathc                     t        | d      S )Nz... c x y -> ... (x y) c)r   xs    =/home/cameronsmith/repos/refactor_flowmap_and_splat/models.py<lambda>r      s    i"<=     c           	      f    t        | d|"t        | j                  d      dz              S |      S )Nz... (x y) c -> ... c x y      ?r   )r   intsize)srcr   s     r   r   r      s2    i,FdedmSXXVX\\^M_I`u stu r   c                 d    t        j                  | t        j                  | ddgf         fd      S )N.r   )torchcat	ones_liker   s    r   r   r      s)    eii5??1S!W:#> ?C r   c                 .    | dd df   d| ddd f   z   z  S )N.r    h㈵> r   s    r   r   r      s%    aCRCj$qRSz/2 r   c                 @    t        j                  | |dz  dz
  dd      S )N      bilinearborder)modepadding_mode)Fgrid_sampler   ys     r   r   r      s    a!A:8T r   c                     t        | j                        dk(  rt        | |      S t        | j                  dd      |j                  dd            j	                  d| j                  d d       S )N   r   r)   r(   )lenshape
grid_samp_flatten	unflattenr0   s     r   r   r      so    3qww<?
1Q 
199UVWX>Z[ZcZcdefgZh@i@s@stuvwv}v}  A  @A  wB  AC r   c                 B    t        t        j                  d||             S )Nb...cij,b...ckj->b...cki)unhomr!   einsum)crdsKs     r   r   r      s    5.HD!QR r   c           	      d    t        t        j                  d|t        |             dd df   |      S )Nr:   .   )projectr!   r<   hom)r=   posesr>   s      r   r   r       s6    %,,7QRWX[\`Xa*bcfgihigici*jlm!o r   c           	      6   d }g }t        t        |       dz
        D ]S  }|j                  t        j                  | |   | |dz                   |j                  t        j
                                U t        j                  |d d  }|j                  |       |S )Nc                     t        |       t        j                  k(  r;t        | d      r.t        j                  j                  | j                  ddd       y y y )Nweightg        relufan_in)anonlinearityr,   )typer   Linearhasattrinitkaiming_normal_rF   )ms    r   init_weights_normalz%make_net.<locals>.init_weights_normal#   sF    7biiq(#''CfS['\ $  r   r)   r    )ranger4   appendr   rL   ReLU
Sequentialapply)dimsrQ   layersinets        r   make_netr[   "   s    ] F3t9Q; !biiQQqS	23bggi ! --
%CII!"Jr   c                   L     e Zd Z fdZi fdZi fdZi fdZi fdZd Z xZS )FlowMapc           	      z    t                    | _        ddlm} |j
                  j                          _        ddlm}  ||j
                  d       _	        t        j                  j                  d       ddlm}  |dd	d
d	ddd      j                          _        t#        j$                  d      }d|v r|d   n|} j                   j'                  |d        j                   j)                         D ]	  }d|_         ddlm}  |t0        j                  j3                  d            j                          _        d _        d j6                  z  }	t9        j:                  d|	       _        t?        |	dd	g       _         t?        |	dz  dd	g       _!        t?        |	dd	g       _"        t?        |	dd	g       _#        dd l$}
t9        jJ                  |
jM                  d j6                  z        t9        jN                  d|	 j6                  z  dd	             _(        t9        jN                  |	d	dd	       _)        t9        jN                  |	d	dd	       _*        t9        jN                  |	d	dd	       _+        t9        jN                  |	ddd	       _,        t9        jN                  |	ddd	       _-        d	 _.        t?        |	d j\                  g       _/         fd _0        d  _1        d\   _2         _3        d _4        t"        jj                  j%                  d d!d"#      j                          _6        y )$Nr   )Raft_Large_Weights)
raft_largeF)weightsprogressz	./gmflow/)GMFlow   r)      swinr3      )feature_channels
num_scalesupsample_factornum_headattention_typeffn_dim_expansionnum_transformer_layersz8./gmflow/gmflow-scale1-mixdata-train320x576-4c3a6e9a.pthmodel)strict)CoTrackerPredictorzB./third_party/co-tracker/checkpoints/cotracker_stride_4_wind_8.pth)
checkpoint@   i   r(      )in_chr@   )padding    c                 4    t        | dj                        S )Nzb ... -> (b o) ...o)r   n_rig)r   selfs    r   r   z"FlowMap.__init__.<locals>.<lambda>^   s    va(<tzzJ r   c                     t        | d      S )Nzb t o ... -> (b o) t ...)r   r   s    r   r   z"FlowMap.__init__.<locals>.<lambda>_   s    &+E"F r   NNzmhamilton723/FeatUpdinov2T)use_norm)7super__init__argstorchvision.models.optical_flowr_   DEFAULT
transformsraft_transformsr`   raft_syspathrS   gmflow.gmflowrc   cudagm_flowr!   loadload_state_dict
parametersrequires_gradcotracker.predictorrq   osjoin
co_trackertime_strider   rL   fmap_downprojr[   	depth_estcorr_weighter_perpoint
static_estmask_scorerconv_modulesrU   PixelNeRFEncoderConv2d
resnet_enc
depth_conv
focal_convtemperature_convaffinities_convaffinities_conv2r{   rig_predictoro_repo_switchcorr_weightsdepthstephub	upsampler)r|   r   r_   r`   rc   rr   ra   paramrq   fdimr   	__class__s   `          r   r   zFlowMap.__init__1   s   	 	G199DDF>(:(B(BUS
 	$(sqRS^_pv  KL  ef  h  m  m  oZZ Z[
)0J)>*W%J$$WU$;\\,,.KEe0CK 	;,  HL  :M  N  S  S  U !!!YYs40!41+.&.Qr!}&=#"DA;/#T"QK0 	l&C&C!DL\L\J\&C&]^`^g^ghklpqu  rB  rB  mB  CD  MN  _O  P		$q15		$q15 iiQq;YYtBq; iiR!<
%tBtzz&:;J
F &/$$*	(=xRVW\\^r   c                 t   | xj                   dz  c_         |d   j                  dd  }d}|d   j                  d d |d   j                  d      c\  }}}d}t        j                  |d   |d   z        d | }	| j
                  j                  r1t        j                  d|d   |d   z  dz
  |      j                         }	| j                  |       t        j                  t        j                  |d   d d d df         |d   d	z  fd      }
t        |d
   |d         }t        j                  |d   ||
fd      }t        |d| j                        }t        j                   | j#                  |j%                  dd      dz  dz         |d      }t        |d| j                  |      |d<   t        j&                  | j)                  |d   j%                  dd            j+                  d||f      dz         }||z   }| j
                  j,                  s| j/                  |d   j%                  dd            j+                  d||f      j1                         j%                  dd      j3                  d      dz   }t        j4                  d      d    j7                         j9                  |      j;                  ||dd      |d<   dx|d   d<   |d   d<   |d d d f   |d   d d z  |d   dd d ddf<   |d d d f   |d   dd d ddf<   |d   d d d df   t=        |d         z   }t?        j@                  |d   |d   d       d   }|t=        |      z  }t        jB                  t        ||d         d d d df   j%                  dd      |j%                  dd      jE                  d      dz  dz
        jG                  d      jI                  ddd      j+                  d||dz
  f      }t        jB                  |d   d d d df   j%                  dd      |j%                  dd      jE                  d      dz  dz
        jG                  d      jI                  ddd      j+                  d||dz
  f      }| jK                  t        j                  |t=        |d         d d dd f   fd            j1                         jM                  d      }t        d| jO                  t=        |d               z  jQ                  d      d       x}}t=        t        j                   t        ||d         j%                  dd      |      j+                  d|j                  d d             }d! }d"g d#}}t;         |ddtS        |dz              d$| jT                  ||dz
  %      jW                         x}}t?        jX                  t        t[        t        |d d dd f   |d         t        |d&|'            d(|j                  d      )      t        t[        t        ||d         t        |d&|'            d(|j                  d      )      t        t[        t        ||d         t        |d&|'            d(|j                  d      )      t[        t        |d d dd f   |d         |      jG                  d      z  jM                  d            d   }t]        |dz
  dd      D ]:  }t        j                  |d d d |f   |d d |dz
  gf   |d d |d f   z  fd      }< t        j                  t        j4                  d*      j9                  |      d+   j_                  |j                  d      ddd      |fd      }t        j                  t`        j>                  jb                  je                  |dd dd df   d,-      |dd ddf   fd      } || jE                  d      z  j+                  d|| jT                  f      jg                  d      }!t=        t        j                   t        |!|d         j%                  dd      |d      j+                  d||f            }!t        j4                  d*      d.   j_                  |||!j                  d      dd      j9                  |!      }"t`        j>                  jb                  ji                  |!dd d*f         |"dd dd df<   |!dd*d f   |"dd ddf<   tk        t        jl                  d/|"d d d df   jo                         |"d d dd f   z  tq        |d d dd f               dd df   |d   d d dd f         |d   d d dd f   z
  }#d0|v rt        t        jB                  t        ||d         j%                  dd      |d0   j%                  dd      jE                  d      dz  dz
  d12      j+                  d||f      d3      }$t        t        jB                  t        |"j%                  dd      |d         j%                  dd      |d0   j%                  dd      jE                  d      dz  dz
  d12      j+                  d||f      d4d*5      }%t;        |%d6|      jo                         t;        |%d7|      z  }&t        jl                  d8|&tq        |$            dd df   }'ts        t        jl                  d9|d   d d df   |'            jM                  dd      |d:<   ||j%                  dd      jg                  d      ju                  d      d      d    }(t        jl                  d/|"tq        |            dd df   })t        jl                  d/%jI                  ddddd*      tq        $jI                  dddd                  dd df   }*t        jB                  |d   j%                  dd      |d0   j%                  dd      jE                  d      dz  dz
  d12      j+                  d||f      jI                  ddddd*      jG                  d      }+t        j                  t        jv                  g d;      j7                         jW                         d.   j_                  |||+j                  d      d      |$d d d d dgf   |$z
  jI                  dddd      fd      jE                  d      },t=        |d         }-|i d<t        |d=| jT                  >      d?t=        |      d@t=        |      dA|#dBt        j                  |!jE                  d      |,fdC      dDt        j                  |)|*fd      dEt        j                  |-|+fd      dF|!j+                  d|      dG|"j+                  dC|      dH|dI|dJ|(d
|d
   dKt        ||d         dJ|(dL|d   z  S )MNr)   rgbr   rs   rs   r(   ,  r   bwd_flowg     @@	depth_inpzb (t s) c x y -> b t (s c) x ysr   r*   r,   z (b t) (s c) x y -> b (t s) c x y)r   bfmapr    dimg333333?r@   
intrinsics.r   r(   .r)   r(   	org_ratio.x_pix-C6?ming       @zb t x o -> (b o) t x 1c                     t        j                  t        j                  t        d      D cg c]  }t        j                  | dz   |dz
  |      ! c}       j                  dd      S c c}w Nr(   g{Gz?r)   r!   stackmeshgridrR   linspacer7   r   en_s       r   r   z!FlowMap.forward.<locals>.<lambda>   V    U^^ejklem=n`aennQsUSTUXSXYZ>[=n-o!p!x!xyz{|!} =n   $A(i  r   g      ?g?c p -> (b o) t p 1 crz   r   t(b s) t p 1 c -> b t (p s) 1 cr   b t c (p s) 1 -> (b s) t p cpr3   r~   r%   epsNNNbtpij,btpj->btpipred_tracksr+   r-   b t c p 1 -> b p t cb t (x y) p 1 -> b p t x yr   b p t x y -> b p s t x yb p t x y -> b p t s x ybksnij,bksj->bksnibij,bksnj->bksnitrack_reprojs)r)   r   r   r   	rig_masks(b o) t xy 1 -> b t o xy 1ry   	res_depthr   flow_from_poselie_crds
world_crdsrgb_crds
lie_perpixpose_perpixeye_surf	poses_allrC   r   	flow_inp_)<r   r5   r   r!   randpermr   overfitr   longget_flowr"   
zeros_likech_fstr   r   r.   interpolater   r7   softplusr   r8   use_gt_intrinsicsr   sigmoidmeaneyefloattor   ch_secgeometryget_world_raysr/   	unsqueezesqueezepermuter   clipr   softmaxr   r{   r   
procrustes	grid_samprR   expandkorniaconversionsrotation_matrix_to_quaternionsumquaternion_to_rotation_matrixrA   r<   inverserB   r;   maxtensor).r|   model_inputoutimsize	low_imresr   r   n_trgtn_samprand_subsetflow_inpr   rgbdffmap_outr   r   focal
corresp_uvrdsr   corresp_surfcorresp_featr   affinity_maskaffinity_mask_unnormget_xpixn_total_samp	samp_amtssample_locsuniform_grid_pixrC   rY   	poses_lier   r   adj_opt_flow
eye_trackspose_trackspose_all_to_all_perpixtrack_surf_reprojsbg_poser   world_crds_tracks
rgb_tracks
lie_tracksr   s.                                                 r   forwardzFlowMap.forwardo   s   		1	5!'',	 '--bq1+e2D2I2I!2L1fnnVAYvay%89'6B99ENN1VAYvay=PQR=RSY,Z,_,_,ak 	k" 99e..{:/Fq!t/LMkZdNefiNijklm;{3F1I>			;u-iA!D>AQAQR==q1CB1Fr1I!J6Wab'1SVZVfVfijkF JJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	I% yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  C  DF  FE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q16qv{;?WXZYZ?[1[K%c!Aai016qvK%c!Aai0 "'*1SbS51&Z9P2QQ
%%k'&:;|;TUYZ[\]VE]"}}VHVAY%?#2#%F%N%NqQR%ST^TfTfghijTkTuTuvwTxyzTz{|T|}  F  F  GI  J  R  R  ST  UV  WX  Y  c  c  de  gh  io  pq  iq  fr  s }}[%83B3%?%G%G!%LZM_M_`abcMdMnMnopMqrsMstuMuv~~  @B  C  K  K  LM  NO  PQ  R  \  \  ]^  `a  bh  ij  bj  _k  l22599l6R]^dReKfghijikgkKl=mnp3qrzz|  B  B  GK  B  L ,5s4;M;MfU`agUhNi;j7j6s6stv6w  zR  ,S  	S*Q]]6-q	+J+R+RSTUV+WXabllmno|  pC  pC  DF  EF  pG  H  I
 ~!$[Y)/1SrAQ=R0STjmqmwmwz{  E  FG  G  *H  *M  *M  *O  	O& ##ix!"~fQi @;Wwz{A|}  ]  `k  `p  `p  qr  `s  ti|F1I >;Wwz{A|}  ]  `k  `p  `p  qr  `s  ti|F1I >;Wwz{A|}  ]  `k  `p  `p  qr  `s  tf1!AB$7	!EkRZZ[\]^_c_chl_c_m	
  vax"%dAuyy%"1"+eAqQRseGnUZ[\]^]_[_U`F`9abc/dud		599Q<??51)<CCEJJqMRTUWXZ[\abcde IIv::XXY^_bcedecefhghfh_hYinrXstyz}  A  @A  A  BD  {D  uE  F  GI  J	#I$7$7$;;FFq!DJJX\\]^_
AMM&IaL*I*Q*QRSTU*VW]cmnxxyz|}  E  |F  G  H
iil>299!F:??SUCVWYZ\]``akl!'!<!<!Z!Z[efijlkljlfl[m!nC2A2I!+CF!3C2I -?AcrcE@R@Z@Z@\]hijklkmim]n@nors{|}~  A  }A  tB  pC  !D  EH  IK  JK  IK  EK  !L0;L0I!AB$0ORR]^eRfghijikgkRlm K' vhvay/I/Q/QRSTU/VWbcpWqWyWyz{|}W~  XI  XI  JL  XM  NO  XO  PQ  XQ  _g ")Aqj)+ADJ!!--{7J7J2b7QRXYZR[0\0d0defgh0iju  wD  kE  kM  kM  NO  PQ  kR  k\  k\  ]_  k`  ab  kb  cd  kd19#;;D9Q&z;RTpstvK%+K8RU[%\%d%d%fgmny  {U  X^  h_  &_"!&.BCYZ]^hZi!jknoqpqoqkq!r#(6HUaIbcdefcfIghz){#|  $B  $B  CD  EF  $GC --a377;??A?FqIJ4P\\"4[XOPSTVUVTVPVW
!LL);K<O<OPQRSTUVWXY<Z[^_i_q_qrstuvwxy_z[{|  ~A  BD  CD  BD  ~D  E==U!3!;!;Aa!@]A[AcAcdefgAhArArsuAvwxAxyzAzIQSS\S\]^`abh_iSjkrkrstuvwxyz{|k}  F  F  GI  J 	YYY 7 = = ? D D F~ V ] ]^_`fgqgvgvwygz{} ~!+AaG!4Z!? H H1Qq Q ""$&&/im 	 +e,-  
	-0LtzzZ
 y)	

 F5M
 \
 uyy*"6"6r"::!FrJ
 J/@#A"E
 uyy(:!6r:
, --b8-
. +//6:/
0 x1
2 3
4 G5
6 K07
8 F<q	:9
: G;
< Z0=
 
 	
r   c                    | xj                   dz  c_         |d   j                  dd  }d}|d   j                  d d |d   j                  d      c\  }}}d}t        j                  |d   |d   z        d | }	| j
                  j                  r1t        j                  d|d   |d   z  dz
  |      j                         }	| j                  |       t        |d   |d         }
t        j                  |d   |
fd      }t        j                  | j                  |j                  dd      d	z  d	z         |d
      j!                  d||f      |d<   t        j"                  | j%                  |d   j                  dd            j!                  d||f      dz         }|
}| j
                  j&                  s| j)                  |d   j                  dd            j!                  d||f      j+                         j                  dd      j-                  d      }t        j.                  d      d    j1                         j3                  |      j5                  ||dd      |d<   d	x|d   d<   |d   d<   |d d d f   |d   d d z  |d   dd d ddf<   |d d d f   |d   dd d ddf<   |d   d d d df   t7        |d         z   }t9        j:                  |d   |d   d       d   }|t7        |      z  }t        j<                  t        ||d         d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }t        j<                  |d   d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }| jE                  t        j                  |t7        |d         d d dd f   fd            j+                         jG                  d      }| jI                  |d   j                  dd            j!                  d||f      }|d   j?                  d      }tK        tM        |d d dgf   |d d dgf         d      }t        j                  |j                  dd      dd
      j!                  d||f      }t        jN                  dt7        |      |jA                  d            }tK        |d      }|jQ                  d      jG                  d      x}}d }dg d}}t5         |ddtS        |d	z              d |j                  d      ||dz
  !      jU                         }t        j                  |D  cg c]%  }  ||  | tS        |tW        |      z  d	z              ' c} d      jX                  }!tK        |d d dd f   d"      |!jU                         d d d d d f   z   }"t        jZ                  |"dk  |"dkD  z  |d d d d d |"j                  d      f   |"      }"t9        j\                  tK        tM        t        |d d dd f   |d         tK        |"d#|$            d%|"j                  d      &      tK        tM        t        ||d         tK        |"d#|$            d%|"j                  d      &      tK        tM        t        ||d         tK        |"d#|$            d%|"j                  d      &      tM        t        |d d dd f   |d         |"      jA                  d      z  jG                  d            d   }#t_        |dz
  dd      D ]:  } t        j                  |#d d d | f   |#d d | dz
  gf   |#d d | d f   z  fd      }#< t        j                  t        j.                  d'      j3                  |#      d(   ja                  |#j                  d      ddd      |#fd      }#t        j                  tb        j8                  jd                  jg                  |#dd dd df   d)*      |#dd ddf   fd      }$||$j?                  d      z  j!                  d||j                  d      f      ji                  d      }%t7        t        j                  t        |%|d         j                  dd      |d
      j!                  d||f            }%t        j.                  d'      d+   ja                  |||%j                  d      dd      j3                  |%      }&tb        j8                  jd                  jk                  |%dd d'f         |&dd dd df<   |%dd'd f   |&dd ddf<   tm        t        jN                  d,|&d d d df   jo                         |&d d dd f   z  tq        |d d dd f               dd df   |d   d d dd f         |d   d d dd f   z
  }'tK        t        j<                  t        ||d         j                  dd      |d   j                  dd      j?                  d      dz  dz
        j!                  d||f      d-      }(tK        t        j<                  t        |&j                  dd      |d         j                  dd      |d   j                  dd      j?                  d      dz  dz
  d./      j!                  d||f      d0d'1      })t5        |)d2|3      jo                         t5        |)d4|3      z  }*t        jN                  d5|*tq        |(            dd df   }+ts        t        jN                  d6|d   d d df   |+            jG                  dd      },|tK        |d7|j                  d      8      tK        |d7|j                  d      8      |t7        |      t7        |      |'|%j!                  d|      |&j!                  d9|      ||#|,|d   t        ||d         |&d d d d d:f   |d   d;z  S c c} w )<Nr)   r   r   r   r(   r   r   r   r   r*   r   r   r    r   r@   r   r   r   r   .r   r   r   r   r   b t c p 1 -> b t p czb t p c, b s c -> b t p szb t p s -> (b s) t p 1c                     t        j                  t        j                  t        d      D cg c]  }t        j                  | dz   |dz
  |      ! c}       j                  dd      S c c}w r   r   r   s       r   r   z"FlowMap.forward_.<locals>.<lambda>H  r   r      r   r   r   zb t s 1 c -> (b s) t 1 1 cr   r   r   r   r3   r~   r%   r   r   r   r   r+   r   r   r   r   r   r   r   r   r   ry   r   
   )r   rig_masks_unnormaffinity_embr   r   r   r   r   r   r   r   r   r   rC   r   ):r   r5   r   r!   r   r   r   r   r   r   r   r"   r.   r   r   r7   r8   r   r   r   r   r   r   r   r   r  r   r  r  r  r/   r  r  r  r   r  r   r   r  r<   r	  r   r   r4   Twherer
  rR   r  r  r  r  r  r  rA   r  rB   r;   )-r|   r  r  r  r  r   r   r  r  r  r   rgbdr   r   r  r   r!  r   r"  r#  r   r<  aff_src_samp_locsaff_sourcesaffinity_emb_lowaff_sim_allpixr%  r$  r&  r'  r(  r*  rY   local_samp_gridsr)  rC   r+  r   r   r,  r-  r.  r/  r0  r   s-                                                r   forward_zFlowMap.forward_   s   		1	5!'',	 '--bq1+e2D2I2I!2L1fnnVAYvay%89'6B99ENN1VAYvay=PQR=RSY,Z,_,_,ak 	k" ;{3F1I>	yy+e,Y7:mmDOODLL1<Mb<PQS<S,TU[aklvvwxz{  }C  zD  EFJJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	 yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  CE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q16qv{;?WXZYZ?[1[K%c!Aai016qvK%c!Aai0 "'*1SbS51&Z9P2QQ
%%k'&:;|;TUYZ[\]VE]"}}VHVAY%?#2#%F%N%NqQR%ST^TfTfghijTkTuTuvwTxyzTz{|T|}  F  F  GI  J  R  R  ST  UV  WX  Y  c  c  de  gh  io  pq  iq  fr  s }}[%83B3%?%G%G!%LZM_M_`abcMdMnMnopMqrsMstuMuv~~  @B  C  K  K  LM  NO  PQ  R  \  \  ]^  `a  bh  ij  bj  _k  l22599l6R]^dReKfghijikgkKl=mnp3qrzz|  B  B  GK  B  L ++K,?,G,G!,LMWWXY[\]cZde
 (6@@D	,q!u*=?PQRTUSVQV?W XYop ==)=)=a)B7PZ[eefgijkqhrs
 &A6JZC[]h]p]pqr]st(8PQ /C-K-KPQ-K-R-W-W\`-W-aa+ ~!$[Y!(1Qs<3C/D"EF\_j_o_opq_ruvy  AB  zB  C  H  H  J 99en%o`ahr!Cc)n9TWY8Y4Z&[%opqrtt 1!AB$ 78TUVfVkVkVmnrswxyz~n~Vkk;q=;q="ABRSTUVWkXcXhXhijXkWkSkBlmxy
 ##ix!"~fQi @;Wwz{A|}  ]  `k  `p  `p  qr  `s  ti|F1I >;Wwz{A|}  ]  `k  `p  `p  qr  `s  ti|F1I >;Wwz{A|}  ]  `k  `p  `p  qr  `s  tf1!AB$7	!EkRZZ[\]^_c_chl_c_m	
  vax"%dAuyy%"1"+eAqQRseGnUZ[\]^]_[_U`F`9abc/dud		599Q<??51)<CCEJJqMRTUWXZ[\abcde IIv::XXY^_bcedecefhghfh_hYinrXstyz}  A  @A  A  BD  {D  uE  F  GI  J	#I$7$7$;;FFq!KL\L\]^L_I`aeefgh
AMM&IaL*I*Q*QRSTU*VW]cmnxxyz|}  E  |F  G  H
iil>299!F:??SUCVWYZ\]``akl!'!<!<!Z!Z[efijlkljlfl[m!nC2A2I!+CF!3C2I -?AcrcE@R@Z@Z@\]hijklkmim]n@nors{|}~  A  }A  tB  pC  !D  EH  IK  JK  IK  EK  !L0;L0I!AB$0ORR]^eRfghijikgkRlm Q]]6(6!9+E+M+MaPQ+RS^_lSmSuSuvwxySz  TE  TE  FH  TI  JK  TK  LM  TM  N  X  X  YZ  \]  ^d  [e  f36
ammF;3F3Fr"3MfUVi,X,`,`abcd,efqr  gA  gI  gI  JK  LM  gN  gX  gX  Y[  g\  ]^  g^  _`  g`-577@yAf:7NPlopr!'4NQW!X!`!`!bciju  wQ  TZ  d[  "["\\*>?UVYZdVefgjkmlmkmgmnell+=k,>WXYZ[X[>\]opqvvwxyz{ !-0L{O_O_`aObc()=>Z]h]m]mno]pq'y)5M)#--b8%//6:)#K0"<q	:!B'$Z0
 
 	
Y &ps   *qc                 R   | xj                   dz  c_         |d   j                  dd  }|d   j                  d d |d   j                  d      c\  }}}d}t        j                  |d   |d   z        d | }| j
                  j                  r1t        j                  d|d   |d   z  dz
  |      j                         }| j                  |       t        |d   |d         }	t        j                  |d   |	fd      }
t        j                  | j                  |
j                  dd      dz  dz         |d	
      j!                  d||f      |d<   t        j"                  | j%                  |d   j                  dd            j!                  d||f      dz         }||	z   }| j
                  j&                  s| j)                  |d   j                  dd            j!                  d||f      j+                         j                  dd      j-                  d      }t        j.                  d      d    j1                         j3                  |      j5                  ||dd      |d<   dx|d   d<   |d   d<   |d d d f   |d   d d z  |d   dd d ddf<   |d d d f   |d   dd d ddf<   |d   d d d df   t7        |d         z   }t9        j:                  |d   |d   d       d   }|t7        |      z  }t        j<                  t        ||d         d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }t        j<                  |d   d d d df   j                  dd      |j                  dd      j?                  d      dz  dz
        jA                  d      jC                  ddd      j!                  d||dz
  f      }| jE                  t        j                  |t7        |d         d d dd f   fd            j+                         jG                  d      }t9        jH                  |d d dd |f   |d d d d |f   |d d d d |f         d   }|}tK        |dz
  dd      D ]:  }t        j                  |d d d |f   |d d |dz
  gf   |d d |d f   z  fd      }< t        j                  t        j.                  d      j3                  |      d   jM                  |j                  d      ddd      |fd      }tO        |d d dd f   |d d d df   jQ                         |d d dd f   z  |d   d d dd f         |d   d d dd f   z
  }d|v rtS        t        j<                  t        ||d         j                  dd      |d   j                  dd      j?                  d      dz  dz
        jA                  d      j!                  d||f      d      }t        jT                  dt5        |d|      jQ                         t5        |d|      z  tW        |            dd df   }tY        t        jT                  d |d   d d df   |            jG                  dd      |d!<   |t7        |      t7        |      ||d   dd"|d   z   z  dd"t7        |      z   z  z
  j[                         j-                         d#z  t        ||d         ||d   d$z  S )%Nr)   r   r   r(   i  r   r   r   r*   r   r   r    r   r@   r   r   r   r   .r   r   r   r   r3   r~   r   r   zbsnij,bksj->bksnizb t x y -> b s t x yr   zb t x y -> b t s x yr   r   r%         $@)r   r   r   r   
zoe_d_lossr   rC   r   ).r   r5   r   r!   r   r   r   r   r   r   r   r"   r.   r   r   r7   r8   r   r   r   r   r   r   r   r   r  r   r  r  r  r/   r  r  r  r   r  r
  rR   r  warpr  r   r<   rB   r;   square)r|   r  r  r  r   r   r  r  r  r   r?  r   r   r  r   r!  r   r"  r#  r   
adj_transfrC   rY   r,  r-  r0  s                             r   rE  zFlowMap.forward_  s   		1	5!'', '--bq1+e2D2I2I!2L1fnnVAYvay%89'6B99ENN1VAYvay=PQR=RSY,Z,_,_,ak 	k" ;{3F1I>	yy+e,Y7:mmDOODLL1<Mb<PQS<S,TU[aklvvwxz{  }C  zD  EFJJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	I% yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  CE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q16qv{;?WXZYZ?[1[K%c!Aai016qvK%c!Aai0 "'*1SbS51&Z9P2QQ
%%k'&:;|;TUYZ[\]VE]"}}VHVAY%?#2#%F%N%NqQR%ST^TfTfghijTkTuTuvwTxyzTz{|T|}  F  F  GI  J  R  R  ST  UV  WX  Y  c  c  de  gh  io  pq  iq  fr  s }}[%83B3%?%G%G!%LZM_M_`abcMdMnMnopMqrsMstuMuv~~  @B  C  K  K  LM  NO  PQ  R  \  \  ]^  `a  bh  ij  bj  _k  l22599l6R]^dReKfghijikgkKl=mnp3qrzz|  B  B  GK  B  L ((!AB{2B)C\RSTUVaRaEbdpqrst  vA  rA  eB  D  EF  G
vax"%dAuyy%"1"+eAqQRseGnUZ[\]^]_[_U`F`9abc/dud		599Q<??51)<CCEJJqMRTUWXZ[\abcde HQqrTN53B3<+?+?+A%!"++MkZfNghijkjlhlNmnoz  |C  pD  EF  GH  GI  EI  pJ  J K' vhvay/I/Q/QRSTU/VWbcpWqWyWyz{|}W~  XI  XI  JL  XM  NO  XO  PQ  XQ  "R  "Z  "Z  []  "^  "h  "h  ij  lm  nt  ku  "v MceJ!&.A&OehnBoBwBwBy  {A  BG  H^  ag  {h  Ch  il  mw  ix  "y  z}  ~@  @  ~@  z@  "A#(6HUaIbcdefcfIghz){#|  $B  $B  CD  EF  $GC y)5M)#K0T+k"::;AtF5M?Q<RRZZ\aacdgg"<q	:$Z0	
 	
 		
r   c                 (
   | xj                   dz  c_         |d   j                  dd  }|d   j                  d d |d   j                  d      c\  }}}| j                  |       t	        |d   |d         }t        j                  |d   |fd      }t        j                  | j                  |j                  dd      dz  dz         |d	      j                  d||f      |d
<   t        j                  | j                  |d
   j                  dd            j                  d||f      dz         }	|	|z   }
| j                  j                  s| j!                  |d
   j                  dd            j                  d||f      j#                         j                  dd      j%                  d      }t        j&                  d      d    j)                         j+                  |
      j-                  ||dd      |d<   dx|d   d<   |d   d<   ||d   z  |d   d<   ||d   d<   t/        t1        |d
   |d   j3                  d            d      }| j5                  t        j                  |d d dgf   j7                  d|dd      |fd            j#                         j9                  d      }t;        j<                  d      d d d   }t?        j@                  |d   |d   d       d   }|t1        |
|d   j3                  d            jC                  d      z  }t?        jD                  |d|d d f   |d d dgf   j7                  d|dd      d|d d f   |d|d d f         d   }t        jF                  d|jI                         tK        |d d dgf   j7                  d|dd                  dd df   }tM        ||d         j9                  dd      }t        jN                  |d   |d   d d dgf         }||d   z
  |j3                  d      z  jQ                         j%                         }t?        j@                  |d   |d   d       d   tS        |
      z  }tU        |d d dd f   |d d d df   jI                         |d d dd f   z  |d   d d dd f         |d   d d dd f   z
  }|tS        |	      tS        |
      ||dz  |d   dd|d   z   z  ddtS        |
      z   z  z
  jQ                         j%                         d z  t	        |d!      ||d"   d#	z  S )$Nr)   r   r   r(   r   r   r   r*   r   r   r    r   r@   r   r   r   r   ).r   r   ).r)   r)   r   r7  r   r   i  r:  .zbtij,btpj->btpipred_visibilityr   g      Y@r%   rG  *   r   )	r   r   r   point_track_lossr   rH  r   rC   r   )+r   r5   r   r   r   r!   r"   r.   r   r   r7   r8   r   r   r   r   r   r   r   r   r   r  r   r   r  r  r   r  r  nparanger  r  r  r
  r<   r  rB   rA   minimumrJ  r  rI  )r|   r  r  r  r   r   r  r   r?  r   r   r  
track_featr   neighb_list	rds_trackeye_surf_trackrC   point_track_surf_reprojpoint_track_reprojvis_maskrO  r   r,  s                           r   rE  zFlowMap.forward_  s&   		1	5!'', '--bq1+e2D2I2I!2L1f 	k" ;{3F1I>	yy+e,Y7:mmDOODLL1<Mb<PQS<S,TU[aklvvwxz{  }C  zD  EFJJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	I% yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  CE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q/4[5M/MK%g./4K%g. !;v+>M@Z@d@deg@h!i  kA  B
22599jA3>O>V>VWYZ`acdf>ghr=stv3wx  A  A  C  H  H  MQ  H  R ii&tt, ++K,F{S_G`aefghi	"Yu[5O5Y5YZ\5]%^%f%fgh%ii##N3{13D$E~VWYZX[V[G\GcGcdfgmnpqsGtux  zE  FG  vG  HH  JV  WZ  [f  gh  Wh  Ji  j  kl  m"',,/@QTUcdeghfidiUjUqUqrtu{|~  @B  VC  RD  #E  FI  JL  KL  JL  FL  #M$%<[=VW\\]^_`a==->!?L]@^_`bcad_d@ef0;}3MMQYQcQcdfQggpprwwy **;w+?L@YZ^_`abeklqerrHQqrTN53B3<+?+?+A%!"++MkZfNghijkjlhlNmnoz  |C  pD  EF  GH  GI  EI  pJ  Jy)5M)/3#K0T+k"::;AtF5M?Q<RRZZ\aacdgg"<3$Z0

 

 
	
r   c           
          |d   j                   d d |d   j                  d      |d   j                   dd  c\  }}} fdt        j                         5   j                  j
                  rdnd _        d|vr j                  |d   d d dd f   j                  dd      j                  t        j                        |d   d d d d	f   j                  dd      j                  t        j                              \  }} ||      j                  d||dz
  f      |d<    j                  j                  rg g }}t        t        j                  d|dz
   j                        j                               D ]  \  }	}
t!        d
|
        j#                  |d   d d d|
d      \  }}|t        j$                  |d   j                   dd  d d d	         dz
  d   j'                         z  }|j)                  |       |j)                  |        j                  j
                  s* j                  j
                  r j*                  dz  dk(  st!        d       t-        dd      j/                  |d   ||d|	z  |
       	 t        j0                  |d      |d<   t        j0                  |d      }t        j0                  |d   t        j2                  |d         j5                  |dz
  d	d	d	      f      |d<   t        j0                  |t        j6                  |      j5                  |dz
  d	d	      f      |d<   d d d        y # 1 sw Y   y xY w)Nr   r(   r)   r   c                    	j                   j                  rwt        |       dkD  ri| d t        |       dz   |d t        |       dz   }}| t        |       dz  d  |t        |       dz  d  }}t        j                   ||       ||      f      S 	j                   j                  r.	j                   j
                  s	j                  | |d      d   }n6| dz  dz   dz  |dz  dz   dz  }} 	j                  | |dgdgdgd	      d
   d   }|t        j                  | j                  dd  d d d         dz
  j                  |       d d d d d f   z  }t        j                  |dd      S )Nr3   r(   rw   )num_flow_updatesr    r      F)attn_splits_listcorr_radius_listprop_radius_listpred_bidir_flow
flow_predsr   r)   r*   T)r,   	antialias)r   r   r4   r!   r"   r   r   r  r5   r  r.   r   )
r   r1   x_lefty_leftx_righty_rightr  r  raftr|   s
          r   rh  zFlowMap.get_flow.<locals>.raft  so   yy  SVAX !*3q619a
Qmv"#CFAIJ-#a&!)*yy$vf"5d776K!LMMyy  ):):JJqBJ7;rT"WcM1R47C-!LL1qcB4bdaev{L|  ~J  K  LN  Ou||AGGBCL2$679==a@aTAQRRC==VdKKr   r   	rgb_larger   r    zdoing video trackingrt   T)	grid_sizegrid_query_framebackward_trackingr      z+writing point track video for visualizationz	./output/d   )save_dir	pad_valueztracks_%02d)videotracks
visibilityfilenamequery_framer   rM  )r5   r   r!   no_gradr   r   n_track_framesr   r7   r  uint8r8   point_track	enumerater   r   printr   r  r   rS   r   r   	visualizer"   r#   r  r   )r|   r  r   r   r  
raft_inp_x
raft_inp_yr   visibilitiesrY   start_frame
pred_trackrs  pred_track_normr  rh  s   `             @@r   r   zFlowMap.get_flow  s   '.44Ra8U9K9P9PQR9ST_`eTfTlTlmompTq1fV	L  ]]_ 	~%)YY%6%6AD, )-(<(<[=UVWXYXZVZ=[=c=cdefg=h=k=klqlwlw=x  {F  GR  {S  TU  VY  WY  VY  TY  {Z  {b  {b  cd  ef  {g  {j  {j  kp  kv  kv  {w  )x%
:*.z**F*P*PQRTUV\]^V^S_*`J' 99((-/K)25>>!F1HTM`M`3a3f3f3h)i 	U+4[A04K@XY[Z[@\hj  hs  GK  1L-
:*4ell;{C[CaCabdbeCfgkikgkCl6mno6oq  6A  6F  6F  6H  +H#**?;L<O<OPZ<[  99,,TYY5F5F499UW<YZ?!"OP&sKUU]hit]u  I  V`  kx  yz  kz  HSU  T	U 05yyQ/GK.!&<!:L 05yy+m:TUZUdUdepq~e  VA  VH  VH  IJ  KL  IL  MO  PR  SU  VV  :W  0XK.3899l5K[K[\hKiKpKpqrstqtuwxzK{=|3}K 12?	~ 	~ 	~s   G$M	 M	C-M		M)__name__
__module____qualname__r   r5  rE  r   __classcell__)r   s   @r   r]   r]   0   s>    4_| (* M
^ )+ I
V )+ ?
B )+ 7
p;~r   r]   )N)5r!   torchvisionr   r  einopsr   r   torchvision.opsopstorch.nnr   r.   numpyrP  r   randomtimer   copyr   
matplotlibr   wandbr	   typingr
   r   r   r   r   r   collectionsr   torchvision.transformsr   r=  r   rS   cotracker.utils.visualizerr   r   r  r  r   rB   r;   r6   r  rA   rI  r[   Moduler]   r&   r   r   <module>r     s       $  $        C C # " + , G  >uC2T
 C	RoM~bii M~r   