
    5g.                        d Z ddlZddlZddlmZ ddlZddlmZmZ ddlm	Z	 ddl
mZ ddlZddlZddlZddlZddlZddlmZ ddlmZ ddlZddlmZ dd	lmZmZmZmZmZmZ dd
l m!Z! ejD                  jG                  d       ddl$m%Z%m&Z& ddl'Z'd Z(ddZ)d Z*d Z+d Z,d Z-d Z.d Z/d Z0 G d dejb                        Z2y)z$Code for pixelnerf and alternatives.    N)nn)	rearrangerepeat)
functional)deepcopy)cm)tqdm)CallableListOptionalTuple	GeneratorDict)defaultdictz./third_party/co-tracker/)
Visualizerread_video_from_pathc                     t        | d      S )Nz... c x y -> ... (x y) c)r   xs    +/home/cameronsmith/repos/flowcams/models.py<lambda>r      s    i"<=     c           	      f    t        | d|"t        | j                  d      dz              S |      S )Nz... (x y) c -> ... c x y      ?r   )r   intsize)srcr   s     r   r   r      s2    i,FdedmSXXVX\\^M_I`u stu r   c                 d    t        j                  | t        j                  | ddgf         fd      S )N.r   )torchcat	ones_liker   s    r   r   r      s)    eii5??1S!W:#> ?C r   c                 .    | dd df   d| ddd f   z   z  S )N.r    h㈵> r   s    r   r   r      s%    aCRCj$qRSz/2 r   c                 >    t        j                  | |dz  dz
  d      S )N      bilinearmode)Fgrid_sampler   ys     r   r   r      s    a!A:> r   c                     t        | j                        dk(  rt        | |      S t        | j                  dd      |j                  dd            j	                  d| j                  d d       S )N   r   r)   r(   )lenshape
grid_samp_flatten	unflattenr/   s     r   r   r      so    3qww<?
1Q 
199UVWX>Z[ZcZcdefgZh@i@s@stuvwv}v}  A  @A  wB  AC r   c                 B    t        t        j                  d||             S )Nb...cij,b...ckj->b...cki)unhomr!   einsum)crdsKs     r   r   r      s    5.HD!QR r   c           	      d    t        t        j                  d|t        |             dd df   |      S )Nr9   .   )projectr!   r;   hom)r<   posesr=   s      r   r   r      s6    %,,7QRWX[\`Xa*bcfgihigici*jlm!o r   c           	      6   d }g }t        t        |       dz
        D ]S  }|j                  t        j                  | |   | |dz                   |j                  t        j
                                U t        j                  |d d  }|j                  |       |S )Nc                     t        |       t        j                  k(  r;t        | d      r.t        j                  j                  | j                  ddd       y y y )Nweight        relufan_in)anonlinearityr,   )typer   Linearhasattrinitkaiming_normal_rE   )ms    r   init_weights_normalz%make_net.<locals>.init_weights_normal!   sF    7biiq(#''CfS['\ $  r   r)   r    )ranger3   appendr   rL   ReLU
Sequentialapply)dimsrQ   layersinets        r   make_netr[       s    ] F3t9Q; !biiQQqS	23bggi ! --
%CII!"Jr   c                   \     e Zd Z fdZi fdZi fdZi fdZd Zd Zd Zd
dZ	d	 Z
 xZS )FlowMapc           	      ~    t                    | _        ddlm} |j
                  j                          _        ddlm}  ||j
                  d       _	        t        j                  j                  d       ddlm}  |dd	d
d	ddd      j                          _        t#        j$                  d      }d|v r|d   n|} j                   j'                  |d        j                   j)                         D ]	  }d|_         ddlm}  |t0        j                  j3                  d            j                          _        |j6                  s	 d}tO        jP                  d|       _)        tU        |dd	g       _+        tU        |dz  ddd	g       _,        tU        |dd	g       _-        tU        |dd	g       _.        tU        |dd	g       _/        d _0        d}tU        |d|g       _1        tU        |dd j`                  g       _2         fd _3        d  _4        t"        jj                  j%                  dd j                  jl                          _7         jn                  jp                  jr                   _:        tO        jv                          jn                  jp                  _9        dd l<}tO        jz                  |j}                  d      tO        j~                  d|dd	              _@        tO        j~                  |d	dd	        _A        tO        j~                  |d	dd	        _B        tO        j~                  |d	dd	        _C        d!}tO        j~                  ||dd	        _D        d"\   _E         _F        d _G        t"        jN                  j                  t#        j                  d	dd#d	      d$%       _J        y )&Nr   )Raft_Large_Weights)
raft_largeF)weightsprogressz	./gmflow/)GMFlow   r)      swinr2      )feature_channels
num_scalesupsample_factornum_headattention_typeffn_dim_expansionnum_transformer_layersz8./gmflow/gmflow-scale1-mixdata-train320x576-4c3a6e9a.pthmodel)strict)CoTrackerPredictorzB./third_party/co-tracker/checkpoints/cotracker_stride_4_wind_8.pth)
checkpointi   @   r(   
   r?   c                 4    t        | dj                        S )Nzb t ... -> (b o) t ...o)r   n_rig)r   selfs    r   r   z"FlowMap.__init__.<locals>.<lambda>e   s    va(@4::N r   c                     t        | d      S )Nzb t o ... -> (b o) t ...)r   r   s    r   r   z"FlowMap.__init__.<locals>.<lambda>f   s    &+E"F r   zintel-isl/MiDaSMiDaS_small)
pretrained)in_ch)padding   NN   T)requires_grad)Ksuper__init__argstorchvision.models.optical_flowr_   DEFAULT
transformsraft_transformsr`   raft_syspathrS   gmflow.gmflowrc   cudagm_flowr!   loadload_state_dict
parametersr   cotracker.predictorrq   osjoin
co_tracker	load_saveprintsegment_anythingsam_model_registrySamPredictorsamto!segment_anything.utils.transformsResizeLongestSideimage_encoderimg_sizeresize_transformr   rL   fmap_downprojr[   	depth_estcorr_weighter_perpoint
static_estmask_scorerconvex_weights_upsamplerrx   mask_level_predictorrig_predictoro_repo_switchhubscratch_netmidasscratchoutput_conv	midas_outIdentityconv_modulesrU   PixelNeRFEncoderConv2d
resnet_enc
depth_conv
focal_convtemperature_convaffinities_convcorr_weightsdepthstep	Parameteronesmask_scores)ry   r   r_   r`   rc   rr   ra   paramrq   r   r   sam_checkpoint
model_typedevicer   fdimn_mask_levelr   affinity_dim	__class__s   `                  r   r   zFlowMap.__init__/   s/   	 	G199DDF>(:(B(BUS
 	$(sqRS^_pv  KL  ef  h  m  m  oZZ Z[
)0J)>*W%J$$WU$;\\,,.KEe0CK 	;,  HL  :M  N  S  S  U ~~! YYs40!41+.&.Qs2a/@&A#"DA;/#T"QK0(0$r!(=%
$,d2l-C$D!%tC4::&>?N
FYY^^$5}UYU^U^UjUjQj^k
zz))55'){{}

&l&C&C!&C&LRYYWZ[_`ajkMlm		$q15		$q15 iiQq;YYtL1E &/$$*	++EJJq2a,@t+Tr   c                 :   | xj                   dz  c_         d   j                  dd  }d   j                  d d d   j                  d      c\  }}}d}t        t	        j
                  d   j                  d                  }| j                         t        d   |d         }	t        j                  d   |	fd      }
t        j                  | j                  |
j                  dd      dz  dz         |d	
      j                  d||f      d<   t        j                  | j!                  d   j                  dd            j                  d||f      dz         }|	|z   }| j"                  j$                  s| j'                  d   j                  dd            j                  d||f      j)                         j                  dd      j+                  d      }t        j,                  d      d    j/                         j1                  |      j3                  ||dd      d<   dxd   d<   d   d<   |d   z  d   d<   |d   d<   t        j4                  | j7                  d   j                  dd            j                  d||f      d      }t9        t;        |d d dgf   d   d d dgf   j=                  d            d      }t        j>                  d||      }t9        t;        d   d   j=                  d            d      }| jA                  t        j                  |d d dgf   jC                  d|dd      |fd            j)                         jE                  d      }|j=                  d      |j=                  d      z  }t        jF                  d   j                  d            d d d   d d d f   jC                  dd      }fd}tI        jJ                  d   d   d       d   }|t;        |d   j=                  d            jM                  d      z  }tI        jN                  t        jP                  |d|j                         d d d d d f   jC                  d|dd      jS                               j                  d|j                        jU                  ddddd      t        jP                  |d d dgf   jC                  d|dd      d|j                         d d d d d f   jC                  d|dd      jS                               j                  d|j                        jU                  ddddd      t        jP                  |d|jV                  d d d d d d d f   jC                  d|ddd      jS                                     d   }t        j>                  d|jY                         t[        |d d dgf   jC                  d|dd                  dd df   }t]        |d         jE                  dd      }|d   z
  d   j=                  d      z  j_                         j+                         }t        j                  t`        jH                  jb                  je                  |dd dd df   d !      |dd ddf   fd      }	 |d d d d |dz  |z  dz  f   }#|tk        |      tk        |      |j                  d||f      |d"z  t        ||      |#d#   d   ||j                  d||f      j                  d||f      d$
z  S )%Nr)   rgbr   r(   pred_tracks	depth_inpr   r   r*   r+   fmapr    dimr?   
intrinsics.r   r(   .r)   r(   	org_ratio.r   r   .r)   r)   zb t c p 1 -> b t p czb t p c, b t q c -> b t p q-C6?minrt   i  c                 D    t        | dd   j                  d            S )Nzb t ... -> b t o ...r   r   rv   )r   r   )r   model_inputs    r   r   z!FlowMap.forward.<locals>.<lambda>   s"    &#9K<V<[<[\^<_` r   r2   zbtpij,btpj->btpi.pred_visibilityr%   )epsg      Y@bwd_flow)
	res_depthr   	poses_liepoint_track_lossr   rB   	flow_inp_r   affinity_embaffinity_sim)8r   r4   r   r   npsqrtget_flowch_fstr!   r"   r-   interpolater   r6   r7   softplusr   r   use_gt_intrinsicsr   sigmoidmeaneyefloatr   r   	normalizer   r   	grid_samp	unsqueezer;   r   expandcliparangegeometryget_world_rayssqueeze
procrustesgatherr   permuteTinverserA   r@   squarekorniaconversionsrotation_matrix_to_quaternionr   rG   ch_secsoftmaxquaternion_to_rotation_matrix)$ry   r   outimsizeb_n_trgtlowres_factorgsr   rgbdr   r   focalr   aff_sourcesaff_sim
track_featr   corr_weights_objneighb_listr   	rds_trackeye_surf_trackrB   point_track_surf_reprojpoint_track_reprojr   r   temperatureaff_sim_allpix
lie_perpixpose_perpixeye_surfadj_opt_flowbg_poses$    `                                  r   forwardzFlowMap.forward}   sk   		1	5!'', '--bq1+e2D2I2I!2L1frww{=166r:;< 	k" ;{3F1I>	yy+e,Y7:mmDOODLL1<Mb<PQS<S,TU[aklvvwxz{  }C  zD  EFJJt{6/B/J/J1Q/OPZZ[\^_`f]ghijjk	I% yy**OOK$7$?$?!$DEOOPQSTU[R\]eegoopqrtuzz  @Bz  CE&+iil4&8&>&>&@&C&CE&J&Q&QRSTZ[\]^&_K%RTTK%g.{</H/Q/4[5M/MK%g./4K%g. {{D$8$8V9L9T9TUVWX9Y$Z$d$defhijpgq$rxy{	,q!u*={=?YZ[]^\_Z_?`?j?jkm?n o  qG  H,,<k;W
 !;v+>M@Z@d@deg@h!i  kA  B
22599jA3>O>V>VWYZ`acdf>ghr=stv3wx  A  A  C  H  H  MQ  H  R'11!4w7H7H7LL ll;}#=#B#B2#FG"MaPTfU\\]_`ef a++K,F{S_G`aefghi	"Yu[5O5Y5YZ\5]%^%f%fgh%ii##^Ak.A.A.CDaPTDT.U.\.\]_`fgijk.l.q.q.st~~  @B  CN  CT  CT  U  ]  ]  ^_  `a  bc  de  fg  h^AqcE299"VBrJ1[M`M`MbcghlmnoscsMtM{M{|~  @F  GI  JK  NL  NQ  NQ  NS  T  ^  ^  _a  bm  bs  bs  t  |  |  }~  @  AB  CD  EF  G-btD1T?Q1R1Y1YZ\]cdfgijl1m1r1r1tu 	 #(,,/A%--/RUVdefhigjejVkVrVrsuv|}  AC  WD  SE  #F  GJ  KM  LM  KM  GM  #N$%<[=VW\\]^_`a0;}3MMQ\]nQoQyQyz|Q}}  G  G  I  N  N  P IIv::XXY^_bcedecefhghfh_hYinrXstyz}  A  @A  A  BD  {D  uE  F  GI  J	T . !BEBJN*+,  y)5M!++B2w7/3"<3$Z0%l3(#--b"R9CCB2wO
 
 	
r   c                 	   | xj                   dz  c_         |d   j                  dd  }|d   j                  d d |d   j                  d      c\  }}}d}| j                  j                  r1t        j                  d|d   |d   z  dz
  |      j                         n t        j                  |d   |d   z        d | }| j                  |       |d   d d d df   t        |d	         z   }	| j                  ||	      \  }
}|j                  d      }dd
lm}  |        | j                  |dD ci c]  }|| j!                  ||          c}z  | j!                  |	      | j!                  |
      | j#                  |      t$              \  }}t        j&                  |dd dd d d d f   j)                         j+                         |fd      }t        j,                  d      j/                  |      |dd dd d d d f<   d|v rt1        t3        ||d         j5                  dd      | j!                  |d         j5                  dd      j                  d            }t7        |d|      }t9        |d|      j;                         t9        |d|      z  }t        j<                  d|t?        |            dd df   }tA        ||d   d d d df         jC                  dd      }t1        t7        tD        d|d         j5                  dd      | j!                  |d         j5                  dd      j                  d            }t7        |d|      }||j                  d      z  jG                  d|| jH                  f      jK                  d      |d<   t7        tL        dtD        j                  d            tD        d d dd f   z  jK                  d      }t7        |dtD        j                  d            }t        j<                  d|t?        |jG                  d|| jH                  f                  dd df   }||
|d   dd |d   z   z  dd |
z   z  z
  jO                         jQ                         d!z  |tD        |||d   d"z  S c c}w )#Nr)   r   r   r(     r   x_pixr    r   )	set_tracer!  r   .r2   r   (b t) c p 1 -> b p t ctb t x y -> b s t x ysb t x y -> b t s x ybsnij,bksj->bksnir?   r   b t o (x y) 1 -> (b o) t 1 x yr   all_pair_pix(b o) c ... -> b c o ...rv   (b o) ... -> b o ...zbrtij,brtxj->brtxi	zoe_depthr%   g     @)r   r1  
zoe_d_lossr   	rig_masks	pts_canonrB   r   ))r   r4   r   r   overfitr!   linspacelongrandpermr   r  get_depth_and_flow_weightsr   pdbr"  adj_pose_estr   r   rand_subsetsr"   detachcloner   r   r   r   r6   r   r   r   r;   rA   r@   r   affinity_maskr7   rx   sumflow_from_poser   r   )ry   r   r  r  r  r  r	  n_samprand_subset
corresp_uvr   r   r  pdb_kr  rB   
eye_tracksall_pair_posesall_pair_surfsall_pair_pix_uncompaffinity_sampleflow_from_pose_compr4  s                           r   forward_zFlowMap.forward_$  sG   		1	5!'', '--bq1+e2D2I2I!2L1f OSyyO`O`ennQvay':1'<VDIIKfkftftu{|}u~  @F  GH  @I  vI  gJ  KR  LR  gS 	k"!'*1SbS51&Z9P2QQ
 "<<[Tl (11!4)$&**;_u8vZ[4::kRSn;U9U8v+v<@JJz<RSWS]S]^cSdeierer  tD  fE  FRS% 		5RaR!,335;;=eDRH))A,//%0c"1"Qqj K' #6(6!9#=#E#Ea#J4::VaboVpKqKyKyz{|}K~  LI  LI  JL  LM  NJ":.FPJ $E*@6JRRTW]^cdz  ~D  XE  EN"\\*=nSQ[_]^abdcdbd^deN").\9RSTUWVWUWSW9X"Y"^"^_`ab"c'	-@`cijkcl(m(u(uvwxy(z{  |F  |F  GR  S`  Ga  |b  |j  |j  kl  mn  |o  |y  |y  z|  |}  ~O'8PSYZO#67P7PQS7T#T"_"_`acdeieoeobp"q"u"uvw"xC !*.9SVcVhVhijVk lo|}~  @A  @B  ~B  pC  !C  I  I  JK  L% 69K9KA9NOLL!5eC@R@RSTVWX\XbXbUc@d<efgjkmlmkmgmn	#K0T+k"::;AtEzNJRRTYY[\__(&"%l3	
 	
 		
; 9ws   -R
c                    | xj                   dz  c_         |d   j                  dd  }|d   j                  d d |d   j                  d      c\  }}}d}| j                  j                  r1t        j                  d|d   |d   z  dz
  |      j                         n t        j                  |d   |d   z        d | }| j                  |       |d   d d d df   t        |d	         z   }	| j                  ||	      \  }
}| j                  j                  s| j                  ||
|	||      |d
<   dg}t        j                  |d|d   z     d d d df         d    g|D cg c]  }|d|z     d     c}z   }t        j                  |d      }t!        | j#                  t        |d               |d         }|D cg c]D  }t        |d   j%                  d      |j%                  d      z        j'                  d      d   F }}|D cg c]!  }| j)                  |      j+                         # }}t-        ||      D cg c]  \  }}|j%                  d      |z   }}}t!        | j/                  t        |d               j1                  d      |d         }|d d }|d d d d d df   }t        j                  |      }t-        |t        |      j%                  d      j3                  d            D cg c]*  \  }}|j5                  dd      j1                  d      |z  , }}}t        j                  |t        |      d d dd f   z  j%                  d      |d d d d d f   j7                  dd|j                  d      dz
  dd      fd      j9                  d      }d}t        j:                  |      dz  }t        j<                  t        ||z   j%                  d            dd      d   dd |d d f   }| j?                  |dD  ci c]  } | | jA                  ||           c} z  | jA                  |	      | jA                  |
      | jC                  |      |      \  }!}"tE        |dz
  dd      D ]F  }t        j                  |"dd |d d d d f   |"d|dz
  gd d d d f   |"d|d d d d d f   z  fd      }"H t        j                  |"dd dd d d d f   jG                         jI                         |"fd      }"t        jJ                  d      jM                  |"      |"dd dd d d d f<   tO        |!d d dd f   |"d d d df   jQ                         |"d d dd f   z  |d
   d d dd f         |d   d d dd f   z
  }#d|v rtS        t!        |!|d         j5                  dd      | jA                  |d         j5                  dd      j%                  d            }$tU        |$d|      }$tW        |"d|      jQ                         tW        |"d|      z  }%t        jX                  d |%t[        |$            dd df   }&t]        |&|d
   d d d df         j9                  dd      }'tS        tU        t^        d!|d   "      j5                  dd      | jA                  |d         j5                  dd      j%                  d            }(tU        |(d|      }(|'|(j%                  d      z  ja                  d|| jb                  f      je                  d      |d#<   t        j                  |d      j%                  d      })tU        |#d$|)j                  d      %      |)d d d df   z  je                  d      }*tU        |"d&|)j                  d      %      }"||
||||||*|*|"d d df   |"|d	   |d
   d'z  S c c}w c c}w c c}w c c}}w c c}}w c c} w )(Nr)   r   r   r(   r   r   r!  r    r   r   all_masks_%izall_masks_%dr   r?   r   r   r   i  gMbP?r$  T)r   
descending.r#  r2   r   r%  r&  r(  r)  r+  r,  r-  r   r.  r/  rv   r0  )r   composite_weightslevel_scoresmask_spatial_scores	is_staticr   rA  flow_from_pose_uncomprB   	rig_posesr   r   )3r   r4   r   r   r5  r!   r6  r7  r8  r   r  r9  r   intrinsics_estr#   r"   r   r   r   maxr   rG   zipr   r  unbindr6   r   r   	rand_likesortr;  r   r   rR   r=  r>  r   r   warpr   r   r   r   r;   rA   r@   r?  r7   rx   r@  )+ry   r   r  r  r  r  r	  rB  rC  rD  r   r   mask_levelsrY   masks	masks_allrT  masks_mask_embeddingsr   r   
mask_scorespatial_maskrS  rR  mask_spatial_scorelevel_scorerQ  r  points_permaskrand_shuffle_noiser<  rF  r  rB   rA  rG  rH  rI  rJ  rK  composite_weights_flatrL  s+                                              r   rM  zFlowMap.forward_h  s~	   		1	5!'', '--bq1+e2D2I2I!2L1f OSyyO`O`ennQvay':1'<VDIIKfkftftu{|}u~  @F  GH  @I  vI  gJ  KR  LR  gS 	k"!'*1SbS51&Z9P2QQ
 "<<[Tl yy**H[H[\ghmnx  zE  FR  ISK,E C^KN-J!KAbqbD!QRSWXY  IT  [U  DE[fguvwgw[xy}[~  [U  U))E!$	4??6+f2E+FGq	R	 x}}ms6;v#6#@#@#CFDTDTUVDW#WY]]bd]efgh}};JKat''*//1KKdghstydz{I`T`z33B7D{{d77{6?R8ST\\ac\deklmeno1"15AaF#	//),	 GJJ]_efr_s_}_}~  `A  `H  `H  IK  `L  GMN$B$6{ 0772>FF2FNQ\\ N N !99|F94Ead4K'K&V&VWX&Y'3AaH'='D'DR9>>Z[K\]^K^_abd'e&gghjjnjnswjnjx 	  ??95d:zz&)4F*F)Q)QRT)U"V\^kopqrstw  yH  zH  yH  IJ  uJ  K**;_u8vZ[4::kRSn;U9U8v+v$(JJz$:4::e;LT]][kMlmy{%
 vax"%wAuyy%BQBq
:KERUWXYZWZV[\]^_R_L`afgjklkmnopqgqarLr9stv/wuw		5RaR!,335;;=eDRH))A,//%0c"1"Qqj hqtnU1SbS5\-A-A-CE!AB$K-OP[\hPijklmlnjnPopq|  ~E  rF  GH  IJ  IK  GK  rL  L K' #6(6!9#=#E#Ea#J4::VaboVpKqKyKyz{|}K~  LI  LI  JL  LM  NJ":.FPJ $E*@6JRRTW]^cdz  ~D  XE  EN"\\*=nSQ[_]^abdcdbd^deN").\9RSTUWVWUWSW9X"Y"^"^_`ab"c'	-@`cijkcl(m(u(uvwxy(z{  |F  |F  GR  S`  Ga  |b  |j  |j  kl  mn  |o  |y  |y  z|  |}  ~O'8PSYZO#67P7PQS7T#T"_"_`acdeieoeobp"q"u"uvw"xC "'+<Q!?!I!I"!M ).9SVlVqVqrsVt u  yO  PQ  RU  SU  RU  PU  yV  !V  \  \  ]^  _% 69O9T9TUV9WX  1'"5! ) 1$7!A#J$Z0%l3#
 
 	
[ [U ~K{N  9ws%   ^79A	^<&_?_8/__
c                    t        j                  |d   |d   d       d   }||z  }t        |j                        dk7  r%||fD cg c]  }|j	                  dd       c}\  }}t        t        ||d   j                  d            d d d df   |j                  d	            j                  d      j                  ddd
d	      }	t        |j                        dk(  rKt        |d|	j                  d      |	j                  d            j                  |	      j                         }n)t        |d      j                  |	      j                         }|j                  d      dk(  r|j                  dd	dd      }t        j                   t#        j$                  |d d dd f   d	|j                  dddd
      d d dd f         t#        j$                  |	d	|j                  dddd
      d d dd f         t#        j$                  |d	|d t        |      dd f               d   }
t        |j                        dk7  r2||
fD cg c]!  }|j'                  d|j                  d d	       # c}\  }}
||
fS c c}w c c}w )Nr!  r   r)   r2   r   r   r   r    r(   r?   zx -> bo t x 1)bor'  zb t o x 1 -> (b o) t x 1)r   r   r3   r4   r6   r   r   r   r   r   r   r   r   r7  r   r   r   r!   r   r7   )ry   r   rD  r   r   rC  rdsr  r   corresp_surf
adj_transfs              r   r;  zFlowMap.adj_pose_est  s   %%k'&:;|;TUYZ[\]U syy>1hWaMb4cQYYq^4c1hz U1C1H1H1L!MaPSQSPSe!TU_UiUijkUlmuuvxy  B  B  CD  EF  GH  IJ  K{  !1$&_XdXiXijkXlo{  pA  pA  BC  pD  3E  3H  3H  IU  3V  3[  3[  3]k%k2LMPPQ]^ccekA!k.@.@Ab.L+((hqtnQ{GYGYZ\]_`bcdGefghihjfjGk)l).lQ{GYGYZ\]_`bcdGefghihjfjGk)l).lQ{SeTWXdTeSefgfhShGi)j+ ,-.

 syy>1\deo[p4qVWQ[[399Ra=5Q4q1hz## 5d 5rs   I0 &I5c                    t        |j                         D cg c]  }d|v  c}      r|S d}d}d}t        j                  |d   j	                  dd      ||f      j                  t        j                        }t        j                  t        j                  t        j                  |||z
  |      t        j                  |||z
  |            d      j                         j	                  dd      d d d f   }d	dgd d }d
dgd t        |       }	t        d      D 
cg c]$  }
t        t        |            D 
cg c]  }
g  c}
& c}
\  }}}t        |      D ]  \  }}|| j                  j!                  |||g      j                         t        j"                  |d         ||gdg}t%        d       t        j&                         5  | j)                  |d      }d d d        t        j                  d   d   j	                  dd      d d d f   j+                         |d   j,                  dd        j/                         j1                  d      j3                  d|d   d   j,                  d d	       }d}|}t        j4                  j6                  j                  |j	                  dd      d d d f   j+                         |      j1                  d      j3                  d|j,                  d d	       j/                         }g }|j	                  dd      D ]e  }|j9                         dk  r't        j:                  g dg      j                         }nt=        j>                  |d          }|jA                  |       g t        jB                  |      j3                  d|j,                  d d	       }d}t        d      D cg c]E  }tD        j<                  jG                  |d d |f   t        jH                  |d d ddf         d      G }}t        |      D ]  \  }}||   jA                  ||           t        j                  |d    |d   j,                  dd        d   }t        j                  |j	                  dd      d d d f   j+                         |d   j,                  dd        j1                  d      j3                  d|j,                  d d	       }tK        t        |      |	      D ]j  \  \  }}}|||   |f   }|||   |f   } t        jB                  | d | t        jL                  tO        d|t        |      z
        g| j,                  dd   j                         f      }!t        jB                  |d | t        jL                  tO        d|t        |      z
        g|j,                  dd   j                         f      }"|"d d d dd f   jQ                         j+                         t        j:                  g d      d   z  |jS                  dd	d      d   jQ                         z   }#||   jA                  |#       ||   jA                  |"       m  t        |      D ]B  \  }}t        j                  ||         t        j                  ||         c|d|z  <   |d|z  <   D y c c}w c c}
w c c}
w # 1 sw Y   xY wc c}w )N	all_masksr      i   	rgb_larger   r)   r    r(      <   r?   .r   )imagepoint_coordspoint_labelsoriginal_sizezquerying samT)multimask_outputr_  r   r   )scale_factorrt   )rF   rF   rF   rF   r   .)         )NNNNr   rO  zmasks_vis_%i)*anykeysr-   r   r6   r   r!   uint8stackmeshgridr6  r   r3   rR   	enumerater   apply_coords_torchr#   r   no_gradr   r   r4   boolr   r7   r   r   r@  tensoropsmasks_to_boxesrS   r"   torchvisionnmsr[  rY  zerosrX  cpur   )$ry   r   rF  grid_slbufsam_slsam_inp_imgs
query_crdsr^  max_masksperlevelr  
masks_vis_
all_masks_keep_idxs_allimg_iimgbatched_inputbatched_outputrp  sfall_masks_hires
all_bboxesmaskboxiou_thresholdrY   	keep_idxsjimg_low	masks_low	max_masksmasks_imasks_i_hirespadded_hirespaddedmask_viss$                                       r   get_segzFlowMap.get_seg  s   +*:*:*<=Qq =>k?Q}}[%=%E%Ea%JFSY?[^^_d_j_jk[[s6#:g0VW\WeWefijpqtjtu|W}!~  @B  C  H  H  J  R  R  ST  UV  W  XY  Z^  X^  _
 qE"1Ib'"33{#34\abc\d,eWX%K@P:Q-RQb-R,e+
:m"<0 .	-IE# "%)%:%:%M%Mj[abhZi%j%o%o%q%*__Z5F%G'-fo	M .!a488M\`83a.anQ&7&@&H&H1&MaPTf&U&[&[&]^ijo^p^v^vwywz^{|  B  B  D  L  L  MN  O  Y  Y  Z[  \j  kl  \m  nu  \v  \|  \|  }  ~  \@  AI
 B&O++77	8I8I!A8NqQUv8V8\8\8^ln7owwxyz  E  E  FG  HQ  HW  HW  XZ  YZ  H[  \  a  a  cIJ!))!A. )88:b=ell4G3H&I&N&N&P#,,T$Z8c!!3() :.889??2A;NOJMkpqrkstfg{**:ac?5??:VWXYZ[V[K\;]^`atIt - 6!a ''	!56
 MM#d)K,>,D,DRS,IJ1MGmmI$5$5a$:1T6$B$H$H$J;W\K]KcKcdfdgKhiqqrst~~  @A  BK  BQ  BQ  RT  ST  BU  VI#&y'=>O#P 	-1i!)A,q.1'	!Q7"YYjy(A%++cRST]^abi^jTjNk  CFmz  nA  nA  BC  BD  nE  CF  CK  CK  CM  (N  Oyy'*9"5ekk#a	RUV]R^H^B_6sahananopoqar6s6x6x6z!{|"1T#d?3779??A%,,~B^_rBssu|  vE  vE  FG  HI  JK  vL  MV  vW  v[  v[  v]  ]1$$X.1$$V,	-K.	-` [) 	@CAaJO++V`abVcJdejepepq{|}q~eGKq()+nQ6F*G	@ > .S,e ba" us0   Z0Z:8	Z5
Z:Z? A
[5Z:?[		c           	      	   |d   j                   d d |d   j                  d      |d   j                   dd  c\  }}}}| j                  :| j                  j                  d      }| j                  j                         }||fS | j                  j                  sv| j                  |d   dz  dz   j                  dd            }	t        j                  |	|d	
      j                  d||f      dz  |d<   dt        | j                  |	      j                  d||f            dz   z  }t        j                  |d   d d d df   j                  dd      |j                  dd      j!                  d      dz  dz
        j#                  d      j%                  ddd      j                  d||dz
  f      }
| j'                  t)        j*                  |
t        |d         d d dd f   fd            j-                         j                  d      }t        j                  t/        ||d         d d d df   j                  dd      |j                  dd      j!                  d      dz  dz
        j#                  d      j%                  ddd      j                  d||dz
  f      }|dd||d d dd f   z
  j                         j-                         dz
  z  z
  z  dt)        j0                  ||d d dd f         dz  j3                         z
  z  }||fS | j                  |d   dz  dz   j                  dd            }	t        j                  |	|d	
      j                  d||f      |d<   t        t        j4                  | j                  |	dz               j                  d||f            dz   }t7        |d   d d d df   j                  dd      |j                  dd      j!                  d            j#                  d      j%                  ddd      j                  d||dz
  f      }
| j'                  t)        j*                  |
t        |d         d d dd f   fd            dz  j-                         j                  d      }||fS )Nr   r(   r)   r   g?r   r   r   r*   r+   rs  r   g     @@r    r   r?   )r4   r   r   r   r   absr   r   r   r6   r-   r   r7   r  r   r.   r   r   r   r   r!   r"   r   r   maximumtanhr   r   )ry   r   rD  r  r  r	  r  r   r   midas_featscorresp_featcorresp_depths               r   r9  z"FlowMap.get_depth_and_flow_weightsW  s   '.44Ra8U9K9P9PQR9ST_`eTfTlTlmompTq1fV ::!**//b/)E**..0L8 l""3 &&**k%&8&;B&>%G%G!%LMK !k&j Q [ [\]_`ag^h ijl lK{ ; E Ea6
 STUYYZE ==V)<QssU)C)K)KAa)PQ[QcQcdefgQhQrQrstQuvwQwxyQyz  C  C  DF  G  O  O  PQ  RS  TU  V  `  `  ab  de  fl  mn  fn  co  pL66uyy,vVabhViOjklmnmokoOpAqrt7uv~~  A  F  F  KO  F  PL MM&vay*A!CRC%*H*P*PQRST*UV`VhVhijklVmVwVwxyVz{|V|}~V~  H  H  IK  L  T  T  UV  WX  YZ  [  e  e  fg  ij  kq  rs  ks  ht  uM'1Q-ad2K1P1P1R0[0[0]^`0`-a+abfginiviv  xE  FK  LM  NO  NP  LP  FQ  jR  ST  jT  iZ  iZ  i\  g\  ]L l"" **k%&8&;B&>%G%G!%LMK !k&j Q [ [\]_`ag^h iK1::dnn[^&DEOOPQSTU[R\]^_``E %[%83B3%?%G%G!%LZM_M_`abcMdMnMnopMqrzz{}~  G  G  HI  JK  LM  N  X  X  YZ  \]  ^d  ef  ^f  [g  hL 77		<PVWbciWjPklmnonplpPqBrsu8vwx||  F  F  H  M  M  RV  M  WLl""r   c           
         |d   j                   dd  }|d   j                   d d |d   j                  d      c\  }}	}
t        j                  |d   |d   z        d d }t        j                  ddd      j                         }t        j                  ||d	   d   z  |gd      }t        j                  d
      d   j                         j                  |      j                  |t        |      ddd      }dx|d<   |d<   |d d d d df   |d<   |d d d d df   |d<   |||d   d d d df   j                  d      dz  }| j                  ||d d d d df   j                  dt        |      ddd      |d d d df   |d d d df   |      \  }}t        |d d d d dd f   ||      |d   z
  }|j!                  d      t#        |d   d d d df         z
  j%                         |d d d df   z  d d d d |f   j'                  ddg      }||j)                  dd      d   z
  dz  }t+        j,                  |d      j                  d      |d    z  j'                  d      d d d f   j                  d|
d      }t        j                  d
      d   j                         j                  |      j                  ||
dd      }dx|d d d d ddf<   x|d d d d ddf<   x}}|d   |d d d d ddf<   |d   |d d d d ddf<   |S )Nr   r   r(   r)   r   d   r   rt  r   r?   NNNr   r   r   r   r!  )r   r!  r    r   r   T)r   keepdimg      $@r   ru  ).r)   )r4   r   r!   r8  r6  r   r  r   r   r   r   r3   r   r;  r   r]  r   r  r  r@  r   r-   softmin)ry   r   r   rD  rC  r   r  xpixr  r  r	  focals	focal_setsample_intrinsicsmodel_input_r  adj_poserA  	flow_errs	est_focalr   cxcys                          r   rW  zFlowMap.intrinsics_est|  s   5!'', '--bq1+e2D2I2I!2L1fnnVAYvay%89$3? ~~b2&++-KKK(@(C!CF KAN	  ))A,~6<<>AA%HOOPQRUV_R`abcdefg>@@'"#4W#=#,T!D]#;'"#,T!D]#;'" #3D[Y`MabcdfefdfbfMgMqMqrsMt%uu!..|Jqbqby<Q<X<XY[\_`i\jkmnpqs<tuz{|}~}{  vA  BN  OP  QS  RS  QS  OS  BT  U`  a( hq12v.x8IJ<X_K`` &--a0J8OPQRTSTRTPT8U1VV[[]`lmnoqpqoqmq`rruvwx  zE  vE  F  J  J  PR  SU  OV  J  W	y}}4}@CCSH	 YYyQ/99"=ioMRRSTUVWX\V\]ddeghnoqr	99Q<	*00255e<CCAfQqQ
688
1Qq78Jq1Qw/82%e,
1Qq7%e,
1Qq7r   c           
          |d   j                   d d |d   j                  d      |d   j                   dd  c\  }}} fdt        j                         5  d _        d|vr؉ j                  |d   d d dd f   j                  dd      j                  t        j                        |d   d d d d	f   j                  dd      j                  t        j                              \  }} ||      j                  d||dz
  f      |d<    j                  j                  rg g }}t        t        j                  d|dz
   j                        j                               D ]  \  }	}
t        d
|
        j!                  |d   d d  j                  j"                  rdnd|
d      \  }}|t        j$                  |d   j                   dd  d d d	         dz
  d   j'                         z  }|j)                  |       |j)                  |        j                  j"                  s* j                  j"                  rډ j*                  dz  dk(  st        d       t-        dd      j/                  |d   ||d|	z  |
       ! t        j0                  |d      |d<   t        j0                  |d      }t        j0                  |d   t        j2                  |d         j5                  |dz
  d	d	d	      f      |d<   t        j0                  |t        j6                  |      j5                  |dz
  d	d	      f      |d<   d d d        y # 1 sw Y   y xY w)Nr   r(   r)   r   c                    t        d| j                         	j                  j                  rwt	        |       dkD  ri| d t	        |       dz   |d t	        |       dz   }}| t	        |       dz  d  |t	        |       dz  d  }}t        j                   ||       ||      f      S 	j                  j                  r.	j                  j                  s	j                  | |d      d   }n6| dz  dz   dz  |dz  dz   dz  }} 	j                  | |dgdgdgd	
      d   d   }|t        j                  | j                  dd  d d d         dz
  j                  |       d d d d d f   z  }t        j                  |dd      S )Nz
doing raftr2   r(       )num_flow_updatesr    r   r~  F)attn_splits_listcorr_radius_listprop_radius_listpred_bidir_flow
flow_predsr   r)   r*   T)r,   	antialias)r   r4   r   r5  r3   r!   r"   r   r   r  r   r-   r   )
r   r0   x_lefty_leftx_righty_rightr  r  raftry   s
          r   r  zFlowMap.get_flow.<locals>.raft  s   ,qww' yy  SVAX !*3q619a
Qmv"#CFAIJ-#a&!)*yy$vf"5d776K!LMMyy  ):):JJqBJ7;rT"WcM1R47C-!LL1qcB4bdaev{L|  ~J  K  LN  Ou||AGGBCL2$679==a@aTAQRRC==VdKKr   r   rr  r   r    zdoing video tracking*   rs  T)	grid_sizegrid_query_framebackward_trackingr     z+writing point track video for visualizationz	./output/r  )save_dir	pad_valueztracks_%02d)videotracks
visibilityfilenamequery_framer   r   )r4   r   r!   r  n_track_framesr   r6   r   r  r7   r   point_trackr  r6  r7  r   r   r5  r  r   rS   r   r   	visualizer"   r#   r   
zeros_like)ry   r   r  r  r	  
raft_inp_x
raft_inp_yr   visibilitiesrY   start_frame
pred_trackr  pred_track_normr  r  s   `             @@r   r   zFlowMap.get_flow  s   '.44Ra8U9K9P9PQR9ST_`eTfTlTlmompTq1fV	L& ]]_ "	~ !D, )-(<(<[=UVWXYXZVZ=[=c=cdefg=h=k=klqlwlw=x  {F  GR  {S  TU  VY  WY  VY  TY  {Z  {b  {b  cd  ef  {g  {j  {j  kp  kv  kv  {w  )x%
:*.z**F*P*PQRTUV\]^V^S_*`J' 99((-/K)25>>!F1HTM`M`3a3f3f3h)i w+4[A04K@XY[Z[@\nrnwnwnnhj  FH1<PT 1@ 1V-
:*4ell;{C[CaCabdbeCfgkikgkCl6mno6oq  6A  6F  6F  6H  +H $**?;L<O<OPZ<[  99,,TYY5F5F499UW<YZ?!"OP&sKUU2=k2JS]7AMZ[Oit V vw$ 05yyQ/GK.!&<!:L 05yy+m:TUZUdUdepq~e  VA  VH  VH  IJ  KL  IL  MO  PR  SU  VV  :W  0XK.3899l5K[K[\hKiKpKpqrstqtuwxzK{=|3}K 12E"	~ "	~ "	~s   G$M	 M	C-M		Mr   )__name__
__module____qualname__r   r  rM  r;  r  r9  rW  r   __classcell__)r   s   @r   r]   r]   .   sO    IU\ (* e
N )+ A
H )+ z
z$PB@T##J#J9~r   r]   )N)3__doc__r!   r  r   r   einopsr   r   torchvision.opsr  torch.nnr   r-   numpyr   r   randomtimer   copyr   
matplotlibr   wandbr	   typingr
   r   r   r   r   r   collectionsr   r   rS   cotracker.utils.visualizerr   r   r   r  r   rA   r:   r5   r   r@   r]  r[   Moduler]   r&   r   r   <module>r     s    *    $  $        C C # + , G  >uC2>
 C	Rol
~bii l
~r   