
    g                        d dl Z d dlZd dlmZ d dl mZ d dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlmZ d dlmZmZm Z m!Z!m"Z"m#Z# d d	l$m%Z% ejL                  jO                  d
       d dl(m)Z)m*Z* d dl+Z+d dl,Z,d dl,m-Z- d Z.ddZ/d Z0d Z1d Z2ddZ3d Z4d Z5d Z6d Z7dZ8dZ9dZ: G d dejv                        Z< G d dejv                        Z=y)    N)nn)	rearrangerepeat)
functional)deepcopy)cm)tqdm)CallableListOptionalTuple	GeneratorDict)defaultdictz./third_party/co-tracker/)
Visualizerread_video_from_path)KMeansc                     t        | d      S )Nz... c x y -> ... (x y) c)r   xs    8/home/cameronsmith/repos/contrastive_pointrack/models.py<lambda>r      s    9Q :;     c           	      f    t        | d|"t        | j                  d      dz              S |      S )Nz... (x y) c -> ... c x y      ?r   )r   intsize)srcr   s     r   r   r       s2    Y-Gijir3sxxXZ|`cOdKez xyz r   c                 d    t        j                  | t        j                  | ddgf         fd      S )N.r   )torchcat	ones_liker   s    r   r   r   !   s)    		1eooaaSk:;R@ r   c                 .    | dd df   d| ddd f   z   z  S )N.r!   gh㈵> r   s    r   r   r   "   s&    !C"H+#rs(!34 r   c                 @    t        j                  | |dz  dz
  ||      S )N      )modepadding_mode)Fgrid_sampler   ypadr*   s       r   r   r   #   s    AMM1a!eaidY\$] r   c                     t        | j                        dk(  rt        | |||      S t        | j                  dd      |j                  dd      ||      j	                  d| j                  d d       S )N   r   r)   r(   )lenshape
grid_samp_flatten	unflattenr.   s       r   r   r   $   s    WZ[\[b[bWcghWhAq#d9S nxyz  zC  zC  DE  GH  zI  KL  KT  KT  UV  XY  KZ  [^  _c  od  on  on  op  rs  ry  ry  z|  {|  r}  o~ r   c                 B    t        t        j                  d||             S )Nb...cij,b...ckj->b...cki)unhomr"   einsum)crdsKs     r   r   r   %   s    %-GD QR r   c           	      d    t        t        j                  d|t        |             dd df   |      S )Nr9   .   )projectr"   r;   hom)r<   posesr=   s      r   r   r   &   s6    gu||4NPUWZ[_W`'abegihigibi'jlmn r   c                 b    | t        j                  t        |             j                  |          S N)r"   randpermr3   tor   s    r   r   r   '   s#    AennSV,//23 r   c           	      6   d }g }t        t        |       dz
        D ]S  }|j                  t        j                  | |   | |dz                   |j                  t        j
                                U t        j                  |d d  }|j                  |       |S )Nc                     t        |       t        j                  k(  r;t        | d      r.t        j                  j                  | j                  ddd       y y y )Nweight        relufan_in)anonlinearityr*   )typer   Linearhasattrinitkaiming_normal_rI   )ms    r   init_weights_normalz%make_net.<locals>.init_weights_normal*   sD    7biiq(#RWW%<%<ahh#\biq%<%r#  r   r)   r!   )ranger3   appendr   rP   ReLU
Sequentialapply)dimsrU   layersinets        r   make_netr_   )   s    s F3t9q=! !biiQa!e56bggi ! --
%CII!"Jr   Tc                        e Zd Zd fd	Zdi fdZdi fdZdi fdZdi fdZdi fdZdi fdZdi fd	Zdi fd
Z xZS )FlowMapNc                    t         |           || _        d| _        d}t	        ||dg      | _        t        j                  |ddd      | _        d}t        j                  ||dd      | _	        t        j                  |ddd      | _
        t        j                  |ddd      | _        t	        |dz  ddg      | _        t        dt        dz  z   | j                  z  d	
      | _        d| _        y )Nr)   @   r?   )padding   r(      r   T)in_chuse_first_pool)super__init__argstime_strider_   	depth_estr   Conv2d
depth_convaffinities_convgeneral_confidence_convgeneral_confidence_conv_staticcorr_weighter_perpoint	ResnetFPNuse_depth_inpimg_encstep)selfrk   fdimaffinity_dim	__class__s       r   rj   zFlowMap.__init__:   s    	 !4q/2))D!Q:!yy|QJ')yyq!Q'G$.0iiaA.N+&.q"a/@&A# !-/(9T=M=M'M^bc 	r   c                    t        j                         r| xj                  dz  c_        |d   j                  dd  }|d   j                  d d |d   j	                  d      c\  }}}d}t        j
                  d|d   j	                  d      dz
  d      j                         }	t        j                  |d	   j	                  d            d d
 }d }
d }|d   } ||      }|x}}t        j                  | j                  |j                  dd      dz  dz         |d      j                  d|j                  d d       }| j                  |j                  dd            j                  d|j                  d d       }t        j                  |d      }t        t!        ||d	   j#                  d                  d d d d |f   }|j%                  d      \  }}|d   d d d d |f   j'                  d      d   d   }t        j(                  d||      }|dz  dz   dz   }||j+                  dd      z  }|d	   d d d|f   }t        j(                  d||      }||z
  |z  j-                         j/                         dz  }||||||dz  S )Nr)   rgbr   r(   rc   rc   r   x_pixi  pred_tracksi'  c                    d}d}| d   }|j                   \  }}}||z  }||z  }||z  }	t        |	|z        }
t        j                  |	t        j                        }d|t        j
                  |	      d |
 <   |j                  ||      }|j                  |d      j                  |d      }t        j                  |d   j                         j                         ||f      S )	N   gffffff?r   r   dtyper)   r   dimNNr4   r   r"   zerosboolrE   viewrepeat_interleaver,   interpolatefloatcudapltimsavepermutecpunumpypdb	set_tracer}   
patch_size
keep_ratioimageCHWnum_patches_hnum_patches_wtotal_patchesnum_keepmask	mask_fullmasked_imagepdb_s                  r   random_maskz%FlowMap.forward_.<locals>.random_mask]       JJc(EkkGAq!OMOM)M9M=:56H;;}EJJ?D=>D.y9:99]M:D ..zq.ASST^deSfI ==9!5!;!;!=!B!B!DaUKKr   c           	      L   t        j                  t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd            } || dz  dz         dz  dz
  S 	Nr   皙?皙?rJ   )
brightnesscontrast
saturationhuer   r(   r)   TColorJitternprandomuniformr}   color_transforms     r   random_color_jitterz-FlowMap.forward_.<locals>.random_color_jittert       mm		8I8I!R8P[][d[d[l[lmnpr[s  AC  AJ  AJ  AR  AR  ST  VX  AY  _a  _h  _h  _p  _p  qt  vy  _z  {O"3r6"9-a/11r   r   bilinearr*   r   pred_visibility).Nb p c, b q c -> b p q-C6?r!   T)r   keepdimb p q, b q d -> b p dg      Y@)imgscontrastive_lossaff_simaffinity_embimgs_premask)r"   is_grad_enabledrw   r4   r   linspacelongrE   r,   r   rv   r6   r7   rp   	normalizech_sec	grid_samp	unsqueezeunbindminr;   sumsquaremean)rx   model_input
track_idxsoutimsizeb_n_trgt	low_imresrand_subsetr   r   img_warpimg_inpimg_inp_premaskfmapr   aff_emb_pertrack	src_feats
warp_featsr   r   attn_weights	warp_crds	pred_crdslosss                             r   forward_zFlowMap.forward_T   s     "499a<9U#))"#.$U+11"15{57I7N7Nq7QA	nnQG(<(A(A"(Ea(GNSSU^^K$>$C$CB$GH%P
	5.	2 u%&x0 %-,/ }}dll7??1a+@3+F+LMv\fhrrstu|  vC  vC  DF  EF  vG  H **DLLA,>?II!TZZXZYZ^\{{<Q7!9\;};U;_;_`b;c#defghijtftu/66q9	*,-a*n=AAaAHKHU,, 7JO
2e# w{{r${??m,Qq^<	LL!8,S	"D(002779#=#(*
 
 	
r   c                    t        j                         r| xj                  dz  c_        d   j                  dd  d   j                  d d d   j	                  d      c\  }}}d}t        j
                  dd   j	                  d      dz
  d      j                         }t        j                  | j                  d   j                  dd      d	z  d	z         d
      j                  dd   j                  d d       }	fd}
d } |
d         \  }} ||      }t        j                  d   |fd      x}}t        j                  | j                  |j                  dd      d	z  d	z         d
      j                  d|j                  d d       }	| j                  |	j                  dd            j                  d|	j                  d d       }t        j                  |d      }t        t!        |d d d df   |j#                  d      d	z  d	z         d         j%                  d      }t'        |d d dd f         j%                  d      d d |f   t'        |      j%                  d      d d |f   }}|}t        j(                  d||      dz  }t        j*                  |j                  dd      t        j,                  t/        |            j1                         d    j3                  |d      j                               }t        j4                         5  t        j                  |d d df   d      }t        j(                  dt'        |dd d dd d df         t'        |            j7                  d      j                  dd      j                  dd      }d d d        ||||||	dz  S # 1 sw Y   xY w)Nr)   r}   r   r(   r~   r   r     r   r   r   c                    g }t        t        |             D ]'  }t        j                  dt        j                  j                         dz   z  dd      t        j                  j                         t        j                  j                         t        j                  j                  ddg      t        j                  j                  ddg      f\  }}}}}|j                  t        j                  ||z  dt        j                  |d|z
   d|z
        gd||z  t        j                  |d|z
   d|z
        gg             * t        j                  dt        j                  |      j                         j                         t        
d         dz  dz
        }t        t!        | |j#                  d      dz  dz         	d         j%                  d	      |fS 
Nr)   r   r!   r   zbij,btxj->btxir   r(   r   rV   r3   r   clipr   randchoicerW   r"   tensorr;   stackr   r   rA   ch_fstr   r   squeezer}   affine_matricesr]   scaletxtyflip_xflip_yr   r   r   s            r   random_affine_warpz+FlowMap.forward.<locals>.random_affine_warp   -   O3s8_ b*,''!RYY^^5Ea5G2HA*Nryy~~O_`b`i`i`n`n`pqsqzqz  rB  rB  DF  GH  CI  rJ  KM  KT  KT  K[  K[  ]_  `a  \b  Kc  +c)bF6&&vq"''RTWXY^W^U_`abg`gJh7ilmotu{o{}  ~E  ~E  FH  KL  MR  KR  IS  TU  V[  T[  ~\  l]  6_  )`  bb %5u{{?7S7X7X7Z7`7`7bdghst{h|d}~d  AB  eB  CI)C	(;(;B(?(B2(EFvayQYYZ\]^gggr   c           	      L   t        j                  t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd            } || dz  dz         dz  dz
  S r   r   r   s     r   r   z,FlowMap.forward.<locals>.random_color_jitter   r   r   r   r   r      r!   .   )r   r   )r   r   r   aff_sim_gridr   r   )r"   r   rw   r4   r   r   r   r,   r   rv   r6   r7   r#   rp   r   r   r   r   r   r   r;   cross_entropyaranger3   r   expandno_gradsoftmax)rx   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   	feat_warpr   r   r   r   dsl_affr  r   s    `                      @r   forwardzFlowMap.forward   s      "499a<9U#))"#.$U+11"15{57I7N7Nq7QA	nnQG(<(A(A"(Ea(GNSSU}}dll;u+=+E+Ea+Kc+QTW+WXZ`gqs}}~  AL  MR  AS  AY  AY  Z\  [\  A]  ^	h	2 0E0BC&x0$)II{5/A(.KA$NN/}}dll7??1a+@3+F+LMv\fhrrstu|  vC  vC  DF  EF  vG  H **DLLA,>?II!TZZXZYZ^\ {{<Q7<"1"#5i6I6I"6Mb6PQS6STU[\]U^_gghjk	%l1QR4&89AA!DQ{]STZ[dTeTmTmnoTpqrs~q~T*	
,, 7JOPRRwq4ell3{CS6T6Y6Y6[\`6a6h6hijkm6n6v6v6xy ]]_ 	iMM,qs"3G<G <<)@PSTWVWTWX[Z[X[P[H\A]^del^mnvv{}v~  I  I  JL  MR  S  ]  ]  ^`  ah  iL	i #((
 
 	
		i 	is   !BN88Oc                 n   t        j                         r| xj                  dz  c_        d   j                  dd  d   j                  d d d   j	                  d      c\  }}}d}t        j
                  dd   j	                  d      dz
  d      j                         }d	 }	fd
}
d } |
d         \  }} ||      }t        j                  d   |fd      x}}t        j                  | j                  |j                  dd      dz  dz         d      j                  d|j                  d d       }| j                  |j                  dd            j                  d|j                  d d       }t        j                  |d      }t        t!        |d d d df   |j#                  d      dz  dz         d         j%                  d      }t'        |d d dd f         j%                  d      d d |f   t'        |      j%                  d      d d |f   }}t        j(                  d||      }t        j*                  |j                  dd      t        j,                  t/        |            j1                         d    j3                  |d      j                               }|||||||dz  S )Nr)   r}   r   r(   r~   r   r   r   c                    d}d}| d   }|j                   \  }}}||z  }||z  }||z  }	t        |	|z        }
t        j                  |	t        j                        }d|t        j
                  |	      d |
 <   |j                  ||      }|j                  |d      j                  |d      }t        j                  |d   j                         j                         ||f      S 	N   g333333?r   r   r)   r   r   r   r   r   s                  r   r   z%FlowMap.forward_.<locals>.random_mask   r   r   c                    g }t        t        |             D ]'  }t        j                  dt        j                  j                         dz   z  dd      t        j                  j                         t        j                  j                         t        j                  j                  ddg      t        j                  j                  ddg      f\  }}}}}|j                  t        j                  ||z  dt        j                  |d|z
   d|z
        gd||z  t        j                  |d|z
   d|z
        gg             * t        j                  dt        j                  |      j                         j                         t        
d         dz  dz
        }t        t!        | |j#                  d      dz  dz         	d         j%                  d	      |fS r   r   r   s            r   r   z,FlowMap.forward_.<locals>.random_affine_warp  r   r   c           	      L   t        j                  t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd            } || dz  dz         dz  dz
  S r   r   r   s     r   r   z-FlowMap.forward_.<locals>.random_color_jitter
  r   r   r   r   r   r   r   r   r!   r   r   r   r   r   r   )r"   r   rw   r4   r   r   r   r#   r,   r   rv   r6   r7   rp   r   r   r   r   r   r   r;   r  r  r3   r   r  )rx   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   s    `                     @r   r   zFlowMap.forward_   s     "499a<9U#))"#.$U+11"15{57I7N7Nq7QA	nnQG(<(A(A"(Ea(GNSSU	54	h	2 0E0BC&x0 %*II{5/A(.KA$NN/ }}dll7??1a+@3+F+LMv\fhrrstu|  vC  vC  DF  EF  vG  H **DLLA,>?II!TZZXZYZ^\{{<Q7 <"1"#5i6I6I"6Mb6PQS6STU[\]U^_gghjk	%l1QR4&89AA!DQ{]STZ[dTeTmTmnoTpqrs~q~T*	,, 7JO wq4ell3{CS6T6Y6Y6[\`6a6h6hijkm6n6v6v6xy *#(
 
 	
r   c                 B   t        j                         r| xj                  dz  c_        d   j                  dd  d   j                  d d d   j	                  d      c\  }}}d}t        j
                  dd   j	                  d      dz
  d      j                         }d	 }	fd
}
d } |
d         \  }} ||      }t        j                  d   |fd      x}}t        j                  | j                  |j                  dd      dz  dz         d      j                  d|j                  d d       }| j                  |j                  dd            j                  d|j                  d d       }|j                         }t        t!        |d d d df   |j#                  d      dz  dz         d         j%                  d      }t'        |d d dd f         j%                  d      d d |f   t'        |      j%                  d      d d |f   }}t        j(                  d||      }t        j*                  |d      }t        j(                  d||d d d|f         }||d d d|f   z
  j-                         j/                         dz  }|||||||dz  S )Nr)   r}   r   r(   r~   r   r   r   c                    d}d}| d   }|j                   \  }}}||z  }||z  }||z  }	t        |	|z        }
t        j                  |	t        j                        }d|t        j
                  |	      d |
 <   |j                  ||      }|j                  |d      j                  |d      }t        j                  |d   j                         j                         ||f      S r  r   r   s                  r   r   z%FlowMap.forward_.<locals>.random_maskF  r   r   c                    g }t        t        |             D ]'  }t        j                  dt        j                  j                         dz   z  dd      t        j                  j                         t        j                  j                         t        j                  j                  ddg      t        j                  j                  ddg      f\  }}}}}|j                  t        j                  ||z  dt        j                  |d|z
   d|z
        gd||z  t        j                  |d|z
   d|z
        gg             * t        j                  dt        j                  |      j                         j                         t        
d         dz  dz
        }t        t!        | |j#                  d      dz  dz         	d         j%                  d	      |fS r   r   r   s            r   r   z,FlowMap.forward_.<locals>.random_affine_warp`  r   r   c           	      L   t        j                  t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd      t        j                  j	                  dd            } || dz  dz         dz  dz
  S r   r   r   s     r   r   z-FlowMap.forward_.<locals>.random_color_jitterg  r   r   r   r   r   r   r   r!   r   r   g      $@r  )r"   r   rw   r4   r   r   r   r#   r,   r   rv   r6   r7   rp   rK   r   r   r   r   r   r;   r  r   r   )rx   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r   s    `                       @r   r   zFlowMap.forward_>  s     "499a<9U#))"#.$U+11"15{57I7N7Nq7QA	nnQG(<(A(A"(Ea(GNSSU	54	h	2 0E0BC&x0 %*II{5/A(.KA$NN/ }}dll7??1a+@3+F+LMv\fhrrstu|  vC  vC  DF  EF  vG  H **DLLA,>?II!TZZXZYZ^\!&&(<"1"#5i6I6I"6Mb6PQS6STU[\]U^_gghjk	%l1QR4&89AA!DQ{]STZ[dTeTmTmnoTpqrs~q~T*	,, 7JOyyb1LL!8,STUVWbSbIcd		!Ak/22::<AACCG *#(
 
 	
r   c                    t        j                         r| xj                  dz  c_        |d   j                  dd  }|d   j                  d d |d   j	                  d      c\  }}}|d   |d   z  }t        j
                  d|d   j	                  d      dz
  |      j                         }	d}
|*t        j                  |d   j	                  d            d d }|d   }t        |d	| j                  
      }t        j                  | j                  |j                  dd      dz  dz         |d      }t        |d| j                  |      |d<   | j                  |d   j                  dd            j                  d||f      }t        j                   |d      }t#        t%        ||d   j'                  d                  d d d d |f   }t(        j*                  j-                  |      }|d d |f   }t        j.                  d||j                  dd            j                  d|j                  dd       }|dz  j1                  d      t        j2                  t5        |            d d d d f   j7                  |d|d      j9                         z
  j;                         |d   d d ||f   d   |d   d d d d |f   d d d f   z  z  }||j=                         dz  |||dz  S )Nr)   r}   r   r(   r   r   r~      b (t s) c x y -> b t (s c) x ysr   r   r    (b t) (s c) x y -> b (t s) c x yr  r   r   r   r   r!   r?      r   ).NN     @@)r   r   r   affinity_emb_unnorm)r"   r   rw   r4   r   r   r   rE   r   rl   r,   r   rv   r6   rp   r7   r   r   r   r   r   r   randintr;   r  eyer3   r  r   r   r   )rx   r   r   r   r   r   r   r   n_sampr   r   r   fmap_outr  r   aff_emb_pertrack_allframe
rand_frameaff_emb_src_featsr   r   s                       r   r   zFlowMap.forward_  s|     "499a<9U#))"#.$U+11"15{57I7N7Nq7QAVAY&nnQM(B(G(G(KA(MvV[[]	 ENN;};U;Z;Z[];^,_`dad,ez e$W&F$JZJZ[==$,,wq!/Ds/JS/P"QSY`jl'3UY]YiYimnoF #22K4G4O4OPQST4UV``abefhndop {{#6A>$*IlKP]D^DhDhikDl,m$nopqrs}o}$~!YY&&v.
5a
lC,, 79JLeLmLmnopqLrs}}  A  B[  Ba  Ba  bc  de  Bf  g
 ###+eiiJ.HaPT.U.\.\]^_abhik.l.q.q.ss{{}  BM  N_  B`  ab  cm  nx  ax  By  zG  BH  KV  Wh  Ki  jk  lm  nx  jx  Ky  z{  |@  z@  KA  BA  B#yy{3(#6	
 
 	
r   c                 h   t        j                         r| xj                  dz  c_        |d   j                  dd  }|d   j                  d d |d   j	                  d      c\  }}}|*t        j
                  |d   j	                  d            d d }t        |d   d d d d dgf   |d   d d dd f   j                  d            j                  d      j                  d	      j                         }t        j                  |d
   d d dd f   |t        j                  |            x}}	|j                  d      d   }|d   }
t        |
d| j                        }
	 d|vst        j                         r| j                  t!        j"                  |d   dz  dz   j%                  dd      |d   dz  dz  |d   dz  dz  fd            x|d<   }t!        j"                  ||d      j'                  d||f      t(        sdndz  x|d<   }dt!        j"                  | j+                  |d         |d      j'                  d||f      dz   z  x}}d|v r	 t9        j:                  |d   |d   d       d   }|t        ||d   j                  d            j                  d      z  }| j=                  |d   j%                  dd            j'                  d||f      }t!        j>                  |d      }tA        t        ||d   j                  d                  }||d
   j                  d	      z  jC                  d      |d
   j                  d	      jC                  d      jE                  d      z  }t        jF                  d|d d |f   |      }t        j                  |j7                         d d |d f   jI                  d	d	|j	                  d	            t        j                  |      |      }| jK                  |d   j%                  dd            j'                  d||f      jM                         jE                  d      }| j                  dk  snt        j                  |      }t        ||d   j                  d            j                  d      |d
   j                  d	      z  }tN        r	 t9        jP                  |d d d dd f   |d d d d df   jI                  d	d	|dz
  d	d	      |d d d d d	f   |d d d d d d d f   z  jE                  d            d   }tW        d       |jI                  d	tY        |      d	d	d	      }n9|d   j	                  d      dz  }t9        jP                  |d d d dd d d |f   jI                  d	|j	                  d      d	d	d	      |d d d d d	d d |f   jI                  d	|j	                  d      d	d	d	      |d d d d d	d d |f   jI                  d	|j	                  d      d	d	d	      |d d d d d d d |d f   z  jE                  d            d   }tS        |dz
  dd	      D ]C  }t        jT                  |d d d d d |f   |d d d d |dz
  gf   |d d d d |d f   z  fd      }E t        jT                  t        jZ                  d       j]                  |      d!   jI                  |j	                  d      |j	                  d      d	d	d	      |fd      }t_        |ja                         d"|      t_        |d#|      z  }t        jF                  d$|tc        t_        |d d d d |f   d%|                  d&d d'f   }te        ||d         jE                  dd      }||d   d d d d d |f   z
  |d
   d d d d d |d f   z  jg                         j%                         ji                         } t        jj                         5  d(\  }!}"t        |d)|!|!*      d d df   }#t        jF                  dtA        |#d&d d |"d d |"f         tA        |#            j'                  d|!|"z  |!|"z  f      j'                  d	|!|!f      }$tA        t        |d   |d   j                  d                  }%|%|d
   j                  d	      z  jC                  d      |d
   j                  d	      jC                  d      jE                  d      z  }%t        jF                  d+|tc        |d d d d |f               d&d d'f   }&|&|d
   d d d d |f   j                  d	      z  jC                  d      |d
   d d d d |f   j                  d	      jC                  d      jE                  d      z  }&d d d        |&%||| |d d df   ||$|||tA        |      d,z  S # 1 sw Y   ,xY w)-Nr)   r}   r   r(   r   r  rig_flow_masksr   r!   r   r   r  r  r   r       r   r   midas_featsd   r  r   	depth_inp
intrinsicsr   r   r     r   z#doing direct pose regression static  r2   NNNb p t x y -> b p s t x yb p t x y -> b p t s x ybpstij,bstpj->bstpib t p c -> b t s p c.r?   )*   r?   b (x y s) c -> b s c x yr/   r   bptij,btpj->btpiworldcrds_pertrackrgb_pertrackrig_pertrack	poses_allpoint_track_losspoint_track_reprojcorr_weightsr   r  r   r  r   depth)6r"   r   rw   r4   r   rE   r   r   r   roundwherer$   r   r   rl   midasr,   r   r6   r7   scratch_model	midas_outrv   softplusro   r   r   r   geometryget_world_raysrp   r   r   r   r   r;   r  rq   sigmoidstatic_solveefficient_procrustesrV   r#   printr3   r   rF   r   inverserA   r@   r   r   r  )'rx   r   r   r   r   r   r   r   rig_samprig_samp_allframer   r)  r"  rA  	res_depth
depth_mask	rds_trackeye_surf_trackr  r   r#  r   r   aff_sim_riggeneral_confgeneral_conf_trackrB   r]   solve_strideposes_all_to_allpoint_track_surf_reprojr?  r>  track_sldslsrc_tracks_0r  r;  r:  s'                                          r   r   zFlowMap.forward_  s     "499a<9U#))"#.$U+11"15{57I7N7Nq7QA ENN;};U;Z;Z[];^,_`dad,ez ,< =aQCi H+VcJdefhihjejJkJuJuvxJy{  E  E  FG  H  Q  Q  RT  U  \  \  ^',{{K@Q4RSTVWVXSX4Y[cejetetu}e~'$<<A<&q) e$W&F$JZJZ[[(E,A,A,C9=AMMS^_dSefhShikSkRtRtuvwxRy  |B  CD  |E  GI  |I  JL  |L  MS  TU  MV  XZ  MZ  []  M]  {^  dn  Eo  :p  pM*;-.]];vS]-^-h-hijlmntku-v  DQx{  WX  .Y  YF#H #Q]]4>>+mB\3]^djt%u%%  AB  DE  FL  CM  &N  OS  &S  !T  TEI +%! ++[-GUaIbdhijkl	"Y{=7Q7[7[\^7_%`%h%hij%kk #22K4G4O4OPQST4UV``abefhndop {{#6A>$*IlKP]D^DhDhikDl,m$n!6EV9W9a9abd9eejjopjqt  AR  uS  u]  u]  ^`  ua  ue  ue  jk  ue  ul  uq  uq  wx  uq  uy  y,, 79I!Z-9XZjk kk8==?1j$3F#G#N#NrSUW^WcWcdfWg#hjojyjy  {B  kC  EL  N 55k&6I6Q6QRSUV6WXccdehikqgrs||~  E  E  JN  E  O99S=Aeool.K|&k-6P6Z6Z[]6^_gghijmx  zK  nL  nV  nV  WY  nZ  Z !55~aqrl7TVdefhlnpopnpepVqVxVxy{|~  @F  GH  @H  IK  LN  WO  Rd  ef  hl  nq  oq  nq  eq  Rr  s{  |}  ~B  CG  HI  JN  |N  sO  RO  QU  QU  Z^  QU  Q_  a  bc  d;<LL"c*or2rBE(7<<R@DHL11>!T12P^R^P^B^3_3f3fgikrkwkwxykz|~  AC  EG  4H3A!T3B3P^R^P^B^3_3f3fgikrkwkwxykz|~  AC  EG  4H/A!T3B3P^R^P^B^/_/f/fgikrkwkwxykz|~  AC  EG  0H:EaDTbVbTbdhFh:i0jkokotxkoky| }~E 6A:q"-  Auyy5ArPQr?TYZ[]^abefaf`gZgThkpqrtuwxwyqykzTzB{}  8Au  A		UYYq\__U34DELLUZZXY]\a\f\fgh\ikmoqsuvx}  BD  F "5==?4NRXY\bch  kE  IO  ]P  P"',,0EGWY\^dftuvxy  |F  vF  gG  I_  ci  _j  Zk  #m  nq  su  tu  su  nu  #v$&={<?XY^^_`bcd!3k-6PQRTXZ[]gQg6h!hlw  yJ  mK  LM  OS  UV  Xb  dh  Lh  mi   i  q  q  s  {  {  }  B  B  D ]]_ 		L*0LHS"#34NQY\defghifijL <<)@&VYZ_\_Z_`ebe`eVeIfBgiop|i}~  I  I  JK  MU  WZ  MZ  [c  eh  [h  Li  j  t  t  uw  yA  BJ  xK  LL!9[-?]A[AeAefhAi#jkL)K8I,J,T,TUW,XX]]bc]dgr  tE  hF  hP  hP  QS  hT  hX  hX  ]^  hX  h_  hd  hd  jk  hd  hl  lL!&/A5#n]^`acm]mNnJo!pqtvxwxvxqx!y#5DU8VWXZ[]gWg8h8r8rsu8v#v&)cacj;?P3QRSUVXbRb3c3m3moq3r3v3v|}3v3~  4D  4D  JK  4D  4L"L		L "4($ 0"4QT":(((#6 0E]
 
 	
		L 		Ls   F"f((f1c                 f   t        j                         r| xj                  dz  c_        |d   j                  dd  }|d   j                  d d |d   j	                  d      c\  }}}|d   |d   z  }t        j
                  d|d   j	                  d      dz
  |      j                         }	d}
t        |d   d d d d dgf   |d   d d dd f   j                  d            j                  d      j                  d	      j                         }t        j                  |d
   d d dd f   |t        j                  |            x}}|j                  d      d   }|*t        j                  |d   j	                  d            d d }|d   }t        |d| j                         }d|vst        j                         r| j#                  |j%                  dd      dz  dz         x|d<   }t'        j(                  | j#                  |j%                  dd      dz  dz         |d      }t        |d| j                   |      |d<   t'        j*                  | j-                  |d   j%                  dd            j/                  d||f      dz         dz   x}}d|v r	 	 t1        |d   |d         dz  }|dk7  j3                         }|d
xx   t        ||d   j                  d      dd      j                  d      j                  d	      j5                         z  cc<   t7        j8                  |d   |d   d       d   }|t        ||d   j                  d            j                  d      z  }	 | j;                  |d   j%                  dd            j/                  d||f      j=                         j?                  d      }t        ||d   j                  d            j                  d      |d
   j                  d	      z  }	 t7        j@                  |d d d dd |	f   |d d d d d	|	f   |d d d d d	|	f   j?                  d            d   }tC        |dz
  dd	      D ]C  }t        jD                  |d d d d d |f   |d d d d |dz
  gf   |d d d d |d f   z  fd      }E t        jD                  t        jF                  d      jI                  |      d   jK                  |j	                  d      |j	                  d      d	d	d	      |fd      }|jK                  d	tO        |      d	d	d	      }tQ        |jS                         d |      tQ        |d!|      z  }t        jT                  d"|tW        tQ        |d d d d |f   d#|                  d$d d%f   }tY        ||d         j?                  dd      }|d
   t        jD                  t        j                  |d d d df         |fd      z  }||d   d d d d d |f   z
  |d d d d d |d f   z  j[                         j%                         j]                         }||||d d df   |d d df   t_        |      t_        |      d&z  }| ja                  |d   j%                  dd            j/                  d||f      }t'        jb                  |d      }t_        t        ||d   j                  d                  } | |d
   j                  d	      z  je                  d      |d
   j                  d	      je                  d      j?                  d      z  }!t        jT                  d'|!d d |f   |!      }"| jg                  |d   j%                  dd            j/                  d||f      j=                         j?                  d      }#t        |#|d   j                  d            j                  d      |d
   j                  d	      z  }$|d   j	                  d      d(z  }%t        j                  |j5                         d d |d f   jK                  d	d	|"j	                  d	            t        j                  |"      |"      }&	 t7        j@                  |d d d dd d d |%f   jK                  d	|"j	                  d      d	d	d	      |d d d d d	d d |%f   jK                  d	|"j	                  d      d	d	d	      |$d d d d d	d d |%f   jK                  d	|"j	                  d      d	d	d	      |&d d d d d d d |%d f   z  j?                  d            d   }'tC        |dz
  dd	      D ]C  }t        jD                  |'d d d d d |f   |'d d d d |dz
  gf   |'d d d d |d f   z  fd      }'E 	 t        jD                  t        jF                  d      jI                  |'      d   jK                  |'j	                  d      |'j	                  d      d	d	d	      |'fd      }'tQ        |'jS                         d |      tQ        |'d!|      z  }(t        jT                  d"|(tW        tQ        |d d d d |f   d#|                  d$d d%f   })tY        |)|d         j?                  dd      }*|*|d   d d d d d |f   z
  |d
   d d d d d |d f   z  j[                         j%                         j]                         }+t_        t        |d   |d   j                  d                  },|,|d
   j                  d	      z  je                  d      |d
   j                  d	      je                  d      j?                  d      z  },t        jT                  d)|'tW        |d d d d |f               d$d d%f   }-|-|d
   d d d d |f   j                  d	      z  je                  d      |d
   d d d d |f   j                  d	      je                  d      j?                  d      z  }-t        jh                         5  d*}.t        |!d+|.|.,      d d df   }/t        jT                  d't_        |/d$d d dd d df         t_        |/            j/                  d|.dz  |.dz  f      j/                  d	|.|.f      }0d d d        ||-|,||'|+|*d d df   |#|"0|||!t_        |      d-z  S # 1 sw Y   ,xY w).Nr)   r}   r   r(   r   r   r~   r'  r!   r   r   r  r  r  r   r   r   r   r  r  r+  g     j@nearestr   )r*   r0   r,  r   r-  r   r2   r0  r1  r2  r3  r4  .r?   )corr_weights_staticpoint_track_loss_staticpoint_track_reproj_staticrB   rQ  rA  r   r/  r8  rc   r6  r7  r9  )5r"   r   rw   r4   r   r   r   r   r   r   rB  rC  r$   r   rE   r   rl   rv   r6   r,   r   rG  ro   r7   r   r   r   rH  rI  rr   rJ  r   rL  rV   r#   r   rF   r  rM  r3   r   rN  r;   rA   r@   r   r   r   rp   r   r   rq   r  )1rx   r   r   r   r   r   r   r   r!  r   r   rO  rP  r   r"  rA  rQ  r+  rR  rS  rT  general_conf_staticgeneral_conf_track_staticstatic_posesr]   static_poses_all_to_allstatic_point_track_surf_reprojstatic_point_track_reprojvis_and_rig_maskstatic_point_track_lossr  r   r#  r   r   rV  rW  rX  rU  rB   rY  rZ  r?  r>  r;  r:  r[  r]  r  s1                                                    r   r   zFlowMap.forward_0  s     "499a<9U#))"#.$U+11"15{57I7N7Nq7QAVAY&nnQM(B(G(G(KA(MvV[[]	 ,< =aQCi H+VcJdefhihjejJkJuJuvxJy{  E  E  FG  H  Q  Q  RT  U  \  \  ^',{{K@Q4RSTVWVXSX4Y[cejetetu}e~'$<<A<&q) ENN;};U;Z;Z[];^,_`dad,ez e$W&F$JZJZ[$(=(=(?-1\\'//!Q:ORU:UX[:[-\\K(}}dll7??1a3H33NQT3T&UW]dnpH"+X7Y]a]m]mqr"sK JJF8K8S8STUWX8Y(Z(d(defijlrhs(twx(xyz{{{	+%!
 {;7q	B3F#Qh--/
-.)ZUbIcImImnpIq  xA  FM  3N  3V  3V  WX  3Y  3a  3a  bd  3e  3j  3j  3l  l. ++[-GUaIbdhijkl	"Y{=7Q7[7[\^7_%`%h%hij%kk $($G$GTZH[HcHcdeghHi$j$u$uvwz{  ~D  zE  %F  %O  %O  %Q  %W  %W  \`  %W  %a(13FTaHbHlHlmoHp(q(y(yz{(|  @K  L]  @^  @h  @h  ik  @l  )l%'<<nQPTVWVXZeMe>fhvwxz~  AD  BD  AD  FQ  xQ  iR  Tm  no  qu  wz  xz  wz  |G  nG  TH  TM  TM  RV  TM  TW   Y  Z[   \vz1b1  aA%))l[\^_acbcac[cNdfrstvwz{~z  zA  tA  gB  EQ  RS  UV  XY  XZ  RZ  E[  g[  N\  ^`  Ca<  a$yyEIIaLOOL,IJZ,[,b,bcoctctuvcw  zF  zK  zK  LM  zN  PR  TV  XZ  -[  ]i  +k  mo   q (..C
ORRPL '-l.B.B.DF`dj&knt  vB  D^  bh  oi  'i#-2\\;PRiknpv  yG  HI  KL  NX  HX  yY  [q  u{  q|  l}  .  @C  EG  FG  EG  @G  .H*(/1OQ\]iQj(k(p(pqrtu(v% +,= >EOO\mnoprqrprnr\sLt  vG  LH  IJ  BK  !K,ETaHbcdfjlmoycyHz,z  O  PQ  SW  YZ  \f  hl  Pl  m  +m  )u  )u  )w  )  )  )A  )F  )F  )H#':+B-Fq!t-L%ac*#I. C #22K4G4O4OPQST4UV``abefhndop {{#6A>$*IlKP]D^DhDhikDl,m$n!6EV9W9a9abd9eejjopjqt  AR  uS  u]  u]  ^`  ua  ue  ue  jk  ue  ul  uq  uq  wx  uq  uy  y,, 79I!Z-9XZjk 55k&6I6Q6QRSUV6WXccdehikqgrs||~  E  E  JN  E  O&k-6P6Z6Z[]6^_gghijmx  zK  nL  nV  nV  WY  nZ  Z %]388<Dkk8==?1j$3F#G#N#NrSUW^WcWcdfWg#hjojyjy  {B  kC  EL  N11>!T12~Q]~B]3^3e3efhjqjvjvwxjy{}  @B  DF  4G3A!T3B3P^R^P^B^3_3f3fgikrkwkwxykz|~  AC  EG  4H"4Qcrc>\>5Q"R"Y"YZ\^e^j^jkl^moqsuwy"z  ~I  JK  MN  PT  Vd  Xd  Vd  fj  Jj  ~k  #k  !q  !q  vz  !q  !{} ~@E 6A:q"-  Auyy5ArPQr?TYZ[]^abefaf`gZgThkpqrtuwxwyqykzTzB{}  8Au  A 		UYYq\__U34DELLUZZXY]\a\f\fgh\ikmoqsuvx}  BD  F "5==?4NRXY\bch  kE  IO  ]P  P"',,0EGWY\^dftuvxy  |F  vF  gG  I_  ci  _j  Zk  #m  nq  su  tu  su  nu  #v$&={<?XY^^_`bcd!3k-6PQRTXZ[]gQg6h!hlw  yJ  mK  LM  OS  UV  Xb  dh  Lh  mi   i  q  q  s  {  {  }  B  B  D yU);[=W=a=abd=efg%4E(F(P(PQS(TTYY^_Y`cn  pA  dB  dL  dL  MO  dP  dT  dT  YZ  dT  d[  d`  d`  fg  d`  dh  h"\\+=uc.YZ\]_iYiJjFklmprtstrtmtu1K@Q4RSTVWYcSc4d4n4noq4rr"%#!#*{;L/MaQRT^N^/_/i/ikm/n/r/rxy/r/z//  FG/  0HH ]]_ 	DH"#34NQY\defghifijL !<<)@&VYZ]\]Z]^a`a^aVaIbBceklxeyz  E  E  FG  IQ  ST  IT  U]  _`  U`  Ha  b  l  l  mo  qy  zB  pC  DL	D "4($ 0"4QT":(((#6 0E]
 
 	
	D 	Ds   A8r''r0c           	      	   t        j                         r| xj                  dz  c_        |d   j                  dd  }|d   j                  d d |d   j	                  d      c\  }}}d}t        j
                  |d   |d   z        d | }	d}
|d   }t        |d| j                  	      }d
|vst        j                         r	 | j                  |j                  dd      dz  dz         x|d
<   }t        |d| j                  |      |d
<   t        j                  | j                  |d
   j                  dd            j                  d||f      dz         dz   x}}|d   d d d df   t%        |d         z   }t'        j(                  |d   |d   d       d   }|t%              z  }t        j*                  t-        ||d         d d d df   j                  dd      |j                  dd      j/                  d      dz  dz
        j1                  d      j3                  ddd      j                  d||dz
  f      }t        j*                  |d
   d d d df   j                  dd      |j                  dd      j/                  d      dz  dz
        j1                  d      j3                  ddd      j                  d||dz
  f      }| j5                  t        j6                  |t%        |d
         d d dd f   fd            j9                         j;                  d      }| j                  dk  snt        j<                  |      }t'        j>                  |d d dd |	f   |d d d d |	f   |d d d d |	f         d   }|}tA        |dz
  dd      D ]:  }t        j6                  |d d d |f   |d d |dz
  gf   |d d |d f   z  fd      }< t        j6                  t        jB                  d      jE                  |      d   jG                  |j	                  d      ddd      |fd      }tI        |d d dd f   |d d d df   jK                         |d d dd f   z  |d   d d dd f         |d   d d dd f   z
  }|j;                  dd      t%        |d         j;                  dd      z
  jM                         jO                         }|t-        ||d         |||t%        |      dz  }tQ        d       |S )Nr)   r}   r   r(   i  r   r~   r  r  r   r   r  r  r   r!   bwd_flowr,  r   r-  r.  r2   r   gɿr   )r@  flow_from_pose_static_lossflow_from_poserB   rA  zjust doing static)8r"   r   rw   r4   r   rE   r   rl   rv   r6   r,   rG  ro   r7   rD  r   rE  rF  r   rH  rI  r-   r   r   r   r   rs   r#   rJ  r   r$   
procrustesrV   r   rF   r  warprN  r   r   rM  rp   r   r   r   r;   rq   rC  rO  r   rL  rT  r   rA   r@   r  )-rx   r   r   r   r   r   r   r   r!  r   r   r   r"  rA  rQ  r)  
corresp_uvrdseye_surfcorresp_surfcorresp_featr@  
adj_transfrB   r]   adj_opt_flowrm  r  r   r#  r   r   rV  rW  rX  rU  rY  rZ  r?  r>  r;  r:  r[  r]  r  s-                                                r   r   zFlowMap.forward_  s     "499a<9U#))"#.$U+11"15{57I7N7Nq7QAnnVAYvay%89'6B	 e$W&F$JZJZ[$(=(=(?15gooaQR>SVY>Y\_>_1``F#h&/;]aeaqaquv&wF#$%JJF@S@[@[\]_`@a0b0l0lmnqrtzp{0|  @A  1A  %B  CD  %D  D	 "'*1SbS51&Z9P2QQ
%%k'&:;|;TUYZ[\]VE]"}}VHVAY%?#2#%F%N%NqQR%ST^TfTfghijTkTuTuvwTxyzTz{|T|}  F  F  GI  J  R  R  ST  UV  WX  Y  c  c  de  gh  io  pq  iq  fr  s }}[%83B3%?%G%G!%LZM_M_`abcMdMnMnopMqrsMstuMuv~~  @B  C  K  K  LM  NO  PQ  R  \  \  ]^  `a  bh  ij  bj  _k  l22599l6R]^dReKfghijikgkKl=mnp3qrzz|  B  B  GK  B  L99S=Aeool.K| ((!AB{2B)C\RSTUVaRaEbdpqrst  vA  rA  eB  D  EF  G
vax"%dAuyy%"1"+eAqQRseGnUZ[\]^]_[_U`F`9abc/dud		599Q<??51)<CCEJJqMRTUWXZ[\abcde HQqrTN53B3<+?+?+A%!"++MkZfNghijkjlhlNmnoz  |C  pD  EF  GH  GI  EI  pJ  J'3'8'8R'@6+V`JaCbCgCghklnCo'o%x%x%z%%  &B""<q	:*D*E]
 	
 	!"#:r   rD   )__name__
__module____qualname__rj   r   r	  __classcell__r{   s   @r   ra   ra   9   su    4 15" P
b 04 ;
x 15" [
z 15" X
t 15" *
X 15" j
X 15" ^
@ 15" z
r   ra   c                   D     e Zd Z	 	 	 	 	 	 	 	 	 	 d fd	ZddZddZ xZS )rt   c                 t   t         |           dd}|| _        || _         ||	      }t	        d|d        t        t        j                  |      ||      | _        |
dk7  rt        j                  |
| j                  j                  j                  j                  d   | j                  j                  j                  | j                  j                  j                  | j                  j                  j                   | j                  j                  j"                        | j                  _        t        j$                         | j                  _        t        j$                         | j                  _        g d|   | _        || _        || _        || _        || _        | j5                  d	t7        j8                  d
d
d
d
      d       | j5                  dt7        j8                  dt6        j:                        d       t        j$                  t        j                  | j*                  dd
            | _        d| _        t        j@                  ddddd| j>                  ffD cg c]  \  }}t        j                  ||d
       c}}      jC                         | _"        t        j@                  dddd| j>                  fD cg c]  }t        j                  ||d
       c}      jC                         | _#        t        j                  |
| j>                  d
      jC                         | _$        y c c}}w c c}w )Nc                 :   | dk(  r(t        j                  t        j                  dd      }|S | dk(  r(t        j                  t        j                  dd      }|S | dk(  r&t        j                  t        j
                  |      }|S | dk(  rd}|S t        d	| z        )
ax  Return a normalization layer
            Parameters:
                norm_type (str) -- the name of the normalization layer: batch | instance | none
            For BatchNorm, we use learnable affine parameters and track running statistics (mean/stddev).
            For InstanceNorm, we do not use learnable affine parameters. We do not track running statistics.
            batchT)affinetrack_running_statsinstanceFgroupnoneNz%normalization layer [%s] is not found)	functoolspartialr   BatchNorm2dInstanceNorm2d	GroupNormNotImplementedError)	norm_typegroup_norm_groups
norm_layers      r   get_norm_layerz*ResnetFPN.__init__.<locals>.get_norm_layer^  s     G#&..NN4T
  j(&..%%e
  g%&..r||=NO
  f$!

  *;iG r   zUsing torchvisionencoder)
pretrainedr  r?   r   )r+   )r   rc            i   latentr)   F)
persistentlatent_scalingr(   r   r  rc   )r  r  )r  rc   r~   r  )r  r(  )%ri   rj   feature_scalerh   rM  getattrtorchvisionmodelsmodelr   rn   conv1rI   r4   kernel_sizestriderd   r+   rY   fcavgpoollatent_size
num_layersindex_interpindex_paddingupsample_interpregister_bufferr"   emptyfloat32r   out_dim
ModuleListr   combs_1combs_2last_conv_up)rx   backboner  r  r  r  r  r  rh   r  rg   r  r  d1d2dr{   s                   r   rj   zResnetFPN.__init__O  s    		2 +,#I.
!8Y7:W[//:j]gh
A:!yy

  ''--a0

  ,,

  ''

  ((!ZZ--:: DJJ 

]]_

7
C$(*.Xu{{1aA'>5Qekk!5==Ae 	 	
 ==IId&&Q/
 }})T\]delnpquq}q}m~I  &A2ryyR';  &A  B  G  G  I}}C2bQUQ]Q]C^%_aryyAq'9%_`eeg))E4<<;@@B &A%_s   $!L/
L5c           
         t        |j                        dkD  r6 | |j                  dd      |      j                  d|j                  d d       S | j                  dk7  rFt        j                  || j                  | j                  dkD  rdnd| j                  dkD  rdnd d	      }|g}| j                  j                  |      }| j                  j                  |      }| j                  j                  |      }|j                  |       | j                  dkD  rS| j                  r| j                  j                  |      }| j                  j                  |      }|j                  |       | j                  dkD  r,| j                  j!                  |      }|j                  |       | j                  d
kD  r,| j                  j#                  |      }|j                  |       | j                  dkD  r,| j                  j%                  |      }|j                  |       | j&                  dk(  rd nd}|d   j                  dd  } | j(                  d    | j*                  d   t        j                  |d   |d   j                  dd  d            |d   z   j                               } | j(                  d    | j*                  d   t        j                  ||d   j                  dd  d            |d   z   j                               } | j(                  d
    | j*                  d
   t        j                  ||d   j                  dd  d            |d   z   j                               } | j(                  d    | j*                  d   t        j                  ||d   j                  dd  d            | j-                  |d         z   j                               }|S )Nr2   r   r)   r(         ?r   areaTscale_factorr*   align_cornersrecompute_scale_factorr?   nearest r   r!   r   r   )r3   r4   r6   r7   r  r,   r   r  r  bn1rK   rW   r  rh   maxpoollayer1layer2layer3layer4r  r  r  r  )rx   r   custom_sizelatentsr  	latent_sz	up_latents          r   r	  zResnetFPN.forward  s   qww<!D1a+$F$P$PQRTUT[T[\^]^T_$``$qt/A/A#'#5#5#;Z_c_q_qtw_wW[  ~B  [_aA#JJQJJNN1JJOOAq??Q""JJ&&q)

!!!$ANN1??Q

!!!$ANN1??Q

!!!$ANN1??Q

!!!$ANN1 $ 1 1Z ?TAJ$$RS)	#DLLOodll1oammGBKPWXZP[PaPabdbePflv6w&x  zA  BD  zE  'E  &K  &K  &M  O	#DLLOodll1oammIgVXkN_N_`b`cNdjt6u&vw~  @B  xC  'C  &I  &I  &K  M	#DLLOodll1oammIgVXkN_N_`b`cNdjt6u&vw~  @B  xC  'C  &I  &I  &K  M	#DLLOodll1oammIgVXkN_N_`b`cNdjt6u&vw{  xI  xI  JQ  RT  JU  xV  'V  &\  &\  &^  `	r   c                 :   t        |j                        dkD  r6 | |j                  dd      |      j                  d|j                  d d       S | j                  dk7  rFt        j                  || j                  | j                  dkD  rdnd| j                  dkD  rdnd d	      }| j                  j                  |      }| j                  j                  |      }| j                  j                  |      }|g}| j                  dkD  rS| j                  r| j                  j                  |      }| j                  j                  |      }|j                  |       | j                  dkD  r,| j                  j!                  |      }|j                  |       | j                  d
kD  r,| j                  j#                  |      }|j                  |       | j                  dkD  r,| j                  j%                  |      }|j                  |       | j&                  dk(  rd nd}|d   j                  dd  }t)        t        |            D ]/  }t        j                  ||   ||n|| j*                  |      ||<   1 t-        j.                  |d      | _        | j0                  j                  d   | j2                  d<   | j0                  j                  d   | j2                  d<   | j2                  | j2                  dz
  z  dz  | _        | j5                  | j0                        S )Nr2   r   r)   r(   r  r   r  Tr  r?   r  r   )r*   r  r   r!   g       @)r3   r4   r6   r7   r  r,   r   r  r  r  rK   r  rh   r  r  rW   r  r  r  r  rV   r  r"   r#   r  r  r   )rx   r   r  r  r  r  r]   s          r   r   zResnetFPN.forward_  s   qww<!		!Q5??1772A;OO$!//#'#5#5#;Z&*&8&83&>dD'+A JJQJJNN1JJOOA #??Q""JJ&&q)

!!!$ANN1??Q

!!!$ANN1??Q

!!!$ANN1??Q

!!!$ANN1 $ 1 1Z ?TAJ$$RS)	s7|$ 	A
(0	k))+	GAJ	 iiQ/!%!2!22!6A!%!2!22!6A"11T5H5H15LMPSSxx$$r   )
resnet34Tr2   r   borderr   r  Tr  r?   rD   )rx  ry  rz  rj   r	  r   r{  r|  s   @r   rt   rt   N  s6     "OCb%f1%r   rt   rD   )r  r   )>r"   r  matplotlib.pyplotpyplotr   r   korniar  einopsr   r   torch.nnr   r,   torchvision.transforms
transformsr   r   r   sysr   timeoscopyr   
matplotlibr   wandbr	   typingr
   r   r   r   r   r   collectionsr   pathrW   cotracker.utils.visualizerr   r   rH  torch_kmeansr   r   r   rA   r:   r5   r   r@   rp  shuffler_   rK  ru   rE  Modulera   rt   r&   r   r   <module>r     s         $ $ "        C C # + , G    
<	z@4]
 	
Rn
3
 Q
bii Q
jv%		 v%r   