
     h>&                        d dl Z d dlmZ d dlmZ d dlZd Z ed      Zdde	de
de
fdZ G d	 d
ej                        Z G d dej                        Z G d dej                        Z G d dej                        Z G d dej                        Z G d dej                        Z G d de      Z G d dej                        Zy)    N)repeatc                       fd}|S )Nc                     t        | t        j                  j                        rt        | t              s| S t        t        |             S N)
isinstancecollectionsabcIterablestrtupler   )xns    A/home/cameronsmith/repos/controll3r/dust3r/croco/models/blocks.pyparsez_ntuple.<locals>.parse   s6    a112:a;MHVAq\""     )r   r   s   ` r   _ntupler      s    # Lr      	drop_probtrainingscale_by_keepc                     |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }| j                  |      j                  |      }|dkD  r|r|j	                  |       | |z  S )z]Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
               r   )r   )shapendim	new_empty
bernoulli_div_)r   r   r   r   	keep_probr   random_tensors          r   	drop_pathr"       sw     BhIIWWQZMDAFFQJ//EKK&11)<M3=9%}r   c                   :     e Zd ZdZddedef fdZd Zd Z xZ	S )DropPathz^Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).
    r   r   c                 F    t         t        |           || _        || _        y r   )superr$   __init__r   r   )selfr   r   	__class__s      r   r'   zDropPath.__init__/   s    h&("*r   c                 Z    t        || j                  | j                  | j                        S r   )r"   r   r   r   r(   r   s     r   forwardzDropPath.forward4   s!    DNNDMM4;M;MNNr   c                 6    dt        | j                  d      dS )Nz
drop_prob=   z0.3f)roundr   r(   s    r   
extra_reprzDropPath.extra_repr7   s    E$..3D9::r   )r   T)
__name__
__module____qualname____doc__floatboolr'   r,   r1   __classcell__r)   s   @r   r$   r$   ,   s&    +% +T +
O;r   r$   c                   H     e Zd ZdZddej
                  ddf fd	Zd Z xZS )MlpzB MLP as used in Vision Transformer, MLP-Mixer and related networksNTr   c                 r   t         |           |xs |}|xs |}t        |      }t        |      }t        j                  |||d         | _         |       | _        t        j                  |d         | _        t        j                  |||d         | _	        t        j                  |d         | _
        y )Nr   biasr   )r&   r'   	to_2tuplennLinearfc1actDropoutdrop1fc2drop2)	r(   in_featureshidden_featuresout_features	act_layerr>   drop
drop_probsr)   s	           r   r'   zMlp.__init__<   s    #2{)8[t_
99[/QH;ZZ
1.
99_laIZZ
1.
r   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r   )rB   rC   rE   rF   rG   r+   s     r   r,   zMlp.forwardI   sH    HHQKHHQKJJqMHHQKJJqMr   )	r2   r3   r4   r5   r@   GELUr'   r,   r8   r9   s   @r   r;   r;   :   s$    L48tWYW^W^eipr /r   r;   c                   &     e Zd Zd fd	Zd Z xZS )	Attentionc                 :   t         |           || _        ||z  }|dz  | _        t	        j
                  ||dz  |      | _        t	        j                  |      | _        t	        j
                  ||      | _	        t	        j                  |      | _
        || _        y )N      r.   r=   )r&   r'   	num_headsscaler@   rA   qkvrD   	attn_dropproj	proj_droprope	r(   dimrZ   rT   qkv_biasrW   rY   head_dimr)   s	           r   r'   zAttention.__init__S   s|    ")#%
99S#'9I.IIc3'	I.	r   c                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  dd      }t        d      D cg c]  }|d d d d |f    c}\  }}	}
| j                  $| j                  ||      }| j                  |	|      }	||	j	                  dd      z  | j                  z  }|j                  d      }| j                  |      }||
z  j	                  dd      j                  |||      }| j                  |      }| j                  |      }|S c c}w )Nr.   r   r\   r   )r   rV   reshaperT   	transposerangerZ   rU   softmaxrW   rX   rY   )r(   r   xposBNCrV   iqkvattns               r   r,   zAttention.forward^   s#   ''1ahhqk!!!Q4>>1;NOYYZ[\]^',Qx0!3q1u:01a 99 		!T"A		!T"AAKKB''4::5|||#~~d#AX  A&..q!Q7IIaLNN1 1s   )EN   Fr   r   r2   r3   r4   r'   r,   r8   r9   s   @r   rQ   rQ   Q   s    	r   rQ   c                   ^     e Zd Zdddddej                  ej
                  df fd	Zd Z xZS )Block      @Fr   Nc                    t         |            |	|      | _        t        ||
||||      | _        |dkD  rt        |      nt        j                         | _         |	|      | _	        t        ||z        }t        ||||      | _        y N)rZ   rT   r]   rW   rY   r   )rH   rI   rK   rL   )r&   r'   norm1rQ   ro   r$   r@   Identityr"   norm2intr;   mlp)r(   r\   rT   	mlp_ratior]   rL   rW   r"   rK   
norm_layerrZ   mlp_hidden_dimr)   s               r   r'   zBlock.__init__t   sx    _
c	H`iuyz	09B),BKKM_
S9_-3R[bfgr   c                     || j                  | j                  | j                  |      |            z   }|| j                  | j                  | j	                  |                  z   }|S r   )r"   ro   rx   r|   rz   )r(   r   rg   s      r   r,   zBlock.forward   sQ    tyyA=>>txx

1677r   	r2   r3   r4   r@   rO   	LayerNormr'   r,   r8   r9   s   @r   rt   rt   r   s*    13e"XZR\\PT	hr   rt   c                   &     e Zd Zd fd	Zd Z xZS )CrossAttentionc                    t         |           || _        ||z  }|dz  | _        t	        j
                  |||      | _        t	        j
                  |||      | _        t	        j
                  |||      | _        t	        j                  |      | _
        t	        j
                  ||      | _        t	        j                  |      | _        || _        y )NrS   r=   )r&   r'   rT   rU   r@   rA   projqprojkprojvrD   rW   rX   rY   rZ   r[   s	           r   r'   zCrossAttention.__init__   s    ")#%
YYsCh7
YYsCh7
YYsCh7
I.IIc3'	I.	r   c                    |j                   \  }}}|j                   d   }	|j                   d   }
| j                  |      j                  ||| j                  || j                  z        j	                  dddd      }| j                  |      j                  ||	| j                  || j                  z        j	                  dddd      }| j                  |      j                  ||
| j                  || j                  z        j	                  dddd      }| j                  $| j                  ||      }| j                  ||      }||j                  dd      z  | j                  z  }|j                  d      }| j                  |      }||z  j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )Nr   r   r   r.   r`   ra   rb   )r   r   rc   rT   permuter   r   rZ   rd   rU   rf   rW   rX   rY   )r(   querykeyvalueqposkposrh   Nqrj   NkNvrl   rm   rn   ro   r   s                   r   r,   zCrossAttention.forward   s   ;;2qYYq\[[^JJu%%a4>>1t~~;MNVVWXZ[]^`abJJsO##AbT^^9KLTTUVXY[\^_`JJu%%a4>>1t~~;MNVVWXZ[]^`ab99 		!T"A		!T"AAKKB''4::5|||#~~d#AX  A&..q"a8IIaLNN1r   rp   rr   r9   s   @r   r   r      s    r   r   c            	       `     e Zd Zdddddej                  ej
                  ddf	 fd	Zd Z xZS )DecoderBlockru   Fr   TNc                    t         |            |	|      | _        t        ||||||      | _        t        ||||||      | _        |dkD  rt        |      nt        j                         | _
         |	|      | _         |	|      | _        t        ||z        }t        ||||      | _        |
r |	|      | _        y t        j                         | _        y rw   )r&   r'   rx   rQ   ro   r   
cross_attnr$   r@   ry   r"   rz   norm3r{   r;   r|   norm_y)r(   r\   rT   r}   r]   rL   rW   r"   rK   r~   norm_memrZ   r   r)   s                r   r'   zDecoderBlock.__init__   s    _
c	H`iuyz	(49W_kt  AE  F09B),BKKM_
_
S9_-3R[bfg)1jor{{}r   c           
      b   || j                  | j                  | j                  |      |            z   }| j                  |      }|| j                  | j	                  | j                  |      ||||            z   }|| j                  | j                  | j                  |                  z   }||fS r   )r"   ro   rx   r   r   rz   r|   r   )r(   r   yrg   yposy_s         r   r,   zDecoderBlock.forward   s    tyyA=>>[[^ttzz!}b"dDQRRtxx

1677!tr   r   r9   s   @r   r   r      s-    13e"XZR\\TX_cEr   r   c                       e Zd ZdZd Zd Zy)PositionGetterz return positions of patches c                     i | _         y r   )cache_positionsr0   s    r   r'   zPositionGetter.__init__   s
    !r   c                 T   ||f| j                   vrSt        j                  ||      }t        j                  ||      }t        j                  ||      | j                   ||f<   | j                   ||f   j	                  d||z  d      j                  |dd      j                         }|S )N)devicer   r   ra   )r   torcharangecartesian_prodviewexpandclone)r(   bhwr   r   r   poss           r   __call__zPositionGetter.__call__   s    !u,,,Qv.AQv.A(-(<(<Q(BD  1%""1Q3',,Q!Q7>>q"aHNNP
r   N)r2   r3   r4   r5   r'   r   r   r   r   r   r      s    '"r   r   c                   0     e Zd ZdZd fd	Zd Zd Z xZS )
PatchEmbedzb just adding _init_weights + position getter compared to timm.models.layers.patch_embed.PatchEmbedc                    t         |           t        |      }t        |      }|| _        || _        |d   |d   z  |d   |d   z  f| _        | j
                  d   | j
                  d   z  | _        || _        t        j                  ||||      | _
        |r ||      nt        j                         | _        t               | _        y )Nr   r   )kernel_sizestride)r&   r'   r?   img_size
patch_size	grid_sizenum_patchesflattenr@   Conv2drX   ry   normr   position_getter)r(   r   r   in_chans	embed_dimr~   r   r)   s          r   r'   zPatchEmbed.__init__   s    X&z*
 $"1+A6zRS}8TU>>!,t~~a/@@IIh	zR\]	-7Jy)R[[]	-/r   c                 *   |j                   \  }}}}t        j                  || j                  d   k(  d| d| j                  d    d       t        j                  || j                  d   k(  d| d| j                  d    d       | j	                  |      }| j                  ||j                  d      |j                  d      |j                        }| j                  r!|j                  d      j                  dd      }| j                  |      }||fS )	Nr   zInput image height (z) doesn't match model (z).r   zInput image width (r   r.   )r   r   _assertr   rX   r   sizer   r   rd   r   )r(   r   rh   rj   HWr   s          r   r,   zPatchEmbed.forward   s    WW
1aa4==++/CA3F]^b^k^klm^n]ooq-rsa4==++/B1#E\]a]j]jkl]m\nnp-qrIIaL""1affQiAHHE<<		!&&q!,AIIaL#vr   c                     | j                   j                  j                  }t        j                  j
                  j                  |j                  |j                  d   dg             y )Nr   ra   )	rX   weightdatar   r@   initxavier_uniform_r   r   )r(   r   s     r   _init_weightszPatchEmbed._init_weights   sC    II!!%%affaggaj"-=&>?r   )      r.   i   NT)r2   r3   r4   r5   r'   r,   r   r8   r9   s   @r   r   r      s    l0	@r   r   )r   FT)r   torch.nnr@   	itertoolsr   collections.abcr   r   r?   r6   r7   r"   Moduler$   r;   rQ   rt   r   r   objectr   r   r   r   r   <module>r      s   "     AJ	
E 
$ 
t 
;ryy ;")) .		 BBII $%RYY %N299 0V @ @r   