
    i                         d dl mZmZmZ d dlZd dlmc mZ d dlm	Z	mZ  G d dej
                  Z G d dej
                  Z G d d	ej
                  ZdS )
    )CallableOptionalUnionN)Tensornnc                   |     e Zd Zdddddej        ddfdededed	ed
ededej        deddf fdZ	dde
de
fdZ xZS )	Attention   T        FNdim	num_headsqkv_bias	proj_bias	attn_drop	proj_drop
norm_layerqk_normreturnc
                 >   t                                                       ||z  dk    s
J d            || _        ||z  | _        | j        dz  | _        t          j        ||dz  |          | _        |r || j                  nt          j                    | _	        |r || j                  nt          j                    | _
        t          j        |          | _        t          j        |||          | _        t          j        |          | _        |	| _        d S )Nr   z$dim should be divisible by num_headsg         bias)super__init__r   head_dimscaler   LinearqkvIdentityq_normk_normDropoutr   projr   rope)selfr   r   r   r   r   r   r   r   r$   	__class__s             J/home/robot-lab/lab/da3_repo/src/depth_anything_3/model/utils/attention.pyr   zAttention.__init__   s    	Y!###%K###"y(]D(
9S#'9993:Mjj///3:Mjj///I..Ic3Y777	I..			    xc                    |j         \  }}}|                     |                              ||d| j        | j                                      ddddd          }|                    d          \  }}	}
|                     |          |                     |	          }	}| j	        | 	                    ||          n|}| j	        | 	                    |	|          n|	}	t          j        ||	|
| j        r| j        j        nd|          }|                    dd                              |||          }|                     |          }|                     |          }|S )Nr      r         r   )	dropout_p	attn_mask)shaper   reshaper   r   permuteunbindr    r!   r$   Fscaled_dot_product_attentiontrainingr   p	transposer#   r   )r%   r)   posr/   BNCr   qkvs              r'   forwardzAttention.forward1   sF   '1ahhqkk!!!Q4>4=IIQQRSUVXY[\^_``**Q--1a{{1~~t{{1~~1!%!6DIIaA!%!6DIIaA**.-@dn&&S
 
 
 KK1%%aA..IIaLLNN1r(   )NN)__name__
__module____qualname__r   	LayerNormintboolfloatModuler   r   r@   __classcell__r&   s   @r'   r	   r	      s          "   	
    I  
     2  f        r(   r	   c            	       V     e Zd Z	 	 ddedeeef         deddf fdZd	edefd
Z	 xZ
S )
LayerScaleh㈵>Fr   init_valuesinplacer   Nc                     t                                                       || _        t          j        |t          j        |          z            | _        d S N)r   r   rO   r   	Parametertorchonesgamma)r%   r   rN   rO   r&   s       r'   r   zLayerScale.__init__G   sD     	\+
3"?@@


r(   r)   c                 X    | j         r|                    | j                  n	|| j        z  S rQ   )rO   mul_rU   r%   r)   s     r'   r@   zLayerScale.forwardQ   s(    %)\Eqvvdj!!!q4:~Er(   )rM   F)rA   rB   rC   rE   r   rG   r   rF   r   r@   rI   rJ   s   @r'   rL   rL   F   s         -1	A AA 5&=)A 	A
 
A A A A A AF FF F F F F F F F Fr(   rL   c                        e Zd Zddej        ddfdedee         dee         dedej        f         d	e	d
e
ddf fdZdedefdZ xZS )MlpNr   Tin_featureshidden_featuresout_features	act_layer.dropr   r   c                    t                                                       |p|}|p|}t          j        |||          | _         |            | _        t          j        |||          | _        t          j        |          | _        d S )Nr   )	r   r   r   r   fc1actfc2r"   r_   )r%   r[   r\   r]   r^   r_   r   r&   s          r'   r   zMlp.__init__V   s     	#2{)8[9[/EEE9;;9_lFFFJt$$			r(   r)   c                     |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|S rQ   )ra   rb   r_   rc   rX   s     r'   r@   zMlp.forwardg   sR    HHQKKHHQKKIIaLLHHQKKIIaLLr(   )rA   rB   rC   r   GELUrE   r   r   rH   rG   rF   r   r   r@   rI   rJ   s   @r'   rZ   rZ   U   s         *.&*.0g% %% "#% sm	%
 CN+% % % 
% % % % % %" F        r(   rZ   )typingr   r   r   rS   torch.nn.functionalr   
functionalr4   r   rH   r	   rL   rZ    r(   r'   <module>rj      s   " - , , , , , , , , ,                  , , , , ,	 , , ,^F F F F F F F F    ")     r(   