o
    i                     @   sJ   d dl Z d dlm  mZ d dlmZmZ e dZG dd dej	Z
dS )    N)Tensornndinov2c                       s   e Zd Zdddddejdddf	dededed	ed
ededejdededdf fddZ	dde
de
fddZde
de
fddZ  ZS )	Attention   FT        Ndim	num_headsqkv_bias	proj_bias	attn_drop	proj_drop
norm_layerqk_norm
fused_attnreturnc                    s   t    || dksJ d|| _|| }|d | _|	| _tj||d |d| _|r/||nt | _	|r:||nt | _
t|| _tj|||d| _t|| _|
| _d S )Nr   z$dim should be divisible by num_headsg         )bias)super__init__r	   scaler   r   LinearqkvIdentityq_normk_normDropoutr   projr   rope)selfr   r	   r
   r   r   r   r   r   r   r   head_dim	__class__ L/data/cameron/da3_repo/src/depth_anything_3/model/dinov2/layers/attention.pyr      s   


zAttention.__init__xc                 C   sL  |j \}}}| |||d| j|| j ddddd}|d |d |d }}	}
| || |	}}	| jd urK|d urK| ||}| |	|}	| jrst	j
||	|
| jrZ| jjnd|d urn|d d d f d| jddnd d}n|| j }||	dd	 }|jd	d
}| |}||
 }|dd|||}| |}| |}|S )Nr      r         r   )	dropout_p	attn_maskr   )shaper   reshaper	   permuter   r   r   r   Fscaled_dot_product_attentiontrainingr   prepeatr   	transposesoftmaxr   r   )r   r%   posr*   BNCr   qkvattnr#   r#   r$   forward/   s:   



zAttention.forwardc           
      C   s   |j \}}}| |||d| j|| j ddddd}|d | j |d |d }}}||dd }	|	jdd}	| |	}	|	| dd|||}| 	|}| 
|}|S )	Nr   r&   r   r'   r(   r+   r,   r-   )r.   r   r/   r	   r0   r   r6   r7   r   r   r   )
r   r%   r9   r:   r;   r   r<   r=   r>   r?   r#   r#   r$   _forwardS   s   "


zAttention._forward)NN)__name__
__module____qualname__r   	LayerNormintboolfloatModuler   r   r@   rA   __classcell__r#   r#   r!   r$   r      sB    	
$r   )loggingZtorch.nn.functionalr   
functionalr1   torchr   	getLoggerloggerrI   r   r#   r#   r#   r$   <module>   s
   

