o
    i                     @   sx   d dl mZmZmZ d dlZd dlm  mZ d dlm	Z	mZ G dd dej
ZG dd dej
ZG dd	 d	ej
ZdS )
    )CallableOptionalUnionN)Tensornnc                       sl   e Zd Zdddddejddfdededed	ed
ededejdeddf fddZ	dde
de
fddZ  ZS )	Attention   T        FNdim	num_headsqkv_bias	proj_bias	attn_drop	proj_drop
norm_layerqk_normreturnc
           
         s   t    || dksJ d|| _|| | _| jd | _tj||d |d| _|r/|| jnt | _	|r;|| jnt | _
t|| _tj|||d| _t|| _|	| _d S )Nr   z$dim should be divisible by num_headsg         bias)super__init__r   head_dimscaler   LinearqkvIdentityq_normk_normDropoutr   projr   rope)
selfr
   r   r   r   r   r   r   r   r!   	__class__ D/data/cameron/da3_repo/src/depth_anything_3/model/utils/attention.pyr      s   


zAttention.__init__xc                 C   s   |j \}}}| |||d| j| jddddd}|d\}}	}
| || |	}}	| j	d ur9| 	||n|}| j	d urF| 	|	|n|	}	t
j||	|
| jrT| jjnd|d}|dd|||}| |}| |}|S )Nr      r         r	   )	dropout_p	attn_mask)shaper   reshaper   r   permuteunbindr   r   r!   Fscaled_dot_product_attentiontrainingr   p	transposer    r   )r"   r'   posr,   BNCr   qkvr%   r%   r&   forward1   s"   *

zAttention.forward)NN)__name__
__module____qualname__r   	LayerNormintboolfloatModuler   r   r=   __classcell__r%   r%   r#   r&   r      s:    	r   c                	       sN   e Zd Z		ddedeeef deddf fdd	Zd
edefddZ	  Z
S )
LayerScaleh㈵>Fr
   init_valuesinplacer   Nc                    s*   t    || _t|t| | _d S N)r   r   rJ   r   	Parametertorchonesgamma)r"   r
   rI   rJ   r#   r%   r&   r   G   s   
zLayerScale.__init__r'   c                 C   s   | j r	|| jS || j S rK   )rJ   mul_rO   r"   r'   r%   r%   r&   r=   Q   s   zLayerScale.forward)rH   F)r>   r?   r@   rB   r   rD   r   rC   r   r=   rF   r%   r%   r#   r&   rG   F   s    

rG   c                       sl   e Zd Zddejddfdedee dee dedejf d	e	d
e
ddf fddZdedefddZ  ZS )MlpNr	   Tin_featureshidden_featuresout_features	act_layer.dropr   r   c                    sV   t    |p|}|p|}tj|||d| _| | _tj|||d| _t|| _d S )Nr   )	r   r   r   r   fc1actfc2r   rW   )r"   rS   rT   rU   rV   rW   r   r#   r%   r&   r   V   s   
	zMlp.__init__r'   c                 C   s6   |  |}| |}| |}| |}| |}|S rK   )rX   rY   rW   rZ   rQ   r%   r%   r&   r=   g   s   




zMlp.forward)r>   r?   r@   r   GELUrB   r   r   rE   rD   rC   r   r   r=   rF   r%   r%   r#   r&   rR   U   s,    rR   )typingr   r   r   rM   Ztorch.nn.functionalr   
functionalr1   r   rE   r   rG   rR   r%   r%   r%   r&   <module>   s   /