o
    i                     @   s   d dl Z d dlZd dlZd dlm  mZ d dlmZ d dlmZ e 	dZ
ejddu Zzer<d dlmZmZ dZneW n eyJ   dZY nw G d	d
 d
ejZG dd deZdS )    N)Tensor)nndinov2XFORMERS_DISABLED)memory_efficient_attentionunbindTFc                       sZ   e Zd Z					ddedededed	ed
eddf fddZddedefddZ  Z	S )	Attention   FT        dim	num_headsqkv_bias	proj_bias	attn_drop	proj_dropreturnNc                    sf   t    || _|| }|d | _tj||d |d| _t|| _tj|||d| _	t|| _
d S )Ng         )bias)super__init__r   scaler   LinearqkvDropoutr   projr   )selfr   r   r   r   r   r   head_dim	__class__ =/data/cameron/moge_repo/moge/model/dinov2/layers/attention.pyr   &   s   
	
zAttention.__init__xc           
      C   s   |j \}}}| |||d| j|| j ddddd}|d\}}}	t|||	|}|dddd|||}| |}| 	|}|S )Nr      r         )
shaper   reshaper   permuter   Fscaled_dot_product_attentionr   r   
r   r!   	attn_biasBNCr   qkvr   r   r    forwardF   s   .

zAttention.forward)r	   FTr
   r
   N)
__name__
__module____qualname__intboolfloatr   r   r2   __classcell__r   r   r   r    r   %   s,     r   c                       s(   e Zd Zddedef fddZ  ZS )MemEffAttentionNr!   r   c           
         s   t s|d ur
tdt |S |j\}}}| |||d| j|| j }t|d\}}}	t	|||	|d}||||g}| 
|}| |}|S )Nz-xFormers is required for using nested tensorsr   r"   )r+   )XFORMERS_AVAILABLEAssertionErrorr   r2   r%   r   r&   r   r   r   r   r   r*   r   r   r    r2   T   s    

zMemEffAttention.forwardr3   )r4   r5   r6   r   r2   r:   r   r   r   r    r;   S   s     r;   )loggingoswarningsZtorch.nn.functionalr   
functionalr(   torchr   	getLoggerloggerenvirongetXFORMERS_ENABLEDxformers.opsr   r   r<   ImportErrorModuler   r;   r   r   r   r    <module>   s&   	
.