
    |2g\
                     z   d dl Z d dlZd dlZd dlmZ d dlmZ  e j                  d      Zej                  j                  d      du Z
	 e
rd dlmZmZ dZ ej                  d       n ej                  d	       e	  G d dej"                        Z G d de      Zy# e$ r d
Z ej                  d       Y <w xY w)    N)Tensor)nndinov2XFORMERS_DISABLED)memory_efficient_attentionunbindTz!xFormers is available (Attention)z xFormers is disabled (Attention)Fz%xFormers is not available (Attention)c                   X     e Zd Z	 	 	 	 	 ddededededededdf fd	Zd
edefdZ xZ	S )	Attentiondim	num_headsqkv_bias	proj_bias	attn_drop	proj_dropreturnNc                 0   t         |           || _        ||z  }|dz  | _        t	        j
                  ||dz  |      | _        t	        j                  |      | _        t	        j
                  |||      | _	        t	        j                  |      | _
        y )Ng         )bias)super__init__r   scaler   LinearqkvDropoutr   projr   )	selfr   r   r   r   r   r   head_dim	__class__s	           M/home/cameronsmith/repos/FeatUp/featup/featurizers/dinov2/layers/attention.pyr   zAttention.__init__%   sx     	")#t^
99S#'9I.IIc3Y7	I.    xc                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|d   | j
                  z  |d   |d   }}}||j                  dd      z  }	|	j                  d      }	| j                  |	      }	|	|z  j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )	Nr      r         )r   )shaper   reshaper   permuter   	transposesoftmaxr   r   r   )
r   r!   BNCr   qkvattns
             r   forwardzAttention.forward8   s    ''1ahhqk!!!Q4>>1;NOWWXY[\^_abdefa&4::%s1vs1va11;;r2&&|||#~~d#AX  A&..q!Q7IIaLNN1r    )   FT        r6   )
__name__
__module____qualname__intboolfloatr   r   r4   __classcell__r   s   @r   r
   r
   $   sp     // / 	/
 / / / 
/& F r    r
   c                   *     e Zd Zddedef fdZ xZS )MemEffAttentionr!   r   c                    t         s|t        d      t        
|   |      S |j                  \  }}}| j                  |      j                  ||d| j                  || j                  z        }t        |d      \  }}}	t        |||	|      }|j                  |||g      }| j                  |      }| j                  |      }|S )Nz-xFormers is required for using nested tensorsr   r#   )	attn_bias)XFORMERS_AVAILABLEAssertionErrorr   r4   r(   r   r)   r   r   r   r   r   )r   r!   rB   r-   r.   r/   r   r0   r1   r2   r   s             r   r4   zMemEffAttention.forwardI   s    !$$%TUU7?1%%''1ahhqk!!!Q4>>1;NOa.1a&q!Q)DIIq!Qi IIaLNN1r    )N)r7   r8   r9   r   r4   r=   r>   s   @r   r@   r@   H   s     F  r    r@   )loggingoswarningstorchr   r   	getLoggerloggerenvirongetXFORMERS_ENABLEDxformers.opsr   r   rC   warnImportErrorModuler
   r@    r    r   <module>rS      s     	    
		8	$ ::>>"56$> ;C!9:89 	;!		 !Hi S  ;HMM9:;s   	3B B:9B: