
    0	hz
                        d dl Z d dlZd dlZd dlmZ d dlmZ  e j                  d      Zej                  j                  d      du Z
	 e
rd dlmZmZ dZne	  G d	 d
ej                         Z G d de      Zy# e$ r dZY *w xY w)    N)Tensor)nndinov2XFORMERS_DISABLED)memory_efficient_attentionunbindTFc                   Z     e Zd Z	 	 	 	 	 ddededededededdf fd	Zdd
edefdZ xZ	S )	Attentiondim	num_headsqkv_bias	proj_bias	attn_drop	proj_dropreturnNc                 0   t         |           || _        ||z  }|dz  | _        t	        j
                  ||dz  |      | _        t	        j                  |      | _        t	        j
                  |||      | _	        t	        j                  |      | _
        y )Ng         )bias)super__init__r   scaler   LinearqkvDropoutr   projr   )	selfr   r   r   r   r   r   head_dim	__class__s	           N/home/cameronsmith/repos/controll3r/MoGe/moge/model/dinov2/layers/attention.pyr   zAttention.__init__%   sx     	")#t^
99S#'9I.IIc3Y7	I.    xc                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|d   | j
                  z  |d   |d   }	}}||j                  dd      z  }
|
j                  d      }
| j                  |
      }
|
|	z  j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )	Nr      r         )r   )shaper   reshaper   permuter   	transposesoftmaxr   r   r   )r   r!   	attn_biasBNCr   qkvattns              r   forwardzAttention.forward8   s    ''1ahhqk!!!Q4>>1;NOWWXY[\^_abdefa&4::%s1vs1va11;;r2&&|||#~~d#AX  A&..q!Q7IIaLNN1r    )   FT        r7   N)
__name__
__module____qualname__intboolfloatr   r   r5   __classcell__r   s   @r   r
   r
   $   sp     // / 	/
 / / / 
/& F r    r
   c                   *     e Zd Zddedef fdZ xZS )MemEffAttentionr!   r   c                    t         s|t        d      t        
|   |      S |j                  \  }}}| j                  |      j                  ||d| j                  || j                  z        }t        |d      \  }}}	t        |||	|      }|j                  |||g      }| j                  |      }| j                  |      }|S )Nz-xFormers is required for using nested tensorsr   r#   )r-   )XFORMERS_AVAILABLEAssertionErrorr   r5   r(   r   r)   r   r   r   r   r   )r   r!   r-   r.   r/   r0   r   r1   r2   r3   r   s             r   r5   zMemEffAttention.forwardI   s    !$$%TUU7?1%%''1ahhqk!!!Q4>>1;NOa.1a&q!Q)DIIq!Qi IIaLNN1r    r8   )r9   r:   r;   r   r5   r?   r@   s   @r   rB   rB   H   s     F  r    rB   )loggingoswarningstorchr   r   	getLoggerloggerenvirongetXFORMERS_ENABLEDxformers.opsr   r   rD   ImportErrorModuler
   rB    r    r   <module>rS      s     	    
		8	$ ::>>"56$> 
C! 	 	!		 !Hi S  s   	A: :BB