
    |2g                     h   d dl Z d dlmZmZ d dlZd dlmZmZ d dlmc m	Z
  G d dej                        Ze j                  j                  d      du Z	 erd dlmZ dZ ej&                  d	       n ej&                  d
       e	  G d de      Zy# e$ r eZdZ ej&                  d       Y )w xY w)    N)CallableOptional)Tensornnc                        e Zd Z	 	 	 	 	 ddedee   dee   dedej                  f   dede	d	df fd
Z
ded	efdZ xZS )	SwiGLUFFNNin_featureshidden_featuresout_features	act_layer.dropbiasreturnc                     t         |           |xs |}|xs |}t        j                  |d|z  |      | _        t        j                  |||      | _        y )N   )r   )super__init__r   Linearw12w3selfr	   r
   r   r   r   r   	__class__s          N/home/cameronsmith/repos/FeatUp/featup/featurizers/dinov2/layers/swiglu_ffn.pyr   zSwiGLUFFN.__init__   sQ     	#2{)8[99[!o*=DI))O\E    xc                     | j                  |      }|j                  dd      \  }}t        j                  |      |z  }| j	                  |      S )Nr   )dim)r   chunkFsilur   )r   r   x12x1x2hiddens         r   forwardzSwiGLUFFN.forward   sD    hhqk1"%Bbwwvr   NNNg        T)__name__
__module____qualname__intr   r   r   Modulefloatboolr   r   r'   __classcell__r   s   @r   r   r      s     *.&*.2FF "#F sm	F
 CN+F F F 
F F r   r   XFORMERS_DISABLED)SwiGLUTzxFormers is available (SwiGLU)zxFormers is disabled (SwiGLU)Fz"xFormers is not available (SwiGLU)c                   r     e Zd Z	 	 	 	 	 ddedee   dee   dedej                  f   dede	d	df fd
Z
 xZS )SwiGLUFFNFusedNr	   r
   r   r   .r   r   r   c                 x    |xs |}|xs |}t        |dz  dz        dz   dz  dz  }t        | 	  ||||       y )Nr            )r	   r
   r   r   )r,   r   r   r   s          r   r   zSwiGLUFFNFused.__init__7   sZ     $2{)8[2Q67!;AAE#+%	 	 	
r   r(   )r)   r*   r+   r,   r   r   r   r-   r.   r/   r   r0   r1   s   @r   r5   r5   6   sw     *.&*.2

 "#
 sm	

 CN+
 
 
 

 
r   r5   )ostypingr   r   warningstorchr   r   torch.nn.functional
functionalr!   r-   r   environgetXFORMERS_ENABLEDxformers.opsr3   XFORMERS_AVAILABLEwarnImportErrorr5    r   r   <module>rH      s    
 %    		 . ::>>"56$> 8'!6756 	8
V 
  8FHMM67	8s   1B B10B1