o
    i                     @   s   d dl Z d dlmZmZ d dlZd dlmZmZ d dlm  m	Z
 G dd dejZe jddu Zzer>d dlmZ dZneW n eyN   eZd	ZY nw G d
d deZdS )    N)CallableOptional)Tensornnc                       sj   e Zd Z					ddedee dee dedejf d	ed
e	ddf fddZ
dedefddZ  ZS )	SwiGLUFFNN        Tin_featureshidden_featuresout_features	act_layer.dropbiasreturnc                    sF   t    |p|}|p|}tj|d| |d| _tj|||d| _d S )N   )r   )super__init__r   Linearw12w3selfr   r	   r
   r   r   r   	__class__ >/data/cameron/moge_repo/moge/model/dinov2/layers/swiglu_ffn.pyr      s
   
	zSwiGLUFFN.__init__xc                 C   s4   |  |}|jddd\}}t|| }| |S )Nr   )dim)r   chunkFsilur   )r   r   x12x1x2hiddenr   r   r   forward   s   

zSwiGLUFFN.forwardNNNr   T)__name__
__module____qualname__intr   r   r   Modulefloatboolr   r   r%   __classcell__r   r   r   r   r      s,    r   ZXFORMERS_DISABLED)SwiGLUTFc                       sX   e Zd Z					ddedee dee dedejf d	ed
e	ddf fddZ
  ZS )SwiGLUFFNFusedNr   Tr   r	   r
   r   .r   r   r   c                    sD   |p|}|p|}t |d d d d d }t j||||d d S )Nr            )r   r	   r
   r   )r*   r   r   r   r   r   r   r   7   s   	
zSwiGLUFFNFused.__init__r&   )r'   r(   r)   r*   r   r   r   r+   r,   r-   r   r.   r   r   r   r   r0   6   s*    r0   )ostypingr   r   warningstorchr   r   Ztorch.nn.functional
functionalr   r+   r   environgetZXFORMERS_ENABLEDZxformers.opsr/   ZXFORMERS_AVAILABLEImportErrorr0   r   r   r   r   <module>   s$   