o
    {i
                     @   s   d dl mZmZmZ d dlm  mZ d dlm	Z	mZ d dl
mZmZ G dd deZG dd dejeZG d	d
 d
ejeZdS )    )CallableListOptionalN)Tensornn)cat_keep_shapesuncat_with_shapesc                   @   s4   e Zd ZdefddZdee dee fddZdS )	ListForwardMixinxc                 C   s   t N)NotImplementedErrorselfr
    r   D/data/cameron/keygrip/volume_dino_tracks/dinov3/layers/ffn_layers.pyforward   s   zListForwardMixin.forwardx_listreturnc                 C   s$   t |\}}}| |}t|||S r   )r   r   r   )r   r   x_flatshapes
num_tokensr   r   r   forward_list   s   
zListForwardMixin.forward_listN)__name__
__module____qualname__r   r   r   r   r   r   r   r   r	      s    r	   c                       sn   e Zd Zddejdddfdedee dee dedejf d	e	d
e
ddf fddZdedefddZ  ZS )MlpN        Tin_featureshidden_featuresout_features	act_layer.dropbiasr   c                    sZ   t    |p|}|p|}tj||||d| _| | _tj||||d| _t|| _d S )Nr"   device)	super__init__r   Linearfc1actfc2Dropoutr!   )r   r   r   r   r    r!   r"   r$   	__class__r   r   r&      s   

zMlp.__init__r
   c                 C   s6   |  |}| |}| |}| |}| |}|S r   )r(   r)   r!   r*   r   r   r   r   r   +   s   




zMlp.forward)r   r   r   r   GELUintr   r   Modulefloatboolr&   r   r   __classcell__r   r   r,   r   r      s.    	r   c                       sv   e Zd Z							ddedee dee deed	ejf  d
ede	deddf fddZ
dedefddZ  ZS )	SwiGLUFFNNr   T   r   r   r   r    .r!   r"   align_tor   c	                    sx   t    |p|}|p|}t|d d }	|	|	 |  }
tj||
||d| _tj||
||d| _tj|
|||d| _d S )N      r#   )r%   r&   r/   r   r'   w1w2w3)r   r   r   r   r    r!   r"   r6   r$   dZswiglu_hidden_featuresr,   r   r   r&   5   s   
zSwiGLUFFN.__init__r
   c                 C   s,   |  |}| |}t|| }| |S r   )r9   r:   Fsilur;   )r   r
   x1x2hiddenr   r   r   r   I   s   


zSwiGLUFFN.forward)NNNr   Tr5   N)r   r   r   r/   r   r   r   r0   r1   r2   r&   r   r   r3   r   r   r,   r   r4   4   s4    
r4   )typingr   r   r   torch.nn.functionalr   
functionalr=   torchr   dinov3.utilsr   r   objectr	   r0   r   r4   r   r   r   r   <module>   s   
