o
    i                     @   sL   d dl Z d dlmZ d dlmZ d dlmZ G dd dejZd
dd	Z	dS )    N)trunc_normal_)weight_normc                       s8   e Zd Z					d fdd	Zdd	 Zd
d Z  ZS )DINOHeadF         Tc                    s`   t    t|d}t||||||d| _| | j ttj	||dd| _
| j
jjd d S )N   )
hidden_dimuse_bnbiasFr   )super__init__max
_build_mlpmlpapply_init_weightsr   nnLinear
last_layerweight_gdatafill_)selfin_dimout_dimr
   nlayersr	   bottleneck_dimZmlp_bias	__class__ =/data/cameron/moge_repo/moge/model/dinov2/layers/dino_head.pyr      s   


zDINOHead.__init__c                 C   sP   t |tjr"t|jdd t |tjr$|jd ur&tj|jd d S d S d S d S )Ng{Gz?)stdr   )
isinstancer   r   r   weightr   init	constant_)r   mr!   r!   r"   r      s   zDINOHead._init_weightsc                 C   s@   |  |}|jtjkrdnd}tjj|dd|d}| |}|S )Ngư>g-q=   )dimpeps)r   dtypetorchfloat16r   
functional	normalizer   )r   xr-   r!   r!   r"   forward$   s
   

zDINOHead.forward)Fr   r   r   T)__name__
__module____qualname__r   r   r4   __classcell__r!   r!   r   r"   r      s    r   FTc                 C   s   | dkrt j|||dS t j|||dg}|r|t | |t   t| d D ]}|t j|||d |rC|t | |t   q,|t j|||d t j| S )Nr   r   r*   )r   r   appendBatchNorm1dGELUrange
Sequential)r   r   r   r	   r
   r   layers_r!   r!   r"   r   ,   s   
r   )NFT)
r/   torch.nnr   torch.nn.initr   Ztorch.nn.utilsr   Moduler   r   r!   r!   r!   r"   <module>   s    