o
    i                     @   s   d dl Z d dlmZmZ d dlZd dlmZmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ e d	Zd
ZG dd dejZ		ddedeegef dedee def
ddZdddZdS )    N)CallableOptional)Tensornn   )	Attention)DropPath)
LayerScale)Mlpdinov2Tc                !       s   e Zd Zddddddddejejeedddfdeded	e	d
e
de
de
de	de	de	dedejf dedejf dedejf dedejf de
de	ddf  fddZddedefddZ  ZS )Blockg      @FT        Ngư>dim	num_heads	mlp_ratioqkv_bias	proj_biasffn_biasdrop	attn_drop	drop_path	act_layer.
norm_layer
attn_class	ffn_layerqk_normln_epsreturnc              
      s   t    |||d| _|||||||||d| _|	r!t||	dnt | _|
dkr.t|
nt | _	|||d| _
t|| }||||||d| _|	rRt||	dnt | _|
dkr_t|
nt | _|
| _d S )N)eps)r   r   r   r   Z	proj_dropr   rope)init_valuesr   )in_featuresZhidden_featuresr   r   bias)super__init__norm1attnr	   r   Identityls1r   
drop_path1norm2intmlpls2Z
drop_path2sample_drop_ratio)selfr   r   r   r   r   r   r   r   r    r   r   r   r   r   r   r   r   Zmlp_hidden_dim	__class__ H/data/cameron/da3_repo/src/depth_anything_3/model/dinov2/layers/block.pyr$      s4   


zBlock.__init__xc                    s   ddt dt f fdd}dt dt f fdd} jr2 jdkr2t|| j|d}t|| jd	}|S  jrQ jd
krQ| ||||d }| || }|S |||||d }||| }|S )Nr4   r   c                    s      j | ||dS )Npos	attn_mask)r(   r&   r%   )r4   r6   r7   r/   r2   r3   attn_residual_funcN   s   z)Block.forward.<locals>.attn_residual_funcc                    s       | S )N)r-   r,   r*   )r4   r8   r2   r3   ffn_residual_funcQ   s   z(Block.forward.<locals>.ffn_residual_funcg?)residual_funcr.   r6   )r;   r.   r   r5   NN)r   trainingr.   "drop_add_residual_stochastic_depthr)   )r/   r4   r6   r7   r9   r:   r2   r8   r3   forwardM   s,   zBlock.forwardr<   )__name__
__module____qualname__r   GELU	LayerNormr   r
   r+   floatboolr   Moduler$   r   r?   __classcell__r2   r2   r0   r3   r      sd    	2r   r   r4   r;   r.   r6   r   c                 C   s   | j \}}}tt|d|  d}tj|| jdd | }| | }	|d ur0|| }||	|d}
n||	}
| d}|
d}
|| }tj|d||
j| j	d|d}|
| S )Nr   device)r6   r   )dtype)alpha)shapemaxr+   torchrandpermrJ   flatten	index_addtorK   view_as)r4   r;   r.   r6   bndsample_subset_sizebrangeZx_subsetresidualZx_flatresidual_scale_factorZx_plus_residualr2   r2   r3   r>   j   s   


r>   c                 C   sJ   | j \}}}tt|d|  d}tj|| jdd | }|| }||fS )Nr   rI   )rM   rN   r+   rO   rP   rJ   )r4   r.   rU   rV   rW   rX   rY   r[   r2   r2   r3   get_branges_scales   s
   r\   )r   N)r   )loggingtypingr   r   rO   r   r   	attentionr   r   r   layer_scaler	   r,   r
   	getLoggerloggerZXFORMERS_AVAILABLErG   r   rE   r>   r\   r2   r2   r2   r3   <module>   s2   
S
 