
    0	h                        d dl  d dlZd dlmZ d dlmc mZ dej                  fdZdej                  fdZ	dej                  fdZ
dej                  j                  dej                  j                  ej                     fd	Zy)
    )*Nmodulec                 T     ddl m  G  fdd j                        }| _         S )Nr   )
checkpointc                   <     e Zd ZW j                  Z fdZ xZS )Fwrap_module_with_gradient_checkpointing.<locals>._CheckpointingWrapperc                 0     t         |   g|ddi|S )Nuse_reentrantF)superforward)selfargskwargs	__class__r   s      </home/cameronsmith/repos/controll3r/MoGe/moge/model/utils.pyr   zNwrap_module_with_gradient_checkpointing.<locals>._CheckpointingWrapper.forward   s!    egoTTETVTT    )__name__
__module____qualname__r   _restore_clsr   __classcell__)r   r   r   s   @r   _CheckpointingWrapperr   	   s    ''	U 	Ur   r   )torch.utils.checkpointr   r   )r   r   r   s   ` @r   'wrap_module_with_gradient_checkpointingr      s)    1U 0 0 U
 -FMr   c                 :    | j                   j                  | _         y N)r   r   )r   s    r   )unwrap_module_with_gradient_checkpointingr      s    ''44Fr   c                 r    t         j                  dk\  sJ d        G d d| j                        }|| _        | S )Nz2.0z"SDPA requires PyTorch 2.0 or laterc                   F    e Zd Zddej                  dej                  fdZy):wrap_dinov2_attention_with_sdpa.<locals>._AttentionWrapperNxreturnc                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }t        j                  |d      \  }}}	t        j                  |||	|      }|j	                  dddd      j                  |||      }| j                  |      }| j                  |      }|S )N      r         )shapeqkvreshape	num_headspermutetorchunbindFscaled_dot_product_attentionproj	proj_drop)
r   r!   	attn_biasBNCr)   qkvs
             r   r   zBwrap_dinov2_attention_with_sdpa.<locals>._AttentionWrapper.forward   s    ggGAq!((1+%%aAt~~qDNN?RS[[\]_`bcefhijCll3*GAq!..q!Q	BA		!Q1%--aA6A		!Aq!AHr   r   )r   r   r   r-   Tensorr    r   r   _AttentionWrapperr       s    	U\\ 	ell 	r   r<   )r-   __version__r   )r   r<   s     r   wrap_dinov2_attention_with_sdpar>      s:    %K'KKF,,  )FMr   bucketr"   c                 P   t         j                  j                  j                  }|j	                         }|j                         }|j                  |       t         j                  j                  ||       t         j                  j                         }|j                  |       |S )N)group)r-   distributedrA   WORLDsizebufferdiv_
all_reducefuturesFuture
set_result)stater?   group_to_use
world_sizegradfuts         r   sync_ddp_hookrP   )   sz    $$**00L""$J==?DIIj	  \ :
--


 CNN4Jr   )typingr-   torch.nnnntorch.nn.functional
functionalr/   Moduler   r   r>   rB   
GradBucketrH   rI   r:   rP   r;   r   r   <module>rX      sy        BII 5bii 5BII &!2!2!=!= %--BVBVW\WcWcBd r   