o
    vi                     @   s~   d dl mZ d dlZd dlmZmZ z
d dlmZ dZW n e	y'   dZY nw de
fdd	Zddd
ddddejfddZdS )    )partialN)
SDPBackendsdpa_kernel)flash_attn_funcTFreturnc                 C   s<   t j rt jjr| jdkrt j| \}}|d | S dS )z
    Returns the compute capability of a given torch device if it's a CUDA device, otherwise returns 0.

    Args:
        device: torch device.

    Returns:
        device_cc (int): compute capability in the SmXXX format (i.e. 90 for Hopper).
    cuda
   r   )torchr   is_availableversiontypeget_device_capability)devicemajorminor r   T/data/cameron/vidgen/cosmos-policy/cosmos_policy/_src/predict2/networks/attention.pyget_device_ccJ   s   
r   g        c              	   C   s  t jt jt jg}|
t jt jfv }t| j}|
|vr!td|
d| |
} ||
}||
}|d ur8| | } |dkrLtrL|rLt	| |||||	dd S |dv r^|r^t
jt
jt
jg}t
j}n)|rst
jt
jt
jg}|dkrot
jnt
j}n|
t jksJ d|
d	t
jg}t
j}|	rtd
zt|dd ttdd}W n ty   t}|g}Y nw | dd} |dd}|dd}||d t jjj| |||||d}W d    n1 sw   Y  |dd }|S )Nzdtype=z is not supported.Z   )qkvsoftmax_scalecausaldeterministicr   )r   d   P   zUnrecognized dtype=.zVDeterministic mode in attention is only supported when Flash Attention 3 is available.T)backendsset_priority_order)r         )r   )	is_causal	dropout_pscale)r	   bfloat16float16float32r   r   NotImplementedErrortoFLASH_ATTN_3_AVAILABLEr   r   CUDNN_ATTENTIONFLASH_ATTENTIONEFFICIENT_ATTENTIONr   r   	TypeError	transposenn
functionalscaled_dot_product_attention
contiguous)r   r   r   Zq_lensZk_lensr#   r   q_scaler   r   dtypesupported_dtypesis_halfZcompute_capZSDPA_BACKENDSZBEST_SDPA_BACKENDZsdpa_kernel_outr   r   r   	attentionZ   s|   





r9   )	functoolsr   r	   torch.nn.attentionr   r   Z!flash_attn_3.flash_attn_interfacer   r*   ModuleNotFoundErrorintr   r%   r9   r   r   r   r   <module>   s&   <