o viF@sdZddlmZddlmZddlmZddlZddlmZddl m Z ddl m Z dd l mZd8d ed ed ededef ddZ   d9d ed ed edeejdBdeejdBdededededBdefddZ   d9d ed ed edeejdBdeejdBdededededBdefddZ      d:d ed ed ededBdedBdedBdedBdedBd edBdeddeefeeeeefBfd!d"Zd ed ed ed#ed$e f d%d&Z   d9d ed ed edeejdBdeejdBdededededBdefd'd(Zd)ed*ed+ededBfd,d-Z . / / 0d;d ed1eeBd2eeBd3eeBd#eeBdeeeeeeeff d4d5Zd ed1ed2ed3ed#ef d6d7ZdS)  z'varlen_tensor_checks..css |] }|dup |dkVqdS)Nrrr:rrrr=s  )NNrr)r r r r2r3csr8r9rr:rrrr=r>css|]}|dkVqdS)rNrr:rrrr=r>zwVariable length Attention requires all 6 of cumulative_seqlen_{Q,KV}, max_seqlen_{Q,KV}, total_seqlen_{Q,KV} to be set.r/zfVariable length Attention only supports sequence-packed memory layout (batch = 1), got query.shape[0]=zDmax_seqlen_Q and max_seqlen_KV must be ints, got type(max_seqlen_Q)=z, type(max_seqlen_KV)=z>Maximum sequence length cannot exceed total, got max_seqlen_Q=z, total_seqlen_Q=z?Maximum sequence length cannot exceed total, got max_seqlen_KV=z, total_seqlen_KV=z@Maximum sequence length cannot be less than 1, got max_seqlen_Q=z, max_seqlen_KV=zBcumulative_seqlen_Q and cumulative_seqlen_KV must both be tensors.zocumulative_seqlen_Q and cumulative_seqlen_KV must be on the same device as QKV, but cumulative_seqlen_Q.device=z, cumulative_seqlen_KV.device=z, query.device=zmcumulative_seqlen_Q and cumulative_seqlen_KV must both be torch.int32 tensors, got cumulative_seqlen_Q.dtype=z, cumulative_seqlen_KV.dtype=zecumulative_seqlen_Q and cumulative_seqlen_KV must both be 1-D tensors, got cumulative_seqlen_Q.dim()=z, cumulative_seqlen_KV.dim()=z_cumulative_seqlen_Q and cumulative_seqlen_KV must match in size, got cumulative_seqlen_Q.shape=z, cumulative_seqlen_KV.shape=r'zmcumulative_seqlen_Q and cumulative_seqlen_KV must contain at least 2 elements, got cumulative_seqlen_Q.shape=)r)rallrany isinstanceinttyperrrtorchint32r() r r r r2r3r4r5r6r7Ztotal_seqlen_QZtotal_seqlen_KVrrrvarlen_tensor_checkss( (       rF is_causal causal_typecCs|r|dus t|tstd|d||kr)|kr)dks,JJ|jd|jdks8J|rZ|tjkr\|jd|jdkr^td|jdd|jdddSdSdS)NznArgument causal_type must be specified as an enum instance of CausalType when is_causal=True, got causal_type=rr.r/zWCausal mask type DontCare is only valid when seqlen_q == seqlen_kv, got query.shape[1]=z, key.shape[1]=)rArrr(r)DontCare)r r r rGrHrrrattention_param_checksOs 2" rJc  Cs|pd}t||||dsdSt|||||||||d sdStt|d} |dvr7| d|dtddS|d } |jd d | } |jd d | } |jd d | } | | ks`| | krr| d | d | d | dtddSdS)NzMulti-Dimensional AttentionrFr-r.zYMulti-Dimensional Attention supports 4-D, 5-D, or 6-D tensors as inputs, got query.dim()=rrr/zlQ, K and V must match in their token layout shapes in multi-dimensional attention, got q_token_layout_shape=z, k_token_layout_shape=z, v_token_layout_shape=Tr0)r r r rrrrr r rnum_dimsZq_token_layout_shapeZk_token_layout_shapeZv_token_layout_shaperrr!multi_dim_attention_tensor_checksdsN    rPparamrOtypenamecsXtrtfddt|DSttr*t|kr*tfddDr*SdS)Nc3s|]}VqdSr9r)r;_)rQrrr=z/check_valid_tuple_or_element..c3s|]}t|VqdSr9)rAr:)rRrrr=s)rAtuplerangerlenr?)rQrOrRr)rQrRrcheck_valid_tuple_or_elements ,rXr%r/F window_sizestridedilationc Cs |dvsJ|d}tdd|jdd|D}t||t}|dur3td|d|d t||t}|durHtd |d |d t||t} | dur]td |d |d t||t} | durrtd|d|d tddt||D}|||| | fS)zF Converts all multi-dimensional parameters to standard types. rKrNcs|]}|VqdSr9rr;srrrr=rTz3multi_dim_attention_param_filter..r/Nz:Parameter 'window_size' must be either an int or tuple of z ints, got window_size=rz5Parameter 'stride' must be either an int or tuple of z ints, got stride=z7Parameter 'dilation' must be either an int or tuple of z ints, got dilation=z;Parameter 'is_causal' must be either a boolean or tuple of z booleans, got is_causal=css$|] \}}|dkr |n|VqdS)r%Nr)r;r<wrrrr=s")r(rUr)rXrBrboolzip) r rYrZr[rGrOtoken_layout_shapeZ window_size_stride_ dilation_Z is_causal_rrr multi_dim_attention_param_filters*      rec Cs<|dvsJ|d}tdd|jdd|D}tdd|Dr3td|d|jd td d|DrDtd |d td dt|||Drctd|d|d|d|jd tdd|Drttd|d tddt||Drtd|d|d tdd|Drtd|d dS)z1 Validates multi-dimensional parameters. rKrNcsr\r9rr]rrrr=rTz3multi_dim_attention_param_checks..r/cs|]}|dkVqdSr/Nrr:rrrr=zAToken layout dimensions must all be >= 2, got token_layout_shape=z (query.shape=z).csrfrgr)r;r_rrrr=rhzgParameter 'window_size' must be either -1 (no sparsity) or >= 2 along every dimension, got window_size=rcss"|] \}}}|||kVqdSr9r)r;r<r_drrrr=s zsThe product of 'window_size' and 'dilation' cannot be greater than the input (token layout shape), got window_size=z , dilation=z, token_layout_shape=cs|]}|dkVqdSrgrr]rrrr=rhz=Parameter 'stride' allows positive integers only, got stride=css|] \}}||kVqdSr9r)r;r_r^rrrr=sz[Parameter 'stride' cannot be greater than window size along any dimension, got window_size=z , stride=csrjrgr)r;rirrrr=rhzAParameter 'dilation' allows positive integers only, got dilation=N)r(rUr)r@rra)r rYrZr[rGrOrbrrr multi_dim_attention_param_checkss@  rk)T)NNTTTN)NNNNNN)r%r/r/F)__doc__collections.abcr functoolsrtypingrrDr-cosmos_policy._src.imaginaire.attention.masksr-cosmos_policy._src.imaginaire.attention.utilsrZ.cosmos_policy._src.imaginaire.attention.varlenrr`rlistrstrr,r1rBrUrFrJrPrCrXrerkrrrrs>             `    3       6  )