
    _hl(                        d dl Z d dlZd dlmZ d dlmc mZ d dlmZ d dl	Z	ddl
m
Z
 d Zd Z G d dej                  Z G d	 d
ej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  ZdS )    N)Optional   )
checkpointc                     t           j                            | j        |           | j        't           j                            | j        d           d S d S )N)stdg        )nninitnormal_weightbias	constant_)lstddevs     O/tmp/cameron/UniRig/src/model/michelangelo/models/modules/transformer_blocks.pyinit_linearr       sL    GOOAH&O)))v
!&#&&&&&     c                 `   t           j        j                            ddd          5  |                     dd          } |                    dd          }|                    dd          }t          j        | ||          }|                    dd          }d d d            n# 1 swxY w Y   |S )NTF)enable_flashenable_mathenable_mem_efficientr      )torchbackendscuda
sdp_kernel	transposeFscaled_dot_product_attention)qkvouts       r   flash_attentionr#   %   s    			'	'Ttbg	'	h	h " "KK1KK1KK1,Q155mmAq!!" " " " " " " " " " " " " " " Js   A/B##B'*B'c                   `     e Zd Zdddej        dej        dedededed	ed
ef fdZ	d Z
 xZS )MultiheadAttentionFflashdevicedtypen_ctxwidthheads
init_scaleqkv_biasr'   c                t   t                                                       || _        || _        || _        t          j        ||dz  |||          | _        t          j        ||||          | _        t          |||||          | _
        t          | j        |           t          | j        |           d S )N   r   r(   r)   r(   r)   )r(   r)   r,   r*   r'   )super__init__r*   r+   r,   r   Linearc_qkvc_projQKVMultiheadAttention	attentionr   )
selfr(   r)   r*   r+   r,   r-   r.   r'   	__class__s
            r   r4   zMultiheadAttention.__init__1   s     	


YueaihvUZ[[[
iuV5III.fEQV^ckpqqqDJ
+++DK,,,,,r   c                     |                      |          }t          | j        |fdd          }|                     |          }|S N F)r6   r   r9   r7   r:   xs     r   forwardzMultiheadAttention.forwardG   s<    JJqMMt~tR77KKNNr   )__name__
__module____qualname__r   r(   r)   intfloatboolr4   rA   __classcell__r;   s   @r   r%   r%   0   s         - - - - {	-
 - - - - - - - - - - -,      r   r%   c                   T     e Zd Zdddej        dej        dededef
 fdZd	 Z	 xZ
S )
r8   Fr&   r(   r)   r,   r*   r'   c                    t                                                       || _        || _        || _        || _        || _        d S N)r3   r4   r(   r)   r,   r*   r'   )r:   r(   r)   r,   r*   r'   r;   s         r   r4   zQKVMultiheadAttention.__init__O   s@    





r   c                    |j         \  }}}|| j        z  dz  }dt          j        t          j        |                    z  }|                    ||| j        d          }t          j        ||d          \  }}}	| j        r?t          |||	          }
|
	                    |
j         d         |
j         d         d          }
nt          j
        d||z  ||z            }|j        }t          j        |                                d                              |          }t          j
        d||	          	                    ||d          }
|
S )Nr0   r   dimr   bthc,bshc->bhtsbhts,bshc->bthcshaper,   mathsqrtviewr   splitr'   r#   reshapeeinsumr)   softmaxrF   type)r:   qkvbsr*   r+   attn_chscaler   r    r!   r"   r   wdtypes                r   rA   zQKVMultiheadAttention.forwardW   s2   9E54:%*DIdi00111hhr5$*b11+c73331a: 		T!!Q**C++cilCIaL"==CC\!1u9a%i F \F]6<<>>r:::??GGF,0&!<<DDRPRSSC
r   )rB   rC   rD   r   r(   r)   rE   rG   r4   rA   rH   rI   s   @r   r8   r8   N   s        jo   %, u{ 3 WZ cg            r   r8   c                        e Zd Zddddddej        dej        deded	ed
edededef fdZ	dej
        fdZdej
        fdZ xZS )ResidualAttentionBlockg      ?TFr-   r.   r'   use_checkpointr(   r)   r*   r+   r,   r-   r.   r'   re   c       	   
      .   t                                                       |	| _        t          ||||||||          | _        t          j        |||          | _        t          ||||          | _	        t          j        |||          | _
        d S )N)r(   r)   r*   r+   r,   r-   r.   r'   r2   )r(   r)   r+   r-   )r3   r4   re   r%   attnr   	LayerNormln_1MLPmlpln_2)r:   r(   r)   r*   r+   r,   r-   r.   r'   re   r;   s             r   r4   zResidualAttentionBlock.__init__m   s     	,&!	
 	
 	
	 LvUCCC	fE:VVVLvUCCC			r   r@   c                     ||                      |                     |                    z   }||                     |                     |                    z   }|S rL   )rg   ri   rk   rl   r?   s     r   _forwardzResidualAttentionBlock._forward   sG    		$))A,,'''1&&&r   c                 `    t          | j        |f|                                 | j                  S rL   )r   rn   
parametersre   r?   s     r   rA   zResidualAttentionBlock.forward   s'    $-!t/@/@$BUVVVr   )rB   rC   rD   r   r(   r)   rE   rF   rG   r4   Tensorrn   rA   rH   rI   s   @r   rc   rc   l   s         $D D D D {	D
 D D D D D D D D D D D D>%,    
W W W W W W W W Wr   rc   c                        e Zd Zddddddej        dej        deded	ed
edede	e         de	e         f fdZ
d Z xZS )MultiheadCrossAttentionTFN)r.   r'   n_data
data_widthr(   r)   r+   r,   r-   r.   r'   rt   ru   c       	            t                                                       || _        || _        || _        |	|n|	| _        t          j        |||||          | _        t          j        | j        |dz  |||          | _	        t          j        ||||          | _
        t          |||||          | _        t          | j        |           t          | j	        |           t          | j
        |           d S )Nr1   r   r2   )r(   r)   r,   rt   r'   )r3   r4   rt   r+   r,   ru   r   r5   c_qc_kvr7   QKVMultiheadCrossAttentionr9   r   )r:   r(   r)   r+   r,   r-   r.   r'   rt   ru   r;   s             r   r4   z MultiheadCrossAttention.__init__   s     	

#-#5%%:9UEuUUUIdouqyxPV^cddd	iuV5III3eF%
 
 
 	DHj)))DIz***DK,,,,,r   c                     |                      |          }|                     |          }t          | j        ||fdd          }|                     |          }|S r=   )rw   rx   r   r9   r7   r:   r@   datas      r   rA   zMultiheadCrossAttention.forward   sM    HHQKKyyt~4y"e<<KKNNr   )rB   rC   rD   r   r(   r)   rE   rF   rG   r   r4   rA   rH   rI   s   @r   rs   rs      s          $$(- - - - {	-
 - - - - - - SM- - - - - -8      r   rs   c                   b     e Zd Zddddej        dej        dededee         f
 fd	Z	d
 Z
 xZS )ry   FN)r'   rt   r(   r)   r,   r'   rt   c                    t                                                       || _        || _        || _        || _        || _        d S rL   )r3   r4   r(   r)   r,   rt   r'   )r:   r(   r)   r,   r'   rt   r;   s         r   r4   z#QKVMultiheadCrossAttention.__init__   sB     	




r   c                    |j         \  }}}|j         \  }}}|| j        z  dz  }dt          j        t          j        |                    z  }	|                    ||| j        d          }|                    ||| j        d          }t          j        ||d          \  }
}| j        r?t          ||
|          }|	                    |j         d         |j         d         d          }nt          j
        d||	z  |
|	z            }|j        }t          j        |                                d                              |          }t          j
        d||          	                    ||d          }|S )Nr   r   rN   rO   r   rQ   rR   rS   )r:   r   kv_r*   r^   rt   r+   r_   r`   r    r!   r"   r   ra   s                  r   rA   z"QKVMultiheadCrossAttention.forward   sV   g5!HFE4:%*DIdi00111FF2udj"--WWRR00{2wB///1: 		T!!Q**C++cilCIaL"==CC\!1u9a%i F \F]6<<>>r:::??GGF,0&!<<DDRPRSSC
r   )rB   rC   rD   r   r(   r)   rE   rG   r   r4   rA   rH   rI   s   @r   ry   ry      s        $d  %, u{ 3 .6sm           r   ry   c                        e Zd Zddddddddeej                 deej                 d	ee         d
ededee         dedede	de	f fdZ
dej        dej        fdZ xZS )ResidualCrossAttentionBlockN         ?TF)rt   ru   mlp_width_scaler-   r.   r'   r(   r)   rt   r+   r,   ru   r   r-   r.   r'   c       
         d   t                                                       ||}t          ||||||||	|
	  	        | _        t	          j        |||          | _        t	          j        |||          | _        t          |||||          | _	        t	          j        |||          | _
        d S )N)	r(   r)   rt   r+   r,   ru   r-   r.   r'   r2   )r(   r)   r+   hidden_width_scaler-   )r3   r4   rs   rg   r   rh   ri   rl   rj   rk   ln_3)r:   r(   r)   rt   r+   r,   ru   r   r-   r.   r'   r;   s              r   r4   z$ResidualCrossAttentionBlock.__init__   s     	J+!!

 

 

	 LvUCCC	LF%HHH	fESboyzzzLvUCCC			r   r@   r|   c                     ||                      |                     |          |                     |                    z   }||                     |                     |                    z   }|S rL   )rg   ri   rl   rk   r   r{   s      r   rA   z#ResidualCrossAttentionBlock.forward   sS    		$))A,,		$8881&&&r   rB   rC   rD   r   r   r(   r)   rE   rF   rG   r4   rq   rA   rH   rI   s   @r   r   r      s        !% %)  !D !D !D &!D $	!D
 !D !D !D SM!D !D !D !D !D !D !D !D !D !DF U\        r   r   c                   l     e Zd Zdddeej                 deej                 dededef
 fdZ	d	 Z
 xZS )
rj   r   )r   r(   r)   r+   r   r-   c                Z   t                                                       || _        t          j        |||z  ||          | _        t          j        ||z  |||          | _        t          j                    | _        t          | j        |           t          | j        |           d S )Nr2   )
r3   r4   r+   r   r5   c_fcr7   GELUgelur   )r:   r(   r)   r+   r   r-   r;   s         r   r4   zMLP.__init__  s     	
IeU-?%?V[\\\	i(: :E&X]^^^GII	DIz***DK,,,,,r   c                 x    |                      |                     |                     |                              S rL   )r7   r   r   r?   s     r   rA   zMLP.forward  s*    {{499TYYq\\22333r   )rB   rC   rD   r   r   r(   r)   rE   rF   r4   rA   rH   rI   s   @r   rj   rj     s        
 ,-	- - -!%,/- -- - &)	-
 #- - - - - -4 4 4 4 4 4 4r   rj   c                        e Zd Zddddddeej                 deej                 deded	ed
edede	de	de	f fdZ
dej        fdZ xZS )Transformerr   TFrd   r(   r)   r*   r+   layersr,   r-   r.   r'   re   c       
            	
 t                                                       | _        | _        || _        t          j        	
f	dt          |          D                       | _        d S )Nc                 @   	 g | ]}t          
	 	  	        S ))	r(   r)   r*   r+   r,   r-   r.   r'   re   )rc   ).0r   r(   r)   r'   r,   r-   r*   r.   re   r+   s     r   
<listcomp>z(Transformer.__init__.<locals>.<listcomp>-  sU         '!)%#1
 
 
  r   )	r3   r4   r*   r+   r   r   
ModuleListrange	resblocks)r:   r(   r)   r*   r+   r   r,   r-   r.   r'   re   r;   s    ```` `````r   r4   zTransformer.__init__  s     	

            v  
 
r   r@   c                 0    | j         D ]} ||          }|S rL   )r   )r:   r@   blocks      r   rA   zTransformer.forward=  s'    ^ 	 	EaAAr   r   rI   s   @r   r   r     s         !$!
 !
 !
 &!
 $	!

 !
 !
 !
 !
 !
 !
 !
 !
 !
 !
 !
 !
 !
F        r   r   )rU   r   torch.nnr   torch.nn.functional
functionalr   typingr   osr   r   r#   Moduler%   r8   rc   rs   ry   r   rj   r   r>   r   r   <module>r      s  .                        				 " " " " " "' ' '
	 	 	       <    BI   <&W &W &W &W &WRY &W &W &WR" " " " "bi " " "J       D' ' ' ' '") ' ' 'T4 4 4 4 4") 4 4 4&' ' ' ' '") ' ' ' ' 'r   