
    qi                         d dl Z d dlZd dlmZ d dlmZ d dlmZ	 d dl
mc mZ ej         G d d                      Z G d dej                  Z G d dej                  ZdS )    Nc                       e Zd ZU dZeed<   dZeed<   ej	        
                    d          Zej	        j        ed<   dZeed	<   d
Zeeef         ed<   dZeed<   edefd            ZdS )
LoRAConfigzConfiguration for LoRA.rankg      ?alphag{Gz?)stddevinit_fnFrslora)axesLlabelreturnc                 p    | j         r!| j        t          j        | j                  z  n| j        | j        z  S N)r	   r   mathsqrtr   selfs    3/home/robot-lab/Pi0.5_yam/src/openpi/models/lora.pyscaling_valuezLoRAConfig.scaling_value   s2    48K[tzDIdi0000TZRVR[E[[    N)__name__
__module____qualname____doc__int__annotations__r   floatnninitializersnormalr   Initializerr	   boolr   tupler   strpropertyr    r   r   r   r      s         !! IIIE5+-?+A+A+A+N+NGR_(NNNFD$D%S/$$$E3\u \ \ \ X\ \ \r   r   c                       e Zd ZU dZeedf         ed<   ej        j	        Z
ej        j        ed<   dZedz  ed<   d Zej        defd	            Zded
eeef         fdZdS )EinsumzTEinsum with LoRA support. Can be used as a drop-in replacement for the Gemma Einsum..shaper   Nlora_configc                    |                      d| j        | j                  | _        | j        x}rt          | j                  t          | j                  }}|j        ||j        d         <   |j        ||j        d         <   |                      d|j        |          | _        |                      d|j        |          | _	        d S d S )Nw   r   lora_alora_b)
paramr   r+   r.   r,   listr   r   w_aw_b)r   configshape_ashape_bs       r   setupzEinsum.setup+   s    Ctz::%%6 	E#DJ//dj1A1AWG&,kGFKN#&,kGFKN#zz(FNGDDDHzz(FNGDDDHHH	E 	Er   eqnc                    |j         }t          j        ||| j                            |                    }| j        x}r|                     |          \  }}t          j        ||| j                            |                    }t          j        ||| j                            |                    }|||j	        z  z   }|S r   )
dtypejnpeinsumr.   astyper,   _make_lora_eqnsr4   r5   r   )	r   r:   xr<   resultr6   eqn_aeqn_bloras	            r   __call__zEinsum.__call__6   s    CDFMM%$8$899%%6 	://44LE5:eQ(>(>??D:eT48??5+A+ABBDdV%999Fr   r   c                    d|v rt          d|           t          j        d|          x}st          d|           |                                \  }}| j        J fd| j        j        D             \  }}| j        j        }                    ||          }|                    ||          }	| d| d|	 }
                    ||          }|	 d| d| }|
|fS )Nr   zL already in eqn: z(.*),(.*)->(.*)zUnsupported einsum eqn: c              3   (   K   | ]}|         V  d S r   r(   ).0rA   rhss     r   	<genexpr>z)Einsum._make_lora_eqns.<locals>.<genexpr>K   s'      BBqCFBBBBBBr   ,z->)
ValueErrorrematchgroupsr,   r   r   replace)r   r:   mlhsouta_labelb_labelr   a_rhsa_outrC   b_rhsrD   rJ   s                @r   r@   zEinsum._make_lora_eqnsC   s   #::7#77888X/555 	?===>>>

S#+++BBBBD,<,ABBB &GU++GU++))))%))GU++))5))C))e|r   )r   r   r   r   r%   r   r   r    r!   zerosr   r#   r,   r   r9   compactr&   rF   r@   r(   r   r   r*   r*   !   s         ^^ c?+-?+@GR_(@@@%)Kd")))	E 	E 	E Z
C 
 
 
 Z
3 5c?      r   r*   c                       e Zd ZU dZeed<   eed<   dZedz  ed<   d Ze	j
        d             Zdej        d	ej        d
eej        ej        f         dz  dej        fdZdS )FeedForwardzFeed forward module.features
hidden_dimNr,   c                    |                      dt          j                            ddd          d| j        | j        f          | _        |                      dt          j                            dd          | j        | j        f          | _        d | _        d | _	        | j
        r|                      d	| j
        j        d| j        | j
        j        f          |                      d
| j
        j        d| j
        j        | j        f          f| _        |                      d| j
        j        | j        | j
        j        f          |                      d| j
        j        | j
        j        | j        f          f| _	        d S d S )Ngating_einsumr
   r   )r   )in_axisout_axis
batch_axis   linear)rb   rc   gating_einsum_lora_agating_einsum_lora_blinear_lora_alinear_lora_b)r2   r    r!   lecun_normalr^   r_   w_gatingw_linearw_gating_loraw_linear_lorar,   r   r   r   s    r   r9   zFeedForward.setup`   so   

O((bT(RRt/
 

 

O((b(AA_dm,
 

 "! 	 

143C3KaQUQ^`d`p`uMvww

*D,<,Dq$JZJ_aeapFq "D 

?D,<,DtX\XhXmFnoo

?D,<,DtGWG\^b^kFlmm"D	 	r   c                    |j         }|                     || j        d         | j        d n%| j        d         d         | j        d         d         f          }t	          j        |          }|                     || j        d         | j        d n%| j        d         d         | j        d         d         f          }||z  }|                     || j        | j                  }|j         |k    sJ |S Nr   r/   )r<   _dotrl   rn   r    gelurm   ro   )r   rA   r<   ff_gate
gate_valueff1activationsoutputss           r   rF   zFeedForward.__call__{   s    ))M!&.DDT5G5J15MtOabcOdefOg4h
 

 WW%%
iiM!&.DDT5G5J15MtOabcOdefOg4h
 

 !3&))K8JKK}%%%%r   rA   r.   lora_weightsr   c           	      2   t          j        ||                    |j                            }||S |t          j        t          j        ||d                             |j                            |d                             |j                            z   S rq   )r=   dotr?   r<   )r   rA   r.   ry   bases        r   rr   zFeedForward._dot   sz    wq!((17++,,Kcgcgaa)?)?)H)HII<XY?KaKabcbiKjKjkkkkr   )r   r   r   r   r   r   r,   r   r9   r    r[   rF   atArrayr%   rr   r(   r   r   r]   r]   X   s         MMMOOO%)Kd")))  6 Z  Z(lbh l28 l528AS;TW[;[ l`b`h l l l l l lr   r]   )r   rN   
flax.linenlinenr    flax.structstruct	jax.numpynumpyr=   openpi.shared.array_typingsharedarray_typingr}   	dataclassr   Moduler*   r]   r(   r   r   <module>r      s    				                   ' ' ' ' ' ' ' ' ' \ \ \ \ \ \ \ \*4 4 4 4 4RY 4 4 4n<l <l <l <l <l") <l <l <l <l <lr   