qi=UdZddlZddlmZmZddlZddlmZddl Z ddl m Z ddl Z ddlmcmZddlmcmZedZdZejGddejZejGdd ejZejGd d ejZejGd d ejZejGddejZeejej dfej!ej dfej!ej dffZ"ee#d<ejGddejZdddZ$dS)z Gemma model implementation from big_vision/models/ppp/gemma.py (with small modifications for NNX compatibility) Used for FAST autoregressive policies. N)Literal TypeAlias)gemma_2b gemma_2b_lorac|dkr tj|dddddddd d d d S|d krLtj|dddddddd d d tjddtjdddd St d|)z+Returns config for specified gemma variant.rii@ư>iTnothing_saveable) variantwidthdepthmlp_dim num_heads num_kv_headshead_dimnorm_eps vocab_sizescan remat_policyrg0@)rankalpha)attnffn) rrrrrrrrrrr lora_configszUnknown variant: )ml_collections ConfigDictlora LoRAConfig ValueError)rs 9/home/robot-lab/Pi0.5_yam/src/openpi/models/gemma_fast.py get_configr%#s*("! ! % 2     /!!("! ! % 2 O4@@@?$???!!     & 222 3 33cJeZdZUeedfed<ejdZdS)Einsum.shapec|j}|dtj|j|}tj|||S)Nw) dtypeparamnn initializers zeros_initr)astypejnpeinsum)selfeqnxr,r+s r$__call__zEinsum.__call__PsP JJsBO6688$* E E L LU S Sz#q!$$$r&N) __name__ __module__ __qualname__tupleint__annotations__r.compactr7r&r$r(r(LsE c?Z%%Z%%%r&r(c.eZdZejdZdS)RMSNormc |j}|dtj|jd}t jt j| t j dd}t j |t j t j |dzz}|d|zz}| |S)NscaleT)axiskeepdimsr r )r,r-r.r/r0r)r2meansquarer1float32asarray reciprocalsqrt)r4r6r,rCvar normed_inputss r$r7zRMSNorm.__call__Ys 7BO$>$>$@$@172;PPhsz!((3;"7"788rDQQQ Asxe 7L7L(M(M$MNN % I ##E***r&N)r8r9r:r.r>r7r?r&r$rArAWs/Z++Z+++r&rAc:eZdZUdZeed<eed<dZdZdZdS)EmbedderzEmbedder module.r embed_dimc|dtj|j|jf|_dS)Ninput_embedding)r-r.r/r0rrQinput_embedding_tabler4s r$setupzEmbedder.setupls>%)ZZ  O & & ( ( _dn -& & """r&c|j|f}|tj|j|jz}|SN)rTr2rLrQr1r,r4r6s r$encodezEmbedder.encodess<  &t , SXdn % % , ,QW 5 55r&c@tj||jjSrX)r2dotrTTrYs r$decodezEmbedder.decodexswq$46777r&N) r8r9r:__doc__r<r=rVrZr^r?r&r$rPrPes[OOONNN    88888r&rPceZdZUdZeed<eed<eed<eed<dZedzed<dZe j dzed<d Z d Z d Z ejdd ZdS) AttentionzAttention module.rrfeaturesrN cache_dtype lora_configc |j|jkrVtjd|j|j|jfdt jddd|j |_ ntj|j|j|jfdt jddd |j |_ tjd |j|j|jfd t jddd|j |_ tj|j|j|jfd t jddd |j |_ dS) N qkv_einsumrD)rr )in_axisout_axis batch_axis)r)nameinit_fnrdq_einsum)r kv_einsumattn_vec_einsum)rrr!r(rbrr.r/ lecun_normalrdrgrnrprqrUs r$rVzAttention.setupsV   . ."k$.$-G!44R"Y_4`` , DOO!K~t}dmD44R"Y]4^^ , DM "[$+T]DMJ 44R"Y_4`` , DN ${>4=$-@"O00bUY0ZZ(    r&cV|jd}dd||z fddf}|jp|j}tj|||}tj|||}tj|jdftj|z} | ||fS)zInitialize KV cacher )rrrr,)r)rcr,r2padr1zerosint32) r4kv cache_size prefill_len pad_widthrck_cachev_cacheidxs r$ _init_cachezAttention._init_cachesgaj ak!9:FFK &1!' '!((;//;;'!((;//;;i SY777+EGW$$r&cP|jddks Jdd|dddf}|jp|j}tj||||}tj||||} |dz} | || fS)zUpdate KV cache with new valuesr z)Only support kv-cache updates of length 1r)r)rcr,jaxlaxdynamic_update_slicer1) r4rxryrr}r~indicesrck_newv_newidx_news r$ _update_cachezAttention._update_cacheswqzQ Kc!fa#&1!' ,,Wahh{6K6KWUU,,Wahh{6K6KWUU'u$$r&Tc :|j}|j|jkr|d|\}} } n/|d|}|d|\} } t ||}||jdzz}t | |} |'|| | |j d\} } } n#|\} } } | | | | | | \} } } | | } } | | | f}tj |d|j}tjd || tj }|j |j d d |j d | j d fkr't!d |j d|j d| j d}tj|dddddddddf||}t$j|d|}tjd|| }tj |d}|d||fS)NzBSD,3KDH->3BSKHz BTD,NDH->BTNHzBSD,2KDH->2BSKH) positionsgrDzB T (K G) H -> B T K G H)KzBTKGH,BSKH->BKGTS)preferred_element_typerr zAttention mask with shape z but shapes for q and k are: z and g BTKGHzB T K G H -> B T (K G) Hz BTNH,NHD->BTD)r,rrrgrnrp _apply_roperrr)reinops rearranger2r3rIr#whererr.softmaxr1rq)r4r6r attn_maskkv_cacher^ deterministicr,qrxryrr}r~logitsbig_neg masked_logitsprobsencodeds r$r7zAttention.__call__s;   . .oo&7;;GAq!! oq11A>>"3Q77DAq Y / / / T]D   Y / / /  $($4$4Q9?2;N$O$O !C''$, !C'$($6$6q!S'7$S$S !C'1'*  Q :d>O P P P/AckZZZ ?qwqz1agaj!'!*E E ErY_rr[\[brrijiprr    )AAAqqq$111,<"=vwOO }266==eDD*0%;;"7,FGG##OW==xGGr&T)r8r9r:r_r<r=rcstrrdr!r"rVrrr.r>r7r?r&r$rara|sNNNMMMMMM"Kt"""*.K4'...   8%%%%%%Z&H&H&HZ&H&H&Hr&raceZdZUdZeed<eed<eed<eed<eed<dZeed<d Ze ed fed <d Z e d zed <e j ejZejed<dZddZd S)BlockzTransformer block.rrrQr hidden_dimdropoutr?. dropout_bdimsNrcdefault_factoryrc t|_t|j|j|j|j|j|j d|_ t|_ tj |j|jd|j d|_|jr&t#j|j|j|_dSd|_dS)Nr)rrrbrrcrdmlpr)rbrrlrdc|SrXr?)r6_s r$zBlock.setup..sQr&)rApre_attention_normrarrrQrrcrgetr pre_ffw_normr! FeedForwardrrrr.DropoutrdroprUs r$rVz Block.setups")))n*^]()--f55     $II#^eY]YjYnYnotYuYu    < ' 4<1CDDDIII&DIIIr&Tc\tj|d}||}|||||||\}}|||}||z }|} ||}||} || |} | | z} | |fS)N) act_batchact_lenact_emb)r.with_logical_constraintrrrrr) r4r6rrrr^rinputs_normalized attn_outputresidualoutputss r$r7zBlock.__call__s  &q*M N N 33A66 $ *;Y S[]cer s s Xii ];; q '' 44 ((;''))G]33W$  r&r)r8r9r:r_r<r=rfloatrr;rcr dataclassesfieldrr rrVr7r?r&r$rrsNNNNNNMMMOOOGU%'M5c?'''"Kt""".?k.?P^Pi.j.j.jL.+jjj'''& ! ! ! ! ! !r&rz bz b _t _k _hz b _t _v _hKVCachecPeZdZUdZeed<eed<eed<eed<eed<eed<eed<eed <eed <eed <d Zeed <dZ e edfed<dZ edzed<dZ e ed<dZeed<ejejZejed<ej ddZdZdS)Modulez gemma model.rrrrrrrrr embed_dtyperrr?.rNrcFrnonerrrTc i} t|j|jd} |%|x} | d<| | x}| d<|| fSg} || ||(| | |t j| d} | |j } | j \}}}|r| S|r|| Jd|;t j |t j dddf}|j d | j d ksJ|j | j f|3tjt j||g}|jd kr|dddddddf}t%||j d }|j |d ||fksJ|j |jd krt(}n@tjt(|j d t/t0j|j}|j|j|j||j|j|j|j |j!d }|j"#d}tj|ddiddddtj$tj$tj$tj$f|j%dd|i|g}|D]}|| ||||| \} }| j&t j&|j ksJ| | d<tOd| } | | d<| r| || fS| | } | | d<| || fS)aEmbed only, or complete forward pass. Args: tokens: Embedded, then and appended to `embedded_prefix`. Can be None. embedded_prefix: Optional prefix that is already embedded. embed_only: Whether to compute embeddings only. pre_logits: If present computes logits from pre_logits and returns. positions: Optional `[B, T]` allows to specify the absolute position of the tokens. mask: Optional attention mask `[B, T, S]`. decode: Whether to use kv-cache. Caller must pass masks and positions. deterministic: Forwarded to all dropout layers. return_prelogits: Whether to return the pre-logits. Returns: If `embed_only=False`, then `(logits, out)` will be returned. If `embed_only=True`, then the embeddings will be returned. If `return_prelogits=True`, then the pre-logits will be returned. embedder)rrQrlN pre_logitsrrhrz5Must explicitly pass positions and mask for decoding.r rfrDr)) prevent_csestatic_argnumspolicy) rrrrQrrrrcrlayersparamsrT)rr) variable_axes split_rngsin_axeslengthparentr final_norm)rlr?)(rPrrr^appendrZr2 concatenater1rr)arangerwr. attentionmake_causal_maskonesndimmaxrrrematrgetattrrcheckpoint_policiesrrrrrrrcrscopepush broadcastrr,rA)r4tokensembedded_prefix embed_onlyrrmaskr^rrreturn_prelogitsoutrr6r batch_sizeseq_lenrrz block_clsblock_kwrblocksblocks r$r7zModule.__call__.sBt$*S]^^^  !$. .AL!%-__Q%7%7 7FS]3;    & HH_ % % %   HHX__V,, - - - OAB ' ' ' HHT% & &%&W" GU  H  (T-=-=G.>-==   7++2239==dAAAgFIq!QWQZ///)/171K/// <<00:w:O1P1PQQD 9>>4AAA &D$*R.11 zj!WjAAAA4:AAA   & &II $ M%s68IJJ I  -,|!/+ -   ** BG'm&*t<<BL", blSz     ) )   ) (  ) )  U UE%8YfmTTKAxxw#)D$4555555I &G & & &q ) )L  $h# # OOA  H (Cr&cX|tjdtjdS)zYConvenience method for initializing all parameters, necessary due to the quirks of linen.)r r rtN)r2rvrwrUs r$initz Module.inits) SYvSY / / /00000r&) NNFNNNFNTF)r8r9r:r_rr=r<rrrr;rcrboolrrrrr rr.r>r7rr?r&r$rrsS LLL JJJ JJJ LLLNNNMMMOOOOOOGU%'M5c?'''"Kt"""D$L#.?k.?P^Pi.j.j.jL.+jjjZ s s s Zs j11111r&ri')max_wavelengthcd|jdz tj|jddztjz}||z}|d|ddddfz }|ddddf}|jtjksJtj|tj|}}tj|dd\}} tj||z| |zz | |z||zzgd} | jtjksJ| S) z0Applies RoPE positions [B, L] to x [B, L, H, D].g@rDrort).NN.r) r)r2rrIr,sincossplitr) r6rrfreq_exponents timescaleradiansrrx1x2ress r$rrsAGBK'3:agbkQ6Fck+Z+Z+ZZN.I "YtT111}%==Gc4l#G =CK ' ' ' 'ww!1!1C Yq!" % % %FB /28b3h.S280CD2 N N NC 9 # # # # Jr&)%r_rtypingrrr flax.linenlinenr.r jax.numpynumpyr2ropenpi.models.loramodelsr!openpi.shared.array_typingshared array_typingatVariantr% typecheckrr(rArPrarr;IntArrayFloatrr=rr?r&r$r s %%%%%%%%  !!!!!!!!!''''''''' - .&4&4&4R%%%%%RY%%% + + + + +bi + + +88888ry888,cHcHcHcHcH cHcHcHL,!,!,!,!,!BI,!,!,!^26"(D.128BHl