
     h:                     2   d dl Z d dlZd dl mZ d dlZd dlZd dlmZmZ d dlm	Z
 d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlmZ d dlmZmZmZmZmZmZ d d	lmZ d
 Z  G d dejB                        Z"d dl m#Z#  G d dejB                        Z$y)    N)nn)	rearrangerepeat)
functional)deepcopy)cm)tqdm)CallableListOptionalTuple	GeneratorDict)defaultdictc           	      6   d }g }t        t        |       dz
        D ]S  }|j                  t        j                  | |   | |dz                   |j                  t        j
                                U t        j                  |d d  }|j                  |       |S )Nc                     t        |       t        j                  k(  r;t        | d      r.t        j                  j                  | j                  ddd       y y y )Nweightg        relufan_in)anonlinearitymode)typer   Linearhasattrinitkaiming_normal_r   )ms    2/home/cameronsmith/repos/controll3r/mlp_helpers.pyinit_weights_normalz%make_net.<locals>.init_weights_normal   sD    7biiq(#RWW%<%<ahh#\biq%<%r#         )rangelenappendr   r   ReLU
Sequentialapply)dimsr    layersinets        r   make_netr.      s    s F3t9q=! !biiQa!e56bggi ! --
%CII!"Jr!   c                   <     e Zd ZdZ	 ddededdf fdZd	dZ xZS )
PositionalEncodingNoFreqFactorzPositionalEncoding module

    Maps v to positional encoding representation phi(v)

    Arguments:
        i_dim (int): input dimension for v
        N_freqs (int): #frequency to sample (default: 10)
    i_dimN_freqsreturnNc                     t         |           || _        |d|z  |z  z   | _        || _        d| j                  dz
  }}dt        j                  ||| j                        z  | _        y )N   r"   )super__init__r1   out_dimr2   torchlinspace
freq_bands)selfr1   r2   r   b	__class__s        r   r7   z'PositionalEncodingNoFreqFactor.__init__'   sb     	
Gu44$,,"1u~~aDLLAAr!   c                     |r|gng }| j                   D ]4  }||z  }|t        j                  |      t        j                  |      gz  }6 t        j                  |d      S )Nr#   dim)r;   r9   sincoscat)r<   vstart_with_vpefreqfvs         r   forwardz&PositionalEncodingNoFreqFactor.forward6   s[     aSbOO 	1DB599R=%))B-00B	1 yy$$r!   )
   )T)__name__
__module____qualname____doc__intr7   rJ   __classcell__r>   s   @r   r0   r0      s7     BB B 
	B%r!   r0   )einsumc                   (     e Zd Zd fd	ZddZ xZS )
CrossAttn_c           
      |   t         |           ||z  }|dz  | _        || _        || _        t        j                  ||d      | _        t        j                  ||dz  d      | _        t        j                  ||      | _	        t        j                  t        j                  |t        d|z              t        j                         t        j                  t        d|z        |            | _        t        j                  | j                  g      | _        t        j                  | j                  g      | _        y )Ng      F)biasr5      )r6   r7   scaleheadschr   r   to_qto_kvprojr(   rP   GELUout	LayerNormln_1ln_2)r<   r[   rZ   dim_head	inner_dimr>   s        r   r7   zCrossAttn_.__init__@   s    u$	%

IIb)%8	YYr9q=u=
IIi,	=="))BAbD	":BGGIryyQTUVWYUYQZ\^G_`LL$''+	LL$''+	r!   c                    | j                  |      }| j                  |      }| j                  | j                  |      }	| j                  |      j	                  dd      \  }
}t        fd|	|
|f      \  }	}
}t        d|	|
      | j                  z  }|j                  |      }t        d||      }t        |d      }| j                  |      |z   }| j                  | j                  |            |z   }|S )	Nr5   r#   r@   c                      t        | d      S )Nzb n (h d) -> (b h) n dh)r   )tri   s    r   <lambda>z$CrossAttn_.forward.<locals>.<lambda>[   s    	!-E K r!   zb i d, b j d -> b i jzb i j, b j d -> b i dz(b h) n d -> b n (h d)rh   )rb   rZ   r\   r]   chunkmaprS   rY   softmaxr   r^   r`   rc   )r<   xy	attn_maskreturn_headsreturn_attnsoftmax_axisx_lny_lnqkrE   simattnr`   ri   s                  @r   rJ   zCrossAttn_.forwardQ   s    yy|yy|JJIIdOzz$%%aR%01KaQRTUYW1a ,a3djj@{{|{,,dA65;iinq hhtyy~&,
r!   )   @   )NFFr#   )rL   rM   rN   r7   rJ   rQ   rR   s   @r   rU   rU   ?   s    ,"r!   rU   )%r9   torchvisionr   kornia	functoolseinopsr   r   torch.nnr   Fnumpynpsysrandomtimeoscopyr   
matplotlibr   wandbr	   typingr
   r   r   r   r   r   collectionsr   r.   Moduler0   rS   rU    r!   r   <module>r      s`        $ $        C C #
%RYY %@ ) )r!   