o
    #i%                     @   s0  d dl mZ d dlmZ d dlmZmZmZmZ d dl	Z	d dl
mZ d dlmZ d dlm  mZ d dlmZmZmZmZ d dl	mZmZ d dlmZ dd	lmZ d
d Zdd Zdd Zdd Z dedefddZ!d%ddZ"dd Z#dd Z$edd Z%G dd  d ej&Z'G d!d" d"e'Z(G d#d$ d$e'Z)dS )&    )abstractmethod)cache)AnyListOptionalTupleN)pack	rearrangereduceunpack)Tensorint32)autocast   )DiagonalGaussianDistributionc                 C   s   | d uS N )vr   r   :/data/cameron/vidgen/VidTok/vidtok/modules/regularizers.pyexists      r   c                  G   s   | D ]
}t |r|  S qd S r   )r   )argsargr   r   r   default   s
   r   c                 C   s   t | g|S r   )r   )tpatternr   r   r   pack_one   s   r   c                 C   s   t | ||d S )Nr   )r   )r   psr   r   r   r   
unpack_one      r   zreturnc                 C   s   |   }| ||    S )z&Round with straight through gradients.)rounddetach)r    zhatr   r   r   	round_ste#   s   r%   h㈵>c                 C   s   | j |d S )N)min)clamplog)r   epsr   r   r   r)   )   r   r)   c                 C   s   |  t |  jddS )Ndim)r)   sum)probr   r   r   entropy-   s   r0   c                 C   s$   t  s| S t|  | t  } | S r   )is_distributeddist
all_reduceget_world_size)r   r   r   r   maybe_distributed_mean1   s
   
r5   c                   C   s   t  o	t  dkS )Nr   )r2   is_initializedr4   r   r   r   r   r1   9   s   r1   c                       sL   e Zd Z fddZdejdeejef fddZe	de
fddZ  ZS )	AbstractRegularizerc                    s   t    d S r   )super__init__self	__class__r   r   r9   ?   s   zAbstractRegularizer.__init__r    r!   c                 C      t  r   NotImplementedError)r;   r    r   r   r   forwardB   s   zAbstractRegularizer.forwardc                 C   r>   r   r?   r:   r   r   r   get_trainable_parametersE   s   z,AbstractRegularizer.get_trainable_parameters)__name__
__module____qualname__r9   torchr   r   dictrA   r   r   rB   __classcell__r   r   r<   r   r7   >   s
    r7   c                       sR   e Zd Zddef fddZdefddZdd	ejde	eje
f fd
dZ  ZS )DiagonalGaussianRegularizerTsamplec                    s   t    || _d S r   )r8   r9   rJ   )r;   rJ   r<   r   r   r9   K   s   

z$DiagonalGaussianRegularizer.__init__r!   c                 c   s    dE d H  d S )Nr   r   r:   r   r   r   rB   O   s   z4DiagonalGaussianRegularizer.get_trainable_parametersNr    c                 C   sR   t  }t|}| jr| }n| }| }t||jd  }||d< ||fS )Nr   kl_loss)rG   r   rJ   modeklrF   r.   shape)r;   r    n_stepsr)   Z	posteriorrK   r   r   r   rA   R   s   
z#DiagonalGaussianRegularizer.forwardTr   )rC   rD   rE   boolr9   r   rB   rF   r   r   rG   rA   rH   r   r   r<   r   rI   J   s    (rI   c                       s  e Zd Z									d0dee dee dee d	ee d
ededededef fddZde	fddZ
d1dededefddZdedefddZdedefddZdedefdd Zdedefd!d"Zd2d$edefd%d&Zd'd( Zed)d*d3ded,ed-edefd.d/Z  ZS )4FSQRegularizerNr           r         ?levelsr-   keep_num_codebooks_dimscaleentropy_loss_weightentropy_loss_annealing_stepsentropy_loss_annealing_factorcommitment_loss_weightdiversity_gammac                    s  t    tj|td}| jd|dd tjtdg|d d  dtd}| jd	|dd || _|| _|| _	|| _
|	| _|
| _t|}|| _|| }|| _|| _t||dk}|dkr`|s`J || _t|t|| | _| j|k}|r{t| j|nt | _|rt|| jnt | _|| _| j  | _| jt| jdd
}| jd|dd | jdtddd tj d| j | jgtj!d| _"d S )N)dtype_levelsF)
persistentr   r+   r   )r-   r]   _basis)project_outimplicit_codebookzerorS      )#r8   r9   rF   tensorr   register_buffercumprodrW   rX   rY   rZ   r[   r\   lencodebook_dimnum_codebookseffective_codebook_dimr   rV   r-   nnLinearIdentity
project_inra   has_projectionsr^   proditemZcodebook_sizeindices_to_codesarangezeroslongZglobal_codebook_usage)r;   rU   r-   rj   rV   rW   rX   rY   rZ   r[   r\   r^   r`   ri   rk   rp   rb   r<   r   r   r9   a   s:   
$
"zFSQRegularizer.__init__r!   c                 C   s   |   S r   )
parametersr:   r   r   r   rB      r   z'FSQRegularizer.get_trainable_parametersMbP?r    r*   c                 C   sN   | j d d|  d }t| j d dkdd}||  }||  | | S )z&Bound `z`, an array of shape (..., d).r   rd   r   g      ?rS   )r^   rF   whereatanhtanh)r;   r    r*   Zhalf_loffsetshiftr   r   r   bound   s   zFSQRegularizer.boundc                 C   s    t | |}| jd }|| S )z5Quantizes z, returns quantized zhat, same shape as z.rd   )r%   r~   r^   )r;   r    	quantized
half_widthr   r   r   quantize   s   
zFSQRegularizer.quantizezhat_normalizedc                 C   s   | j d }|| | S Nrd   r^   )r;   r   r   r   r   r   _scale_and_shift      
zFSQRegularizer._scale_and_shiftr$   c                 C   s   | j d }|| | S r   r   )r;   r$   r   r   r   r   _scale_and_shift_inverse   r   z'FSQRegularizer._scale_and_shift_inversec                 C   s6   |j d | jks
J | |}|| j jddtS )z.Converts a `code` to an index in the codebook.r+   r,   )rN   ri   r   r`   r.   tor   )r;   r$   r   r   r   codes_to_indices   s   
zFSQRegularizer.codes_to_indicesTindicesc                 C   sh   |j dt| j k}t|d}|| j | j }| |}| jr$t|d}|r+| |}|r2t|d}|S )zInverse of `codes_to_indices`.   z... -> ... 1z... c d -> ... (c d)b ... d -> b d ...)ndimintrV   r	   r`   r^   r   ra   )r;   r   ra   is_img_or_videoZcodes_non_centeredcodesr   r   r   rs      s   




zFSQRegularizer.indices_to_codesc                 C   s4   || j kr| jS | j| j }||| j  || j   S r   )rY   rX   rZ   )r;   rO   startr   r   r   calculate_entropy_loss_weight   s   
z,FSQRegularizer.calculate_entropy_loss_weightFenabled      Y@inv_temperaturerO   c                 C   s  |j dk}|rt|d}t|d\}}|jd | jks)J d| j d|jd  | |}t|d| jd}tjd	d
dw |j	}|
 }|}| |}| |}	| jdks[| jdkrdtd|| j }
|
 | jdd}t|d}t| }t|dd}t|}t| }|| j|  }tj|| dd}| }n	| j } } }}||}W d   n1 sw   Y  t|d}| |}|rt||d}t|d}t|	|d}	| jst|	d}	|| | || j  }|t |	|dfS )z
        einstein notation
        b - batch
        n - sequence (or flattened spatial dimensions)
        d - feature dimension
        c - number of codebook dim
           zb d ... -> b ... dzb * dr+   zexpected dimension of z but found dimension of zb n (c d) -> b n c d)ccudaFr   r   z... i d, j d -> ... i jr,   zb n ... -> (b n) ...z... c d -> c dmeannone)	reductionNzb n c d -> b n (c d)r   zb * cz... 1 -> ...)r   aux_loss)!r   r	   r   rN   r-   ro   rj   rF   r   r]   floatr   r   rX   r[   einsumrb   softmaxr0   r   r
   r5   r\   Fmse_lossr#   rc   typera   r   rV   r   rG   )r;   r    r   rO   r   r   
orig_dtypeZoriginal_inputr   r   distancer/   Zper_sample_probsZper_sample_entropyZavg_probZcodebook_entropyZentropy_aux_lossZcommit_lossoutr   r   r   r   rA      sL   
	
,








zFSQRegularizer.forward)	Nr   NNrS   r   rT   rS   rT   )rx   rP   )r   r   )rC   rD   rE   r   r   r   rQ   r   r9   r   rB   r   r~   r   r   r   r   rs   r   r   rA   rH   r   r   r<   r   rR   _   sN    	
5&rR   )r&   )*abcr   	functoolsr   typingr   r   r   r   rF   torch.distributeddistributedr2   torch.nnrl   Ztorch.nn.functional
functionalr   einopsr   r	   r
   r   r   r   Ztorch.cuda.ampr   distributionsr   r   r   r   r   r%   r)   r0   r5   r1   Moduler7   rI   rR   r   r   r   r   <module>   s0    

