
    _hd                         d Z ddlZddlmZmZmZmZ ddlmZ 	 ddedeej	        eej	                 f         f         deej	                 d	eej	                 d
e
de
f
dZ G d dej        j                  ZdS )z
Adapted from: https://github.com/openai/guided-diffusion/blob/22e0df8183507e13a7813f8d38d51b072ca1e67c/guided_diffusion/nn.py#L124
    N)CallableIterableSequenceUnion)versionFfunc.inputsparamsflaguse_deepspeedc                     |rX|rddl } |j        j        | g|R  S t          |          t          |          z   }t	          j        | t          |          g|R  S  | | S )a  
    Evaluate a function without caching intermediate activations, allowing for
    reduced memory at the expense of extra compute in the backward pass.
    :param func: the function to evaluate.
    :param inputs: the argument sequence to pass to `func`.
    :param params: a sequence of parameters `func` depends on but does not
                   explicitly take as arguments.
    :param flag: if False, disable gradient checkpointing.
    :param use_deepspeed: if True, use deepspeed
    r   N)	deepspeedcheckpointing
checkpointtupleCheckpointFunctionapplylen)r   r	   r
   r   r   r   argss          G/tmp/cameron/UniRig/src/model/michelangelo/models/modules/checkpoint.pyr   r      s    "   	E59*5dDVDDDDV}}uV}},!'c&kkADAAAAtV}    c                       e Zd Zed             Zed             Ze e            d                         Ze e            d                         ZdS )r   c                      t          j        t          j                  t          j        d          k    r t          j                            d          S t          j        j                                        S )N2.5.0cudadevice_type)r   parsetorch__version__amp
custom_fwdr    r   r   _get_fwd_decoratorz%CheckpointFunction._get_fwd_decorator<   sS    =*++w}W/E/EEE9''F';;;:>,,...r   c                      t          j        t          j                  t          j        d          k    r t          j                            d          S d } | S )Nr   r   r   c                 L    t           j        j                            |           S )Nbwd)r   r   r!   
custom_bwdr'   s    r   r)   z9CheckpointFunction._get_bwd_decorator.<locals>.custom_bwdH   s    z~00S0999r   )r   r   r   r    r!   r)   )r)   s    r   _get_bwd_decoratorz%CheckpointFunction._get_bwd_decoratorC   sR    =*++w}W/E/EEE9''F';;;: : :r   c                     || _         t          |d |                   | _        t          ||d                    | _        t	          j                    5   | j         | j         }d d d            n# 1 swxY w Y   |S )N)run_functionlistinput_tensorsinput_paramsr   no_grad)ctxr,   lengthr   output_tensorss        r   forwardzCheckpointFunction.forwardL   s     ( gvg//VWW..]__ 	B 	B-S-s/@AN	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	Bs   A//A36A3c                 .   d | j         D             | _         t          j                    5  d | j         D             } | j        | }d d d            n# 1 swxY w Y   t          j                            || j         | j        z   |d          }| ` | `~d|z   S )Nc                 \    g | ])}|                                                     d           *S )T)detachrequires_grad_.0xs     r   
<listcomp>z/CheckpointFunction.backward.<locals>.<listcomp>Z   s.    XXXQXXZZ66t<<XXXr   c                 8    g | ]}|                     |          S r#   )view_asr9   s     r   r<   z/CheckpointFunction.backward.<locals>.<listcomp>_   s"    FFFqaiillFFFr   T)allow_unused)NN)r.   r   enable_gradr,   autogradgradr/   )r1   output_gradsshallow_copiesr3   input_gradss        r   backwardzCheckpointFunction.backwardW   s     YXcFWXXX   	? 	? GFC4EFFFN-S-~>N	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? n)) 00	 * 
 
 k))s   AAAN)__name__
__module____qualname__staticmethodr$   r*   r4   rF   r#   r   r   r   r   ;   s        / / \/   \    \ * *  \* * *r   r   )F)__doc__r   typingr   r   r   r   	packagingr   Tensorboolr   rA   Functionr   r#   r   r   <module>rQ      s   ,   6 6 6 6 6 6 6 6 6 6 6 6         
3elHU\,BBCC
DU\" U\" 	
    8/* /* /* /* /*0 /* /* /* /* /*r   