
    qi                    .   d dl mZ d dlZd dlmZ d dlZd dlZd dlmZ d dl	m
Z
 d dlZd dlmZ d dlmc mZ d dlmZ d dlmc mZ d dlmc mZ d dlmc mZ d0dZd1dZ 	 d2d3dZ!d4dZ" G d d e          Z# G d! d"ej$                  Z%ej&        '                    e%d#$          ej(         G d% d&ej&        j)                                          Z*ej&        '                    e%d#'           G d( d)ej&        j)                              Z+d5d+Z,d6d/Z-dS )7    )annotationsN)Protocol)epath)array_typingcheckpoint_direpath.Path | strkeep_period
int | None	overwriteboolresumereturn"tuple[ocp.CheckpointManager, bool]c                  t          j        |                                           } d}|                                 r]|rC|                                  |                     dd           t          j        d|             n|rd}nt          d|  d          |                     dd           t          j
        | t                      t          j                    t          j                    dt          j        d|dt          j        d	
                              }|r9t          |                                          dv rt          j        d           d}||fS )NFT)parentsexist_okzWiped checkpoint directory zCheckpoint directory zJ already exists. Use --overwrite or --resume to indicate how to handle it.assetstrain_stateparams   i   )timeout_secs)max_to_keepr	   createasync_options)item_handlersoptions) )r   zSCheckpoint directory exists, but does not contain any checkpoints. Aborting resume.)r   PathresolveexistsrmtreemkdirlogginginfoFileExistsErrorocpCheckpointManagerCallbackHandlerPyTreeCheckpointHandlerCheckpointManagerOptionsAsyncOptionstuple	all_steps)r   r	   r   r   resumingmngrs         </home/robot-lab/Pi0.5_yam/src/openpi/training/checkpoints.pyinitialize_checkpoint_dirr2      s    Z//7799NH  
	!!###   ===LG~GGHHHH 	HH!0 0 0 0  
 555 %''688133
 

 ,#*===	
 
 
  D$  E$..**++z99jkkk>    checkpoint_managerocp.CheckpointManagerstatetraining_utils.TrainStatedata_loader_data_loader.DataLoaderstepintc                    dfd}t          j                    5  t          |          \  }}d d d            n# 1 swxY w Y   ||d|id}|                     ||           d S )N	directory
epath.Pathc                                                     }|j        }|&|j        !t          j        | |j        z  |           d S d S d S N)data_config
norm_statsasset_id
_normalizesave)r=   rA   rB   r8   s      r1   save_assetszsave_state.<locals>.save_assetsG   sX    !--// +
!k&:&FOI(<<jIIIII "!&F&Fr3   r   r   )r=   r>   )atdisable_typechecking_split_paramsrE   )r4   r6   r8   r:   rF   r   r   itemss     `     r1   
save_staterK   A   s    J J J J J J 
	 	"	" 3 3+E22V3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 "V$ E
 D%(((((s   :>>c                    ~t          j                    5  t          |          \  }}|                     ||d|id          }d d d            n# 1 swxY w Y   t	          |d         |d                   S )Nr   r   r   )rJ   r   )rG   rH   rI   restore_merge_params)r4   r6   r8   r:   r   r   restoreds          r1   restore_staterQ   Y   s     			 	"	" 	
 	
+E22V%--*#V,  . 
 
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 -0(82DEEEs   /AAA
assets_dirrC   str&dict[str, _normalize.NormStats] | Nonec                    t          j        |           |z  }t          j        |          }t	          j        d|            |S )NzLoaded norm stats from )r   r   rD   loadr$   r%   )rR   rC   norm_stats_dirrB   s       r1   load_norm_statsrX   n   sD    Z
++h6N00JL;>;;<<<r3   c                      e Zd ZddZdS )Callbackr=   r>   r   Nonec                    d S r@   r   )selfr=   s     r1   __call__zCallback.__call__v   s      r3   N)r=   r>   r   r[   )__name__
__module____qualname__r^   r   r3   r1   rZ   rZ   u   s        ::::::r3   rZ   c                  (    e Zd ZdZddZdd	Zd
 ZdS )r)   ziA CheckpointHandler for calling an arbitrary function asynchronously. Only for saving, not for restoring.r=   r>   argsCallbackSavec                b    t          j                    dk    r|                    |           d S d S )Nr   )jaxprocess_indexcallbackr]   r=   rc   s      r1   rE   zCallbackHandler.save|   s6    !##MM)$$$$$ $#r3   r   list[futures.Future]c                b   K   t          j        t          j        | j        ||                    gS r@   )future%CommitFutureAwaitingContractedSignalsasyncio	to_threadrE   ri   s      r1   
async_savezCallbackHandler.async_save   s-      <W=NtyZcei=j=jkkllr3   c                     t          d          )Nz(CallbackHandler does not support restore)NotImplementedError)r]   rc   kwargss      r1   rN   zCallbackHandler.restore   s    !"LMMMr3   N)r=   r>   rc   rd   )r=   r>   rc   rd   r   rj   )r_   r`   ra   __doc__rE   rp   rN   r   r3   r1   r)   r)   y   sZ        ss% % % %m m m mN N N N Nr3   r)   T)for_savec                      e Zd ZU ded<   dS )rd   rZ   rh   N)r_   r`   ra   __annotations__r   r3   r1   rd   rd      s           r3   rd   )for_restorec                      e Zd ZdS )CallbackRestoreN)r_   r`   ra   r   r3   r1   rz   rz      s        03r3   rz   +tuple[training_utils.TrainState, at.Params]c                    | j         | j         }t          j        | d           }n| j        }t          j        | i           }||fS )N
ema_paramsr   )r~   dataclassesreplacer   )r6   r   r   s      r1   rI   rI      sQ    #!!)%DAAA!)%;;;r3   r   r   dict[str, at.Params]c                    | j         rt          j        | |d                   S t          j        | |d                   S )Nr   r}   r   )r   r   r   rM   s     r1   rO   rO      sD     M";6(;KLLLL{6(3CDDDDr3   )
r   r   r	   r
   r   r   r   r   r   r   )r4   r5   r6   r7   r8   r9   r:   r;   r@   )
r4   r5   r6   r7   r8   r9   r:   r
   r   r7   )rR   r   rC   rS   r   rT   )r6   r7   r   r{   )r   r7   r   r   r   r7   ).
__future__r   rn   concurrent.futuresfuturesr   r$   typingr   etilsr   rf   orbax.checkpoint
checkpointr'   orbax.checkpoint.futurerl   openpi.sharedr   rG   openpi.shared.normalizeshared	normalizerD   openpi.training.data_loadertrainingr8   _data_loaderopenpi.training.utilsutilstraining_utilsr2   rK   rQ   rX   rZ   AsyncCheckpointHandlerr)   rc   register_with_handler	dataclassCheckpointArgsrd   rz   rI   rO   r   r3   r1   <module>r      s   " " " " " "  $ $ $ $ $ $                  



       ( ( ( ( ( ( ( ( ( , , , , , , , , , , , , , , , 2 2 2 2 2 2 2 2 2 . . . . . . . . .* * * *Z) ) ) )8 	F F F F F*   ; ; ; ; ;x ; ; ;N N N N Nc0 N N N $??    38*    @? TBB 3 3 3 3 3ch- 3 3 CB 3   E E E E E Er3   