o
    ?߱i~                     @   s   d dl Z d dlmZmZ d dlmZ d dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ G d	d
 d
eZdS )    N)ABCabstractmethod)Optional)CheckpointConfig	JobConfig)INTERNAL)ImaginaireModel)callback)easy_ioc                   @   s  e Zd ZdZ	d$dededeej fddZ	e
ded	ejjd
ejjjdejjdeddfddZe
			d%ded	eejj d
eejjj deejj def
ddZedd Zedd Zedd Zedd Zd&ddZdedB fddZdeddfdd Zd!eddfd"d#ZdS )'AbstractCheckpointerz^The checkpointer class. Supports checkpoint saving/loading to both local disk or object store.Nconfig_checkpoint
config_job	callbacksc                 C   s:  || _ || _|jj| _|jj| _tj|jd| _	tj|jd| _
|j| _|jp*d}ts9ddlm} |r9||}|| _|j| _|j| _d| _|j| _|j| _t|dg | _|j| _|jjrwd| _tjdddd	|jj d
i|jjdd nd| _|jjrd| _tjdddd	|jj d
i|jjdd dS d| _dS )zConstructor of the checkpointer.

        Args:
            config_checkpoint (CheckpointConfig): The config object for the checkpointer.
        checkpointsNr   )download_checkpointkeys_to_skip_loadingZ_ckpt_s3_loaders3z
s3://ckpt/s3:///)backendpath_mappings3_credential_path)keybackend_argsZ_ckpt_s3_saver)r   r   save_to_object_storeenabledload_from_object_storeospathjoin
path_local_local_dirname_object_store_dirnamestrict_resume	load_pathr   3cosmos_predict2._src.imaginaire.utils.checkpoint_dbr   load_training_stateonly_load_scheduler_statesave_threadverbosekeys_not_to_resumegetattrr   broadcast_via_filesystemload_s3_backend_keyr
   set_s3_backendbucketcredentialssave_s3_backend_key)selfr   r   r   r$   r    r3   [/data/cameron/vidgen/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/checkpointer/base.py__init__    sT   




zAbstractCheckpointer.__init__model	optimizer	schedulergrad_scaler	iterationreturnc                 C      d S Nr3   )r2   r6   r7   r8   r9   r:   r3   r3   r4   savec   s   	zAbstractCheckpointer.savec                 C   r<   r=   r3   )r2   r6   r7   r8   r9   r3   r3   r4   loadn   s   zAbstractCheckpointer.loadc                 C      | j r| jj jS dS )z+Get the bucket name for saving checkpoints.N)r   r   r/   r2   r3   r3   r4   save_bucketx      z AbstractCheckpointer.save_bucketc                 C   r@   )z,Get the bucket name for loading checkpoints.N)r   r   r/   rA   r3   r3   r4   load_bucket}   rC   z AbstractCheckpointer.load_bucketc                 C       | j rd| j d| j S | jS Nr   r   )r   rB   r"   r!   rA   r3   r3   r4   save_dirname   
   z!AbstractCheckpointer.save_dirnamec                 C   rE   rF   )r   rD   r"   r!   rA   r3   r3   r4   load_dirname   rH   z!AbstractCheckpointer.load_dirnamec                 C   s   | j r
| j   dS dS )zFinalize the checkpointer.N)r(   r   rA   r3   r3   r4   finalize   s   zAbstractCheckpointer.finalizec                 C   s@   d}t j| jd}tj| | jdrtj| | jd }|S )zGet the file name of the latest saved checkpoint. If it doesn't exist, return None.

        Returns:
            checkpoint_file (str | None): file name of the latest saved checkpoint.
        Nlatest_checkpoint.txtbackend_key)	r   r   r   rI   r
   existsr-   r?   strip)r2   checkpoint_filecheckpoint_pathr3   r3   r4   _read_latest_checkpoint_file   s
   z1AbstractCheckpointer._read_latest_checkpoint_filerP   c                 C   s0   | d}t j| jd}tj||| jd dS )zTrack the file name of the latest saved checkpoint.

        Args:
            checkpoint_file (str): file name of the latest saved checkpoint.
        
rK   rL   N)r   r   r   rG   r
   dumpr1   )r2   rP   contentrQ   r3   r3   r4   _write_latest_checkpoint_file   s   

z2AbstractCheckpointer._write_latest_checkpoint_filerQ   c                 C   s$   t j| | jdstd| dS )zIf the file checkpoint_path does not exist, raise an error.

        Args:
            checkpoint_path (str): full path to the checkpoint.
        rL   zFile not found (object store): N)r
   rN   r-   FileNotFoundError)r2   rQ   r3   r3   r4   _check_checkpoint_exists   s   z-AbstractCheckpointer._check_checkpoint_existsr=   )NNN)r;   N) __name__
__module____qualname____doc__r   r   r   r	   CallBackGroupr5   r   r   torchoptim	Optimizerlr_schedulerLRScheduleramp
GradScalerintr>   r?   propertyrB   rD   rG   rI   rJ   strrR   rV   rX   r3   r3   r3   r4   r      sf    
C


	




r   )r   abcr   r   typingr   r^   &cosmos_predict2._src.imaginaire.configr   r   %cosmos_predict2._src.imaginaire.flagsr   %cosmos_predict2._src.imaginaire.modelr   %cosmos_predict2._src.imaginaire.utilsr	   -cosmos_predict2._src.imaginaire.utils.easy_ior
   r   r3   r3   r3   r4   <module>   s   