o
    9ie2                     @   s<  U d Z ddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
 ddlmZmZ ddlZddlmZmZ ddlmZmZ ddlmZ d	Zd
edefddZd
edefddZd
edefddZd
edefddZG dd dejZdedefddZ eee!e f Z"G dd deZ#G dd de#Z$G dd de#Z%e$e%B Z&ee'd< d e(e defd!d"Z)G d#d$ d$eeZ*G d%d& d&e*Z+G d'd( d(e*Z,e+e,B Z-ee'd)< G d*d+ d+ejZ.i Z/e0ee.f e'd,< 	 d-e.fd.d/Z1d0d1d
ed2edefd3d4Z2ej3d5edefd6d7Z4ej3d8d1d
ed2edefd9d:Z5e5Z6dS );a  Database of released checkpoints.

The database maps checkpoint internal URIs to public URIs and associates metadata (e.g.
experiment name).

## Usage

Register a checkpoint:

```python
CheckpointConfig(
    uuid="0e8177cc-0db5-4cfd-a8a4-b820c772f4fc",
    s3=CheckpointDirS3(
        uri="s3://bucket/path/to/checkpoint",
    ),
    hf=CheckpointDirHf(
        repository="org/repo",
        revision="revision",
        subdirectory="path/to/checkpoint",
    ),
).register()
```

Checkpoints can be referenced by UUID, S3 URI, or local path. Optionally, use `get_checkpoint_uri` to validate and normalize the URI.

```python
# S3 URI
checkpoint_uri = get_checkpoint_uri("s3://bucket/path/to/checkpoint")
# UUID
checkpoint_uri = get_checkpoint_uri("0e8177cc-0db5-4cfd-a8a4-b820c772f4fc")
# Local path
checkpoint_uri = get_checkpoint_uri("/path/to/checkpoint", check_exists=True)
```

When the checkpoint is loaded, call 'download_checkpoint':

```python
from cosmos_predict2._src.imaginaire.flags import INTERNAL

if not INTERNAL:
    from cosmos_predict2._src.imaginaire.utils.checkpoint_db import download_checkpoint

    checkpoint_uri = download_checkpoint(checkpoint_uri)

load_checkpoint(checkpoint_uri)
```
    N)ABCabstractmethod)Path)	Annotated	TypeAlias)Selfoverride)EXPERIMENTAL_CHECKPOINTSINTERNAL)logz1.3.5checkpoint_urireturnc                 C   s*   z
t t|  W dS  ty   Y dS w )z!Return True if the URI is a UUID.TF)uuidUUIDstr
ValueErrorr    r   ]/data/cameron/vidgen/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py_is_uuidR   s   r   c                 C   s   d| v pt |  S )z'Return True if the URI is a local path.z://)r   r   r   r   r   _is_path[   s   r   c                 C   s"   |  d} | dr| d} | S )zNormalize checkpoint URI./s3://z/model)rstrip
startswithremovesuffixr   r   r   r   normalize_uri`   s   


r   c                 C   s6   t | } | dr| dddd } d|  } | S )zSanitize checkpoint URI.r   r      zs3://bucket/)r   r   removeprefixsplitr   r   r   r   sanitize_urih   s
   

r    c                   @   s4   e Zd ZU dZejdddZejedZ	ee
d< dS )_CheckpointUriz%Config for checkpoint file/directory.forbidTextrafrozendefault_factorymetadataN)__name__
__module____qualname____doc__pydantic
ConfigDictmodel_configFielddictr(   __annotations__r   r   r   r   r!   q   s
   
 r!   uric                 C   s"   |  dstd|  dt| S )zValidate and normalize S3 URI.r   zInvalid S3 URI: z. Must start with 's3://')r   r   r   )r3   r   r   r   _validate_s3_uri}   s   
r4   c                   @   s   e Zd ZU dZeed< dS )_CheckpointS3zConfig for checkpoint on S3.r3   N)r)   r*   r+   r,   S3Urir2   r   r   r   r   r5      s   
 r5   c                   @      e Zd ZdZdS )CheckpointFileS3z!Config for checkpoint file on S3.Nr)   r*   r+   r,   r   r   r   r   r8          r8   c                   @   r7   )CheckpointDirS3z&Config for checkpoint directory on S3.Nr9   r   r   r   r   r;      r:   r;   CheckpointS3cmd_argsc                 C   s|   ddt  dg| }tjd}|r|d|g tt|  t	j
|dd t	jg |ddttjd	d
iB d S )zRun Hugging Face CLI download command and return the local path.

    Uses a newer Hugging Face CLI version to download checkpoint. The dependency
    version is very old and not robust.
    Zuvxzhf>=downloadZHF_TOKENz--tokenT)textz--quietZHF_HUB_OFFLINE1)r?   env)_MINIMUM_HF_CLI_VERSIONosenvirongetextendr   infoshlexjoin
subprocess
check_callcheck_outputr1   strip)r=   cmdtokenr   r   r   _hf_download   s   *rP   c                   @   sX   e Zd ZU dZeed< 	 eed< 	 dZedB ed< 	 edefddZdefd	d
Z	dS )_CheckpointHfz&Config for checkpoint on Hugging Face.
repositoryrevisionN_pathr   c                 C   s   d S )Nr   selfr   r   r   	_download   s   z_CheckpointHf._downloadc                 C   s   | j du r
|  | _ | j S ).Download checkpoint and return the local path.N)rT   rW   rU   r   r   r   r>      s   

z_CheckpointHf.download)
r)   r*   r+   r,   r   r2   rT   r   rW   r>   r   r   r   r   rQ      s   
 rQ   c                   @   s.   e Zd ZU dZeed< 	 edefddZdS )CheckpointFileHfz+Config for checkpoint file on Hugging Face.filenamer   c                 C   s6   | j ddd| j| jg}t|}tj|sJ ||S )rX   --repo-typemodel
--revision)rR   rS   rZ   rP   rC   pathexists)rV   r=   r^   r   r   r   rW      s   zCheckpointFileHf._downloadN)r)   r*   r+   r,   r   r2   r   rW   r   r   r   r   rY      s   
 rY   c                   @   s^   e Zd ZU dZdZeed< 	 dZeedf ed< 	 dZ	eedf ed< 	 e
defd	d
ZdS )CheckpointDirHfz0Config for checkpoint directory on Hugging Face. subdirectoryr   .includeexcluder   c                 C   s   t | jpdg}t | j}| jr*||fD ]}t|D ]\}}tj| j|||< qq| jddd| j	g}|r=|
dg| |rG|
dg| t|}| jrVtj|| j}tj|s`J ||S )rX   *r[   r\   r]   z	--includez	--exclude)listrc   rd   rb   	enumeraterC   r^   rI   rR   rS   rF   rP   r_   )rV   rc   rd   patternsipatternr=   r^   r   r   r   rW      s,   
zCheckpointDirHf._downloadN)r)   r*   r+   r,   rb   r   r2   rc   tuplerd   r   rW   r   r   r   r   r`      s   
 r`   CheckpointHfc                   @   s   e Zd ZU dZejdddZeed< 	 eed< 	 ej	e
dZe
ed< 	 d	Zed	B ed
< 	 d	Zed	B ed< 	 eed< 	 eed< 	 edefddZdefddZededed	B fddZededefddZdd Zd	S )CheckpointConfigzConfig for checkpoint.r"   Tr#   r   namer&   r(   N
experimentconfig_files3hfr   c                 C   s   | j  d| j dS )zReturn full name for debugging.())rn   r   rU   r   r   r   	full_name#  s   zCheckpointConfig.full_namec                 C   s(   t r| jjS td| j  | j S )rX   zDownloading checkpoint )r
   rq   r3   r   rG   ru   rr   r>   rU   r   r   r   r>   (  s   
zCheckpointConfig.downloadr3   c                 C   s   t |}t|dS )z:Return checkpoint config for URI if found, otherwise None.N)r   _CHECKPOINTSrE   )clsr3   r   r   r   maybe_from_uri0  s   zCheckpointConfig.maybe_from_uric                 C   s&   |  |}|du rtd| d|S )zDReturn checkpoint config for URI if found, otherwise raise an error.NCheckpoint '`' not found. Set 'export COSMOS_EXPERIMENTAL_CHECKPOINTS=1' to include experimental checkpoints.)rx   r   )rw   r3   rV   r   r   r   from_uri6  s   

zCheckpointConfig.from_uric                 C   s   t |  dS )zRegister checkpoint config.N)register_checkpointrU   r   r   r   register@  s   zCheckpointConfig.register)r)   r*   r+   r,   r-   r.   r/   r   r2   r0   r1   r(   ro   rp   r<   rl   propertyru   r>   classmethodr   rx   r{   r}   r   r   r   r   rm     s2   
 	rm   rv   checkpoint_configc                 C   sL   t s
| jjdv r
dS | j| jjfD ]}|tv rtd| d| t|< qdS )zZRegister checkpoint config.

    DEPRECATED: Use 'CheckpointConfig.register' instead.
    )znvidia/Cosmos-Experimentalz$nvidia-cosmos-ea/Cosmos-ExperimentalNry   z' already registered.)r	   rr   rR   r   rq   r3   rv   r   )r   r3   r   r   r   r|   I  s   
r|   F)check_existsr   c                C   s   t | } t|  }dur|jjS | dr| S t| r8|r6t|  	 }|
 s2td| dt|} | S tr<| S td|  d)z&Validate and normalize checkpoint URI.Nhf://ry   z' does not exist.rz   )r   rm   rx   rq   r3   r   r   r   
expanduserabsoluter_   r   r   r
   )r   r   
checkpointcheckpoint_pathr   r   r   get_checkpoint_uriZ  s"   

r   checkpoint_hfc                 C   s~   |  dsJ d|  | d}|d}t|dk r$td|  dd|d d }d|dd  }t|d|d	 S )
Nr   zNot a HuggingFace URI: r      z Invalid HuggingFace URI format: z1. Expected format: hf://org/repo/path/to/file.pth   main)rR   rS   rZ   )r   r   r   lenr   rI   rY   r>   )r   Zhf_pathpartsZrepo_idrZ   r   r   r   _download_hf_checkpointo  s    


r   Tc                C   sX   t r| S t|  }dur| S | drt| S |r*tj| s*t	d|  d| S )a  Download a checkpoint by URI and return the local path.

    This should only be used when the checkpoint is loaded. If you just need a
    URI, use 'get_checkpoint_uri' instead.

    Downloaded checkpoints are cached, so calling this multiple times will
    return the same path.

    Supports:
    - Checkpoint UUID: 0e8177cc-0db5-4cfd-a8a4-b820c772f4fc
    - S3 URI: s3://bucket/path/to/checkpoint
    - HuggingFace URI: hf://org/repo/path/to/file.pth
    - Local path: /path/to/checkpoint
    Nr   zCheckpoint path z does not exist.)
r
   rm   rx   r>   r   r   rC   r^   r_   r   )r   r   r   r   r   r   download_checkpoint  s   
r   )7r,   	functoolsrC   rH   rJ   r   abcr   r   pathlibr   typingr   r   r-   typing_extensionsr   r   %cosmos_predict2._src.imaginaire.flagsr	   r
   %cosmos_predict2._src.imaginaire.utilsr   rB   r   boolr   r   r   r    	BaseModelr!   r4   AfterValidatorr6   r5   r8   r;   r<   r2   rf   rP   rQ   rY   r`   rl   rm   rv   r1   r|   r   	lru_cacher   r   Zget_checkpoint_pathr   r   r   r   <module>   sR   0		,=