
    qi"              
       z   U d dl Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlmZ dZdZ ej        e          Zdej        fdZddded	edej        fd
Zdedej        ddfdZdedej        ddfdZdej        defdZ dej        ddfdZ!dej        ddfdZ"dededede#fdZ$ ej%        d           e$ddd           ej%        d           e$ddd           ej%        d           e$ddd          iZ&e'ej(        e#f         e)d <   d!ej        dej        defd"Z*dS )#    NOPENPI_DATA_HOMEz~/.cache/openpireturnc                      t          j        t          j        t          t
                                                                                              } |                     dd           t          |            | S )NTparentsexist_ok)
pathlibPathosgetenv_OPENPI_DATA_HOMEDEFAULT_CACHE_DIR
expanduserresolvemkdir_set_folder_permission)	cache_dirs    7/home/robot-lab/Pi0.5_yam/src/openpi/shared/download.pyget_cache_dirr      sa    RY'8:KLLMMXXZZbbddIOOD4O0009%%%    F)force_downloadurlr   c                   t           j                            |           }|j        dk    rNt	          j        |           }|                                st          d|            |                                S t                      }||j
        z  |j                            d          z  }|                                }d}|                                r|st          ||          rd}n|S 	 |                    d          }t          j        |          5  t#          |           |rZt$                              d|            |                                rt+          j        |           n|                                 |                                st$                              d|  d	|            |                    d
          }	|j        dk    r|j
        dk    rt1          | |	fi | nt3          | |	fi | t+          j        |	|           t#          |           ddd           n# 1 swxY w Y   n+# t6          $ r}
d|  d| d}t7          |          |
d}
~
ww xY w|S )a  Download a file or directory from a remote filesystem to the local cache, and return the local path.

    If the local file already exists, it will be returned directly.

    It is safe to call this function concurrently from multiple processes.
    See `get_cache_dir` for more details on the cache directory.

    Args:
        url: URL to the file to download.
        force_download: If True, the file will be downloaded even if it already exists in the cache.
        **kwargs: Additional arguments to pass to fsspec.

    Returns:
        Local path to the downloaded file or directory. That path is guaranteed to exist and is absolute.
     File not found at /FTz.lockzRemoving expired cached entry: zDownloading  to z.partialgszopenpi-assetsNz>Local file permission error was encountered while downloading zA. Please try again after removing the cached data using: `rm -rf z*`)urllibparseurlparseschemer	   r
   existsFileNotFoundErrorr   r   netlocpathstrip_should_invalidate_cachewith_suffixfilelockFileLock_ensure_permissionsloggerinfois_dirshutilrmtreeunlink_download_gsutil_download_fsspecmovePermissionError)r   r   kwargsparsedr&   r   
local_pathinvalidate_cache	lock_pathscratch_pathemsgs               r   maybe_downloadr?       s   " \""3''F }|C  {{}} 	@#$>$>$>???||~~IV]*V[->->s-C-CCJ##%%J   	5iLL 	# ***733	y)) 	0 	0	*** (JjJJKKK$$&& (M*----%%'''$$&& 0@3@@J@@AAA)55jAA =D((V]o-M-M$S,AA&AAAA$S,AA&AAAL*555#J////	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	02  * * *]S ] ]NX] ] ] 	 c"")* s=   .)H; DH/#H; /H33H; 6H37H; ;
I#II#r9   c           
         t          j        d          *t                              d           t	          | |fi | dS |                    dd           t          j        dddd|  d	t          |          gd
           dS )z^Download a file or directory from GCS using gsutil if available, otherwise fall back to gcsfs.gsutilNzggsutil not found, falling back to gcsfs. This may fail if GCP credentials are not configured correctly.Tr   z-mcpz-rz/*)check)	r0   whichr-   warningr4   r   
subprocessrunstr)r   r9   r7   s      r   r3   r3   l   s    |H%u	
 	
 	
 	j33F333TD111N	4tZZZZA     r   c                 4   t          j        j        | fi |\  }}|                    |           }|d         dk    p&|d         dk    o|d                             d          x}r|                    |           }n|d         }t          j        |ddd	
          5 }t          j        	                    d          }	|	
                    |j        | ||          }
|
                                sut          d g |                    d          |D                       }|                    ||j        z
             t#          j        d           |
                                u|                    ||j        z
             ddd           dS # 1 swxY w Y   dS )zWDownload a file from a remote filesystem to the local cache, and return the local path.type	directorysizer   namer   iBTi   )totalunit
unit_scaleunit_divisor   )max_workers)	recursivec              3   p   K   | ]1}|                                 |                                j        V  2d S N)is_filestatst_size).0fs     r   	<genexpr>z#_download_fsspec.<locals>.<genexpr>   s?      mmAabajajalalmqvvxx/mmmmmmr   *N)fsspeccore	url_to_fsr.   endswithdutqdm
concurrentfuturesThreadPoolExecutorsubmitgetdonesumrglobupdatentimesleep)r   r9   r7   fs_r.   r/   
total_sizepbarexecutorfuturecurrent_sizes               r   r4   r4   {   s   K!#0000EB773<<Dv,+-c$v,!2C2bVH]H]^aHbHbdv "UU3ZZ

&\
	$4d	S	S	S )W[%88Q8GGjFKK++-- 	mm9]:;K;KC;P;P9]R\9]mmmmmLKKtv-...JqMMM ++-- 	 	J'((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s   C%FFFr&   target_permissionc                     |                                  j        |z  |k    r t                              d|  d           dS |                     |           t                              d|  d|            dS )znchmod requires executable permission to be set, so we skip if the permission is already match with the target.z	Skipping z+ because it already has correct permissionsNzSet r   )rY   st_moder-   debugchmod)r&   rx   s     r   _set_permissionr}      s    yy{{..2CCCRRRRSSSJJ !!!
LL555"35566666r   folder_pathc                 n    t          | t          j        t          j        z  t          j        z             dS )z7Set folder permission to be read, write and searchable.N)r}   rY   S_IRWXUS_IRWXGS_IRWXO)r~   s    r   r   r      s(    K!<t|!KLLLLLr   c                 8   dt           j        ddfd}dt           j        ddfd} ||            t          j        t	          |                     D ]H\  }}}t          j        |          }|D ]}||z  } ||           |D ]}	||	z  }
t          |
           IdS )zSince we are sharing cache directory with containerized runtime as well as training script, we need to
    ensure that the cache directory has the correct permissions.
    r&   r   Nc                     t                      }|                     |          }|}|j        D ]}t          ||z             ||z  }d S rW   )r   relative_topartsr   )r&   r   relative_pathmoving_pathparts        r   3_setup_folder_permission_between_cache_dir_and_pathzP_ensure_permissions.<locals>._setup_folder_permission_between_cache_dir_and_path   s^    !OO	((33!' 	- 	-D";#5666%,KK	- 	-r   	file_pathc                 j   t           j        t           j        z  t           j        z  t           j        z  t           j        z  t           j        z  }|                                  j        dz  r9t          | |t           j	        z  t           j
        z  t           j        z             dS t          | |           dS )zLSet all files to be read & writable, if it is a script, keep it as a script.@   N)rY   S_IRUSRS_IWUSRS_IRGRPS_IWGRPS_IROTHS_IWOTHrz   r}   S_IXUSRS_IXGRPS_IXOTH)r   file_rws     r   _set_file_permissionz1_ensure_permissions.<locals>._set_file_permission   s    ,-<t|KdlZ]a]ii>>#e+ 	0Iw'='Lt|'[\\\\\Iw/////r   )r	   r
   r   walkrH   r   )r&   r   r   rootdirsfiles	root_pathfiler   dirdir_paths              r   r,   r,      s    
-', -SW - - - -0 0 0 0 0 0 87===WSYY// - -dEL&&	 	, 	,D!D(I  ++++ 	- 	-C 3H"8,,,,	-- -r   yearmonthdayc                     t          j         | ||t           j                  }t          j        |                                          S )z.Get the mtime of a given date at midnight UTC.)tzinfo)datetimeUTCro   mktime	timetuple)r   r   r   dates       r   
_get_mtimer      s7    T5#hlCCCD;t~~''(((r   z-openpi-assets/checkpoints/pi0_aloha_pen_uncapi        z$openpi-assets/checkpoints/pi0_libero   zopenpi-assets/checkpoints/   _INVALIDATE_CACHE_DIRSr   c                 2   |                                 sJ d|             t          |                    |                     }t                                          D ]9\  }}|                    |          r|                                j        |k    c S :dS )zPInvalidate the cache if it is expired. Return True if the cache was invalidated.r   F)r#   rH   r   r   itemsmatchrY   st_mtime)r   r9   r   patternexpire_times        r   r(   r(      s     AA AZ A AAA
..y99::M 6 < < > > = ==='' 	=??$$-<<<<	= 5r   )+concurrent.futuresre   r   loggingr   r	   rer0   rY   rF   ro   urllib.parser   r*   r_   fsspec.generictqdm_loggable.autoautord   r   r   	getLogger__name__r-   r
   r   rH   boolr?   r3   r4   intr}   r   r,   floatr   compiler   dictPattern__annotations__r(    r   r   <module>r      s          				  				                  ! ! ! ! ! ! ' % 		8	$	$w|     8= I I I I I7< I I I IX# 7< d    )# )7< )d ) ) ) )&7', 73 7 7 7 7M M M M M M
-gl -t - - - -B)S ) )3 )5 ) ) ) ) BJ>??DRSUWAXAXBJ566

4A8N8NBJ+,,jjq!.D.D3 RZ./    ', SW      r   