o
    ?߱i                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlZd dl	m
Z
mZ d dlmZ 	 defddZdefdd	Ze  e  e  Zg d
Zg dZdedefddZdd ZdefddZdd Zdd ZdefddZdd Zdd Zddded efd!d"Z dS )#    N)Path)FLAGSVERBOSE)logreturnc                   C   s
   dt jv S )NRANK)osenviron r
   r
   M/data/cameron/vidgen/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.pyenable_distributed   s   
r   c                   C   s   t jdddkS )Nr   0)r   r	   getr
   r
   r
   r   is_rank0#   s   r   )z3cosmos_predict2._src.imaginaire.utils.checkpoint_dbz'cosmos_predict2._src.imaginaire.trainerz*.callbacks.*)z*._*z
projects.*z!cosmos_predict2._src.imaginaire.*recordc                 C   s   | d j dkr)| d }tD ]}t||r dS qtD ]}t||r& dS qdS t| s0dS | d }tD ]}t||rA dS q6tD ]}t||rO dS qDdS )NlevelCRITICALnameTF)r   _LOGGER_INCLUDEfnmatch_LOGGER_EXCLUDEr   _rank0_only_filter)r   module_namepatr
   r
   r   _console_filter4   s.   
r   c                   C   sb   t j  t jjtjtrdndttrt jnt	dd ts/t
jt
jd tj  td d S d S )NDEBUGINFOF)r   formatfiltercatch)r   ignore)r   loggerremoveaddsysstdoutr   _LOGGER_FORMATr   r   loggingbasicConfigERRORloguruwarningsfilterwarningsr
   r
   r
   r   _init_log_consoleL   s   


r-   
output_dirc              	   C   sZ   | d }| d }t d|  t jj|ddttddd t jj|dd	tt jddd d S )
Nzconsole.logz	debug.logzLog saved to wr   TF)moder   r   r   Zenqueuer   r   )r   infor!   r#   r&   r   r   )r.   Zconsole_pathZ
debug_pathr
   r
   r   _init_log_files]   s*   	
r2   c                  C   s   ddl m}  |   d S )Nr   distributed)%cosmos_predict2._src.imaginaire.utilsr4   initr3   r
   r
   r   _init_distributedu   s   r7   c                  C   s@   dd l m}  ddlm} | r|  |  r|   d S d S )Nr   )parallel_state)Ztorch.distributedr4   Zmegatron.corer8   Zis_initializedZdestroy_model_parallelZdestroy_process_group)distr8   r
   r
   r   _cleanup_distributed{   s   r:   c                    s>   dd l dd l    fdd}t| d S )Nr   c                     s\   t d   j g} | D ]} d|j  }|| t d|  qd S )NzStopping profilerzprofile.zProfile saved to )r   r1   stop	renderersZSessionRendererZoutput_file_extension
write_textoutput)r<   ZrendererZoutput_pathr.   Zprofilerpyinstrumentr
   r   stop_profiler   s   
z%_init_profiler.<locals>.stop_profiler)r@   Zpyinstrument.renderersZProfilerstartatexitregister)r.   rA   r
   r?   r   _init_profiler   s   rE   c                   C   s(   t jdd t  t rt  dS dS )zInitialize environment.ZTOKENIZERS_PARALLELISMfalseN)r   r	   
setdefaultr-   r   r7   r
   r
   r
   r   init_environment   s
   
rH   c                   C   s   t  rt  dS dS )zClean up environment.N)r   r:   r
   r
   r
   r   cleanup_environment   s   
rI   F)profilerJ   c                C   s@   | j ddd t sdS t|  tt  |rt|  dS dS )zInitialize output directory.T)parentsexist_okN)mkdirr   r2   r   debugr   rE   )r.   rJ   r
   r
   r   init_output_dir   s   rO   )!rC   r   r'   r   r$   r+   pathlibr   r*   Z%cosmos_predict2._src.imaginaire.flagsr   r   r5   r   boolr   r   Zget_datetime_formatZget_machine_formatZget_message_formatr&   r   r   dictr   r-   r2   r7   r:   rE   rH   rI   rO   r
   r
   r
   r   <module>   s2   
	