o
    ?߱i                     @   s|   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ dZ	dZ
e jd ddefdd	Ze jd ddefd
dZdS )    N)distributedlog)easy_io   i )global_stepr   c          	   	   #   sB    j jj} j jj}|rtj jjdt	
 dkr#tjdd t	
  fdd}td  tjsDtjdd td}}|||  }|dksWJ d	tjjtjjjtjjjgtjj|||d
| j jj j jj j jj j jjd}||_|V  W d    d S 1 sw   Y  d S t }d V  d S )NZtorch_tracer   Texist_okc                    s   dt | j }tj|}tj|stj|dd td| j  t	
 } jjjv r:| | d d tdt	
 | dd	 d S )
N
iteration_Tr   zDumping traces at step /rankz_trace.json.gzzFinished dumping traces in .2f seconds)strstep_numospathjoinexistsmakedirsr   infotime	monotonictrainer	profilingtarget_ranksexport_chrome_trace)profZcurr_trace_dir_nameZcurr_trace_dirbeginconfigrankZ	trace_dir Y/data/cameron/vidgen/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/profiling.pytrace_handler0   s    z-maybe_enable_profiling.<locals>.trace_handlerz*Profiling active. Traces will be saved at    z=profile_freq must be greater than or equal to warmup + active)waitwarmupactive)
activitiesscheduleon_trace_readyrecord_shapesprofile_memory
with_stackwith_modules)r   r   enable_profilingprofile_freqr   r   r   job
path_localr   get_rankr   r   r   r   TORCH_TRACE_WARMUPtorchprofilerprofileProfilerActivityCPUCUDAr(   record_shaper+   r,   r-   r   
contextlibnullcontext)	r   r   r.   r/   r"   r%   r&   r$   torch_profilerr    r   r!   maybe_enable_profiling#   s@   


"
r>   c             
   #   s     j jj}|rh j jjrdntj jjdt	
 dkr&tjdd tj	
 G  fddd}td  || j jj}z|V  W d S  tjjyg } z|jdd	 W Y d }~d S d }~ww d V  d S )
Nzs3://rundirmemory_snapshotr   Tr   c                       s6   e Zd ZdedefddZd
def fddZd	S )z4maybe_enable_memory_snapshot.<locals>.MemoryProfilerr   freqc                 S   s    t jjjtd || _|| _d S )N)max_entries)r4   cudamemory_record_memory_historyMEMORY_SNAPSHOT_MAX_ENTRIESr   r@   )selfr   r@   r    r    r!   __init__f   s   
z=maybe_enable_memory_snapshot.<locals>.MemoryProfiler.__init__Fexit_ctxc                    s   |  j d7  _ |s| j | j dkrd S |s| j }d| }n| j d }d| d}tj|} jjjsBtj|sBtj	|dd t
d|  t } jjjv rettjj | d d	 t
d
t | dd d S )Nr#   r   r	   _exitTr   z Dumping memory snapshot at step r
   z_memory_snapshot.picklez$Finished dumping memory snapshot in r   r   )r   r@   r   r   r   r   r   save_s3r   r   r   r   r   r   r   r   dumpr4   rB   rC   	_snapshot)rF   rH   Z	curr_stepdir_nameZcurr_snapshot_dirr   r   r   Zsnapshot_dirr    r!   stepl   s&   

 z9maybe_enable_memory_snapshot.<locals>.MemoryProfiler.stepN)F)__name__
__module____qualname__intrG   boolrO   r    rN   r    r!   MemoryProfilere   s    rU   z2Memory profiler active. Snapshot will be saved at )rH   )r   r   enable_memory_snapshotrJ   r   r   r   r0   r1   r   r2   r   r4   r   r   r/   rB   OutOfMemoryErrorrO   )r   r   Zenable_snapshotrU   r5   er    rN   r!   maybe_enable_memory_snapshotX   s&   



rY   )r;   r   r   r4   %cosmos_predict2._src.imaginaire.utilsr   r   -cosmos_predict2._src.imaginaire.utils.easy_ior   r3   rE   contextmanagerrS   r>   rY   r    r    r    r!   <module>   s   4