o
    vi                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ G d	d
 d
eZdS )    N)Tensor)EveryN)ImaginaireModel)ImaginaireTrainer)log)
rank0_only)easy_ioc                       s   e Zd ZdZdddddededef fd	d
Z	ddedee	e
jf dee	e
jf de
jdeddf fddZedededee	ef dee	ef dededdfddZ  ZS )	IterSpeeda	  
    Args:
        hit_thres (int): Number of iterations to wait before logging.
        save_s3 (bool): Whether to save to S3.
        save_s3_every_log_n (int): Save to S3 every n log iterations, which means save_s3_every_log_n n * every_n global iterations.
       T
   )	hit_thressave_s3save_s3_every_log_nr   r   r   c                   sH   t  j|i | d | _d| _|| _|| _|| _| jj| _	t | _
d S )Nr   )super__init__timehit_counterr   r   r   	__class____name__namelast_hit_time)selfr   r   r   argskwargsr    V/data/cameron/vidgen/cosmos-policy/cosmos_policy/_src/predict2/callbacks/iter_speed.pyr   &   s   
zIterSpeed.__init__r   model
data_batchoutput_batchloss	iterationreturnNc                    s   | j | jk r;td| d| j d  d| j d| ddt | j dd	 |  j d7  _ t | _tj	  d S t
 ||||| d S )
Nz
Iteration z: Hit counter:    /z	 | Loss: .4fz	 | Time: .2fs)r   r   r   infoitemr   r   torchcudasynchronizer   on_training_step_end)r   r   r   r   r    r!   r   r   r   r-   0   s"   

zIterSpeed.on_training_step_endtrainerc           
      C   s   | j d u rt   | _ d S t   }|| j  | j | j }t| d|dd| d tjr?t|d|}	tj||	d|d || _ | j	rc|| j
| j  dkret||d	d
| j d|dd d S d S d S )Nz : iter_speed r&   z seconds per iteration | Loss: r%   sample_counter)ztimer/iter_speedr/   )stepr   )
iter_speedr!   zs3://rundir/z/iter_09dz.yaml)r   every_n	step_sizer   r(   r)   wandbrungetattrr   r   r   dumpr   )
r   r.   r   r   r   r    r!   Zcur_timer1   r/   r   r   r   every_n_implF   s2   


"zIterSpeed.every_n_impl)r   )r   
__module____qualname____doc__intboolr   r   dictstrr*   r   r-   r   r   r9   __classcell__r   r   r   r   r	      sB    $

r	   )r   r*   r5   r   /cosmos_policy._src.imaginaire.callbacks.every_nr   #cosmos_policy._src.imaginaire.modelr   %cosmos_policy._src.imaginaire.trainerr   #cosmos_policy._src.imaginaire.utilsr   Z/cosmos_policy._src.imaginaire.utils.distributedr   +cosmos_policy._src.imaginaire.utils.easy_ior   r	   r   r   r   r   <module>   s   