o
    vi                     @   sl   d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	m
Z
mZ ddlmZmZ G dd deZdS )	z
Extended Trainer for Cosmos Policy with epoch tracking.

This trainer extends the base ImaginaireTrainer to add:
- Epoch tracking and sampler epoch setting for proper distributed sampling
    N)ImaginaireModel)ImaginaireTrainer)distributedlogmisc)maybe_enable_memory_snapshotmaybe_enable_profilingc                       sF   e Zd ZdZ fddZdedejjj	dejjj	ddfd	d
Z
  ZS )CosmosPolicyTrainera  
    Extended Trainer for Cosmos Policy.

    Adds special handling for:
    - Epoch tracking to properly set dataloader sampler epochs (needed for distributed training)
    - Simplified initial validation check (removes run_validation_on_start requirement)
    c                    s   t  | d S )N)super__init__)selfconfig	__class__ ;/data/cameron/vidgen/cosmos-policy/cosmos_policy/trainer.pyr   +   s   zCosmosPolicyTrainer.__init__modeldataloader_traindataloader_valreturnNc                 C   sH  |j d| jjjd}|| jjj | j  || jj| jj	\}}t
jjdi | jjj}| j  | j||||}d}td| jjj  | jjjdkr[t| jjj|}	n| jjjdkre|}	n
td| jjj td | jj||d	 | jjjr|dkr| jjjr| j|||d	 d
}
t| j|dM}t| j|d4}d}	 |j| t |}	 | j!| zRz8| "d) | j#j$d| jjj%j&d
d t'|}W d   n1 sw   Y  W d   n1 sw   Y  W n t(y   Y W | j)| nw W | j)| n| j)| w || jjj*krd}
nt+j |dd}| jj,|||d	 | jj-|||d	 |j.s@|	/  |	j.sHJ d|j.sPJ d| j0|	||||||d\}}}| jj1|||||d	 |dkrpq|d7 }|| jj2j3 dkr| jj4|||||d	 | jj5|||||d	 | jjjr|| jjj6 dkr| j|||d	 t78| jjj9 | j#:| |r|;  |r|;  q|d7 }|
rnqW d   n	1 sw   Y  W d   n	1 sw   Y  t<d || jj2j3 dkr| jj4|||||d	 | jj=||d	 | j>  t?  | j@  dS )a  The training function.

        Args:
            model (ImaginaireModel): The PyTorch model.
            dataloader_train (torch.utils.data.DataLoader): The training data loader.
            dataloader_val (torch.utils.data.DataLoader): The validation data loader.
        cuda)memory_formatr   zDistributed parallelism mode: ddpfsdpz&Unknown distributed parallelism mode: zStarting training...)	iterationF)global_stepTr   dataloading)profile_cudaN)devicez"model_ddp is not in training mode.zmodel is not in training mode.)r   grad_accum_iter   zDone with training.)r   )Ator   trainerr   on_train_start	callbackson_optimizer_init_startinit_optimizer_scheduler	optimizer	schedulertorchamp
GradScalergrad_scaler_argson_optimizer_init_endcheckpointerloadr   criticaldistributed_parallelismr   parallel_model_wrapperr   
ValueErrorinforun_validationrun_validation_on_startvalidater   r   sampler	set_epochiteron_before_dataloadingtraining_timerstraggler_detectorprofile_sectionstraggler_detectionanalyze_dataloadingnextStopIterationon_after_dataloadingmax_iterr   on_training_step_starton_training_step_batch_starttrainingtraintraining_stepon_training_step_batch_end
checkpoint	save_itersaveon_training_step_endvalidation_itersignalalarmtimeout_periodgenerate_reportstepsuccesson_train_endfinalizebarrier
on_app_end)r   r   r   r   r'   r(   grad_scalerr   r   	model_ddp_end_trainingtorch_profilermemory_profilerepochdataloader_train_iter
data_batchoutput_batchlossr   r   r   rH   .   s   




 		

 <A 
G
zCosmosPolicyTrainer.train)__name__
__module____qualname____doc__r   r   r)   utilsdata
DataLoaderrH   __classcell__r   r   r   r   r	   "   s    r	   )rg   rP   r)   torch.utils.data#cosmos_policy._src.imaginaire.modelr   %cosmos_policy._src.imaginaire.trainerr   #cosmos_policy._src.imaginaire.utilsr   r   r   -cosmos_policy._src.imaginaire.utils.profilingr   r   r	   r   r   r   r   <module>   s   