o
    ?ß±iQ  ã                   @   s4   d dl Z d dlmZ d dlmZ G dd„ deƒZdS )é    N)ÚEveryN)Úlogc                       s4   e Zd ZdZddœdef‡ fdd„Zdd„ Z‡  ZS )	ÚManualGarbageCollectiona´  
    Disable auto gc and manually trigger garbage collection every N iterations
    It is super useful for large scale training to reduce gpu sync time!
    Can reach 50% speedup.

    It is important to note that this callback only disables gc in main process and have auto gc enabled in subprocesses.

    We start disable gc after warm_up iterations to avoid disabling gc in subprocesses, such as dataloader, which can cause OOM
    é   )Úwarm_upr   c                   s*   d|d< t ƒ j|i |¤Ž d| _|| _d S )NFÚbarrier_after_runr   )ÚsuperÚ__init__ÚcounterÚwarm)Úselfr   ÚargsÚkwargs©Ú	__class__© ú]/data/cameron/vidgen/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/callbacks/manual_gc.pyr	   !   s   
z ManualGarbageCollection.__init__c                 C   sT   ~~~~~|  j d7  _ | j | jk rd S | j | jkr#t ¡  t d¡ t d¡ d S )Né   zGarbage collection disabled)r
   r   ÚgcÚdisabler   ÚcriticalÚcollect)r   ÚtrainerÚmodelÚ
data_batchÚoutput_batchÚlossÚ	iterationr   r   r   Úevery_n_impl(   s   

z$ManualGarbageCollection.every_n_impl)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr	   r   Ú__classcell__r   r   r   r   r      s    
r   )r   Ú1cosmos_predict2._src.imaginaire.callbacks.every_nr   Ú%cosmos_predict2._src.imaginaire.utilsr   r   r   r   r   r   Ú<module>   s   