# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import gc

import torch

from imaginaire.callbacks.every_n import EveryN
from imaginaire.utils import log, misc


def trigger_manual_gc(source=None):
    # thresholds = [2000, 20, 20]
    # # gc.set_threshold(*thresholds)
    # counts = gc.get_count()
    # for gen in [2, 1]:
    #     if counts[gen - 1] > thresholds[gen]:
    #         with misc.timer(f'manual garbage collection gen{gen} (triggered by {source})'):
    #             gc.collect(gen)
    #             torch.cuda.empty_cache()
    #         break
    
    if gc.isenabled():
        log.debug('Automatic garbage collection is enabled => skipping manual call')
        return
    
    with misc.timer(f'manual garbage collection gen2 (triggered by {source})'):
        gc.collect(2)
    with misc.timer(f'cuda empty cache (triggered by {source})'):
        torch.cuda.empty_cache()


class ManualGarbageCollection(EveryN):
    """
    Disable auto gc and manually trigger garbage collection every N iterations
    It is super useful for large scale training to reduce gpu sync time!
    Can reach 50% speedup.

    It is important to note that this callback only disables gc in main process and have auto gc enabled in subprocesses.

    We start disable gc after warm_up iterations to avoid disabling gc in subprocesses, such as dataloader, which can cause OOM
    """

    def __init__(self, *args, warm_up: int = 5, **kwargs):
        kwargs["barrier_after_run"] = False
        super().__init__(*args, **kwargs)

        self.counter = 0
        self.warm = warm_up

    def every_n_impl(self, trainer, model, data_batch, output_batch, loss, iteration):
        del trainer, model, data_batch, output_batch, loss
        self.counter += 1
        if self.warm <= 0:
            return  # keep (never disable) automatic GC
        if self.counter < self.warm:
            return
        if self.counter == self.warm:
            gc.disable()
            log.critical('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@')
            log.critical('Automatic garbage collection has been disabled!')
            log.critical('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@')

        # NOTE(bvh): vanilla cosmos would just do gen1 here but this is not strong enough
        # OTOH, gen2 can take a couple seconds, so check thresholds and run dynamically
        # gc.collect(1)
        trigger_manual_gc(source='every_n_impl')