import os
import sys

from custom.utils.sync.aws import aws_s3_ls

s3_path_east = 's3://tri-ml-datasets'
s3_path_west = 's3://tri-ml-sandbox-16011-us-west-2-datasets'

script = 'python externals/vidar/scripts/datasets/sync/sync_from_raw.py'
prefix = f'cv_datasets/processed/LBM'


datasets = aws_s3_ls(f'{s3_path_east}/{prefix}')
for dataset in datasets:
    command = f'{script} ' \
            f'{s3_path_east}/{prefix}/{dataset} ' \
            f'{s3_path_west}/{prefix}/{dataset} ' \
            f'--num_proc 24 --verbose'
    os.system(command)