import os import sys from tqdm import tqdm from anydata.sync.sync_utils import aws_s3_sync, aws_s3_cp DATASETS = [ 'AgiBotWorldAlpha', 'AgiBotWorldBeta', 'ArgoVerse2sync', 'AriaSynthetic', 'Assembly101', 'BlendedMVG', 'CO3Dv2', 'DDAD', 'DIODE', 'DL3DV10K', 'DROID101', 'DTU', 'EgoDex', 'EgoVerse', 'HumanoidEveryday', 'Hypersim', 'IODAcore', 'KITTI', 'Kubric4D', 'Kubric5D', 'LBM', 'LLFF', 'LyftL5', 'MIPNeRF360', 'MVImgNet', 'MVImgNet2', 'NuScenes', 'NYUv2', 'OmnidataStarter', 'PD4D', 'PDnvs25', 'PDnvs89', 'PDv1', 'PDv2', 'RealEstate10K', 'RH20T', 'RTMV', 'ScanNet', 'ScanNetPP', 'Syn4D', 'TartanAir', 'TartanGround', 'VKITTI2', 'WaymoPerception', 'WildRGBD', ] EXTRAS_OUROBOROS = "--exclude '*proximity*' " \ "--exclude '*virtual*' " \ "--exclude '*point_cache*' " \ "--exclude '*point_cloud*' " \ "--exclude '*normals*' " \ "--exclude '*instance*' " \ "--exclude '*base_color*' " \ "--exclude '*roughness*' " \ "--exclude '*rgb_*' " \ "--exclude '*depth_*' " \ "--exclude '*intrinsics_*' " \ "--exclude '*optical_flow_*' " S3_ROOT = 's3://tri-ml-sandbox-16011-us-west-2-datasets/cv_downloaded' S3_ROOT_PROCESSED = 's3://tri-ml-sandbox-16011-us-west-2-datasets/cv_processed' S3_ROOT_LBM = 's3://robotics-manip-lbm' LOCAL_ROOT = '/data/cv_downloaded' LOCAL_ROOT_PROCESSED = '/data/cv_processed' if __name__ == "__main__": datasets = DATASETS if len(sys.argv) > 1: if '/' in sys.argv[1]: sk, st = sys.argv[1].split('/') datasets = datasets[int(st)::int(sk)] else: datasets = [d for d in sys.argv[1:] if d in datasets] progress = tqdm(datasets, ncols=96) for dataset in progress: progress.set_description(dataset) if os.path.exists(f'{LOCAL_ROOT}/{dataset}'): continue if os.path.exists(f'{LOCAL_ROOT_PROCESSED}/{dataset}'): continue ####### AgiBotWorldAlpha if dataset == 'AgiBotWorldAlpha': path = 'AgiBotWorldAlpha/observations/352/648564' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'AgiBotWorldAlpha/parameters/352/648564' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'AgiBotWorldAlpha/proprio_stats/352/648564' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'AgiBotWorldAlpha/task_info/task_352.json' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### AgiBotWorldBeta ####### ArgoVerse2sync if dataset == 'ArgoVerse2sync': path = 'ArgoVerse2sync/01bb304d7bd835f8bbef7086b688e35e__Summer_2019' aws_s3_sync(f'{S3_ROOT_PROCESSED}/{path}', f'{LOCAL_ROOT_PROCESSED}/{path}') ####### AriaSynthetic if dataset == 'AriaSynthetic': path = 'AriaSynthetic/10005' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### Assembly101 if dataset == 'Assembly101': path = 'Assembly101/nusar-2021_action_both_9011-b06b_9011_user_id_2021-02-01_154253' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'Assembly101/AssemblyHands' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'Assembly101/AssemblyPoses' extras = "--exclude '*' --include '*nusar-2021_action_both_9011-b06b_9011_user_id_2021-02-01_154253*'" aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=extras) ####### BlendedMVG if dataset == 'AriaSynthetic': path = 'BlendedMVG/BlendedMVSPP/000000000000000000000012' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### CO3Dv2 if dataset == 'CO3Dv2': path = 'CO3Dv2/bottle/34_1411_4529' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'CO3Dv2/bottle/frame_annotations.jgz' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### DDAD if dataset == 'DDAD': path = 'DDAD/000008' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=EXTRAS_OUROBOROS) ####### DIODE if dataset == 'DIODE': path = 'DIODE/train/outdoor/scene_00008/scan_00087' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'DIODE/intrinsics.txt' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### DL3DV10K if dataset == 'DL3DV10K': path = 'DL3DV10K/images_480/4K/005f630cb145af1a20586ce1648173ce9558feccd310d103a15296cb6819d53f' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'DL3DV10K/colmap_480/4K/005f630cb145af1a20586ce1648173ce9558feccd310d103a15296cb6819d53f/005f630cb145af1a20586ce1648173ce9558feccd310d103a15296cb6819d53f' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'DL3DV10K/images_960/10K/9725175383bd86faf7a20f73b65fd2b98e52bcda91b3d8db1c02251d2189f638/9725175383bd86faf7a20f73b65fd2b98e52bcda91b3d8db1c02251d2189f638' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### DROID101 if dataset == 'DROID101': path = 'DROID101/raw/AUTOLab/success/2023-07-27/Thu_Jul_27_08:46:17_2023' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### DTU if dataset == 'DTU': path = 'DTU/scan101' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### EgoDex if dataset == 'EgoDex': path = 'EgoDex/part1/add_remove_lid' extras = "--exclude '*' --include '*80.*'" aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=extras) ####### EgoVerse if dataset == 'EgoVerse': path = 'EgoVerse/2025-09-22-02-36-16-000000' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### HumanoidEveryday if dataset == 'HumanoidEveryday': path = 'HumanoidEveryday/Loco_manipulation/pick_up_a_caution_sign_stand_and_walk_to_put_it_agaist_wall/episode_1' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### Hypersim if dataset == 'Hypersim': path = 'Hypersim/ai_001_008' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'Hypersim/metadata_camera_parameters.csv' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### IODAcore if dataset == 'IODAcore': path = 'IODAcore/tri301-D_20190930_121854-d004-ML_OP_MDC_Diverse_Camera_360_Data_collection-data-09-30-2019_13-38-24.tlog.1014.24' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=EXTRAS_OUROBOROS) ####### KITTI if dataset == 'KITTI': path = 'KITTI/2011_10_03/2011_10_03_drive_0047_sync' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'KITTI/2011_10_03/calib_cam_to_cam.txt' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'KITTI/2011_10_03/calib_imu_to_velo.txt' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'KITTI/2011_10_03/calib_velo_to_cam.txt' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### Kubric4D if dataset == 'Kubric4D': path = 'Kubric4D/scn00006' aws_s3_sync(f'{S3_ROOT_PROCESSED}/{path}', f'{LOCAL_ROOT_PROCESSED}/{path}') ####### Kubric5D if dataset == 'Kubric5D': path = 'Kubric5D/scn00012' aws_s3_sync(f'{S3_ROOT_PROCESSED}/{path}', f'{LOCAL_ROOT_PROCESSED}/{path}') ####### LBM if dataset == 'LBM': path = 'efs/data/tasks/BimanualPutRedBellPepperInBin/riverway/sim/bc/teleop/2025-01-02T14-21-19-05-00/diffusion_spartan/episode_87/processed' aws_s3_sync(f'{S3_ROOT_LBM}/{path}', f'{LOCAL_ROOT}/LBM/{path}') ####### LLFF if dataset == 'LLFF': path = 'LLFF/flower' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### LyftL5 if dataset == 'LyftL5': path = 'LyftL5/test_scene00044' aws_s3_sync(f'{S3_ROOT_PROCESSED}/{path}', f'{LOCAL_ROOT_PROCESSED}/{path}') ####### MIPNeRF360 if dataset == 'MIPNeRF360': path = 'MIPNeRF360/counter' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### MVImgNet if dataset == 'MVImgNet': path = 'MVImgNet/103/00001a7b' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### MVImgNet2 if dataset == 'MVImgNet2': path = 'MVImgNet2/mvi2_07.tar/141/57077ca7' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### NuScenes if dataset == 'NuScenes': path = 'NuScenes/samples__scene-0782' aws_s3_sync(f'{S3_ROOT_PROCESSED}/{path}', f'{LOCAL_ROOT_PROCESSED}/{path}') ####### NYUv2 if dataset == 'NYUv2': path = 'NYUv2/sync/bathroom_0007' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### OmnidataStarter if dataset == 'OmnidataStarter': path = 'OmnidataStarter/depth_zbuffer/depth_zbuffer__hm3d__00001-UVdNNRcVyV1' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'OmnidataStarter/point_info/point_info__hm3d__00001-UVdNNRcVyV1' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'OmnidataStarter/rgb/rgb__hm3d__00001-UVdNNRcVyV1' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### PD4D if dataset == 'PD4D': path = 'PD4D/PD_trex_Point_Cache_Final_2_17_2023__scene_000003' aws_s3_sync(f'{S3_ROOT_PROCESSED}/{path}', f'{LOCAL_ROOT_PROCESSED}/{path}') ####### PDnvs25 if dataset == 'PDnvs25': path = 'PDnvs25/scene_000005' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=EXTRAS_OUROBOROS) ####### PDnvs89 if dataset == 'PDnvs89': path = 'PDnvs89/scene_000009' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=EXTRAS_OUROBOROS) ####### PDv1 if dataset == 'PDv1': path = 'PDv1/pd_phase2_010_01-scene_000005' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=EXTRAS_OUROBOROS) ####### PDv2 if dataset == 'PDv2': path = 'PDv2/scene_000020' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=EXTRAS_OUROBOROS) ####### RealEstate10K if dataset == 'RealEstate10K': path = 'RealEstate10K/train/0005c41463fe5e01' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'RealEstate10K/geometry/train/0005c41463fe5e01.txt' aws_s3_cp(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### RH20T if dataset == 'RH20T': path = 'RH20T_full/cfg1/task_0001_user_0001_scene_0001_cfg_0001_human' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path.replace("_full","")}') path = 'RH20T_full/cfg1/task_0001_user_0001_scene_0001_cfg_0001' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path.replace("_full","")}') path = 'RH20T_full/cfg1_depth/task_0001_user_0001_scene_0001_cfg_0001' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path.replace("_full","")}') path = 'RH20T_full/cfg1/calib' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path.replace("_full","")}') ####### RTMV if dataset == 'RTMV': path = 'RTMV/abo/falling_amazon_berkeley_scenes/00003' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### ScanNet if dataset == 'ScanNet': path = 'ScanNet/scene0000_00' extras = "--exclude '*color_640*'" aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=extras) ####### ScanNetPP if dataset == 'ScanNetPP': path = 'ScanNetPP/00a231a370' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### Syn4D if dataset == 'Syn4D': path = 'Syn4D/RGBD/arena/exr_layers/depth/seq_000000_6' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'Syn4D/RGBD/arena/exr_layers/image/seq_000000_6' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'Syn4D/RGBD/caption' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') path = 'Syn4D/RGBD/caption_global' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### TartanAir if dataset == 'TartanAir': path = 'TartanAir/abandonedfactory/Hard/P002' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### TartanGround if dataset == 'TartanGround': path = 'TartanGround/AbandonedFactory2/Data_omni/P0001' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') ####### VKITTI2 if dataset == 'VKITTI2': path = 'VKITTI2' extras = "--exclude '*' --include '*/Scene01/clone/*'" aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}', extras=extras) ####### WaymoPerception if dataset == 'WaymoPerception': path = 'WaymoPerception/train__scene00010' aws_s3_sync(f'{S3_ROOT_PROCESSED}/{path}', f'{LOCAL_ROOT_PROCESSED}/{path}') ####### WildRGBD if dataset == 'WildRGBD': path = 'WildRGBD/box/scenes/scene_004' aws_s3_sync(f'{S3_ROOT}/{path}', f'{LOCAL_ROOT}/{path}') #######