# Created by BVH, Jun 2026. # OLD vidar-era MAnyView (4-view) DVS checkpoint (s28c v2), run through the modern vidar # inference pipeline (custom/eval/infer_vidar.py). Based on the s28c training config # custom/experiment/examples_vidar/rdvs4_anyview40h.py, with the modern vidar plumbing # mirrored from custom/experiment/examples_vidar/rwm3_drive40h.py. # NOTE(bvh): this is a vidar-era ckpt; depending on parity you may want to pass # legacy_network_behavior / legacy_logistics_behavior at inference (not set here). import os from hydra.core.config_store import ConfigStore from custom.experiment.template import template_any4d_2b, S3_PRETRAINED_ROOT, S3_OUTPUT_ROOT ################################################################# job = dict( # this becomes JobConfig project='a4d2', group='debug', name='vidar_rdvs4_drive', prepend_datetime=True, # becomes {now:%m-%d-%H-%M}_{config.job.name} local_root='', # empty = _DEFAULT_LOCAL_ROOT (/any4d on DGX, /tmp/a4d2 on SM) s3_root=S3_OUTPUT_ROOT, # default from template.py; set '' to disable S3 sync ) wandb = dict( enabled=True, entity='tri', project='a4d2', num_validation_logs=5, ) any4d_config = dict( # this is part of Any4DConfig, which inherits from Predict2Video2WorldModelConfig transforms='default', # NOTE(bvh): reference_camera now separate per dataset dataloader='vidar_flex', # was 'basile' (deleted in 68f6120a refactor); vidar_flex is the current flex dataloader for VidarDataset data_library='vidar', # required so a4d_model picks dl_module.vidar_to_any4d (not anydata_to_any4d) vae='a4d_vae', video_entries=[ # first entry = base / pretrained dict( # key: (channel_start, channel_end, in_proj_init, out_proj_init) # NOTE: there will be T*H*W tokens with this information rgb0=(0, 16, 'load', 'load'), # do not change rgb0_input_mask=(16, 17, 'load', None), # do not change rgb0_output_mask=(17, 18, 'load', None), # do not change cams0=(18, 50, 'zero/load', 'zero/load'), cams0_input_mask=(50, 51, 'zero/load', None), cams0_output_mask=(51, 52, 'zero/load', None), ), # later entries = new viewpoints dict( # key: (channel_start, channel_end, in_proj_init, out_proj_init) rgb1=(0, 16, 'copy:rgb0/load', 'copy:rgb0/load'), rgb1_input_mask=(16, 17, 'copy:rgb0_input_mask/load', None), rgb1_output_mask=(17, 18, 'copy:rgb0_output_mask/load', None), cams1=(18, 50, 'copy:cams0/load', 'copy:cams0/load'), cams1_input_mask=(50, 51, 'copy:cams0_input_mask/load', None), cams1_output_mask=(51, 52, 'copy:cams0_output_mask/load', None), ), dict( # key: (channel_start, channel_end, in_proj_init, out_proj_init) rgb2=(0, 16, 'copy:rgb1/load', 'copy:rgb1/load'), rgb2_input_mask=(16, 17, 'copy:rgb1_input_mask/load', None), rgb2_output_mask=(17, 18, 'copy:rgb1_output_mask/load', None), cams2=(18, 50, 'copy:cams1/load', 'copy:cams1/load'), cams2_input_mask=(50, 51, 'copy:cams1_input_mask/load', None), cams2_output_mask=(51, 52, 'copy:cams1_output_mask/load', None), ), dict( # key: (channel_start, channel_end, in_proj_init, out_proj_init) rgb3=(0, 16, 'copy:rgb1/load', 'copy:rgb1/load'), rgb3_input_mask=(16, 17, 'copy:rgb1_input_mask/load', None), rgb3_output_mask=(17, 18, 'copy:rgb1_output_mask/load', None), cams3=(18, 50, 'copy:cams1/load', 'copy:cams1/load'), cams3_input_mask=(50, 51, 'copy:cams1_input_mask/load', None), cams3_output_mask=(51, 52, 'copy:cams1_output_mask/load', None), ), ], num_views=4, video_concat_mode='view', video_proj_mode='per_view', view_timestep_mode='per_view', block_gate_fix=True, # s28c v2 was trained with the block_gate_fix flag enabled loss_weights=dict( rgb0=1.0, rgb1=1.0, rgb2=1.0, rgb3=1.0, ), harmonize_streams=True, harmonize_frames=True, load_modals=['rgb', 'cams'], # 'language', 'action', ], task_probs=dict( cross_modal=0.0, dyn_view_synth=1.0, # DVS only forecast=0.0, pose_est=0.0, inv_dyn=0.0, policy=0.0, world_model=0.0, ), use_views='rand2', shuffle_cams2views=True, # for 4-view train_directives=dict( ), val_directives=dict( tasks='dyn_view_synth', # DVS only at val remove_cond=True, ), data_val_overrides=dict(frame_stride=1), track_metrics=dict( rgb0=['psnr', 'ssim'], rgb1=['psnr', 'ssim'], rgb2=['psnr', 'ssim'], rgb3=['psnr', 'ssim'], ), train_visuals_interval=99, train_visuals_detail=1, val_visuals_detail=1, visuals_quality=8, viz_input_blacklist=[], # show cams viz_mask_border_width=0, # overridden because DVS is temporally invariant val_num_steps=35, ) ################################################################# S3_PRETRAINED_PREFIX = r's3://tri-ml-sandbox-16011-us-west-2-datasets/cosmos-predict-2/checkpoints' model = dict( # this makes up parts of Predict2Video2WorldModelConfig and Predict2ModelManagerConfig # official s28c v2 (4-view DVS, vidar-era): dit_path='s3://tri-ml-sandbox-16011-us-west-2-datasets/sagemaker/cosmos-predict2/a4d2/debug/01-03-14-18_s28c_dvs4_av40h_b2/model/iter_000059000_007552000.pt', text_encoder_path='', # disable (s28c was trained text-free) vae_path=f'{S3_PRETRAINED_PREFIX}/nvidia/Cosmos-Predict2-2B-Video2World/tokenizer/tokenizer.pth', fsdp_shard_size=8, ############ run_validation=True, skip_first_validation='delay', # to check VRAM validation_iter=500, # frequency / interval of validation runs max_iter=60000, # total number of training steps grad_accum_iter=2, context_parallel_size=1, device_monitor=0, manual_gc_iter=288, manual_gc_warm_up=-1, # never disable automatic GC to be safe (weird VRAM issue) ############ state_t=11, # for noise level; = latent # frames for now tokenizer_chunk_duration=41, # for VAE; = raw # frames for now ) checkpoint = dict( save_iter=1000, s3_folder='s3://tri-ml-sandbox-16011-us-west-2-datasets/sagemaker/cosmos-predict2/', early_sanity_check=5, ) optimizer = dict( lr=3.0e-5, ) scheduler = dict( warm_up_steps = [500], cycle_lengths = [model['max_iter']], f_start=[0.01], f_max=[1.0], f_min=[0.05], ) dataset_train = dict( config='custom/config/vidar/train/multi/av_ike4_40h_32g.yaml', num_workers=4, batch_size=2, ) dataset_val = dict( config=dict( ### DRIVING (vidar val DVS yamls) DDAD='custom/config/vidar/val/dvs4/ddad_ike4_40h.yaml', Argo='custom/config/vidar/val/dvs4/argo_ike4_40h.yaml', Lyft='custom/config/vidar/val/dvs4/lyft_ike4_40h.yaml', Waymo='custom/config/vidar/val/dvs4/waymo_ike4_40h.yaml', ), num_workers=0, # can crash debugger otherwise batch_size=1, ) metrics = None # dict() # disable because fragile ################################################################# cs = ConfigStore.instance() this_config = template_any4d_2b( job, wandb, any4d_config, model, checkpoint, optimizer, scheduler, metrics, dataset_train, dataset_val) # NOTE(bvh): unique name to avoid ConfigStore collision with custom/experiment/infotech/rdvs4_drive.py # (same basename => same default 'any4d_rdvs4_drive'; both get imported by the custom.experiment scan). experiment_name = 'any4d_vidar_rdvs4_drive' cs.store( group='experiment', package='_global_', name=experiment_name, node=this_config, )