# BVH, May 2026. # AnyView ECCV rebuttal: 2-view DVS training (LR) on GCDKubric4D + GCDPD4D, vidar pipeline. # Modernized from the older train_anyview_ccg_LR.py, structurally aligned with # examples_anydata/rdvs4_robot.py. Stays 2-view; legacy_a4d_{network,logistics}=2 # for pre-cfc1f99e (2026-03-31) parity (resume from old ckpt trained under the # t_embedder loop-leak bug). import os from hydra.core.config_store import ConfigStore from custom.experiment.template import template_any4d_2b, S3_PRETRAINED_ROOT, S3_OUTPUT_ROOT ################################################################# job = dict( # this becomes JobConfig project='a4d2', group='eccv', name='dvs2', prepend_datetime=True, # becomes {now:%m-%d-%H-%M}_{config.job.name} local_root='', # empty = _DEFAULT_LOCAL_ROOT (/any4d on DGX, /tmp/a4d2 on SM) s3_root=S3_OUTPUT_ROOT, # default from template.py; set '' to disable S3 sync ) wandb = dict( enabled=True, entity='tri', project='a4d2_eccv', num_validation_logs=5, ) any4d_config = dict( # this is part of Any4DConfig, which inherits from Predict2Video2WorldModelConfig transforms='default', # required for data_library='vidar' dataloader='vidar_flex', # vidar pipeline (was 'basile' pre-68f6120a) data_library='vidar', legacy_a4d_network=2, # now renamed to legacy_network_behavior / legacy_logistics_behavior legacy_a4d_logistics=2, vae='a4d_vae', video_entries=[ # first entry = base / pretrained dict( # key: (channel_start, channel_end, in_proj_init, out_proj_init) # NOTE: there will be T*H*W tokens with this information rgb0=(0, 16, 'load', 'load'), # do not change rgb0_input_mask=(16, 17, 'load', None), # do not change rgb0_output_mask=(17, 18, 'load', None), # do not change cams0=(18, 50, 'zero/load', 'zero/load'), cams0_input_mask=(50, 51, 'zero/load', None), cams0_output_mask=(51, 52, 'zero', None), ), # later entries = new viewpoints dict( # key: (channel_start, channel_end, in_proj_init, out_proj_init) rgb1=(0, 16, 'copy:rgb0/load', 'copy:rgb0/load'), rgb1_input_mask=(16, 17, 'copy:rgb0_input_mask/load', None), rgb1_output_mask=(17, 18, 'copy:rgb0_output_mask/load', None), cams1=(18, 50, 'zero/load', 'zero/load'), cams1_input_mask=(50, 51, 'zero/load', None), cams1_output_mask=(51, 52, 'zero', None), ), ], lowdim_adaln_entries=dict( ), num_views=2, video_concat_mode='view', video_proj_mode='per_view', view_timestep_mode='per_view', # NOTE: legacy bug block_gate_fix=True, # should be enabled in every new training run train_cond_aug_sigma_range=(0.0, 0.0), # disabled because new feature val_cond_aug_sigma=0.0, # disabled because new feature loss_weights=dict( rgb0=1.0, # no rgb1 since predicted view is always rgb0 in DVS ), harmonize_streams=True, harmonize_frames=True, load_modals=['rgb', 'cams'], # NOTE(bvh): pre-refactor flat fields `override_zero_origin` and # `override_temporal_stride_val` are deprecated; their semantics now live in # data_train_overrides / data_val_overrides (consulted by both AnyData and # vidar pipelines). Kept here as historical refs only: # override_zero_origin=True, # makes more sense for 1->1 DVS setting? # override_temporal_stride_val=1, data_train_overrides=dict(zero_origin=True), data_val_overrides=dict(frame_stride=1, zero_origin=True), use_views='all', shuffle_cams2views=False, # AV train pairs are already canonical L/R; no shuffle task_probs=dict( cross_modal=0.0, dyn_view_synth=1.0, # DVS only forecast=0.0, pose_est=0.0, inv_dyn=0.0, policy=0.0, world_model=0.0, ), train_directives=dict( num_pred_views=1, ), val_directives=dict( tasks='dyn_view_synth', num_pred_views=1, remove_cond=True, ), track_metrics=dict( rgb0=['psnr', 'ssim', 'lpips'], # no rgb1 since predicted view is always rgb0 ), train_visuals_interval=99, train_visuals_detail=1, val_visuals_detail=1, visuals_quality=8, viz_input_blacklist=[], viz_extra_modes=[], viz_mask_border_width=0, # DVS is temporally invariant val_num_steps=35, ) ################################################################# model = dict( # pretrained / vanilla cosmos: # dit_path=f'{S3_PRETRAINED_ROOT}/nvidia/Cosmos-Predict2-2B-Video2World/model-480p-10fps.pt', # AnyView HR: dit_path='s3://tri-ml-sandbox-16011-us-west-2-datasets/sagemaker/cosmos-predict2/a4d2/cvpr/10-31-19-46_officialHR2v/model/iter_000008700_001670400.pt', text_encoder_path='', # uncomment to disable vae_path=f'{S3_PRETRAINED_ROOT}/nvidia/Cosmos-Predict2-2B-Video2World/tokenizer/tokenizer.pth', fsdp_shard_size=8, ############ run_validation=True, skip_first_validation='delay3', # to check VRAM validation_iter=1000, # frequency / interval of validation runs # max_val_iter=3, # limit validation steps per dataset for faster iteration max_iter=40000, # total number of training steps # NOTE: ^ AV paper says: 30k LR + 10k HR grad_accum_iter=1, context_parallel_size=1, device_monitor=0, manual_gc_iter=288, manual_gc_warm_up=-1, # never disable automatic GC to be safe (weird VRAM issue) ############ state_t=11, # for noise level; = latent # frames for now tokenizer_chunk_duration=41, # for VAE; = raw # frames for now ) checkpoint = dict( save_iter=1000, early_sanity_check=5, ) optimizer = dict( # lr=5.0e-5, # legacy entry loss averaging lr=2e-5, # new active entry loss averaging ) scheduler = dict( warm_up_steps=[500], cycle_lengths=[model['max_iter']], f_start=[0.01], f_max=[1.0], f_min=[0.05], ) dataset_train = dict( config='custom/config/vidar/anyview_eccv/mixed/train_2v_16gpu.yaml', num_workers=4, # per GPU batch_size=8, # per GPU ) dataset_val = dict( # quant_v4_LR: LR proxy of cvpr_eval_qnt*_HR with mult-of-64 alignment # (driving + egoexo split into 2 subcats each, robotics merged into 1). config=dict( ### 4D AssembHandsQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/assemblyhands.yaml', # 64 DyCheckM5Q4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/dycheckm_5seq.yaml', # 64 EgoExoIDEQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/egoexo4d_ID_evenT.yaml', # 32 EgoExoIDOQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/egoexo4d_ID_oddT.yaml', # 32 EgoExoOODEQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/egoexo4d_OOD_evenT.yaml', # 32 EgoExoOODOQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/egoexo4d_OOD_oddT.yaml', # 32 Kub4DDirQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/kubric4d_dir.yaml', # 64 Kub4DGradQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/kubric4d_grad.yaml', # 64 Kubric5DQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/kubric5d.yaml', # 64 ### DRIVING ArgoLFRFQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/argoverse_LF_RF.yaml', # 32 ArgoFLFRQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/argoverse_FL_FR.yaml', # 32 DDADLFRFQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/ddad_LF_RF.yaml', # 32 DDADFLFRQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/ddad_FL_FR.yaml', # 32 LyftLFRFQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/lyftl5_LF_RF.yaml', # 32 LyftFLFRQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/lyftl5_FL_FR.yaml', # 32 PD4DQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/pd4d.yaml', # 64 PD4DDirQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/pd4d_dir.yaml', # 64 PD4DGradQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/pd4d_grad.yaml', # 64 WaymoLFRFQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/waymo_LF_RF.yaml', # 32 WaymoFLFRQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/waymo_FL_FR.yaml', # 32 ### ROBOTICS DROIDIDQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/droid_ID.yaml', # 64 DROIDOODQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/droid_OOD.yaml', # 64 LBMQ4 = 'custom/config/vidar/anyview_eccv/quant_v4_LR/lbmv12.yaml', # 64 ), num_workers=0, batch_size=1, ) metrics = dict( gt_source='gt', modes=['r','rpv','apv'], # ,'ipv'], rgb=dict(), unroll=True, # re-enabled after lowdim guards in custom/eval/metrics.py:unroll_a4d (2026-05-07) ) ################################################################# cs = ConfigStore.instance() this_config = template_any4d_2b( job, wandb, any4d_config, model, checkpoint, optimizer, scheduler, metrics, dataset_train, dataset_val) # Use the filename (without extension) as the experiment name experiment_name = 'any4d_' + os.path.splitext(os.path.basename(__file__))[0] cs.store( group='experiment', package='_global_', name=experiment_name, node=this_config, )