# Created by BVH, Jun 2026. # Cams-free twin of rcdvs11_gaia.py (rncdvs = real non-camera-conditioned DVS): 11-view dynamic # view synthesis on GAIA driving data WITHOUT explicit camera conditioning. The GAIA rig is fixed # across samples, so poses are learnable implicitly from the view index alone; dyn_view_synth still # runs at 100% probability (unified_anydrive only requires camera data when cams entries exist). # Channel layout (rgb 0-16, masks 16-18) matches the first 18 channels of the cams config, so the # MAnyView warm-start projections copy cleanly through surgery. # NOTE: GAIA lives on the AD2 bucket (gaia-e2e-wfm-datasets), so needs the ad2-s3 AWS profile. import os from hydra.core.config_store import ConfigStore from custom.experiment.template import template_any4d_2b, S3_PRETRAINED_ROOT, S3_OUTPUT_ROOT ################################################################# job = dict( project='a4d2', group='example', name='rncdvs11_gaia', prepend_datetime=True, local_root='', s3_root=S3_OUTPUT_ROOT, ) wandb = dict( enabled=True, entity='tri', project='a4d2', num_validation_logs=5, ) any4d_config = dict( dataloader='unified_anydrive', # CAM_ORDER fixes the gaia view ordering (front first) vae='a4d_vae', video_entries=[ # entries 0-3 mirror rdvs4_drive so the 4-view MAnyView ckpt loads cleanly dict( rgb0=(0, 16, 'load', 'load'), rgb0_input_mask=(16, 17, 'load', None), rgb0_output_mask=(17, 18, 'load', None), ), dict( rgb1=(0, 16, 'copy:rgb0/load', 'copy:rgb0/load'), rgb1_input_mask=(16, 17, 'copy:rgb0_input_mask/load', None), rgb1_output_mask=(17, 18, 'copy:rgb0_output_mask/load', None), ), dict( rgb2=(0, 16, 'copy:rgb1/load', 'copy:rgb1/load'), rgb2_input_mask=(16, 17, 'copy:rgb1_input_mask/load', None), rgb2_output_mask=(17, 18, 'copy:rgb1_output_mask/load', None), ), dict( rgb3=(0, 16, 'copy:rgb1/load', 'copy:rgb1/load'), rgb3_input_mask=(16, 17, 'copy:rgb1_input_mask/load', None), rgb3_output_mask=(17, 18, 'copy:rgb1_output_mask/load', None), ), # entries 4-10 = new viewpoints, copy from the last trained/available view (rgb3) dict( rgb4=(0, 16, 'copy:rgb3/load', 'copy:rgb3/load'), rgb4_input_mask=(16, 17, 'copy:rgb3_input_mask/load', None), rgb4_output_mask=(17, 18, 'copy:rgb3_output_mask/load', None), ), dict( rgb5=(0, 16, 'copy:rgb3/load', 'copy:rgb3/load'), rgb5_input_mask=(16, 17, 'copy:rgb3_input_mask/load', None), rgb5_output_mask=(17, 18, 'copy:rgb3_output_mask/load', None), ), dict( rgb6=(0, 16, 'copy:rgb3/load', 'copy:rgb3/load'), rgb6_input_mask=(16, 17, 'copy:rgb3_input_mask/load', None), rgb6_output_mask=(17, 18, 'copy:rgb3_output_mask/load', None), ), dict( rgb7=(0, 16, 'copy:rgb3/load', 'copy:rgb3/load'), rgb7_input_mask=(16, 17, 'copy:rgb3_input_mask/load', None), rgb7_output_mask=(17, 18, 'copy:rgb3_output_mask/load', None), ), dict( rgb8=(0, 16, 'copy:rgb3/load', 'copy:rgb3/load'), rgb8_input_mask=(16, 17, 'copy:rgb3_input_mask/load', None), rgb8_output_mask=(17, 18, 'copy:rgb3_output_mask/load', None), ), dict( rgb9=(0, 16, 'copy:rgb3/load', 'copy:rgb3/load'), rgb9_input_mask=(16, 17, 'copy:rgb3_input_mask/load', None), rgb9_output_mask=(17, 18, 'copy:rgb3_output_mask/load', None), ), dict( rgb10=(0, 16, 'copy:rgb3/load', 'copy:rgb3/load'), rgb10_input_mask=(16, 17, 'copy:rgb3_input_mask/load', None), rgb10_output_mask=(17, 18, 'copy:rgb3_output_mask/load', None), ), ], lowdim_adaln_entries=dict(), num_views=11, video_concat_mode='view', video_proj_mode='per_view', view_timestep_mode='per_view', block_gate_fix=True, train_cond_aug_sigma_range=(0.001, 0.01), val_cond_aug_sigma=0.001, loss_weights=dict( rgb0=1.0, rgb1=1.0, rgb2=1.0, rgb3=1.0, rgb4=1.0, rgb5=1.0, rgb6=1.0, rgb7=1.0, rgb8=1.0, rgb9=1.0, rgb10=1.0, ), harmonize_streams=True, harmonize_frames=True, data_train_overrides=dict( subsample=None, single_sample='random', ), data_val_overrides=dict( frame_stride=1, subsample=20, single_sample='first', ), # load_modals=['rgb', 'language'], # unified_anydrive ignores load_modals; cams absence is # already enforced by having no cams entries above (fixed rig, poses implicit in the view index) # use_views='all', # anydrive dataloader does not support anyway # shuffle_cams2views=False, # anydrive dataloader does not support anyway reference_view=0, # no-op without cams (kept for symmetry with rcdvs11_gaia) ref_view_rel_per_timestep=False, anydrive_cams2views=False, # NOTE(2026-06-15): identity cam->view mapping (not per-dataset reorder); # fixes the camera-ordering mistake. REQUIRED for all DVS11_gaia runs (NOT rwm11). task_probs=dict( cross_modal=0.0, dyn_view_synth=1.0, # DVS only forecast=0.0, pose_est=0.0, inv_dyn=0.0, policy=0.0, world_model=0.0, ), # Fixed split: 5 input views -> 6 novel views (num_pred_views = num_views - 5). train_directives=dict( num_pred_views=6, ), val_directives=dict( tasks='dyn_view_synth', num_pred_views=6, ), action_multiplier=3.0, track_metrics=dict( rgb0=['psnr', 'ssim', 'lpips'], rgb1=['psnr', 'ssim', 'lpips'], rgb2=['psnr', 'ssim', 'lpips'], rgb3=['psnr', 'ssim', 'lpips'], rgb4=['psnr', 'ssim', 'lpips'], rgb5=['psnr', 'ssim', 'lpips'], rgb6=['psnr', 'ssim', 'lpips'], rgb7=['psnr', 'ssim', 'lpips'], rgb8=['psnr', 'ssim', 'lpips'], rgb9=['psnr', 'ssim', 'lpips'], rgb10=['psnr', 'ssim', 'lpips'], ), train_visuals_interval=99, train_visuals_detail=1, val_visuals_detail=1, visuals_quality=8, viz_input_blacklist=[], viz_extra_modes=[], viz_mask_border_width=2, val_num_steps=35, ) ################################################################# model = dict( # Warm-start from AnyDrive GAIA 11 (s49, the active 16n rwm11_gaia run) on the gaia bucket; # surgery subsets the per-view proj to the rgb+mask channels (cams/world-model channels dropped). # NOTE(2026-06-11): old robotics-bucket s28c path was a cross-account mistake (gaia acct cannot read it). dit_path='s3://gaia-e2e-wfm-datasets/any4d/a4d2/sm_ad2/06-10-08-13_s49_rwm11_gaia53_b2_r416_16n/model/iter_000007000_001408000.pt', text_encoder_path='', # text-free, matching s28c vae_path=f'{S3_PRETRAINED_ROOT}/nvidia/Cosmos-Predict2-2B-Video2World/tokenizer/tokenizer.pth', fsdp_shard_size=8, ############ run_validation=True, skip_first_validation='delay3', validation_iter=500, max_val_iter=3, max_iter=60000, grad_accum_iter=1, context_parallel_size=1, device_monitor=0, manual_gc_iter=288, manual_gc_warm_up=-1, ############ state_t=11, tokenizer_chunk_duration=41, ) checkpoint = dict( save_iter=1000, early_sanity_check=5, ) optimizer = dict( lr=1e-5, ) scheduler = dict( warm_up_steps=[500], cycle_lengths=[model['max_iter']], f_start=[0.01], f_max=[1.0], f_min=[0.05], ) dataset_train = dict( config=[ 'custom/config/anydata/web/infotech/gaia_ike11_train.yaml', ], num_workers=4, batch_size=1, # 11 views x 41 frames is VRAM-heavy; raise if it fits ) dataset_val = dict( config=dict( GAIA = 'custom/config/anydata/web/infotech/gaia_ike11_val.yaml', ), num_workers=0, batch_size=1, ) metrics = None ################################################################# cs = ConfigStore.instance() this_config = template_any4d_2b( job, wandb, any4d_config, model, checkpoint, optimizer, scheduler, metrics, dataset_train, dataset_val) experiment_name = 'any4d_' + os.path.splitext(os.path.basename(__file__))[0] cs.store( group='experiment', package='_global_', name=experiment_name, node=this_config, )