o jR#@sdZddlZddlZddlZddlZddlmZddlmmZ dD]Z e ej vr2e e ej e <q#ddej d_ ddej d_d dej d_ejdd dd lmZdd lmZmZmZmZmZmZmZmZmZmZd ZdZ GdddeZ!e"dkre#ej$%rdndZ#e!ddddd&e#'Z(e)dde(*DZ+e,de+de-dddd&e#Z.e-dd&e#dZ/e0 e(e.e/Z1Wdn1swYe12D]\Z3Z4e4dure,de3de5e4j6qdSdS) usDinoVolumeQuery with DA3 geometric features fused in. Architecture: 1. DINOv3 backbone → (B, D_dino, 28, 28) patches 2. DA3-LARGE backbone (frozen) → last-layer patch tokens at (B, D_da3=2048, 28, 28) 3. Project DA3 patches: 1×1 conv 2048 → 256 4. Concat: (D_dino + 256, 28, 28) 5. Fusion: 2 or 3 conv layers with GELU, 3×3 kernels → (D_dino, 28, 28) 6. Upsample to (D_dino, 56, 56) 7. Refine to (d_feat=32, 56, 56) — F_feat 8. Existing query-MLP volume scoring DA3 features carry geometric/depth priors. The fusion module learns to combine DINO's semantic features with DA3's geometric features. N)depth_anything_3.utils.export!depth_anything_3.utils.pose_aligncOdSNakrr2/data/cameron/para/libero/model_dino_da3_fusion.pyr rcOrrrrrrr r r rcOrrrrrrr r r z/data/cameron/da3_repo/src)DepthAnything3) DinoVolumeQueryN_WINDOW N_HEIGHT_BINSN_GRIPPER_BINS N_ROT_BINSD_FEATD_SINZD_SINTIMG_SIZE PRED_SIZEz/data/cameron/da3_large_weightsics>eZdZdZeddddfdd Zdd Zd d d ZZS)DinoVolumeQueryDA3zN1view query-MLP with DA3 geometric features fused in via concat + conv layers.T) da3_weights da3_proj_dim fusion_layers freeze_da3c stj|i|t|}|jj|_~|r&|jD]}d|_q|j d|_ t j |j |dd|_ |j|} g} t|D]'} | dkrF| n|j} |j} | t j | | ddd| |dkre| t q>t j| |_||_||_dS)NFi) kernel_sizer)r padding)super__init__r from_pretrainedmodelbackbone da3_backbone parameters requires_gradevalZ da3_token_dimnnConv2dda3_proj embed_dimrangeappendGELU Sequentialfusionrr)selfrrrrargskwargsfullpZfused_inlayersiZin_chZout_ch __class__rr r$.s,        zDinoVolumeQueryDA3.__init__c Cs`|j\}}}}|d}tjrtjntj}t5tj|j j |d|j |dt t |j dgddd\}} Wdn1sDwYWdn1sSwY|d} t| t tfrg| d } | ddd f} t |j d d } | d} | jd| } t| d }| |}| dd| | ||f}||||| jdd d dd }|S)zRun DA3 backbone, return last-layer patch tokens reshaped to (B, 2048, H_p, W_p). DA3 expects (B, S=1, 3, H, W) input. r) device_typedtypeNZ out_layers) Zsaddle_balanced)Z cam_tokenZexport_feat_layersZref_view_strategyrn_storage_tokensg?r!r)shape unsqueezetorchcudais_bf16_supportedbfloat16float16no_gradautocastdevicetyper(listgetattr isinstancetupleintreshapepermute contiguousfloat)r5rgbB_ZH_inZW_inxZautocast_dtypeZfeatsZ_auxlastZ n_storageZn_prefixZn_patchZHp_da3ZWp_da3patchesrrr _extract_da3_patchesOs2    &z'DinoVolumeQueryDA3._extract_da3_patchesNc'Cs|jd}|j}|j}||\}}|jdd\} } ||} | jdd| | fkr6tj| | | fddd} || } tj || gdd} | | } tj| |j |j fddd}| |}|jdd\}}|j r|d||jd|d}|d ||jd|d}tj||jd }||dd||f}|tj ||gd d}n||}|d||||||}||j}|d||d ||d }|}|jD]}|||}q||}||||}||}||}|jd kr ||||d |j }n||}|dd|j!f} |d|j!|j!|j"f}!|d|j!|j"df}"t#d| |}#t#d|!|j$}$t#d|"|j}%|#d|$d d |%d d d }&|&|||dS)NrbilinearF)sizemode align_cornersr)dim).r).r)rOrDper_axisr!.zbtc, bchw -> bthwzbtc, zc -> btzzbtc, tc -> btr) volume_logitsgripper_logitsrotation_logits pixel_feats)%rFn_windowd_model_extract_dino_featuresr`F interpolater.rHcatr4 pred_sizerefineuse_eef image_sizelongclamparangerO input_projrGexpandrV t_cond_projt_sinblocks final_normviewq_head grip_head rotation_moderot_head n_rot_binsd_featd_sin_zeinsumz_sin)'r5rZ start_pixkp_zyxr[TdZ patch_dinoclsZH_dinoZW_dinoZ patch_da3r.fusedfeat_upF_featHWsxsyb_idxeef_featq_inq_in_btcond_tcond_bthblkpenult q_spatialgripperrotationq_Fq_zq_tscore_yxscore_zscore_trhrrr forwardssn      ""           zDinoVolumeQueryDA3.forwardr) __name__ __module__ __qualname____doc__DA3_WEIGHTS_DEFAULTr$r`r __classcell__rrr<r r+s!$r__main__rIcpui1d_pcar)rlrurrccs|] }|jr|VqdSr)r*numel).0r9rrr srz Trainable: ,r!z z: )7rossystypesrHtorch.nnr,torch.nn.functional functionalronmodules ModuleTypeexportZalign_poses_umeyamaZbatch_align_poses_umeyamapathinsertZdepth_anything_3.apir model_dino_volume_queryrrrrrrrrrrrZDA3_INPUT_SIZErrrOrI is_availabletor+msumr)n_tprintrandrZsprMoutitemsr vrTrFrrrr sP   0   "