o iD@sddlmZddlZddlmZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZddlmZmZmZdd lmZd d ZGd d d ejZGdddejZdS)) annotationsN)Dict) DictConfig OmegaConf) create_object)pose_encoding_to_extri_intri)apply_metric_scalingcompute_alignment_maskcompute_sky_maskleast_squares_scale_scalarsample_tensor_for_quantileset_sky_regions_to_max_depth)affine_inverseas_homogeneousmap_pdf_to_opacity)get_extrinsic_from_camraycCs t|S)N)rcreate)Zcfg_objr8/data/cameron/da3_repo/src/depth_anything_3/model/da3.py _wrap_cfg$s rcseZdZdZdZd/fdd Zddgdddfd0ddZd1ddZd2ddZd3d$d%Z d4d&d'Z  d5d6d)d*Z d7d-d.Z Z S)8DepthAnything3Neta  Depth Anything 3 network for depth estimation and camera pose estimation. This network consists of: - Backbone: DinoV2 feature extractor - Head: DPT or DualDPT for depth prediction - Optional camera decoders for pose estimation - Optional GSDPT for 3DGS prediction Args: preset: Configuration preset containing network dimensions and settings Returns: Dictionary containing: - depth: Predicted depth map (B, H, W) - depth_conf: Depth confidence map (B, H, W) - extrinsics: Camera extrinsics (B, N, 4, 4) - intrinsics: Camera intrinsics (B, N, 3, 3) - gaussians: 3D Gaussian Splats (world space), type: model.gs_adapter.Gaussians - aux: Auxiliary features for specified layers NcsPtt|tjr |ntt||_t|tjr|ntt||_d\|_ |_ |durKt|tjr5|ntt||_ t|tjrD|ntt||_ d\|_ |_ |dur|durt|tjra|ntt||_ |j j d}t|tjr|j|ksJd|d|j||_ dS|d|ksJd|d|dtt||_ dSdSdS)zY Initialize DepthAnything3Net with given yaml-initialized configuration. NNNzgs_head.out_dim should be z, got Z output_dimz!gs_head output_dim should set to )super__init__ isinstancennModulerrbackboneheadcam_deccam_enc gs_adaptergs_headZd_inZout_dim)selfZnetr r!r"r$r#Z gs_out_dim __class__rrrBs4         zDepthAnything3Net.__init__Fsaddle_balancedx torch.Tensor extrinsicstorch.Tensor | None intrinsicsexport_feat_layerslist[int] | Noneinfer_gsbool use_ray_poseref_view_strategystrreturnDict[str, torch.Tensor]c Cs"|dur*tj|jjdd||||jdd}Wdn1s$wYnd}|j||||d\} } |jd|jd} } tj|jjdd.|| | | } |r]|| | | } n| | | | | } |rr| | | | | |||} Wdn1s|wY| | } | | || | | _ | S)a1 Forward pass through the network. Args: x: Input images (B, N, 3, H, W) extrinsics: Camera extrinsics (B, N, 4, 4) intrinsics: Camera intrinsics (B, N, 3, 3) feat_layers: List of layer indices to extract features from infer_gs: Enable Gaussian Splatting branch use_ray_pose: Use ray-based pose estimation ref_view_strategy: Strategy for selecting reference view Returns: Dictionary containing predictions and auxiliary features NF)Z device_typeenabled) cam_tokenr.r3)torchZautocastdevicetyper"shaper_process_depth_head_process_ray_pose_estimation_process_camera_estimation_process_gs_head_process_mono_sky_estimation_extract_auxiliary_featuresZaux)r%r)r+r-r.r0r2r3r9featsZ aux_featsHWoutputrrrforwardds,  zDepthAnything3Net.forwardrHcCsd|vr|St|jdd}|dkr|S|dkr|S|j|}|dkr:tjd|d|jd}||}n|}t|d }t |jd ||d \|_}|S) zProcess mono sky estimation.sky333333?Z threshold 順rrNr<Gz?NZ max_depth) r rJsumdepthnumelr;randintr<quantiler )r%rH non_sky_mask non_sky_depthidx sampled_depth non_sky_max_rrrrCs"       z.DepthAnything3Net._process_mono_sky_estimationheightintwidthcCstd|vrd|vrt|j|j|jjd|jjd\}}}t|}|ddddddddf}tddd|jd|jd d d  |j }|dddddfd ||ddddddf<|ddddd fd ||ddddd d f<|dddddf|d |dddddd f<|ddddd f|d |ddddd d f<|`|`||_ ||_ |S) z=Process ray pose estimation if ray pose decoder is available.rayray_confr8Nrrr?) rrarbr>rr;Zeyerepeatclonetor<r+r-)r%rHr^r`Zpred_extrinsicZpred_focal_lengthsZpred_principal_pointsZpred_intrinsicrrrr@s&    42222z.DepthAnything3Net._process_ray_pose_estimationrElist[torch.Tensor]rFrGcCs|j|||ddS)z3Process features through the depth prediction head.r)patch_start_idx)r )r%rErFrGrrrr?sz%DepthAnything3Net._process_depth_headcCsZ|jdur+||dd}d|vr|`d|vr|`t|||f\}}t||_||_|S)z>Process camera pose estimation if camera decoder is available.Nr:rrarb)r!rarbrrr+r-)r%rErFrGrHZpose_encZc2wZixtrrrrAs  z,DepthAnything3Net._process_camera_estimation in_imagesc Cs|jdus |jdur |S|dddusJd|dd}|dd} |dur,| dus0Jd|} t|}| dur>t| } |j|||d|d} | j} | j} |j|| |jt| | ||f| d }||_|S) z=Process 3DGS parameters estimation if 3DGS head is available.NrTz&must provide MV depth for the GS head.r+r-z5must process camera info first if GT is not availabler)rErFrGrkZimages)r+r-ZdepthsZ opacities raw_gaussiansZ image_shapeZ gt_extrinsics) r$r#getrZraw_gsZ raw_gs_confrTrZ gaussians)r%rErFrGrHrlr+r-Zctx_extrZctx_intrZgt_extrZgs_outsrmZ densitiesZgs_worldrrrrBs@    z"DepthAnything3Net._process_gs_head feat_layers list[int]c Csrt}t|t|ks Jt||D]$\}}||jd|jd||j||j|jdg}||d|<q|S)z1Extract auxiliary features from specified layers.rrr:Z feat_layer_)rlenzipZreshaper> PATCH_SIZE) r%rErorFrGZ aux_featuresZfeatZ feat_layerZ feat_reshapedrrrrDs z-DepthAnything3Net._extract_auxiliary_features)NNNNr)r*r+r,r-r,r.r/r0r1r2r1r3r4r5r6)rHr6r5r6)rHr6r^r_r`r_r5r6)rErjrFr_rGr_r5r6) rErjrFr_rGr_rHr6r5r6r)rErjrFr_rGr_rHr6rlr*r+r,r-r,r5r6) rErjrorprFr_rGr_r5r6)__name__ __module__ __qualname____doc__rsrrIrCr@r?rArBrD __classcell__rrr&rr(s&% 7    8rcs\eZdZdZd%fdd Zddgddd fd&ddZd'ddZd'ddZ d(d)d#d$ZZ S)*NestedDepthAnything3Neta  Nested Depth Anything 3 network with metric scaling capabilities. This network combines two DepthAnything3Net branches: - Main branch: Standard depth estimation - Metric branch: Metric depth estimation for scaling alignment The network performs depth alignment using least squares scaling and handles sky region masking for improved depth estimation. Args: preset: Configuration for the main depth estimation branch second_preset: Configuration for the metric depth branch anyviewrmetriccs"tt||_t||_dS)z Initialize NestedDepthAnything3Net with two branches. Args: preset: Configuration for main depth estimation branch second_preset: Configuration for metric depth branch N)rrrda3 da3_metric)r%r{r|r&rrrDs  z NestedDepthAnything3Net.__init__NFr(r)r*r+r,r-r.r/r0r1r2r3r4r5r6c CsJ|j|||||||d}||} ||| }||| }||| }|S)an Forward pass through both branches with metric scaling alignment. Args: x: Input images (B, N, 3, H, W) extrinsics: Camera extrinsics (B, N, 4, 4) - unused intrinsics: Camera intrinsics (B, N, 3, 3) - unused feat_layers: List of layer indices to extract features from infer_gs: Enable Gaussian Splatting branch use_ray_pose: Use ray-based pose estimation ref_view_strategy: Strategy for selecting reference view Returns: Dictionary containing aligned depth predictions and camera parameters )r.r0r2r3)r}r~_apply_metric_scaling_apply_depth_alignment_handle_sky_regions) r%r)r+r-r.r0r2r3rH metric_outputrrrrIPs    zNestedDepthAnything3Net.forwardrHrcCst|j|j|_|S)z0Apply metric scaling to the metric depth output.)rrTr-)r%rHrrrrrvs z-NestedDepthAnything3Net._apply_metric_scalingc Cst|jdd}|dksJd|j|}t|dd}t|d}t|j||j|j|}|j|}|j|} t | |} |j| 9_|j dddddd d f| 9<d |_ | |_ |S) z2Apply depth alignment using least squares scaling.rKrLrMz)Insufficient non-sky pixels for alignmentrN)Z max_samplesrfNrdr)r rJrS depth_confr r;rWr rTr r+Z is_metricitem scale_factor) r%rHrrXZ depth_conf_nsZdepth_conf_sampledZ median_confZ align_maskZ valid_depthZvalid_metric_depthrrrrrs       & z.NestedDepthAnything3Net._apply_depth_alignmenti@ sky_depth_deffloatc Cs|t|jdd}|j|}|dkr#tjd|d|jd}||}n|}tt|d|}t |j|j ||d\|_|_ |S) z4Handle sky regions by setting them to maximum depth.rKrLrNrrOrPrQrR) r rJrTrUr;rVr<minrWr r) r%rHrrrXrYrZr[r\rrrrs    z+NestedDepthAnything3Net._handle_sky_regions)r{rr|rrt)rHr6rr6r5r6)r)rHr6rr6rrr5r6) rurvrwrxrrIrrrryrrr&rrz4s & %rz) __future__rr;Ztorch.nnrZaddictr omegaconfrrdepth_anything_3.cfgrZ&depth_anything_3.model.utils.transformrZ depth_anything_3.utils.alignmentrr r r r r Zdepth_anything_3.utils.geometryrrrZ depth_anything_3.utils.ray_utilsrrrrrzrrrrs