o i4@sddlTddlmZddlmZddlmZddlZddlZddl m Z ddl m m Z ddlZddlZddlZddlZddlZddlmZddlmZmZmZd d lmZmZmZd d lmZmZm Z Gd d d e j!Z"dS))*)Number)partial)PathN)hf_hub_download)normalized_view_plane_uvrecover_focal_shiftangle_diff_vec3)wrap_dinov2_attention_with_sdpa'wrap_module_with_gradient_checkpointing)unwrap_module_with_gradient_checkpointing) DINOv2EncoderMLP ConvStackcseZdZUeed<eed<eed<eed<eed<eed<dddddd d gfdee e fdee e fdee e fdee e fd ee e fdee e fd e d de e ffdd ZedejfddZedejfddZedefddZejdefddZed4dee eeefdeee e fddfddZddZd d!Zd"d#Zd$ejdejfd%d&Z d'ejd(ee ej!fdee ejffd)d*Z"e#  + , ,  ,d5d'ejd(e d-e d.ed/ed0eee$ejfd1edee ejffd2d3Z%Z&S)6 MoGeModelencoderneck points_head mask_head scale_headonnx_compatible_modeNlinearii normal_head remap_output)rsinhexpsinh_expnum_tokens_rangec  stt|| rtd| ||_||_tdi||_t di||_ |dur3t di||_ |dur?t di||_ |durKt di||_ |durYtdi||_dSdS)Nz8The following deprecated/invalid arguments are ignored: )superr__init__warningswarnrrrrrrrrrrr) selfrrrrrrrrZdeprecated_kwargs __class__r (/data/cameron/moge_repo/moge/model/v2.pyr"s  zMoGeModel.__init__returncCt|jSN)next parametersdevicer%r r r(r.;zMoGeModel.devicecCr*r+)r,r-dtyper/r r r(r1?r0zMoGeModel.dtypecCs t|ddS)N_onnx_compatible_modeF)getattrr/r r r(rCs zMoGeModel.onnx_compatible_modevaluecCs||_||j_dSr+)r2rr)r%r4r r r(rGs pretrained_model_name_or_path model_kwargscKsvt|r |}n td |ddd|}tj|ddd}|d}|dur)|||d i|}|j|dd d |S) a  Load a model from a checkpoint file. ### Parameters: - `pretrained_model_name_or_path`: path to the checkpoint file or repo id. - `compiled` - `model_kwargs`: additional keyword arguments to override the parameters in the checkpoint. - `hf_kwargs`: additional keyword arguments to pass to the `hf_hub_download` function. Ignored if `pretrained_model_name_or_path` is a local path. ### Returns: - A new instance of `MoGe` with the parameters loaded from the checkpoint. modelzmodel.pt)Zrepo_idZ repo_typefilenameZcpuT)Z map_locationZ weights_only model_configNF)strictr )rexistsrtorchloadupdateZload_state_dict)clsr5r6Z hf_kwargsZcheckpoint_pathZ checkpointr9r7r r r(from_pretrainedLs  zMoGeModel.from_pretrainedcC|jdSr+)r init_weightsr/r r r(rBmzMoGeModel.init_weightscCs:|j|jdD]}t||rt||q dS)Nrrr)renable_gradient_checkpointingrhasattrr3)r%headr r r(rEps   z'MoGeModel.enable_gradient_checkpointingcCrAr+)renable_pytorch_native_sdpar/r r r(rHwrCz$MoGeModel.enable_pytorch_native_sdpapointscCs|jdkr |S|jdkrt|}|S|jdkr6|jddgdd\}}t|}tj|||gdd}|S|jdkrW|jddgdd\}}tjt|t|gdd}|Std |j) Nrrrrr dimrzInvalid remap output type: )rr<rsplitrcat ValueError)r%rIZxyzr r r( _remap_pointszs     zMoGeModel._remap_pointsimage num_tokenscs|j\}}|j|j}}}||d||d}} t|tjr3|| }} n t|t| }} j||| dd\} ddddgt dD];} t | d| |d| |||d} | ddd d |d d d } | dur| | <qRtj| | gdd | <qRfd d d D\} }}tdr| nd}fdd | ||fD\} }}| dur| dddd} | } |dur| dddd}tj|d d }|dur|d}|dur|d}| |||d}dd|D}|S)N?T)Zreturn_class_tokenr)widthZheight aspect_ratior1r.rr rJrKc3s0|]}t|rt|dndVqdS)rJN)rFr3).0rG)featuresr%r r( s.z$MoGeModel.forward..rDrc3s4|]}|durtj|fddddndVqdS)NZbilinearF)modeZ align_cornersZ antialias)FZ interpolate)rXv)img_himg_wr r(rZs2rInormalmask metric_scalecSi|] \}}|dur||qSr+r rXkr]r r r( z%MoGeModel.forward..)shaper.r1 isinstancer<TensorroundZlongrrangerZpermute unsqueezeexpandconcatrrFrrQr\ normalizesqueezeZsigmoidritems)r%rRrSZ batch_size_r.r1rWZbase_hZbase_wZ cls_tokenlevelZuvrIrbrcrd return_dictr )rYr^r_r%r(forwardsH  "      zMoGeModel.forward Tresolution_levelforce_projection apply_maskfov_xuse_fp16c s|dkrd}|d}nd}|j|j|jd}|jdd\} } | | } | | } |dur?|j\} }t| |d|| }tj |jj tj |oM|jtj kd |j ||d Wdn1sbwYfd d d D\}}}}t dd|||||g\}}}}}tj |jj tjd|dur|dk}nd}|dur3|durt||\}}n4| d| ddtttj||j|jdd}|jdkr|d|jd}t|||d\}}|dd| dd| |dd| dd}}tj||tjd|j|jdtjd|j|jd}|d|d7<|dur,||ddkM}|d}nd\}}|rG|durGtjj||d}|durm|dur]||dddddf9}|durm||ddddf9}|r|dur|durt|d|tjnd}|durt||tjnd}|durt|d|t|nd}Wdn 1swY|||||d}dd|D}|rdd|D}|S)a User-friendly inference function ### Parameters - `image`: input image tensor of shape (B, 3, H, W) or (3, H, W) - `num_tokens`: the number of base ViT tokens to use for inference, `'least'` or `'most'` or an integer. Suggested range: 1200 ~ 2500. More tokens will result in significantly higher accuracy and finer details, but slower inference time. Default: `'most'`. - `force_projection`: if True, the output point map will be computed using the actual depth map. Default: True - `apply_mask`: if True, the output point map will be masked using the predicted mask. Default: True - `fov_x`: the horizontal camera FoV in degrees. If None, it will be inferred from the predicted point map. Default: None - `use_fp16`: if True, use mixed precision to speed up inference. Default: True ### Returns A dictionary containing the following keys: - `points`: output tensor of shape (B, H, W, 3) or (H, W, 3). - `depth`: tensor of shape (B, H, W) or (H, W) containing the depth map. - `intrinsics`: tensor of shape (B, 3, 3) or (3, 3) containing the camera intrinsics. r`TrF)r1r.Nry) device_typer1enabled)rSc3s|] }|dVqdSr+)get)rXrgoutputr r(rZsz"MoGeModel.infer..racSst|tjr |S|Sr+)rkr<rlfloat)xr r r(sz!MoGeModel.infer..)rr1rTr r)r.r1)focal).r).NN)NN) intrinsics).N)rIrdepthrcrbcSrer+r rfr r r(rh*riz#MoGeModel.infer..cSsi|] \}}||dqS)r)rsrfr r r(rh-s)rLrotor1r.rjrintr<ZautocasttypeZfloat16rxmapZfloat32r ZtanZdeg2radZ as_tensorndimrputils3dZptZintrinsics_from_focal_centerZtensorZcloneZdepth_map_to_point_mapwhereinfZ zeros_likert)r%rRrSrzr{r|r}r~Zomit_batch_dimZoriginal_heightZoriginal_widthZarearWZ min_tokensZ max_tokensrIrbrcrdZ mask_binaryrshiftruZfxZfyrrrwr rr(infersr   $"  8 62    "&-zMoGeModel.inferr+)NryTTNT)'__name__ __module__ __qualname__r__annotations__rrboolDictstrAnyLiteralListrr"propertyr<r.r1rsetter classmethodUnionrIObytesOptionalr@rBrErHrlrQZ LongTensorrxZinference_moderr __classcell__r r r&r(rs       4 ,8 r)#typingZnumbersr functoolsrpathlibrr#r<Ztorch.nnZnnZtorch.nn.functionalZ functionalr\Z torch.utilsZtorch.utils.checkpointZ torch.ampZ torch.versionrZhuggingface_hubrZutils.geometry_torchrr r Zutilsr r rmodulesrrrZModulerr r r r(s$