}g@ddlZddlZddlZddlZddlmZddlmZddlZddl Z ddl m Z ddl Z ddlZddlZddlmZddlmZmZddlmZddlmZmZddlmcmZddlZdd lmZddlZddlZddlZddl Z dd l!m"Z"ejFejHd d dZ%dYdZ&ddl'Z'e'jPdZ)e)jUdde+ddde)jUdde+ddde)jUdddd e)jUd!d"d#d e)jUd$dd e)jUd%dd e)jUd&d'e+d()e)jUd*e+d)e)jUd+d,e,d-d./e)jUd0d1e,d2d3/e)jUd4e,d5d6/e)jUd7e,dd8/e)jUd9e-d:d;/e)jUdd#dd?@e)jUdAe,dBdC/e)jUdDe-d-dE/e)jUdFdddG@e)jUdHe+dddIe)jUdJe,d-dK/e)jUdLe,d-dM/e)jUdNd#ddO@e)jUdPd#ddQ@e)jUdRdddS@e)jUdTdddU@dZdVZ.e.Z/ej`jcde&e/e/jde/jfjhe/jfjjsdndWe/jfjlXy)[N)trange) rearrange)deepcopy) make_grid)rrepeat)ch_sec)autocast GradScaler)glob) rasterization)rrzcuda:0)devicec t|tr@|jDcic]$\}}|tt j |&c}}S|j Scc}}wN) isinstancedictitemsto_gputorchtensorcuda)obkvs :/home/cameronsmith/repos/multivid_point_track_sfm/train.pyrrsKjY[]aNbrxxzJtq!6%,,q/22JqhjhohohqqJs)A!dc  d}g}tjj|jj|j j }tj|jjdz|jj|jj} ttjjj| |jjt!|jj"|jjdd} d} t%|jj&d D]n} | dk(r$t)| \} }t/| t/|}} d }|j  }||| |j | }|j1D]1\}}t3j4||j7i| ||z }3t3j4d|j7i| |j9|j;|j=| dk(rt?j>|_tj@5d}| |zdk(rB | |dzzdk(s|j jC| }tEjFd|| |d| }ddd| |zdk(r| r|jjHrqt-d|jJtjL| |j jOdtPjRjU|jJd|jjVr| dzdk(rtj@5|j jC| }ddd| d|d<d| vr| d| dc|d<|d<tjL|j1Dcic]L\}}tY|tjZk(s#t]|j^dkDs<||jaNc}}d|jjbdt-d|jjdr9| dzdk(r1tjL|dd|jjbd| dz } qy#t*$rxt-d ttjjj| |jjt!d |jjdd} YwxYw#1swYxYw#1swYxYwcc}}w)Ncd}i}d|vr|dkDs|ddz|d<d|vr|dkDs|ddz|d<|dzd k(r t||tjd |d d i| tjd|d di| |S)Nc tjtt|Dcgc]}||dd||j fc}Scc}wr)rstackrangelenlong)xyis rz0train_flowmap..loss_fn..s:5;;E#a&M'Rq!Aadiik/(:'RS'Rs"Apoint_track_loss2g@zmetrics/tracks_errpoint_track_loss_staticzmetrics/tracks_err_static rzest/fx intrinsics)rrrrstepzest/fy)rrr/)printwandblog) model_outgt model_inputmodelr.rayslossess rloss_fnztrain_flowmap..loss_fnsS  *R^ghz^{|^vFZ?[ $ 1tBwluwPmQRUmUfMhFi 7A:eF4(4  8[6w?@dK 8[6w?@dK )lrparamsr/)num_trgtn_skipsfrT) batch_size num_workersshuffle pin_memoryFitting)desczdone w datasetgr-lossz Saving to )r.model_state_dictz checkpoint.ptrr,c2w gt_intrinsicsgt_posesz/data/cameron/pose_exps/poses_z.ptzexported poses affinity_embz$/data/cameron/feat_exp_exps/featexp_z _oursaff.pt)3roptimAdamargsr;r6 parametersdataMultiImageFoldervid_lenr>r?iterutils DataLoaderr@min n_workersr n_train_stepsnext StopIterationr0rrr1r2itembackwardr. zero_gradtimeno_gradforward_allptsvis wandb_summary save_modelsave_dirsave state_dictospathjoin export_posestypeTensorr"shapedetachname export_feats)run train_dataset until_img until_vid until_saverN single_datar9 losses_aggdataset dataloaderr.step_r5 ground_truth total_lossoutr8 loss_namerG wandb_imgsrrs r train_flowmaprs0dJ KK   CII4H4H4J  KE  ! !CHH,<,[^[c[c[f[f gGU[[%%00SXXEXEXfijmjrjrj|j|~A~F~F~Q~QgRgkw{0|}J D..Y?5 !8-1*-=*l)/{(;VL=QK IIk "lK $G%||~ OIt IIy$))+.T : $ J   6:??,-D9ejjl5??+< 7DIIK38 ]]_ aJI~q Q)A+&)SYY5M5Mk5Z,,ak&>{&K# K +L 9C  #+V[J\]hix]y%IS_S5I JJ h1tAw ?TY\]^]d]dYefgYg!((* hmKLOLTLTLYLYKZZ]k^ _ " # 88 T#Xq[ JJs>*.RSVS[S[S`S`Raal,m n ak5! &' !1!1!R=-A UU($U U &U =A=T>=T>U U zsimple training job) descriptionz-nz--nameFzwandb training name)rmdefaultrequiredhelpz-cz --init_ckptzIFile for checkpoint loading. If folder specific, will use latest .pt filez-oz--online store_true)ractionz-sz --save_modelTz--viserz--save_opt_visz-dz --datasethydrant)rmrz --imgpathz-bz --batch_sizer/z,number of videos/sequences per training step)rmrrz-vz --vid_lenz*video length or number of images per batchz --n_workersrFz number of workers per dataloaderz --until_savez number of steps until model savez--lrg-C6?z learning ratez--n_train_stepsgחAz --overfitz$Whether to overfit on a single scene)rrrz --until_imgr)z%Number of steps until image summary. z--sfz5Image resolution scale factor (fractional is cheaper)z --load_savezVWhether to load the previously saved data if overfitting (to avoid running flow again)z --splat_srczsplat src pt filez --time_stridezYNumber of frames to flatten in the encoder like a temporal convolution (to save memory). z--n_skipz@Number of frames to skip between adjacent frames in dataloader. z--use_gt_intrinsicszZWhether to use GT intrinsics instead of predicting them. Useful for pretraining scene rep.z --point_trackzWhether to use point trackingz--export_poseszExport poses when overfittingz--export_featszExport feats when overfittingcRtj|}tj}t j }t d|tjdd|jrdnd|jd}tjjdd |jz|_ tj|jd tj tj"j%|jd tj tj"j%|jd ||_||_|j(rt)j*|_g}d}d|_t dt1j2|j5|_|j8tj"j;tj"j=|j8r |j8nUt?tAtj"j%|j8dtj"jB} |j6jEtGjH|dd|S|S)Nzuser=cameronsmithbusinessbiasingonlinedisabledz /tmp/wandb)entityprojectmoderqdir.z/tmp/T)exist_okz checkpoint*zvideo*z(/data/cameron/monocular_ests/pets_dogs/*zdummy multivid testz*.pt)keyrIF)strict)'parser parse_argsargparse Namespacegetpassgetuserr0r1initrrqrslog_coderfrimakedirsrgrjrkrPviser ViserServer viser_serverrzmodelsFlowMaprr6 init_ckptisfile expandusermaxr getctimeload_state_dictrloadrhr) rPvalselfuserrs imgfoldersglob_str ckpt_filecurrent_model_dictloaded_state_dictrrnew_state_dicts rmake_runrs   T "D    D ?? D E$. **29VZVaVa(gqw{xAxAHR TC IIsDII%DMKK t, JJrww||DMM=9: JJrww||DMM845DIDJ zzU%6%6%84$ J9HDL  %**,DJ ~~!&(ggnnRWW5G5G5W&XDNN^abfgigngngsgstxuCuCDJhKcLRTRYRYRbRb_c JJ & &uzz)'<=O'PX] & ^ K4Kr:i,)rwrvru)rrNN)NF)7rir`rr1tqdmreinopsrrcgeometrycopyrnumpynppiqakorniatorchvision.utilsrrrrtorch.cuda.ampr r torch.nn.functionalnn functionalFrr rRrnerfviewgsplatr inverseonesrrrArgumentParserr add_argumentstrintfloatrrsautogradset_detect_anomalyrzrPrwoverfitrur:rrsh'$/  jejj12q~B  -B CDBK`aDSuS^_DU<HDlKIu\B$eLID3yAKc"5Dc!AopD3q>jkMs1:\]NC=_`FdH%3s3x_UKlHnoMBYZJS8z{)4 SopOT,Lkl$eLNmn$eLNmn2hj!!%( c#++)<)