o
    ô+üij€  ã                   @   s¼  d Z ddlZddlZej dej e¡¡ ddlZddlm	  m
Z ddlmZ ddlmZmZ ddlZddlmZ ddlZddlZddlZddlZddlmZmZmZmZmZ ddlZ ej e¡Z!e j" #dej $e!d¡¡Z%e j" &e%¡Z'e%j( )e'¡ e'j*Z*e'j+Z+e'j,Z,e'j-Z-e'j.Z.dZ/d	Z0d
Z1dZ2dZ3dd„ Z4dd„ Z5dd„ Z6dd„ Z7dd„ Z8dd„ Z9dd„ Z:dd„ Z;efdd„Z<dd „ Z=d!d"„ Z>	%d/d&d'„Z?d(d)„ Z@d*d+„ ZAd,d-„ ZBeCd.krÜeBƒ  dS dS )0aH  Train PARA on smith300 arm data.

Adapted from panda_streaming/train_panda_para.py. Same loss heads (volume,
gripper, rotation), same wandb EEF heatmap visualizations -- just point at
the smith300 data loader and zero the gripper loss weight (we did not
record the gripper position on this dataset; setting weight to 0 keeps the
head wired up so it can be re-enabled once gripper data exists, without
modifying the model).

Usage:
  cd /data/cameron/para/para_mac
  CUDA_VISIBLE_DEVICES=9 MUJOCO_GL=egl \
  DINO_REPO_DIR=/data/cameron/keygrip/dinov3 \
  DINO_WEIGHTS_PATH=/data/cameron/.cache/torch/hub/checkpoints/dinov3_vits16plus_pretrain_lvd1689m-4057cbaa.pth \
  python train_smith300_para.py \
    --data_dir /data/cameron/mac_robot_datasets/dataset_20260501_180125 \
    --run_name smith300_para_v0 \
    --epochs 500 --batch_size 4
é    N)Ú
DataLoaderÚrandom_split)ÚPath)ÚSmith300TrajectoryDatasetÚN_WINDOWÚproject_to_pixelÚDEFAULT_SMITH300_XMLÚEEF_BODY_NAMEÚmodelzmodel.pyiÀ  ç      ð?g      @g      à?é
   c                 C   sF   t jt j}}| | || d   dd¡}|td   ¡  dtd ¡S )Nç:Œ0âŽyE>r   é   )Úmodel_moduleÚ
MIN_HEIGHTÚ
MAX_HEIGHTÚclampÚN_HEIGHT_BINSÚlong)Zheight_valuesÚmin_hÚmax_hÚ
normalized© r   ú4/data/cameron/para/para_mac/./train_smith300_para.pyÚdiscretize_heightD   s   r   c                 C   sö   | j \}}}}}|d d …d d …df  ¡  d|d ¡}|d d …d d …df  ¡  d|d ¡}	| d|d ¡}
g }t|ƒD ]7}| d d …|f  |d¡}|
d d …|f ||  |	d d …|f |  |d d …|f   ¡ }| t ||¡¡ q<t 	|¡ 
¡ S )Nr   r   éÿÿÿÿ)Úshaper   r   ÚrangeÚreshapeÚappendÚFÚcross_entropyÚtorchÚstackÚmean)Zpred_volume_logitsÚtrajectory_2dZtarget_height_binsÚBÚNÚNhÚHÚWÚpxÚpyZh_binÚlossesÚtZlogits_flatZ
target_idxr   r   r   Úcompute_volume_lossJ   s   &&@r/   c                 C   sL   t tjƒ}t tjƒ}| | || d   dd¡}|td   ¡  dtd ¡S )zQMap continuous gripper q (rad) to bin indices using model_module.MIN/MAX_GRIPPER.r   r   r   )Úfloatr   ÚMIN_GRIPPERÚMAX_GRIPPERr   ÚN_GRIPPER_BINSr   )ÚvaluesZmin_gZmax_gr   r   r   r   Údiscretize_gripperW   s   

r5   c                 C   s>   t |ƒ}| j\}}}|  || |¡}| || ¡}t ||¡S )znCE over N_GRIPPER_BINS bins.  pred_gripper_logits: (B, N, Ng).
    target_gripper: (B, N) continuous q in rad.)r5   r   r   r    r!   )Zpred_gripper_logitsZtarget_gripperÚtarget_binsr&   r'   ZNgÚlogitsÚtargetr   r   r   Úcompute_gripper_loss_   s
   r9   c                 C   sd   t jtj| jt jd}t jtj| jt jd}| | || d   dd¡}|td   	¡  dtd ¡S )N©ÚdeviceÚdtyper   r   r   )
r"   Útensorr   ÚMIN_ROTr;   Úfloat32ÚMAX_ROTr   Ú
N_ROT_BINSr   )Zeuler_valuesÚmin_rÚmax_rr   r   r   r   Údiscretize_rotationi   s   rD   c                 C   s   t |ƒ}| j\}}}}g }tdƒD ]/}| d d …d d …|d d …f  || |¡}	|d d …d d …|f  || ¡}
| t |	|
¡¡ qt |¡ 	¡ S )Né   )
rD   r   r   r   r   r    r!   r"   r#   r$   )Zpred_rotation_logitsZtarget_eulerr6   r&   r'   Ú_ÚNrr-   Úaxisr7   r8   r   r   r   Úcompute_rotation_lossp   s   ( rI   c                 C   sn   | j dd}| jd }tjdd|| jd}|| }tjtj| jtjd}tjtj	| jtjd}|||  | S )z…Decode (B, N, 3, Nr) rotation logits into (B, N, 3) euler in rad
    using the per-axis bin centers (inverse of discretize_rotation).r   ©Údimr   r   ©r;   r:   )
Úargmaxr   r"   Úlinspacer;   r=   r   r>   r?   r@   )Úrotation_logitsZ	pred_binsrG   Úbin_centersr   rB   rC   r   r   r   Údecode_pred_euler{   s   
rQ   c                 C   s*  | j \}}}}}tj||d| jd}tj||| jtjd}t|ƒD ]T}| d d …|f }	|	jdd\}
}|
 |d¡jdd}|| }|| }| 	¡ |d d …|df< | 	¡ |d d …|df< |	tj
|| jdd d …||f jdd|d d …|f< q!tjtj}}tjddt| jd}|| ||  | }||fS )Né   rL   r:   r   rJ   r   r   )r   r"   Úzerosr;   r   r   Úmaxr   rM   r0   Úaranger   r   r   rN   r   )Úvolume_logitsr&   r'   r(   r)   r*   Úpred_2dZpred_h_binsr.   Úvol_tZ
max_over_hrF   Zflat_idxr,   r+   r   r   rP   Zpred_heightr   r   r   Úextract_pred_2d_and_height‡   s    6rY   c               	   C   sð  ddl }ddlm} |j |¡}| |¡}| ||jjt	¡}|j
}g g g }}	}
| D ]Ñ\}}t tj |d¡¡}tj|d tjd}|jd }d|jv oQd	|jv }|rhtj|d tjd}tj|d	 tjd}|D ]‘}tt|d
 ƒ|jd d ƒ}tt|d ƒ|d ƒD ]u}|rœ| t||df ƒ¡ |	 || ¡ nOtj|tjd}||d|…f |dt|jd |ƒ…< ||jd|…< | ||¡ |j|  ¡ }|j|  ¡ }|g d¢ }| t|d ƒ¡ |	 | |¡  d¡¡ |dkrú|
 t||df ƒ¡ q…qjq+t !|¡}t !|	¡}	|
r+tt|
ƒƒtt"|
ƒƒ}}dt"d|| ƒ }||8 }||7 }d}nd\}}d}t| ¡ ƒt| "¡ ƒ|||	jdd #¡ |	j"dd #¡ dœ}|rad|d›d|d›dt$|
ƒ› dnd}t%d|d d ›d|d! d ›d"|› ƒ |S )#a5  Compute joint height/rotation/gripper ranges across one or more
    datasets. Each entry in `data_dirs_with_episodes` is a (data_dir,
    episodes_list) pair. Datasets with `eef_pos`/`eef_euler` saved
    (UMI capture path) skip arm-FK; datasets without (smith300 capture
    path) run FK on q_motors[:, :nq].r   N©ÚRotationz
joints.npzÚq_motors©r<   r   Úeef_posÚ	eef_eulerÚendÚstartrR   )r   rR   rE   r   Úxyzé   é   gš™™™™™©?gü©ñÒMbP?T)g      ð¿r   F©rH   )Ú
min_heightÚ
max_heightÚmin_gripperÚmax_gripperÚmin_rotÚmax_rotzrecorded range [z+.3fz, z] (z	 samples)zPLACEHOLDER (not recorded)zStats: height=[rf   ú.4frg   z], gripper=)&ÚmujocoÚscipy.spatial.transformr[   ÚMjModelÚfrom_xml_pathÚMjDataÚ
mj_name2idÚmjtObjÚ
mjOBJ_BODYr	   ÚnqÚnpÚloadÚosÚpathÚjoinÚasarrayÚfloat64r   ÚfilesÚminÚintr   r   r0   rS   ÚqposÚ
mj_forwardÚxposÚcopyÚxquatÚ	from_quatÚas_eulerÚarrayrT   ÚtolistÚlenÚprint) Zdata_dirs_with_episodesÚ
mujoco_xmlrm   ZRotÚmj_modelÚmj_dataÚeef_idÚn_qposZheightsZeulersZgrippersÚdata_dirÚepisodesÚjointsr\   Ún_recorded_motorsZhas_saved_eefr^   r_   ÚepÚep_endÚidxÚqÚposÚ	quat_wxyzZ	quat_xyzwÚgminÚgmaxÚpadZgripper_recordedÚstatsÚmsgr   r   r   Úcompute_dataset_stats_fastœ   sz   


ÿ$€ñþ

üÿÿÿÿrŸ   c           	      C   sP   t | ||ƒ}|du rdS |d | | }|d | | }tt|ƒƒtt|ƒƒfS )zProject 3D world point to pixel coords at image_size resolution.
    Mirrors the panda version but takes the calibration as args (per-dataset).Nr   r   )r   r   Úround)	Zpos_3dÚ
image_sizeÚT_cam_worldÚK_origÚimg_wÚimg_hÚpixÚuÚvr   r   r   Úproject_world_to_pixelâ   s   r©   c                 C   sD   |   ¡  tj¡}|d  |t|ƒ 9  < |d  |t|ƒ 9  < |S )z?Anisotropic K for projection at image_size x image_size raster.r   r   )rƒ   Úastyperv   r|   r0   )r£   r¤   r¥   r¡   ÚKr   r   r   Ú_scale_K_to_imageí   s   r¬   ç{®Gáz¤?rR   Fc              
   C   sö  ddl m} | d|¡ ¡ }	ttj|tjd||ƒ}
|
du r dS | jdd… \}}t	t
|
d ƒƒt	t
|
d ƒƒ}}d|  krD|k rSn dS d|  krR|k sUdS  dS g d¢}td	ƒD ]›}tj|tjd||	dd…|f   }t|||ƒ}|du r|q]t	t
|d ƒƒt	t
|d ƒƒ}}|| }|réd
}t|ƒD ]N}|d dkr¢q™|| }|d | }t	t
||||   ƒƒt	t
||||   ƒƒf}t	t
||||   ƒƒt	t
||||   ƒƒf}t | ||||tj¡ q™q]t | ||f||f||tj¡ q]dS )zÇDraw 3 colored axes (X=red, Y=green, Z=blue, BGR convention) at the
    EEF origin showing orientation. `dashed=True` for predicted overlay so
    GT vs pred read off side-by-side at the same origin.r   rZ   rb   r]   NrR   r   ))r   r   éÿ   ©r   r®   r   )r®   r   r   rE   é   )rn   r[   Ú
from_eulerÚ	as_matrixr   rv   r{   r|   r   r   r    r   Úcv2ÚlineÚLINE_AA)Zvis_bgrZ	origin_3dZ	euler_xyzr¢   ÚK_imageÚlengthÚ	thicknessÚdashedÚScipyRÚRZ
origin_pixÚhÚwÚu0Úv0Z
colors_bgrrH   Ztip_3dZtip_pixÚu1Úv1ÚcolorÚnÚiÚt0Út1Úp0Úp1r   r   r   Údraw_pose_axesõ   sH   ÿ"ÿÿ$"00ù 	îrÉ   c                 C   sú  g }d}| d   ¡  ¡ }| d   ¡  ¡ }t| d jd ƒ}t|ƒD ]E}| d |   ¡  ¡ }|jdd… \}	}
|du r[| ¡  tj¡}|ddd…f  |	9  < |ddd…f  |	9  < |d  tj	¡ ¡ }d	| v rï| d	 |  
¡   ¡  ¡ }|| ¡  }| ¡ d
kr‡|| ¡  }t |¡}||d< t |d |d  dd¡}|d  tj	¡}t | ¡ |j¡\}}d|  krº|
k rïn n3d|  krÆ|	k rïn n't |t|ƒt|ƒfdtjdd¡ t |dt|ƒd t|ƒd ftjddd¡ | d |   ¡  ¡  tj¡}t|||ƒ}|durtt|d ƒƒtt|d ƒƒfnd}|durÖ|\}}d|  kr/|
k rÖn n¥d|  kr=|	k rÖn n—t |||fddd¡ t |d|d |d ftjddd¡ | ¡ }d|d< t|||ƒ}|durtt|d ƒƒtt|d ƒƒfnd}|durÖ|\}}d|  kr–|
k rÖn n>d|  kr¤|	k rÖn n0t |||fddd¡ t |||f||fdd¡ t |d|d d›|d |d ftjddd¡ d| v rDd | v rDzD| d |   ¡  ¡  tj¡}| d |   ¡  ¡  tj¡}| d  |   ¡  ¡  tj¡}t|||||d!dd"d# t|||||d!dd$d# W n tyC } ztd%|› d&|› d$d' W Y d}~nd}~ww t |d(|› d)tjddd¡ t |d(|› d)tjdd*d¡ | |¡ q!tj|dd+}tj ||› d,|d › d-S ).a.  Horizontal strip: one tile per timestep with heatmap + GT/pred annotations.
    Pulls per-sample T_cam_world (= 'world_to_camera') and K (= cam_K_norm
    scaled by image_size) from the batch dict so the strip works correctly
    for samples drawn from any dataset (e.g. mixed smith300 + UMI training).NÚworld_to_cameraÚ
cam_K_normÚrgb_frames_rawr   rR   r   r®   Úpred_heatmapr   ).r   gš™™™™™á?gÍÌÌÌÌÌÜ?r¯   é   Úpredr°   gffffffÖ?Útrajectory_3drd   )r®   r®   r®   r   Zeefg        )r   r®   r®   )r®   r®   r   zh=z.3fé   g333333Ó?Úgt_eulerÚ
pred_eulerr­   F)r·   r¸   r¹   Tzaxes draw failed t=ú: ©Úflushzt=)r°   é   )é   rØ   rØ   re   z: t=0..)Úcaption)!ÚcpuÚnumpyr   r   r   rƒ   rª   rv   r|   Úuint8Údetachr~   rT   Ú
zeros_likeÚclipÚunravel_indexrM   r³   Ú
drawMarkerÚMARKER_CROSSÚputTextÚFONT_HERSHEY_SIMPLEXr   r    Úcircler´   rÉ   Ú	ExceptionrŠ   r   ÚconcatenateÚwandbÚImage)ÚsampleZ
split_nameZtilesr¶   r¢   rË   Ún_windowr.   Úframer)   r*   ÚvisZheatZheat_rgbZpred_yZpred_xr^   r¦   Úptr§   r¨   Z
ground_posZgpixZgptZugZvgZeef_3drÒ   rÓ   ÚexcÚstripr   r   r   Úbuild_wandb_strip  s   
0ÿ
ÿ.
8
ÿ.
8"
ÿÿ
ÿ$€ÿrñ   c              
   C   sˆ  |   ¡  t ¡ ² |d dd…  |¡}|d dd…  |¡}|dd…ddd…f }tt }|| }| ¡  dtd ¡}| |||d\}	}
}}
t|ƒd }g }t	|	j
d ƒ}t|ƒD ]2}|	d|f }tj| d¡dd |j
¡}|jddd }tj|d	 ttfd
ddd }| |¡ qZ|d d |d d |d d |d d |t |¡|d d |d d dœW  d  ƒ S 1 s½w   Y  dS )zBRun model on first sample in batch, return dict for visualization.Úrgbr   r   r%   N©Úquery_pixelsr   rJ   )NNÚbilinearF)ÚsizeÚmodeÚalign_corners)r   r   rÌ   rÐ   Útrajectory_eulerrÊ   rË   )rÌ   rÐ   r%   rÒ   rÓ   rÍ   rÊ   rË   )Úevalr"   Úno_gradÚtoÚ	PRED_SIZEÚ
IMAGE_SIZEr   r   rQ   r   r   r   r    Úsoftmaxr   rT   Úinterpolater   r#   )r
   Úbatchr;   rò   Útraj_2dÚstart_kpÚscaleÚtraj_2d_scaledrô   rV   rF   rO   rÓ   Zpred_heatmapsrë   r.   rX   Z	vol_probsZhmZhm_upr   r   r   Úbuild_vis_sampleq  sD   
ÿÿÿ





÷$ær  c            B         sx	  t  ¡ } | jddddd | jddd | jd	d
dd | jdtdd | jdtdd | jdtdd | jdtdd | jddd | jddd | jdd
d | jdtdd | jdttdd  | jd!td"d#d  | jd$td | jd%ttd&t› d'd  |  ¡ }t	 
t	j ¡ rŒd(nd)¡‰td*ˆ› dd+ t|jƒ}g }|D ]V}d }d,D ]}tj ||¡}tj |¡rº|} nq§|d u rÆtd-|› ƒ‚t|ƒ}t |¡}	W d   ƒ n1 sÚw   Y  | ||	d. f¡ td/|› d0t|	d. ƒ› d1ƒ q¡t||jd2}
|
d3 t_|
d4 t_|
d5 t_|
d6 t_ |
d7 t_!|
d8 t_"d9d9|
d3 gt_#dd|
d4 gt_$t%d:|j&› ƒ}|j'ddd; t|d< d=ƒ}tj(|
|d>d? W d   ƒ n	1 sWw   Y  d9d@l)m*}m+} g g }}|D ]v\}}t,||j-|j|j. |j/dA}t,||j-|jdB|j/dA}t|ƒ}t0dt||j1 ƒƒ}|| }dCt2t3|ƒƒdD  }t	 4¡  5|¡}t	j6||dE 7¡ }| |||d |… ƒ¡ | ||||d … ƒ¡ tdFt%|ƒj8› dG|› dH|› dI|› ƒ qkt|ƒdkrí||ƒn|d9 }t|ƒdkrü||ƒn|d9 }tdJt|ƒ› dHt|ƒ› dK|j.rdLndM› dd+ t9||j:dd9dddN}t9||j:dBd9ddO‰t;ˆƒg‰t<|j/dP =ˆ¡‰|j>rRˆj? @¡ D ]}dB|_AqEtdQdd+ tBjCtDdRdS„ ˆ @¡ ƒ|jEddT}tBjFjG||jHdU}tIjJ|jK|j&|jLi tM|ƒ¥|
¥tNtOtPdVœ¥dW tdXƒ}d9‰‡‡fdYdZ„‰ ‡ ‡‡‡fd[d\„} tQ|jHƒD ]}!ˆ R¡  g g g g f\}"}#}$}%|D ]¿}&|&d]  =ˆ¡}'|&d^  =ˆ¡}(|&d_  =ˆ¡})|&d`  =ˆ¡}*|&da  =ˆ¡}+tStT },|(|, }-tU|)d d …d d …d>f ƒ}.|(d d …d9d d …f }/|- V¡  Wd9tSd ¡}0ˆ|'|/|0db\}1}2}3}tX|1|-|.ƒ}4tY|2|*ƒ}5tZ|3|+ƒ}6tN|4 tO|5  tP|6  }7| [¡  |7 \¡  t	j]j^ _ˆ @¡ dc¡ | `¡  |" |7 a¡ ¡ |# |4 a¡ ¡ |$ |5 a¡ ¡ |% |6 a¡ ¡ ˆd7 ‰|jbd9kroˆ|jb d9kro| |&ƒ q±| `¡  tc d|"¡}8ˆ e¡  g g }9}:t	 f¡   ˆD ]•}&|&d]  =ˆ¡}'|&d^  =ˆ¡}(|&d_  =ˆ¡})|&d`  =ˆ¡}*|&da  =ˆ¡}+tStT },|(|, }-tU|)d d …d d …d>f ƒ}.|(d d …d9d d …f }/|- V¡  Wd9tSd ¡}0ˆ|'|/|0db\}1}2}3}tX|1|-|.ƒ}4tY|2|*ƒ}5tZ|3|+ƒ}6tN|4 tO|5  tP|6  };|9 |; a¡ ¡ tg|1ƒ\}<}|<|- jhddde d¡  a¡ }=|: |=¡ qŠW d   ƒ n	1 s+w   Y  tc d|9¡}>tc d|:¡}?|!|8tc d|#¡tc d|$¡tc d|%¡|>|?| i¡ d9 dfœ}@tI j|@¡ |!dg d9krutdh|!di›dj|8dk›dl|>dk›dm|?dn›do	dd+ ˆ k¡ | k¡ |!dpœ|
¥}A|>|k r–|>}t	 li |A¥dq|>i¥|dr ¡ |!ds d9kr¥t	 l|A|dt ¡ qžtI m¡  tdu|dk›ƒ tdv|› ƒ d S )wNz
--data_dirTú+z™One or more dataset dirs (each with joints.npz, meta.json, rgb_overlay/episodes.json, rgb_NNNNNN.jpg). Multiple dirs are concatenated for joint training.)ÚrequiredÚnargsÚhelpz
--run_nameZsmith300_para_test)Údefaultz--no_color_augÚ
store_truezYDisable color-channel-permutation + HSV/brightness augmentation on the model input frame.)Úactionr
  z--epochsiô  )Útyper  z--batch_sizeé   z--lrg-Cëâ6?z--frame_strider   z--wandb_modeZonlinez--wandb_projectZpara_smith300z--freeze_backbone)r  z--val_splitg333333Ã?z--vis_everyzo(legacy) Epoch-level vis cadence. Kept for back-compat; step-level vis (--vis_every_steps) is the primary path.)r  r  r
  z--vis_every_stepséd   zøLog a fresh wandb strip every N optimization steps. Each call uses the CURRENT train batch (different samples each time) plus a fresh val batch, instead of the same cached pair, so the strip cycles through the dataset and reflects recent gradients.z--mujoco_xmlz
--n_windowzRPrediction window length (number of future timesteps the model predicts). Default zc. Affects volume_head depth, MLP head outputs, and dataset window slicing -- threaded through both.ÚcudarÚ   zDevice: rÕ   )zrgb_overlay/episodes.jsonzepisodes.jsonzno episodes.json under r‘   z
  dataset rÔ   z	 episodes)r‹   rf   rg   rh   ri   rj   rk   r   zcheckpoints/)ÚparentsÚexist_okzdataset_stats.jsonr½   rR   )Úindent)ÚSubsetÚConcatDataset)Úframe_strider‹   Úaugment_colorrë   Fé*   i'  )Ú	generatorz  z: train=z, val=z, total=zCombined: train=z, color_aug=ÚOFFZON)Ú
batch_sizeÚshuffleÚnum_workersÚ
pin_memoryÚ	drop_last)r  r  r  r  )rë   zBackbone frozenc                 S   s   | j S )N)Úrequires_grad)Úpr   r   r   Ú<lambda>  s    zmain.<locals>.<lambda>)ÚlrÚweight_decay)ÚT_max)ÚVOLUME_LOSS_WEIGHTÚGRIPPER_LOSS_WEIGHTÚROTATION_LOSS_WEIGHT)ÚprojectÚnamer÷   ÚconfigÚinfc                      s:   zt ˆ d ƒW S  ty   tˆƒˆ d< t ˆ d ƒ Y S w )Nr   )ÚnextÚStopIterationÚiterr   )Úval_iter_for_visÚ
val_loaderr   r   Ú_next_val_batch*  s   þzmain.<locals>._next_val_batchc              
      sÊ   z_z2t ˆ| ˆƒ}t ˆˆ ƒ ˆƒ}t|dƒ}t|dƒ}dˆi}|dur%||d< |dur-||d< t |¡ W n tyQ } ztdˆ› d|› d	d
 W Y d}~nd}~ww W ˆ ¡  dS W ˆ ¡  dS ˆ ¡  w )zBuild + log a wandb strip from the current train batch and a fresh
        val batch. Cycles through samples over time. Restores model.train().ÚtrainÚvalÚstepNzvis/train_stripzvis/val_stripzstep-vis err at step=rÔ   TrÕ   )r  rñ   rè   Úlogræ   rŠ   r4  )Zcurrent_train_batchZtrain_sampleZ
val_sampleZtrain_stripZ	val_stripZlog_payloadÚe)r3  r;   Úglobal_stepr
   r   r   Ú_maybe_log_step_vis1  s&   

$€ÿÿþz!main.<locals>._maybe_log_step_visrò   r%   rÐ   Útrajectory_gripperrù   ró   r   r   rJ   )Úepochz
train/lossztrain/vol_lossztrain/grip_lossztrain/rot_losszval/losszval/pixel_errorr$  r   zEpoch Z4dz	 | train=rl   z | val=z
 | px_err=z.1fr+   )Zmodel_state_dictZoptimizer_state_dictr<  Úval_losszbest.pthé2   z
latest.pthzDone! Best val loss: zCheckpoints: )nÚargparseÚArgumentParserÚadd_argumentr   r0   ÚVIS_EVERY_EPOCHSr   r   Ú
parse_argsr"   r;   r  Úis_availablerŠ   Úlistr   rx   ry   rz   ÚexistsÚFileNotFoundErrorÚopenÚjsonrw   r   r‰   rŸ   r‹   r   r   r   r1   r2   r>   r@   ZMIN_POSZMAX_POSr   Zrun_nameÚmkdirÚdumpÚtorch.utils.datar  r  r   r  Zno_color_augrë   rT   Z	val_splitÚabsÚhashÚ	GeneratorÚmanual_seedÚrandpermrˆ   r+  r   r  r0  ÚTrajectoryHeatmapPredictorrü   Zfreeze_backboneZdinoÚ
parametersr!  ÚoptimÚAdamWÚfilterr$  Úlr_schedulerÚCosineAnnealingLRÚepochsrè   ÚinitZwandb_projectZ
wandb_modeÚvarsr'  r(  r)  r   r4  rý   rþ   r   r   r   r/   r9   rI   Ú	zero_gradÚbackwardÚnnÚutilsÚclip_grad_norm_r6  ÚitemZvis_every_stepsrv   r$   rú   rû   rY   ÚnormÚget_last_lrr7  Ú
state_dictÚsaveÚfinish)Br"  ÚargsZ	data_dirsZdirs_with_episodesZddirZepi_pathÚcandÚcpÚfÚep_datar   Zckpt_dirr  r  Ztrain_subsetsZval_subsetsrF   Zds_augZds_cleanZn_totalZn_valZn_trainÚseedÚgenÚidxsZtrain_dsZval_dsZtrain_loaderÚparamÚ	optimizerÚ	schedulerZbest_val_lossr:  r<  Ztrain_lossesZtrain_vol_lossesZtrain_grip_lossesZtrain_rot_lossesr  rò   r  Ztraj_3dZ	traj_gripZ
traj_eulerr  r  Zheight_binsr  rô   rV   Zgripper_logitsrO   Zvol_lossZ	grip_lossZrot_lossÚlossZ
train_lossZ
val_lossesZval_pixel_errsZv_lossrW   Úerrr=  Z
val_px_errZlog_dictZ	ckpt_datar   )r3  r;   r9  r
   r1  r2  r   Úmain  sÊ  
ÿÿ
ÿ
ÿ
ÿÿ
þ
ÿ ÿ





ÿ
üü*ÿÿ
ÿ
ÿ

ÿýþÿ

ÿþ€


ÿ

ÿþæÿ


ø
ÿÿÿýý
€rt  Ú__main__)r­   rR   F)DÚ__doc__Úsysrx   ry   ÚinsertÚdirnameÚ__file__r"   Ztorch.nn.functionalr^  Ú
functionalr    Ztorch.optimrT  rL  r   r   rÛ   rv   Úpathlibr   r?  rè   rI  r³   Údata_smith300_parar   r   r   r   r	   Úimportlib.utilÚ	importlibZ
_local_dirÚutilÚspec_from_file_locationrz   Z_specÚmodule_from_specr   ÚloaderÚexec_modulerR  r   r3   rA   rý   rþ   r'  r(  r)  rB  r   r/   r5   r9   rD   rI   rQ   rY   rŸ   r©   r¬   rÉ   rñ   r  rt  Ú__name__r   r   r   r   Ú<module>   sh    ÿ
F	
ÿ&V,  "
ÿ