*i hdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZddlmZddlmZmZmZmZmZddlZddlZddlmZejGddZd ejd ejfd Zd ejd ejd ejfdZdejd ejfdZ GddZ!ded eee"effdZ#dZ$dede"de%deee%e%fd ee&e'e"ff dZ(dede"de%d ee'fdZ)de%de%d e%fdZ*dZ+deee"efd e%d!ed"ee"d ee"ejff d#Z,d$ee"e"fd%e"d e"fd&Z-d'eejd e%d(e.d e/fd)Z0Gd*d+Z1d,e"d-e"d e%fd.Z2d,e"d-e"d e%fd/Z3d,e"d-e"d e"fd0Z4d,e"d-e"d e"fd1Z5d2ed3e"d4e"d dfd5Z6d6ed!ed7e"d8eed df d9Z7d!ed8eed ee"fd:Z8dAde&d ee"effd?Z: dBd8eed!ed3ee"d4ee"d df d@Z;dS)CuExport converted Raiden episodes to WebDataset sharded .tar format. Each sample in a shard contains: {uuid}.{cam}_t{idx}.jpg — camera images at specified time indices (JPEG quality 95) {uuid}.lowdim.npz — windowed lowdim arrays (T × D each key) {uuid}.metadata.json — episode / sample metadata {uuid}.language_instructions.json — language annotations Alongside the shards the following files are written: preprocessing_config.yaml — full config snapshot manifest.jsonl — one JSON line per shard: {"shard": ..., "num_sequences": N} stats.json — per-key statistics (mean/std/min/max + percentiles) processing_metadata.json N)ThreadPoolExecutor)Path)AnyDictListOptionalTuple)tqdmceZdZUdZeed<dZeed<dZeed<dZ eed<e j d Z e eed <dZeed <d Zeed<dZeed<dZee eed<e j e Zeeefed<dZeeeefed<dZeed<dZeed<dZeed<dZeed<dZeed<dZ eed<d Z!eed!<d"Z"eed#<d$Z#eed%<d&Z$eed'<dS)(ShardifyConfigz2Parameters controlling the shardification process. output_dirpast_lowdim_stepsfuture_lowdim_stepsstridec ddgS)Nrr-/home/robot-lab/raiden_cmu/raiden/shardify.pyzShardifyConfig.<s "ar)default_factory image_indicesmax_padding_leftmax_padding_rightcopypadding_strategyN camera_namescamera_name_mapresize_images_size_ jpeg_qualityF use_depthfilter_still_samplesg?still_thresholdT fail_on_nanrmax_episodes_to_processdsamples_per_shard num_workersPstats_reservoir_size stats_stride)%__name__ __module__ __qualname____doc__r__annotations__rintrr dataclassesfieldrrrrr strr!rdictr"rr#r r%r&boolr'r(floatr)r*r,r.r0r2rrrr r )s<<s!!!! FCOOO 1{0QQQM49QQQcs"c""")-L(49%,,,&7k&7&M&M&MOT#s(^MMM59sCx1888L#It"'$&&&!OU!!!K#%S%%%!s   K!'#&&&L#rr vreturnc|ddtj|ddtj}}|tj|z }|tj|||zz }|tj|z }tj||}tj|||gdS)u(Convert a (6,) rot6d vector to a (3, 3) rotation matrix via Gram-Schmidt. Inverse of ``_rot9_to_rot6d``: reconstructs the full rotation matrix from its first two rows. Args: v: (6,) float array — [R[0,:], R[1,:]]. Returns: (3, 3) float64 rotation matrix. Nrraxis)astypenpfloat64linalgnormdotcrossstack)r?a1a2b1b2b3s r _rot6d_to_matrRdsrrU\\"* % %q1v}}RZ'@'@B binnR  B bfRnnr! !B binnR  B "b  B 8RRLq ) ) ))rxyzrot6dctjdtj}t||ddddf<|tj|dddf<|S)uFBuild a 4×4 rigid-body transform from a (3,) position and (6,) rot6d.dtypeNr)rFeyerGrRrE)rSrTTs r_build_transformr[xsY q ###Ae$$Abqb"1"fIzz"*%%Abqb!eH Hrrot9c||jdddz}|dddddf|jdddzS)ulConvert a row-major flattened 3×3 rotation matrix to the 6D representation. Uses the first two rows of R: [R[0,:], R[1,:]] = [R00,R01,R02, R10,R11,R12] → (6,). Matches vla_foundry's ``matrix_to_rot_6d``: ``R[:2, :].flatten()``. Args: rot9: (..., 9) float array — row-major flattened 3×3. Returns: (..., 6) float array. Nr)rr.)rB)reshapeshape)r\mats r_rot9_to_rot6drbsY ,,tz#2#/ 0 0C sBQBz? " "4:crc?T#9 : ::rc^eZdZdZd dededefdZdejdd fd Zde e e ffd Z d S) _StatsAccumulatorzyIncrementally tracks mean/std/min/max and a reservoir for percentiles. Handles data of shape (T, D) per sample. r/rZDreservoir_sizec||_||_d|_tj||ftj|_tj||ftj|_tj||ftj tj|_ tj||ftj tj|_ tj|||ftj |_ d|_||_dS)NrrW)rZrenrFzerosrG_mean_M2fullinf_min_maxfloat32_res_res_n _res_size)selfrZrerfs r__init__z_StatsAccumulator.__init__sXq!fBJ777 8QF"*555GQFBF"*=== GQFRVG2:>>> Hna32:FFF  'rsampler@Nc|tj}|xjdz c_||jz }|xj||jz z c_|xj|||jz zz c_tj|j||_tj|j ||_ |j |j kr>|tj |j |j <|xj dz c_ dSttjd|j}||j kr)|tj |j |<dSdS)zAdd one (T, D) sample.rrN)rErFrGrhrjrkminimumrnmaximumrorrrsrprqr8randomrandint)rtrvxdeltajs rupdatez_StatsAccumulator.updates MM"* % % ! DJ edfn$  EQ^,,Jty!,, Jty!,, ; ' '%+]]2:%>%>DIdk " KK1 KKKKBI%%a0011A4>!!%}}RZ88 ! "!rc |jdkriStj|jt |jdz dz }|jd}|j|z }|j|j|dzzzd}tj|t |j|jzdz dz }|j d}|j d}|j d|j }|d|j} tj| gdd} tj|gdd} dtjd t$f fd id  |d  |d  |d |d |jd |d |j d |j d | dd | dd | dd | dd | dd | dd | dd | dd | d | d | d | d|jt'|j dS) zDReturn a stats dict compatible with the reference stats.json format.rrrCr^Nr)rr^r$bcarrr@c^|jdkr|Sfd|DS)Nrc&g|] }|Srr).0row_to_lists r z@_StatsAccumulator.finalize.._to_list..s!111cHHSMM111r)ndimtolist)rrs rrz,_StatsAccumulator.finalize.._to_lists6x1}}zz||#1111S111 1rmeanstdminmaxmean_per_timestepstd_per_timestepmin_per_timestepmax_per_timestep percentile_1 percentile_2 percentile_5 percentile_95r percentile_98rV percentile_99rpercentile_1_per_timesteppercentile_2_per_timesteppercentile_5_per_timestep)percentile_95_per_timesteppercentile_98_per_timesteppercentile_99_per_timestepcountpercentile_sample_count)rhrFsqrtrkrrjrsumrZrnrrorqrrr_re percentilendarrayrr8) rt std_per_ts global_meanr} M2_combined global_std global_min global_maxresflat pcts_global pcts_per_tsrs @rfinalizez_StatsAccumulator.finalizesf 6Q;;IWTXDFQJ(:(::;; joo1o--  [(x$&5!8"3388a8@@ W[3tv/BA+F+FFGG Y]]]** Y]]]** i $+ &{{2tv&&mD*?*?*?aHHH mC)>)>)>QGGG  2"* 2 2 2 2 2 2 2  HH[)) 88J''  88J''  88J''  $*!5!5   4 4    3 3   3 3  HH[^44  HH[^44  HH[^44  XXk!n55  XXk!n55  XXk!n55  (+a.)A)A (+a.)A)A! " (+a.)A)A# $+3(;q>*B*B*2(;q>*B*B*2(;q>*B*BV'*4;'7'7-    r)r/) r3r4r5r6r8rurFrrrr;rrrrrrdrds ( (# (# (s ( ( ( (9RZ9D9999$4 $sCx.4 4 4 4 4 4 rrdep_dircp|dz }t|d}|s"t|d}|std|g}|D]Q}t|d5}|t j|dddn #1swxYwYR|S)z>Load all per-frame lowdim pkl files from an episode directory.lowdim??????????.pklz ?????????.pklzNo lowdim .pkl files in rbN)sortedglobFileNotFoundErroropenappendpickleload)r lowdim_dir pkl_filesframespfs r_load_episode_framesrs("Jz'78899I =:???;;<< I G: G GHHH F ** !T]] *a MM&+a.. ) ) ) * * * * * * * * * * * * * * * Ms6(B**B. 1B. r$ camera_name frame_idxresizec`|dz |z |ddz }|sdStjt|}|dS|(|\}}tj|||ftj}tjd|tjtg\}} t| dfS)u~Return (image_bytes, "jpg") for a frame, optionally resizing. The converter saves frames with ``cv2.imwrite`` in BGR order. JPEG encoding via cv2 converts BGR→YCbCr internally, so PIL (used by WebDataset dataloaders) decodes the JPEG back as correct RGB — no manual channel flip needed. Returns: (bytes, "jpg"), or None if the file is absent. rgb010d.pngN) interpolationz.jpgjpg) existscv2imreadr;rINTER_LANCZOS4imencodeIMWRITE_JPEG_QUALITY _JPEG_QUALITYbytes) rrrrpathimg_bgrh_outw_out_bufs r _load_rgbr s E>K 'Y*A*A*A*A AD ;;==tjT##Gt  u*WuenCDVWWW \&'C,Dm+T U UFAs ::u rc|dz |z |ddz }|sdStj|d}tjd|\}}t |S)zReturn 16-bit PNG bytes for a depth frame, or None if absent. Loads the uint16 mm depth stored by the converter as a ``.npz`` and re-encodes it as a 16-bit greyscale PNG (lossless, widely supported). depthrz.npzNr)rrFrrrr)rrrrdepth_mmrrs r_load_depth_pngr&sl G k )y,C,C,C,C CD ;;==twt}}W%H \&( + +FAs ::ridxn_framescDtdt||dz S)Nrr)rr)rrs r _clamp_framer9s q#c8a<(( ) ))ryamr anchor_idxconfigoutput_cam_namesc  *+,|jdz|jz}t*t}|j+*+fdt |j |jdzD,dt dtdtdttj f,fd }i}|dd d }||d d d d f|d |d<t|d d d df|d |d<|d d ddf|d|d<|j dd krL|d d ddf|d|d<t|d d ddf|d|d<|d d dd f|d|d<|dd d } | | d d d d f|d|d<t| d d d df|d|d<| d d ddf|d|d<| j dd krL| d d ddf|d|d<t| d d ddf|d|d<| d d dd f|d|d<|dd d} | 9| d d d df|d |<| j ddkr| d d ddf|d!|<|d"d d} | 9| d d d df|d#|<| j ddkr| d d ddf|d$|<} *fd%|j D} |D]9}t|j|}| d&i|}|Qtj|tj'}tj|d t| ddf|d(|<g}| D]}|d)i|}| tjd*tj'}|tj|tj'tj||d+|<;|j}d,D]}d-|d.|d}d-|d.|d}||vr|||}|||}tjt5||}d/D]C}|dkr%d0|d.|d}d0|d.|d}d0|d.|d1}d0|d.|d2} n$d-|d.|d}d-|d.|d}d-|d.|d1}d-|d.|d2} ||vrW||}!||}"tj|!}#tj|"}$t |!j d D]}%t5|!|%|"|%}&||&z}'|'d d d ftj|#|%<|'d d3d d ftj|$|%<|#||<|$|| <Etj|t>'}(d4|(d |j<tj|t>'})d4|)|jdzd <|(|d5<|)|d6<|S)7z2Build windowed (T, D) arrays for one anchor frame.rc<g|]}t|zzSrr)roffsetrrhss rrz(_build_window_arrays..Ls<  Z&1*,a00rr:startlengthr@c g}D]`}||}|dS|tj||||ztjatj|S)NrW)getrrFasarrayrprL)r:rrrowsfivalr window_idxs r_collectz&_build_window_arrays.._collectQs| S SB*..''C{tt KK 3uuv~'=#>bjQQQ R R R Rx~~ractionrNrzrobot__action__poses__left::__xyz __rot_6d zrobot__action__grippers__left::_handzrobot__action__poses__right::z robot__action__grippers__right:: actual_poseszrobot__actual__poses__left::zrobot__actual__grippers__left::zrobot__actual__poses__right::z robot__actual__grippers__right::jointsz%robot__actual__joint_position__left::z&robot__actual__joint_position__right:: action_jointsz&robot__desired__joint_position__left::z'robot__desired__joint_position__right::c6g|]}t|zSrr)rirrhs rrz(_build_window_arrays..s'WWWQj1na88WWWr intrinsicsrW intrinsics. extrinsicsrV extrinsics.)leftrightrobot__actual__poses__z::)ractualrobot__action__poses____xyz_relative__rot_6d_relativer^T past_mask future_mask) rrlen_ROBOTrranger;r8rrFrrbr`r _reverse_mapr"rrrptilerYrrGrLrHinvr[ empty_likerEflattenrir=)-rrrrrZRrout action_seqactual_poses_seq joints_seqact_joints_seq anchor_frameimg_frame_indicescam_namesrc_camKK_arrext_rowsrextanchor_iside anc_xyz_key anc_rot_keyanc_xyz anc_rot6d T_anc_invsrcxyz_keyrot_keyout_xyzout_rotxyz_seq rot6d_seqrel_xyz rel_rot6drT_tT_relrrrhrrs-`` @@@r_build_window_arraysr<@so  1$v'AAA F AA AV55v7QTU7UVVJ C"*9M"$C(Ar**J7A!!!QqS&7I 31 3 3 34:H qqq!B$w ; ;  61 6 6 67;EQQQ2X:N 6a 6 6 67  A " $ $ *%LWWWWW&BVWWW$;;v5x@@   \2 . . 2 27 ; ; =Jq 333E,.Gd c"344a;--C(h(( )# ? ?B*..r2266w??C{fQbj111 OOBJs"*=== > > > >(*(:(: $( $ $%%'H!%%%%?t??q??? BtBBqBBB c ! ! k"8, $X. IMM"27I"F"FGG ' % %ChC4CC1CCCF4FF1FFFL4LL1LLLO4OO1OOOC4CC1CCCF4FF1FFFL4LL1LLLO4OO1OOOc!!'lGG ImG,,G i00I7=+,, J J&wqz9Q<@@!C"2A2q5\00<< $RaR!V}4466==bjII ! "CL$CLL7 %<$'''I,0I((()(1D)))K26K(1,../ C $C  Jrr"out_namecN|D]\}}||kr|cS|S)z=Return the source camera name for a given output camera name.)items)r"r=r1dsts rrrs=#))++S (??JJJ  Orr thresholdc |dS||dzd}t|dkrdS||ddf}||ddf}tttj|ddddf|z dttj|ddddf|z d}||kS) zGReturn True if the future EE trajectory barely moves (below threshold).NFrrrrrrC)rrr>rFrHrI)rrrAfuture anchor_xyz_l anchor_xyz_rmax_moves r _is_stillrGsu  Q(( )F 6{{auj!A#o.Lj"R%/0L binnVAAAqsF^l:nCCGGIIJJ binnVAAAr"uH- <1nEEIIKKLLH i rcjeZdZdZdedefdZdeee fddfdZ d d Z d d Z de efd ZdS) _ShardWriterz*Writes samples into sequential tar shards. shard_dirr,cL||_||_d|_g|_g|_dS)Nr) _shard_dir_sps _shard_idx_buf _shard_counts)rtrJr,s rruz_ShardWriter.__init__s,#% ,. (*rfilesr@Nc|j|t|j|jkr|dSdSN)rOrrrM_flush)rtrQs raddz_ShardWriter.addsF  ty>>TY & & KKMMMMM ' &rc@|jr|dSdSrS)rOrTrts rclosez_ShardWriter.closes' 9  KKMMMMM  rc d|jdd}|j|z }tj|d5}|jD]m}|D]V\}}tj|}t||_| |tj |Wn dddn #1swxYwY|j t|j|xjdz c_g|_dS)Nshard_06dz.tarw)namer)rNrLtarfilerrOr?TarInforsizeaddfileioBytesIOrPr)rtr]rtfrvfnamedatainfos rrTz_ShardWriter._flush s911111% \$ $ $ 7) 7 7#)<<>>77KE4"?666D #D DIJJtRZ%5%566667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 !!#di..111 1 sA6B//B36B3c>dt|jDS)NcJg|] \}}tjd|d|d!S)rZr[)shard num_sequences)jsondumps)rrrhs rrz/_ShardWriter.manifest_lines..sJ   1 J!1!!1!1!1AFF G G   r) enumeraterPrWs rmanifest_linesz_ShardWriter.manifest_liness.  !$"455    r)r@N)r3r4r5r6rr8rurr;rrUrXrTrrorrrrIrIs44+$+3++++c5j)d      S       rrIbucketprefixc|ddz}|||d}|ddS)zEReturn the number of objects under s3://bucket/prefix/, or 0 if none./r)BucketPrefixMaxKeysKeyCountr)rstriplist_objects_v2r) s3_clientrprqprefix_with_slashresps r_s3_prefix_existsr}%sI c**S0  $ $F;LVW $ X XD 88J " ""rc2|ddz}d}|d}|||D]O}d|dgD}|r+||d|i|t |z }P|S) zDDelete all objects under s3://bucket/prefix/. Returns count deleted.rsrryrtruc"g|] }d|di S)Keyr)ros rrz%_s3_delete_prefix..2s!GGGE1U8$GGGrContentsObjects)rtDelete)rx get_paginatorpaginaterdelete_objectsr)rzrprqr{deleted paginatorpageobjectss r_s3_delete_prefixr,s c**S0G''(9::I""&9J"KK$$GGdhhz2.F.FGGG  $  $ $FIw;O $ P P P s7|| #G Nrc 4ddl}|j|jjd}|dd|}|ddz}|dz}|d}d} |||D]]} | dgD]D} | d } || t|dz} | ||| d | | d z } E^td | d|d|d|S)zZCopy all objects under prefix/ to prefix_backup_/ and return the backup prefix.rNz %Y%m%d_%H%M%Srs_backup_ryrrr)rtr)rt CopySourcerrz Backed up z object(s) to s3://) datetimenowtimezoneutcstrftimerxrrrr copy_objectprint)rzrprqrts backup_prefixr{backup_with_slashrcopiedrobjsrc_keydst_keys r_s3_backup_prefixr9shOOO    x04 5 5 > > O OB}}S))77277M c**S0%+''(9::I F""&9J"KK  88J++  C%jG''#6G2H2H2J2J*KKG  ! !&,W== "    aKFF  M M MF M M] M M MNNN rctdtd|d|dttdtdtdt td}|d vr d d d d |Std S)zPrompt the user what to do when the S3 destination already has data. Returns one of: ``"delete"``, ``"backup"``, ``"cancel"``. z& S3 destination already contains data:z s3://rsz* [d] Delete existing shards and overwritez, [b] Back up existing shards then overwritez [c] CancelTzChoice [d/b/c]: )dbcdeletebackupcancelz Please enter d, b, or c.)rinputstriplower)rprqchoices r_prompt_s3_overwriterQs  3444 &F & &V & & &''' GGG 6777 8999 . GGG,)**002288:: _ $ $!x@@H H *+++ ,r local_dir s3_bucket s3_prefixc ddl}|d}t|d}d|D}t dt |d|d|d|D]R}|d||}|t|||t d |St d dS) z;Upload all files in local_dir to s3://s3_bucket/s3_prefix/.rNs3*c:g|]}||Sr)is_file)rrs rrz upload_to_s3..js% - - -1 -Q - - -rz Uploading z files to s3://rs zUpload complete.) boto3clientrrglobrr relative_to upload_filer;)rrrrrrQrkeys r upload_to_s3rdsLLL d  B 9??3'' ( (E - - - - -E Js5zz J J) J Ji J J JKKK 77Q]]95577 s1vvy#... j3jj rrJoutput_dir_str episode_dirscddl}t||}id|ddd|jd|jd|jd t |jd d d |jd |jd|j d|j d|d|j d|j d|j rt |j ndd|jd|j|jdd}t#|dz d5}|||ddddddS#1swxYwYdS)z5Write preprocessing_config.yaml alongside the shards.rNr!compute_statisticsTr)r'rr image_formatrr*rrr.r r rr#r,r()rvalidation_episodes_pathzpreprocessing_config.yamlr\F)default_flow_style sort_keys)yaml_resolve_output_cam_namesr)r'rlistrr*rrr.r rr#r,r(rrdump)rJrrrrrcfgrs r_write_preprocessing_configrxsKKK0FF ( d  v)   ;   v9   f233     "6#A  F3  V5  v)  n  F3  V5    $d6#<=== # $ V5% & 61' (-$(+   C. i55s ; ;Dq #qUd CCCDDDDDDDDDDDDDDDDDDs C22C69C6cjtjS|D]}t|dz d}|s*t |dd5}t j|}dddn #1swxYwYt|di}fd|DcSgS)z/Return the ordered list of output camera names.Nrrrrr cFg|]}j||Sr)r"r)rrrs rrz-_resolve_output_cam_names..s,CCCQ&**1a00CCCr) r!rrrrrrrkeys)rrrrrframesrc_camss` rrrs&F'(((DDFX-334DEEFF    )A, % % #KNNE # # # # # # # # # # # # # # # ,3388::;;CCCC(CCCCCC IsBB B rfdata_dirc ddl}ddlm}t|dz }t d|Ddd}|s"t d ||jd d }d |D |gt z}||d dd}||vr|n fd|D}g} |D]D} t d| D} | | | fE| S)zUse fzf to select one or more converted tasks. Returns a list of ``(task_dir, episode_dirs)`` tuples for each selected task. rN) fzf_select processedc3K|]E}|td|DA|VFdS)c3lK|]/}||dz V0dS metadata.jsonNis_dirrreps r z2select_processed_task...sT46o%--//rN)ranyiterdirrrs rrz(select_processed_task..s|  xxzz :;))++        rc4|jSrS)statst_mtime)rs rrz'select_processed_task..saffhh'rT)rreversezNo converted tasks found in rz*** ALL TASKS ***c |i|]9}|jdtd|Dd|:S) (c3pK|]1}||dz -dV2dS)rrNrrs rrz3select_processed_task...sEeeRYY[[eb?FZEbEbEdEde!eeeeeerz episode(s)))r]rrrs r z)select_processed_task..s]  6ssceeaiikkeeeeesssuvrzShardify task(s)> z[Tab: toggle select | Enter: confirm | Select '*** ALL TASKS ***' to shardify everything)promptmultiheaderc g|] }| Srr)rrlabelss rrz)select_processed_task..s;X;X;X!F1I;X;X;Xrc3pK|]1}||dz -|V2dSrrrs rrz(select_processed_task..s_  yy{{ !#_ 4<<>>       r) sys raiden.utilsrrrrrexitrr) rrrbase task_dirs _ALL_LABELchoicesselected chosen_dirsresulttask_dirrrs @rselect_processed_taskrs JJJ'''''' >>K 'D  \\^^    ( '   I  3T33444 $JFlT&\\)Gz#l H *X55));X;X;X;Xx;X;X;XK F00  &&((      x.//// Mrargsc"#$|\}$}}|d"|d#|d}|d}|d}|d}|d}|d} |j} |j| z} |j| z} td td | $z  | z } td td | #d z $z z  | z }| |jks ||jkrd d iS|jr]tj"#$fd t|j |jd zD}t||j|j rd diSt"$||}|j r|D]m\}}t|tjrN|jjdkr>tj|rt+d|d|d$nn/t)d|Drd diSi}t/t1j}|jD]}t7$|z#}d|}t9||D][\}}t;| |||j}||\}}|||d|d|d|<|jr!tA| ||}||||d|d|d<\tCj"}tj#|fi||$||d<#$fd|jD}||d|d$d$|j|td $|jz tK#d z $|jztM| tM||#| d kp|d k|d } tOj(| )||d<d |p|gi}!tOj(|!)||d!<d|||d"S)#uBuild one sample for the shard writer (runs in a worker thread). Args: args: (ctx, t, config, output_cam_names) — packed as a tuple so ThreadPoolExecutor.map can call this as a plain function. Returns: A dict with key ``"filtered"`` set to ``"padding"``, ``"still"``, or ``"nan"`` when the sample is dropped, or ``None`` when it should be written. Kept samples also carry ``"sample_files"``, ``"lowdim"``, ``"episode_id"``, and ``"lang"`` keys. rr src_cam_names episode_id language_tasklanguage_promptcontrolrrrfilteredpaddingc g|]_}tjt|zdtjdtj`S)rrrW)rFrrrrirp)rrrrts rrz!_build_sample..sp      <Ax889==hQS UU*   rstillrz NaN in key 'z ' at episode z frame c3K|]T}t|tjo5|jjdko%tj|VUdS)rN) isinstancerFrrXkindisnanr)rrs rrz _build_sample..+sq   sBJ ' ' $ #% $ !!##      rnanrN.rz .depth.pngz .lowdim.npzc6g|]}t|zSrr)rrrrs rrz!_build_sample..Js' J J Jl1q5(++ J J Jr_t04d) r sample_idanchor_timestepanchor_relative_idximage_timestepslowdim_start_timesteplowdim_end_timestep past_paddingfuture_paddingr!original_episode_length is_paddedrz.metadata.jsonoriginalz.language_instructions.json)r sample_filesrr)*rrrrrrr'rFrLrrGr(r<r)r?rrrXrrr ValueErrorvaluesr;uuiduuid4rrziprr#r&rrbrcsavez_compressedgetvaluerr8rlrmencode)%rctxrrrrrrrrrleft_frames_neededright_frames_neededleft_pad right_padwindow_actionsrrrr sample_uuidimg_idx abs_framesuffixr%out_camrrimg_extrrimg_ts sample_metalangrrrs% @@@r _build_sampler1s(,$CF$ ]F:H(M\"J(M+,O)nG ]F A 1A5 4q81Q 2Q 67771<=>>HA#a!41 q8H!IJJJaOPQQI&)))Y9Q-Q-QI&&" )       --v/IA/M        ^V%=v?U V V )( ("&!V5E F FF'   HC#rz** sy~/D/D8C==$$&&$OsOOOOAOO    }}       ' & &&(Ldjll##K' X X Wh77 W #M3C D D X X GWvw 6;TUUF!% WNQ JJgJJJJJJK X'CC$RWLK!N!N'!N!NF!N!N!NO X *,,C&&v&&&03 LK,,,-K J J J JV5I J J JF #;;j;;A;;;%7!!$QF,D(D!E!E"8a<V5O1OPPH i..(#+\2Y]K48:k3J3J3Q3Q3S3SLK///0 9M: ;D@D  AA fhhK<<<= $   rc@tj}jdz }|dd|r|rddl}|d}t |||rt ||}|dkrtddS|d krAtd t|||td t|||n9|d kr3td t|||} td | dt|} tj | j dkr| dj } t| @|r |rd|d|ntj} t!|| | jdzjz} t'|j} i}d}d}d}d}d}tdt+| dg}i}d}d}t-| dD]/\}}|dz s1td|dt+| d|jd|dz }N t3|}nI#t4$r<}td|dt+| d|jd||dz }Yd}~d}~wwxYw|d}t7|dz 5}t9j|}dddn #1swxYwY|dpg}|dpg} |d}!|||t+|fd@Dt|ddt|dd|j|d d!|| |!d" |s| s|!r|| d#}"|!r|!|"d<|"||j<|t+|z }td|dt+| d$|jd%t+|d& 1|rd'|d(nd}#td)t+|d*|#d'|d+d,|D}$tj |$td-t+|$d.t+|d/j d0i}%@fd1|$D}&tCj"25}'tG|'$tJ|&t+|&d3d4}(|(D]})|)d5d6kr|dz }|)d5d7kr|dz }'|)d5d8kr|dz }9| &|)d9|dz }|)d:}*|%|*ddz|%|*<|dz }|('||z|z| j(;|j)zdkr|)d<}+|+*D]\},}-tW|-tXj-s |-j.t^ks |-j0d=kr<|,1d>s|,1d?rg|-2| d@3tXj4}.|.j5d}/|,|vrtm| |/j7||,<||,8|. dddn #1swxYwYtdAts|%*D]\}0}1tdB|0d|1| :|dCz ;dD<| =dDz|rd}2t}dE|?D}3|dFz }4dG|D}5|*D]t\}0}"|"ds|"ds#|5|0}|;|dFz }6|6@sUtjB|6|4|0z dH|2dz }2ut7|dIz dJ5}7t9jC||7d=Kdddn #1swxYwYtdL|3dM|2dNtdOdP|*D}8|8*D]\},}9|9dQd}1|9dRg}:|9dSg};tW|:tr|:|;kndT}<|dXtjDdYdZ| Dt+| d[tjFtjHtjItjKd\||||t|>dd]d^}?tj|?d_d`<t7|daz dJ5}7t9jC|?|7d=Kdddn #1swxYwYtdb|dc| j(dd||z|zde|df|dg|dh|>didjtdk||r|rt|||dSdSdS)laConvert a list of converted episode directories to sharded WebDataset format. Args: episode_dirs: Paths to episode directories, each containing ``lowdim/``, ``rgb/``, and ``metadata.json``. config: Shardification parameters. s3_bucket: If set, upload the shards directory to this S3 bucket. s3_prefix: S3 key prefix for upload (e.g. ``yam_datasets/task_name``). shardsT)parentsexist_okrNrrz Cancelled.rzBacking up existing shards...zDeleting existing shards...rz Deleted z object(s).zs3://rsrzLoading z episode(s)...rz [z] SKIP z: no metadata.jsonz: event_markersaudio_segments audio_fullc:g|]}tj|Sr)rr")rr,rs rrz run_shardify..s6"""!!7AA"""rrrrleader) rrrrrrrrr6r7r8)r6r7z] rz frames)z, z skippedzLoaded z episodesz frames total cFg|]}t|dD]}||fS)r)r)rr"rs rrz run_shardify..sIs:1G1G,-arz Processing z anchors from z% episodes (globally shuffled, stride=z)...c"g|] \}}||f Srr)rr"rrrs rrz run_shardify..s'KKKQ#q&"23KKKr) max_workersanchor)totalunit dynamic_ncolsrrrr rr)rrjrr^rr rz Samples per episode:rzmanifest.jsonl c3*K|]}|d dVdS)r6rNr)rr?s rrzrun_shardify..0s,XXaQEWXXXXXXXraudioc,i|]}|d|dS)rrr)rr"s rrz run_shardify..4s#PPPSL)3x=PPPr) dirs_exist_okzsubtask_index.jsonr\)indentu ✓ subtask_index.json (z episode(s) with event_markers, z with audio/)z Computing statistics...c>i|]\}}||Sr)r)rraccs rrz run_shardify..Is3 (SS\\^^rrrrFz [!] min==maxz: n=z stats.jsonz1.0z%Y-%m-%dT%H:%M:%Sc,g|]}t|Sr)r;rs rrz run_shardify..\s111SVV111r)r num_episodes)python_versionplatformhostname)total_samples_createdpadding_samples_filteredstill_samples_filterednan_samples_filteredelapsed_seconds)metadata_version created_at source_datar environment processingrr zprocessing_metadata.jsonz Done: u samples → z shards filtered: z (pad=z still=z nan=z ) elapsed: z.0frzOutput: )Ntimer mkdirrrr}rrrrrrzshuffler*rr;rrrrIr,rrnrr]rrrrlrrrrrr.r mapr1rU set_postfixrNr2r?rrFrrXr=r startswithr_rErpr`rdr0rrrX write_textjoinrorrrshutilcopytreerrr9asdictrversionrNsocket gethostnameroundr)Arrrrt_startrJ_boto3_s3rrhepsrrZwriterstats_accumulators total_samplesfiltered_paddingfiltered_still filtered_nan stats_counter ep_contexts subtask_indexskipped total_framesrrrer"_mf_ep_metamarkerssegmentsr8entryskip_msgall_workep_sample_counts work_argsexecutorpbarrrrrr sample_arrreep_idrn_audio_episodesn_marker_episodes audio_root ep_dir_by_id src_audiorstatsrmnmxsameflagelapsed proc_metarsA ` @r run_shardifyrisikkG!H,I OOD4O0003Y3mmD!! S)Y 7 7 3))Y??F!!l###8##5666!#y)<<<3444!#y)<<<<8##3444%c9i@@11111222 |  C N3 %))2F2231==  $" $' ''I''' " # #  6>3GGG 1$v'AAA )V%= > >F79MNLM -SXX - - -... K&(MGLsA&&,K,K 6(0022  LLLCHHLLV[LLL M M M qLG  )&11FF     ???CHH??V[??A?? @ @ @ qLG HHHH ay &?* + + &sy~~H & & & & & & & & & & & & & & &,,//52<< 0117R\\,//   KK""""#3""""%\%5%5or%J%J!K!K#&|'7'78I2'N'N#O#O$k#<< 8<M +! !6#66!;;#H-F$*LLNNCCS)#rz::%$9,,1 $>>-88%CNN)==%%%([[B%7%7%>%>rz%J%J &,Q/&8886G !1f&A77.s3+3/66zBBBBI$ C+C+C+C+C+C+C+C+C+C+C+C+C+C+C+CZ "###/557788%% u #5##E##$$$$ LLNNN!!--dii8M8M8O8O.P.PSW.WXXX XX=+?+?+A+AXXXXX( QPKPPP )//11 " "LE5*+ uyy/F/F !%%e,,F~(I##%%  OIzE'9 N N N N  !   )22C 8 8 2A ImQq 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2  >): > >. > > >    %&&&,>,D,D,F,FE++--++Qgq!! UU5"   UU5"  %b$//:rRxxU#'/R )3))E)4))**** i,& , , %ikkG#G!m$78811S111HH  $V,,!k )++*,,  &3(8&4$0$Wa00   I,),F,=(>(>Ih % i44c : :*a )Qq))))***************  #= # #v/@ # #%6E # #  # #)7 # #>J # #" # # #  Y !!!6Y6Y 9555556666ssI J2JJ8KK K G,[[[a99a=a=0fff.jjj)rf)NN)r=rGrIr}rrrrrrrr1rrrrrs     11111133333333333333  22222222t*RZ*BJ****( "* RZ BJ     ; ; ; ; ; ;*Z Z Z Z Z Z Z Z D  $tCH~*>       U38_ %  eE3J : D s s xPU    &*c*S*S**** W c3h WW W3i W  #rz/ WWWWt$sCx.CC $ 25 BG      .$ $ $ $ $ $ $ $ X##c#c####  c c    cc0,,c,c,,,,& D S S T    ("D"D "D"Dt* "D  "D"D"D"DJ *.t* #Y.11C1T%d4j@P:Q5R1111ry y #s(^yyyy~ $# S6S6t*S6 S6}S6} S6  S6S6S6S6S6S6r