# Copyright 2026 Toyota Research Institute. All rights reserved. import os import numpy as np from glob import glob from anydata.utils.read import read_image, read_depth, read_json from anydata.utils.write import write_json, write_jpg_from_mp4, write_lowdim, write_labels from anydata.converters.utils import run, add_key_to_dict, fill_metadata, parse_dst_seq, frame_name, crawl, prepare_lowdim from anydata.converters.utils import extract_mp4 from anydata.sync.sync_utils import remove_path import json import numpy as np # import s3fs import pandas as pd import projectaria_tools.core.mps as mps from pandas.errors import DtypeWarning from projectaria_tools.core import data_provider from projectaria_tools.core.mps.utils import get_nearest_pose from projectaria_tools.core.stream_id import StreamId ####################################################### def get_depth(filename): depth = read_depth(filename, div=1000) return depth def get_mask(filename): return read_image(filename, '1') def resize_intrinsics(K, dim, DIM): K_new = np.copy(K) K_new[0] *= (DIM[1] / dim[1]) K_new[1] *= (DIM[0] / dim[0]) return K_new ####################################################### def get_sequences(args): takes = read_json(f'{args.src}/takes.json') seqs = glob(f'{args.src}/takes/*') fseqs = [] args.takes = dict() for seq in seqs: bseq = os.path.basename(seq) for take in takes: if take['take_name'] == bseq: fseqs.append(seq) args.takes[seq] = take break return fseqs def parse_sequence(seq, args): return parse_dst_seq(seq, args) ####################################################### def process_sequence(i, seq, dst, args): take_info = args.takes[seq] local_root = seq ### Initialize lists and dicts cameras = [] for cam, val in take_info['frame_aligned_videos'].items(): if cam.startswith('cam'): cameras.append(cam) if cam.startswith('aria'): cameras.extend([c for c in list(val.keys()) if c != 'et']) num_frames = {cam: dict() for cam in cameras} resolution = {cam: dict() for cam in cameras} labels, lowdim = [], {} dense_labels = ['rgb'] #@############################################ ### LANGUAGE PROMPT annos = dict() for split in ['train', 'val']: with open(os.path.join(args.src, f'annotations/keystep_{split}.json'), 'r') as fd: anno = json.load(fd) annos.update(anno['annotations']) if take_info['take_uid'] in annos: key_steps = annos[take_info['take_uid']]['segments'] key_steps = sorted(key_steps, key=lambda x: x['start_time']) prompt = np.array([step['step_description'] for step in key_steps]) else: prompt = np.array(['']) # EGO-CAMERAS timesync = pd.read_csv(os.path.join( args.src, take_info['capture']['root_dir'], 'timesync.csv')) start_idx = take_info['timesync_start_idx'] + 1 end_idx = take_info['timesync_end_idx'] trajectory = mps.read_closed_loop_trajectory(os.path.join( args.src, take_info['capture']['root_dir'], 'trajectory/closed_loop_trajectory.csv')) for aria_key in take_info['frame_aligned_videos']: if not aria_key.lower().startswith('aria'): continue # take-level calibration vrs_provider = data_provider.create_vrs_data_provider( os.path.join(args.src, take_info['root_dir'], f'{aria_key}_noimagestreams.vrs') ) device_calib = vrs_provider.get_device_calibration() miss_pose_idx = set() # iterate over all views (slam-left, slam-right, rgb, et) on the device for view, cam_info in take_info['frame_aligned_videos'][aria_key].items(): # ignore eye-tracking stream if view == 'et': continue dense = {label: dict() for label in dense_labels} video = extract_mp4(f'{seq}/{cam_info["relative_path"]}', 0, 110) from PIL import Image for i in range(video.shape[0]): if i > 100: break frame = frame_name(i) dense['rgb'][frame] = video[i] stream_id = StreamId(cam_info['stream_id']) stream_label = vrs_provider.get_label_from_stream_id(stream_id) cam_calib = device_calib.get_camera_calib(stream_label) intrinsics = cam_calib.get_projection_params() if intrinsics.shape[0] == 15: f, cx, cy, k1, k2, k3, k4, k5, k6, p1, p2, s1, s2, s3, s4 = intrinsics intrinsics = np.array([f, f, cx, cy, k1, k2, p1, p2, k3, k4, k5, k6, s1, s2, s3, s4]) elif intrinsics.shape[0] == 8: fx, fy, cx, cy, k1, k2, k3, k4 = intrinsics intrinsics = np.array([fx, fy, cx, cy, k1, k2, k3, k4]) else: raise ValueError('Invalid intrinsics', intrinsics.shape) # Scale intrinsics to the proper resolution (2880 -> 1408) if view == 'rgb': intrinsics[:4] *= 1408 / 2880 T_device_cam = cam_calib.get_transform_device_camera() stream_name = os.path.basename(cam_info['relative_path']).strip('.mp4') for i, idx in enumerate(range(start_idx, end_idx + 1)): # end_idx is inclusive according to doc if i > 100: break frame = frame_name(i) timestamp_ns = timesync.iloc[idx][f'{stream_name}_capture_timestamp_ns'] pose = get_nearest_pose(trajectory, timestamp_ns) T_world_device = pose.transform_world_device T_world_cam = T_world_device @ T_device_cam extrinsics = T_world_cam.to_matrix() ######## LOWDIM RGB prepare_lowdim(lowdim, dst, view, frame) ######## INTRINSICS + EXTRINSICS filename_lowdim = add_key_to_dict(lowdim, f'{dst}/lowdim/{view}/{frame}.npz') lowdim[filename_lowdim]['extrinsics'] = extrinsics lowdim[filename_lowdim]['intrinsics'] = intrinsics ######## WRITE LABELS write_labels(dst, view, args.storage, dense, labels, resolution, num_frames) # EXO-CAMERAS gopro_calibs = mps.read_static_camera_calibrations(os.path.join( args.src, take_info['root_dir'], 'trajectory/gopro_calibs.csv')) for gopro_cam in gopro_calibs: view = gopro_cam.camera_uid dense = {label: dict() for label in dense_labels} video = extract_mp4(f'{seq}/frame_aligned_videos/{view}.mp4', 0, 110) for i in range(video.shape[0]): if i > 100: break frame = frame_name(i) dense['rgb'][frame] = video[i] intrinsics = gopro_cam.intrinsics extrinsics = gopro_cam.transform_world_cam.to_matrix() if intrinsics.shape[0] == 15: f, cx, cy, k1, k2, k3, k4, k5, k6, p1, p2, s1, s2, s3, s4 = intrinsics intrinsics = np.array([f, f, cx, cy, k1, k2, p1, p2, k3, k4, k5, k6, s1, s2, s3, s4]) if intrinsics.shape[0] == 8: fx, fy, cx, cy, k1, k2, k3, k4 = intrinsics intrinsics = np.array([fx, fy, cx, cy, k1, k2, k3, k4]) else: raise ValueError('Invalid intrinsics', intrinsics.shape) for i in range(video.shape[0]): if i > 100: break frame = frame_name(i) ######## LOWDIM RGB prepare_lowdim(lowdim, dst, view, frame) ######## INTRINSICS + EXTRINSICS filename_lowdim = add_key_to_dict(lowdim, f'{dst}/lowdim/{view}/{frame}.npz') lowdim[filename_lowdim]['extrinsics'] = extrinsics lowdim[filename_lowdim]['intrinsics'] = intrinsics ######## WRITE LABELS write_labels(dst, view, args.storage, dense, labels, resolution, num_frames) ######## WRITE LOWDIM write_lowdim(args, dst, labels, num_frames, lowdim) ############ METADATA filename = f'{dst}/metadata.json' seq_metadata = fill_metadata( args=args, info=dict( name='EgoExo4D', tags=['real','dynamic'], raw_id=seq.replace(f'{args.src}/', ''), ), labels=labels, cameras=cameras, resolution=resolution, num_frames=num_frames, framerate=10, rgb=dict(extension='jpg'), intrinsics=dict(model='fisheye'), extrinsics=dict(transform='cam2world',metric=True), depth=dict(extension='npz',metric=True,sparse=True), semantic=None, action=None, language=None, specific=None, ) write_json(filename, seq_metadata) return dst ####################################################### if __name__ == '__main__': converter = os.path.basename(__file__) run(converter, get_sequences, parse_sequence, process_sequence) ####################################################### # from projectaria_tools.core.mps.utils import filter_points_from_confidence, filter_points_from_count # print("Loading and filtering point cloud ... be patient ...") # point_cloud = mps_data_provider.get_semidense_point_cloud() # # Filter the point cloud by inv depth and depth and load # threshold_invdep = 5e-4 # threshold_dep = 5e-4 # filtered_point_cloud = filter_points_from_confidence(point_cloud, threshold_invdep, threshold_dep) # # Downsampling the data for web viewing # downsampled_points_cloud = filter_points_from_count(filtered_point_cloud, 500_000) # # Retrieve point positions # points_position = np.stack([it.position_world for it in downsampled_points_cloud]) # log_point_cloud(points_position, "world/point_cloud") # distance_std_threshold = 5e-4 # inverse_distance_std_threshold = 5e-4 # global_points_path = '/data/cv_downloaded/EgoExo4D/takes/fair_cooking_09_2/trajectory/semidense_points.csv.gz' # points = mps.read_global_point_cloud(global_points_path) # filtered_points = filter_points_from_confidence(points, inverse_distance_std_threshold, distance_std_threshold) # filtered_points = filter_points_from_count(filtered_points, 500_000) # points = np.stack([point.position_world for point in filtered_points], 0) # # points = torch.tensor(points).permute(1, 0).unsqueeze(0).float() # import sys # sys.exit() # ######## DEPTH # if 'pointcloud' not in labels: labels.append('pointcloud') # filename_pointcloud_out = f'{dst}/pointcloud.npz' # write_npz(filename_pointcloud_out, dict(pointcloud=points)) # ######## DEPTH # if 'depth' not in labels: labels.append('depth') # camany = Camera( # K=torch.tensor(intrinsics).unsqueeze(0).float(), # Twc=torch.tensor(extrinsics).unsqueeze(0).float(), # hw=resolution_rgb) # depth = camany.project_pointcloud(points)[0,0].numpy() # filename_depth_out = f'{dst}/depth/{view}/{frame}.npz' # write_npz(filename_depth_out, dict(depth=depth)) # ######## DEPTH # if 'depth' not in labels: labels.append('depth') # camany = Camera( # K=torch.tensor(intrinsics).unsqueeze(0).float(), # Twc=torch.tensor(extrinsics).unsqueeze(0).float(), # hw=resolution_rgb) # depth = camany.project_pointcloud(points)[0,0].numpy() # filename_depth_out = f'{dst}/depth/{view}/{frame}.npz' # write_npz(filename_depth_out, dict(depth=depth))