# Copyright 2026 Toyota Research Institute. All rights reserved. import os import cv2 import numpy as np import torchvision import torch from glob import glob from scipy.spatial.transform import Rotation as R from anydata.utils.read import read_numpy, read_yaml, read_image, read_depth from anydata.utils.write import write_image, write_npz, write_json from anydata.converters.utils import run, add_key_to_dict, fill_metadata, parse_dst_seq, frame_name from anydata.utils.geometry import invert_extrinsics ####################################################### def get_rgb(path): rgb = np.array(read_image(path)) return rgb def get_depth(path): depth = torch.tensor(np.load(path, allow_pickle=True)['depth']) depth[torch.abs(depth) > 1e6] = 0.0 return depth def get_intrinsics(cfg): params = [] for key in ['poly', 'inv_poly', 'center', 'affine', 'max_fov']: val = cfg[key] if isinstance(val, list): params.extend(val) else: params.append(val) params = torch.tensor(params).squeeze() return np.squeeze(params.numpy()) def get_pose(cfg, cam2world): cam2rig = np.array(cfg['pose']).reshape((6, 1)) vec = torch.tensor(cam2rig).unsqueeze(0)[..., 0] rot = torch.tensor(R.from_rotvec(vec[:, :3]).as_matrix()) trans = vec[:, 3:].unsqueeze(-1) T = torch.cat([rot, trans], -1) T = torch.cat([T, torch.tensor([[[0.0, 0.0, 0.0, 1.0]]])], 1).float() # T = T[0].numpy() @ cam2world # return T # pose_fix = np.array([ # [1, 0, 0, 0], # [0, -1, 0, 0], # [0, 0, -1, 0], # [0, 0, 0, 1], # ]) pose_fix = np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]) pose = T[0].numpy() # pose = pose @ cam2world print() print() print(cam2world) print(pose) # cam2world = pose_fix @ cam2world print(pose) # pose = invert_extrinsics(pose) pose = pose_fix @ pose # @ pose_fix # pose = invert_extrinsics(pose) pose = cam2world @ pose # pose = pose @ cam2world # pose = invert_extrinsics(pose) # pose = pose @ cam2world # pose = invert_extrinsics(pose) print(pose) print() print() return pose # cam2world = pose_fix @ cam2world # T = invert_extrinsics(cam2world) # return T # T = T[0].numpy() @ invert_extrinsics(cam2world) # return invert_extrinsics(T) # T = invert_extrinsics(T[0].numpy()) @ cam2world) # return invert_extrinsics(T) ####################################################### def get_sequences(args): seqs = glob(f'{args.src}/**/config.yaml', recursive=True) seqs = [os.path.dirname(seq) for seq in seqs] return seqs def parse_sequence(seq, args): return parse_dst_seq(seq, args) ####################################################### def process_sequence(i, seq, dst, args): ### Initialize lists and dicts cameras = sorted([os.path.basename(v).split('_')[0] for v in glob(f'{seq}/poses/*.npz')]) num_frames = {cam: dict() for cam in cameras} resolution = {cam: dict() for cam in cameras} labels, lowdim = [], {} import yaml # import Edict cfg = f'{seq}/config.yaml' cfg = yaml.safe_load(open(cfg)) cfg = cfg['cameras'] ############ LOOP OVER CAMERAS for c, cam in enumerate(cameras): ### Get filenames filenames = sorted(glob(f'{seq}/{cam}/*.png')) filename_rgbs = [f for f in filenames if 'depth' not in f] filename_depths = sorted(glob(f'{seq}/{cam}/*.npz')) poses = np.load(f'{seq}/poses/{cam}_poses.npz')['cam2world'] print(np.load(f'{seq}/poses/{cam}_poses.npz')['convention']) ######## RGB FILENAMES for i, filename_rgb in enumerate(filename_rgbs): frame = frame_name(i) if i > 5: break ######## RGB if 'rgb' not in labels: labels.append('rgb') rgb = get_rgb(filename_rgb) filename_rgb_out = f'{dst}/rgb/{cam}/{frame}.jpg' write_image(filename_rgb_out, rgb) ######## LOWDIM RGB filename_lowdim = add_key_to_dict(lowdim, f'{dst}/lowdim/{cam}/{frame}.npz') lowdim[filename_lowdim]['camera'] = cam lowdim[filename_lowdim]['timestep'] = int(frame) print(c, i) ######## INTRINSICS + EXTRINSICS if 'intrinsics' not in labels: labels.append('intrinsics') if 'extrinsics' not in labels: labels.append('extrinsics') filename_lowdim = add_key_to_dict(lowdim, f'{dst}/lowdim/{cam}/{frame}.npz') lowdim[filename_lowdim]['extrinsics'] = get_pose(cfg[c], poses[i]) lowdim[filename_lowdim]['intrinsics'] = get_intrinsics(cfg[c]) resolution[cam]['rgb'] = rgb.shape[:2] num_frames[cam]['rgb'] = len(filename_rgbs) ######## DEPTH FILENAMES for i, filename_depth in enumerate(filename_depths): frame = frame_name(i) if i > 5: break ######## DEPTH if 'depth' not in labels: labels.append('depth') depth = get_depth(filename_depth) print(depth.min(), depth.max()) filename_depth_out = f'{dst}/depth/{cam}/{frame}.npz' write_npz(filename_depth_out, dict(depth=depth)) resolution[cam]['depth'] = depth.shape[:2] num_frames[cam]['depth'] = len(filename_depth) # ######## EXTRINSICS FILENAMES # for i, filename_extrinsic in enumerate(filename_extrinsics): # frame = frame_name(i) # ######## INTRINSICS + EXTRINSICS # if 'intrinsics' not in labels: labels.append('intrinsics') # if 'extrinsics' not in labels: labels.append('extrinsics') # extrinsics = np.genfromtxt(filename_extrinsic).astype(np.float32).reshape((4, 4)) # filename_lowdim = add_key_to_dict(lowdim, f'{dst}/lowdim/{cam}/{frame}.npz') # lowdim[filename_lowdim]['extrinsics'] = extrinsics # lowdim[filename_lowdim]['intrinsics'] = intrinsics # num_frames[cam]['extrinsics'] = len(filename_extrinsics) # ######## DEPTH FILENAMES # for i, filename_depth in enumerate(filename_depths): # frame = frame_name(i) # ######## DEPTH # if 'depth' not in labels: labels.append('depth') # depth = cv2.imread(filename_depth, cv2.IMREAD_ANYDEPTH).astype(np.float32) / 1000.0 # filename_depth_out = f'{dst}/depth/{cam}/{frame}.npz' # write_npz(filename_depth_out, dict(depth=depth)) # resolution[cam]['depth'] = depth.shape[:2] # num_frames[cam]['depth'] = len(filename_depths) ######## WRITE LOWDIM for key, val in lowdim.items(): write_npz(key, val) print(labels) ############ METADATA filename = f'{dst}/metadata.json' seq_metadata = fill_metadata( args=args, info=dict( name='ScanNet', tags=['real'], raw_id=seq.replace(f'{args.src}/', ''), ), labels=labels, cameras=cameras, resolution=resolution, num_frames=num_frames, framerate=10, rgb=dict(extension='jpg'), intrinsics=dict(model='pinhole'), extrinsics=dict(transform='cam2world',metric=True), depth=dict(extension='npz',metric=True,sparse=True), semantic=None, action=None, language=None, specific=None, ) write_json(filename, seq_metadata) return dst ####################################################### if __name__ == '__main__': converter = os.path.basename(__file__) run(converter, get_sequences, parse_sequence, process_sequence) ####################################################### # #!/usr/bin/env python3 # from camviz import Draw # from camviz.utils.cmaps import jet # from vidar.utils.read import read_image # from scipy.spatial.transform import Rotation as R # from easydict import EasyDict as Edict # from scipy.io import loadmat # import numpy as np # import torch # import torchvision # # from omnimvs.utils.common import * # # from omnimvs.utils.geometry import applyTransform # # from omnimvs.utils.ocam import OcamModel # # from omnimvs.dataset import makeSphericalRays # # import open3d as o3d # import glob # import yaml # import cv2 # import sys # import os # from PIL import Image # import numpy as np # from vidar.datasets.utils.misc import invert_pose # from tqdm import tqdm # sys.path.append('/home/yukinorikurahashi/ml_ws/omnimvs') # # PATH_CONFIG = 'images/w1024_h1024_fov220/config.yaml' # # PARENT_DIR = 'images/w1024_h1024_fov220/abandoned-room' # BASE_DIR = 'images/w1024_h1024_fov220/' # BASE_DIR = 'abandoned-room-20250807T024606Z-1-001/' # PROJECT_NAME = 'abandoned-room' # INDEX = 1 # OUTPUT_DIR = 'output' # def read_image(path): # img = Image.open(path) # if img.mode != 'RGB': # img = img.convert('RGB') # return img # def get_rgb(path): # rgb = read_image(path) # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 # return rgb # def get_depth(path, i): # depth = torch.tensor(np.load(path)['depth']) # depth[torch.abs(depth) > 1e6] = 0.0 # return depth # def get_intrinsics(cfg): # params = [] # for key in ['poly', 'inv_poly', 'center', 'affine', 'max_fov']: # val = cfg[key] # if isinstance(val, list): # params.extend(val) # else: # params.append(val) # params = torch.tensor(params).squeeze() # return np.squeeze(params.numpy()) # def get_pose(cfg, cam2world): # cam2rig = np.array(cfg['pose']).reshape((6, 1)) # vec = torch.tensor(cam2rig).unsqueeze(0)[..., 0] # rot = torch.tensor(R.from_rotvec(vec[:, :3]).as_matrix()) # trans = vec[:, 3:].unsqueeze(-1) # T = torch.cat([rot, trans], -1) # T = torch.cat([T, torch.tensor([[[0.0, 0.0, 0.0, 1.0]]])], 1).float() # T = T[0].numpy() @ cam2world # return invert_pose(T) # def main(): # src = '/data/cv_datasets/OmniMMT/w1024_h1024_fov220/20250813' # dst = '/data/processed/omnicam2' # print(src, dst) # from glob import glob # folders1 = glob(f'{src}/*') # print(folders1) # for f1 in folders1: # cfg = f'{f1}/config.yaml' # cfg = yaml.safe_load(open(cfg)) # cfg = Edict(cfg) # cfg = cfg.cameras # for i in range(len(cfg)): # files = glob(f'{f1}/cam{i+1}/*.png') # files = [f.split('/')[-1][:-4] for f in files] # files = [f for f in files if not f.startswith('depth_')] # poses = np.load(f'{f1}/poses/cam{i+1}_poses.npz', i)['cam2world'] # for n in tqdm(sorted(files)): # n = int(n) # cfg_i = cfg[i] # dst_i = f'{dst}/{f1.replace("/", "__")}' # rgb = get_rgb(f'{f1}/cam{i+1}/{n:05d}.png') # depth = get_depth(f'{f1}/cam{i+1}/depth_{n:05d}.npz', i) # from vidar.utils.write import write_image, write_npz # write_image(f'{dst_i}/rgb/cam{i}/{n:05d}.jpg', rgb[0]) # write_npz(f'{dst_i}/depth/cam{i}/{n:05d}.npz', dict(data=depth)) # intrinsics = get_intrinsics(cfg_i) # pose = get_pose(cfg_i, poses[n-1]) # write_npz(f'{dst_i}/lowdim/cam{i}/{n:05d}.npz', # dict(shape=(rgb.shape[-2], rgb.shape[-1]), intrinsics=intrinsics, pose=pose)) # import sys # sys.exit() # parent_dir = os.path.join(BASE_DIR, PROJECT_NAME) # config_path = os.path.join(parent_dir, 'config.yaml') # config_yaml = yaml.safe_load(open(config_path)) # conf = Edict(config_yaml) # camera_config = conf.cameras # for i in range(4): # cfg = camera_config[i] # intrinsics = get_intrinsics(cfg) # pose = get_pose(cfg) # np.savez_compressed(f'lowdim{i}.npz', intrinsics=params, pose=pose) # ##################################################### # from vidar.geometry.camera import Camera # cams = [] # for i in range(4): # lowdim = np.load(f'lowdim{i}.npz', allow_pickle=True) # intrinsics = torch.tensor(lowdim['intrinsics']).unsqueeze(0) # pose = torch.tensor(lowdim['pose']).unsqueeze(0) # cam = Camera(K=intrinsics, hw=(1024, 1024), Tcw=pose, geometry='omni') # cams.append(cam) # ocams = [] # xyzs = [] # rgbs = [] # for i in range(4): # ocam = OcamModel() # ocam.setConfig(camera_config[i]) # ocams.append(ocam) # cam_dir = os.path.join(parent_dir, f'cam{i+1}') # path_img_cam = os.path.join(cam_dir, f'{INDEX:05d}.png') # rgb = read_image(path_img_cam) # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 # width , height = rgb.shape[-1], rgb.shape[-2] # path_depth = os.path.join(cam_dir, f'depth_{INDEX:05d}.npz') # depth = torch.tensor(np.load(path_depth)['depth'][i]) # depth = depth.view(1, -1) # max_depth = 1e9 # depth[torch.abs(depth) > max_depth] = 0.0 # # xs, ys = np.meshgrid(range(width), range(height)) # # p = np.concatenate((xs.reshape((1, -1)), ys.reshape((1, -1))), axis=0) # # print('rays1a', p.shape, p.mean()) # # rays = ocams[i].pixelToRay(p) # # print('rays1b', rays.shape, rays.mean()) # # rays = torch.tensor(rays).float() # # print('rays1c', rays.shape, rays.mean()) # # rays[torch.isnan(rays)] = 0.0 # # print('rays1d', rays.shape, rays.mean()) # # rays = torch.nn.functional.normalize(rays, dim=0, p=2) # # print('rays1e', rays.shape, rays.mean()) # # rays = rays.float() # # print('rays1f', rays.shape, rays.mean()) # # xyz = rays * depth # # xyz = applyTransform(ocam.cam2rig, xyz) # # xyz = xyz.view(1, 3, 1024, 1024).float() # xyz2 = cams[i].reconstruct_depth_map(depth.view(1, 1, 1024, 1024), to_world=True) # xyzs.append(xyz2) # # xyzs.append(xyz) # rgbs.append(rgb) # draw = Draw((1600, 900)) # draw.add3Dworld('wld') # for i in range(4): # draw.addBufferf(f'pts{i}', xyzs[i][0]) # draw.addBufferf(f'rgb{i}', rgbs[i][0]) # cnt = 0 # while draw.input(): # if draw.RETURN: # cnt = (cnt + 1) % 4 # print(cnt) # draw.halt(100) # draw.clear() # # draw['wld'].size(1).color('red').points(f'pts{cnt}',f'rgb{cnt}') # draw['wld'].size(2).color('blu').points('pts0','rgb0') # draw['wld'].size(2).color('blu').points('pts1','rgb1') # draw['wld'].size(2).color('gre').points('pts2','rgb2') # draw['wld'].size(2).color('yel').points('pts3','rgb3') # # draw['wld'].size(2).color('red').points('pts0','rgb0') # # draw['wld'].size(2).color('blu').points('pts1') # ,'rgb1') # # draw['wld'].size(2).color('gre').points('pts2') # ,'rgb2') # # draw['wld'].size(2).color('yel').points('pts3') # ,'rgb3') # draw.update(30) # if __name__ == '__main__': # main() # # # from vidar.utils.config import Config # # # from easydict import EasyDict as Edict # # # from omnimvs.utils.dbhelper import loadDBConfigs # # # import numpy as np # # # import torch # # # import torchvision # # # from camviz import Draw # # # from camviz.utils.cmaps import jet # # # from vidar.utils.read import read_image # # # # path = '/data/mmt_data9-20250730T184449Z-1-001/mmt_data9' # # # # depth = '/data/mmt_data9-20250730T184449Z-1-001/mmt_data9/output/data_00001.npz' # # # # image = '/data/mmt_data9-20250730T184449Z-1-001/mmt_data9/cam1/00001.png' # # # # path = 'abandoned-room-20250806T034543Z-1-001/abandoned-room' # # # path = 'apartment-interior-scene-20250806T170043Z-1-001/apartment-interior-scene' # # # rgbs, depths = [], [] # # # for i in [1,2,3,4]: # # # rgb = f'{path}/cam{i}/00003.png' # # # depth = f'{path}/cam{i}/depth_00003.npz' # # # rgbs.append(rgb) # # # depths.append(depth) # # # data = Edict( # # # omnimvs_sweep_min_depth=0.1, # # # ) # # # ocams = loadDBConfigs('config', path, data)[1] # # # from vidar.utils.write import write_image # # # from vidar.utils.viz import viz_depth # # # from omnimvs.utils.geometry import applyTransform, inverseTransform # # # xyzs = [] # # # for i in range(4): # # # width, height = 1024, 1024 # # # xs, ys = np.meshgrid(range(width), range(height)) # # # p = np.concatenate((xs.reshape((1, -1)), ys.reshape((1, -1))), axis=0) # # # tr = ocams[i].cam2rig # # # p = applyTransform(tr, p) # # # rays = ocams[i].pixelToRay(p) # # # rays = torch.tensor(rays).view(1, 3, 1024, 1024) # # # rays[torch.isnan(rays)] = 0.0 # # # rgb = read_image(rgbs[i]) # # # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255 # # # depth = torch.tensor(np.load(depths[i])['depth'][:1]).unsqueeze(0) # # # depth = depth / 1000 # # # # depth = 1.0 / depth # # # # rgb = torch.nn.functional.interpolate(rgb, size=(1024, 1024), mode='bilinear', align_corners=None) # # # # depth = torch.nn.functional.interpolate(depth, size=(1024, 1024), mode='nearest', align_corners=None) # # # rgbs[i] = rgb # # # depths[i] = depth # # # xyz = rays * depth # # # write_image(f'rgb{i}.png', rgb[0]) # # # write_image(f'depth{i}.png', viz_depth(depth[0])) # # # print(i) # # # # tr = ocams[i].cam2rig # # # # xyz = applyTransform(tr, xyz[0].view(3, -1).numpy()) # # # # xyz = torch.tensor(xyz).view(1, 3, 1024, 1024) # # # # print(T.shape) # # # xyzs.append(xyz) # # # draw = Draw((1600, 900)) # # # draw.add3Dworld('wld') # # # for i in range(4): # # # draw.addBufferf(f'pts{i}', xyzs[i][0]) # # # draw.addBufferf(f'rgb{i}', rgbs[i][0]) # jet(rays[0,2])) # # # cnt = 0 # # # while draw.input(): # # # if draw.RETURN: # # # cnt = (cnt + 1) % 4 # # # print(cnt) # # # draw.halt(100) # # # draw.clear() # # # draw['wld'].size(1).color('red').points(f'pts{cnt}',f'rgb{cnt}') # # # # draw['wld'].size(2).color('blu').points('pts1','rgb1') # # # # draw['wld'].size(2).color('gre').points('pts2','rgb2') # # # # draw['wld'].size(2).color('yel').points('pts3','rgb3') # # # # draw['wld'].size(2).color('red').points('pts0','rgb0') # # # # draw['wld'].size(2).color('blu').points('pts1') # ,'rgb1') # # # # draw['wld'].size(2).color('gre').points('pts2') # ,'rgb2') # # # # draw['wld'].size(2).color('yel').points('pts3') # ,'rgb3') # # # draw.update(30) # # # # from vidar.utils.write import write_image # # # # from vidar.utils.viz import viz_depth # # # # write_image('depth.png', viz_depth(depth)) # # # # bbb = ocams[0].rayToPixel(aaa) # # ###################### # # #!/usr/bin/env python3 # # from camviz import Draw # # from camviz.utils.cmaps import jet # # from vidar.utils.read import read_image # # from scipy.spatial.transform import Rotation as R # # from easydict import EasyDict as Edict # # from scipy.io import loadmat # # import numpy as np # # import torch # # import torchvision # # from omnimvs.utils.common import * # # from omnimvs.utils.geometry import applyTransform # # from omnimvs.utils.ocam import OcamModel # # from omnimvs.dataset import makeSphericalRays # # import open3d as o3d # # import glob # # import yaml # # import cv2 # # import sys # # import os # # from PIL import Image # # import numpy as np # # from vidar.datasets.utils.misc import invert_pose # # from tqdm import tqdm # # sys.path.append('/home/yukinorikurahashi/ml_ws/omnimvs') # # # PATH_CONFIG = 'images/w1024_h1024_fov220/config.yaml' # # # PARENT_DIR = 'images/w1024_h1024_fov220/abandoned-room' # # BASE_DIR = 'images/w1024_h1024_fov220/' # # BASE_DIR = 'abandoned-room-20250807T024606Z-1-001/' # # PROJECT_NAME = 'abandoned-room' # # INDEX = 1 # # OUTPUT_DIR = 'output' # # def read_image(path): # # img = Image.open(path) # # if img.mode != 'RGB': # # img = img.convert('RGB') # # return img # # def get_rgb(path): # # rgb = read_image(path) # # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 # # return rgb # # def get_depth(path, i): # # depth = torch.tensor(np.load(path)['depth']) # # depth[torch.abs(depth) > 1e6] = 0.0 # # return depth # # def get_intrinsics(cfg): # # params = [] # # for key in ['poly', 'inv_poly', 'center', 'affine', 'max_fov']: # # val = cfg[key] # # if isinstance(val, list): # # params.extend(val) # # else: # # params.append(val) # # params = torch.tensor(params).squeeze() # # return np.squeeze(params.numpy()) # # def get_pose(cfg): # # cam2rig = np.array(cfg['pose']).reshape((6, 1)) # # vec = torch.tensor(cam2rig).unsqueeze(0)[..., 0] # # rot = torch.tensor(R.from_rotvec(vec[:, :3]).as_matrix()) # # trans = vec[:, 3:].unsqueeze(-1) # # T = torch.cat([rot, trans], -1) # # T = torch.cat([T, torch.tensor([[[0.0, 0.0, 0.0, 1.0]]])], 1).float() # # T = T[0].numpy() # # return invert_pose(T) # # def main(): # # src = '/data/omni/20250813_bedroom-realistic-20250814T181046Z-1-001' # # dst = '/data/processed/omnicam' # # from glob import glob # # folders1 = glob(f'{src}/*') # # for f1 in folders1: # # cfg = f'{f1}/config.yaml' # # cfg = yaml.safe_load(open(cfg)) # # cfg = Edict(cfg) # # cfg = cfg.cameras # # for i in range(len(cfg)): # # files = glob(f'{f1}/cam{i+1}/*.png') # # files = [f.split('/')[-1][:-4] for f in files] # # files = [f for f in files if not f.startswith('depth_')] # # for n in tqdm(files): # # n = int(n) # # cfg_i = cfg[i] # # dst_i = f'{dst}/{f1.replace("/", "__")}' # # rgb = get_rgb(f'{f1}/cam{i+1}/{n:05d}.png') # # depth = get_depth(f'{f1}/cam{i+1}/depth_{n:05d}.npz', i) # # from vidar.utils.write import write_image, write_npz # # write_image(f'{dst_i}/rgb/cam{i}/{n:05d}.jpg', rgb[0]) # # write_npz(f'{dst_i}/depth/cam{i}/{n:05d}.npz', dict(data=depth)) # # intrinsics = get_intrinsics(cfg_i) # # pose = get_pose(cfg_i) # # write_npz(f'{dst_i}/lowdim/cam{i}/{n:05d}.npz', # # dict(shape=(rgb.shape[-2], rgb.shape[-1]), intrinsics=intrinsics, pose=pose)) # # import sys # # sys.exit() # # parent_dir = os.path.join(BASE_DIR, PROJECT_NAME) # # config_path = os.path.join(parent_dir, 'config.yaml') # # config_yaml = yaml.safe_load(open(config_path)) # # conf = Edict(config_yaml) # # camera_config = conf.cameras # # for i in range(4): # # cfg = camera_config[i] # # intrinsics = get_intrinsics(cfg) # # pose = get_pose(cfg) # # np.savez_compressed(f'lowdim{i}.npz', intrinsics=params, pose=pose) # # ##################################################### # # from vidar.geometry.camera import Camera # # cams = [] # # for i in range(4): # # lowdim = np.load(f'lowdim{i}.npz', allow_pickle=True) # # intrinsics = torch.tensor(lowdim['intrinsics']).unsqueeze(0) # # pose = torch.tensor(lowdim['pose']).unsqueeze(0) # # cam = Camera(K=intrinsics, hw=(1024, 1024), Tcw=pose, geometry='omni') # # cams.append(cam) # # ocams = [] # # xyzs = [] # # rgbs = [] # # for i in range(4): # # ocam = OcamModel() # # ocam.setConfig(camera_config[i]) # # ocams.append(ocam) # # cam_dir = os.path.join(parent_dir, f'cam{i+1}') # # path_img_cam = os.path.join(cam_dir, f'{INDEX:05d}.png') # # rgb = read_image(path_img_cam) # # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 # # width , height = rgb.shape[-1], rgb.shape[-2] # # path_depth = os.path.join(cam_dir, f'depth_{INDEX:05d}.npz') # # depth = torch.tensor(np.load(path_depth)['depth'][i]) # # depth = depth.view(1, -1) # # max_depth = 1e9 # # depth[torch.abs(depth) > max_depth] = 0.0 # # # xs, ys = np.meshgrid(range(width), range(height)) # # # p = np.concatenate((xs.reshape((1, -1)), ys.reshape((1, -1))), axis=0) # # # print('rays1a', p.shape, p.mean()) # # # rays = ocams[i].pixelToRay(p) # # # print('rays1b', rays.shape, rays.mean()) # # # rays = torch.tensor(rays).float() # # # print('rays1c', rays.shape, rays.mean()) # # # rays[torch.isnan(rays)] = 0.0 # # # print('rays1d', rays.shape, rays.mean()) # # # rays = torch.nn.functional.normalize(rays, dim=0, p=2) # # # print('rays1e', rays.shape, rays.mean()) # # # rays = rays.float() # # # print('rays1f', rays.shape, rays.mean()) # # # xyz = rays * depth # # # xyz = applyTransform(ocam.cam2rig, xyz) # # # xyz = xyz.view(1, 3, 1024, 1024).float() # # xyz2 = cams[i].reconstruct_depth_map(depth.view(1, 1, 1024, 1024), to_world=True) # # xyzs.append(xyz2) # # # xyzs.append(xyz) # # rgbs.append(rgb) # # draw = Draw((1600, 900)) # # draw.add3Dworld('wld') # # for i in range(4): # # draw.addBufferf(f'pts{i}', xyzs[i][0]) # # draw.addBufferf(f'rgb{i}', rgbs[i][0]) # # cnt = 0 # # while draw.input(): # # if draw.RETURN: # # cnt = (cnt + 1) % 4 # # print(cnt) # # draw.halt(100) # # draw.clear() # # # draw['wld'].size(1).color('red').points(f'pts{cnt}',f'rgb{cnt}') # # draw['wld'].size(2).color('blu').points('pts0','rgb0') # # draw['wld'].size(2).color('blu').points('pts1','rgb1') # # draw['wld'].size(2).color('gre').points('pts2','rgb2') # # draw['wld'].size(2).color('yel').points('pts3','rgb3') # # # draw['wld'].size(2).color('red').points('pts0','rgb0') # # # draw['wld'].size(2).color('blu').points('pts1') # ,'rgb1') # # # draw['wld'].size(2).color('gre').points('pts2') # ,'rgb2') # # # draw['wld'].size(2).color('yel').points('pts3') # ,'rgb3') # # draw.update(30) # # if __name__ == '__main__': # # main()