#!/usr/bin/env python3 from camviz import Draw from camviz.utils.cmaps import jet from vidar.utils.read import read_image from scipy.spatial.transform import Rotation as R from easydict import EasyDict as Edict from scipy.io import loadmat import numpy as np import torch import torchvision # from omnimvs.utils.common import * # from omnimvs.utils.geometry import applyTransform # from omnimvs.utils.ocam import OcamModel # from omnimvs.dataset import makeSphericalRays # import open3d as o3d import glob import yaml import cv2 import sys import os from PIL import Image import numpy as np from vidar.datasets.utils.misc import invert_pose from tqdm import tqdm sys.path.append('/home/yukinorikurahashi/ml_ws/omnimvs') # PATH_CONFIG = 'images/w1024_h1024_fov220/config.yaml' # PARENT_DIR = 'images/w1024_h1024_fov220/abandoned-room' BASE_DIR = 'images/w1024_h1024_fov220/' BASE_DIR = 'abandoned-room-20250807T024606Z-1-001/' PROJECT_NAME = 'abandoned-room' INDEX = 1 OUTPUT_DIR = 'output' def read_image(path): img = Image.open(path) if img.mode != 'RGB': img = img.convert('RGB') return img def get_rgb(path): rgb = read_image(path) rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 return rgb def get_depth(path, i): depth = torch.tensor(np.load(path)['depth']) depth[torch.abs(depth) > 1e6] = 0.0 return depth def get_intrinsics(cfg): params = [] for key in ['poly', 'inv_poly', 'center', 'affine', 'max_fov']: val = cfg[key] if isinstance(val, list): params.extend(val) else: params.append(val) params = torch.tensor(params).squeeze() return np.squeeze(params.numpy()) def get_pose(cfg, cam2world): cam2rig = np.array(cfg['pose']).reshape((6, 1)) vec = torch.tensor(cam2rig).unsqueeze(0)[..., 0] rot = torch.tensor(R.from_rotvec(vec[:, :3]).as_matrix()) trans = vec[:, 3:].unsqueeze(-1) T = torch.cat([rot, trans], -1) T = torch.cat([T, torch.tensor([[[0.0, 0.0, 0.0, 1.0]]])], 1).float() T = T[0].numpy() @ cam2world return invert_pose(T) def main(): src = '/data/cv_datasets/OmniMMT/w1024_h1024_fov220/20250813' dst = '/data/processed/omnicam2' print(src, dst) from glob import glob folders1 = glob(f'{src}/*') print(folders1) for f1 in folders1: cfg = f'{f1}/config.yaml' cfg = yaml.safe_load(open(cfg)) cfg = Edict(cfg) cfg = cfg.cameras for i in range(len(cfg)): files = glob(f'{f1}/cam{i+1}/*.png') files = [f.split('/')[-1][:-4] for f in files] files = [f for f in files if not f.startswith('depth_')] poses = np.load(f'{f1}/poses/cam{i+1}_poses.npz', i)['cam2world'] for n in tqdm(sorted(files)): n = int(n) cfg_i = cfg[i] dst_i = f'{dst}/{f1.replace("/", "__")}' rgb = get_rgb(f'{f1}/cam{i+1}/{n:05d}.png') depth = get_depth(f'{f1}/cam{i+1}/depth_{n:05d}.npz', i) from vidar.utils.write import write_image, write_npz write_image(f'{dst_i}/rgb/cam{i}/{n:05d}.jpg', rgb[0]) write_npz(f'{dst_i}/depth/cam{i}/{n:05d}.npz', dict(data=depth)) intrinsics = get_intrinsics(cfg_i) pose = get_pose(cfg_i, poses[n-1]) write_npz(f'{dst_i}/lowdim/cam{i}/{n:05d}.npz', dict(shape=(rgb.shape[-2], rgb.shape[-1]), intrinsics=intrinsics, pose=pose)) import sys sys.exit() parent_dir = os.path.join(BASE_DIR, PROJECT_NAME) config_path = os.path.join(parent_dir, 'config.yaml') config_yaml = yaml.safe_load(open(config_path)) conf = Edict(config_yaml) camera_config = conf.cameras for i in range(4): cfg = camera_config[i] intrinsics = get_intrinsics(cfg) pose = get_pose(cfg) np.savez_compressed(f'lowdim{i}.npz', intrinsics=params, pose=pose) ##################################################### from vidar.geometry.camera import Camera cams = [] for i in range(4): lowdim = np.load(f'lowdim{i}.npz', allow_pickle=True) intrinsics = torch.tensor(lowdim['intrinsics']).unsqueeze(0) pose = torch.tensor(lowdim['pose']).unsqueeze(0) cam = Camera(K=intrinsics, hw=(1024, 1024), Tcw=pose, geometry='omni') cams.append(cam) ocams = [] xyzs = [] rgbs = [] for i in range(4): ocam = OcamModel() ocam.setConfig(camera_config[i]) ocams.append(ocam) cam_dir = os.path.join(parent_dir, f'cam{i+1}') path_img_cam = os.path.join(cam_dir, f'{INDEX:05d}.png') rgb = read_image(path_img_cam) rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 width , height = rgb.shape[-1], rgb.shape[-2] path_depth = os.path.join(cam_dir, f'depth_{INDEX:05d}.npz') depth = torch.tensor(np.load(path_depth)['depth'][i]) depth = depth.view(1, -1) max_depth = 1e9 depth[torch.abs(depth) > max_depth] = 0.0 # xs, ys = np.meshgrid(range(width), range(height)) # p = np.concatenate((xs.reshape((1, -1)), ys.reshape((1, -1))), axis=0) # print('rays1a', p.shape, p.mean()) # rays = ocams[i].pixelToRay(p) # print('rays1b', rays.shape, rays.mean()) # rays = torch.tensor(rays).float() # print('rays1c', rays.shape, rays.mean()) # rays[torch.isnan(rays)] = 0.0 # print('rays1d', rays.shape, rays.mean()) # rays = torch.nn.functional.normalize(rays, dim=0, p=2) # print('rays1e', rays.shape, rays.mean()) # rays = rays.float() # print('rays1f', rays.shape, rays.mean()) # xyz = rays * depth # xyz = applyTransform(ocam.cam2rig, xyz) # xyz = xyz.view(1, 3, 1024, 1024).float() xyz2 = cams[i].reconstruct_depth_map(depth.view(1, 1, 1024, 1024), to_world=True) xyzs.append(xyz2) # xyzs.append(xyz) rgbs.append(rgb) draw = Draw((1600, 900)) draw.add3Dworld('wld') for i in range(4): draw.addBufferf(f'pts{i}', xyzs[i][0]) draw.addBufferf(f'rgb{i}', rgbs[i][0]) cnt = 0 while draw.input(): if draw.RETURN: cnt = (cnt + 1) % 4 print(cnt) draw.halt(100) draw.clear() # draw['wld'].size(1).color('red').points(f'pts{cnt}',f'rgb{cnt}') draw['wld'].size(2).color('blu').points('pts0','rgb0') draw['wld'].size(2).color('blu').points('pts1','rgb1') draw['wld'].size(2).color('gre').points('pts2','rgb2') draw['wld'].size(2).color('yel').points('pts3','rgb3') # draw['wld'].size(2).color('red').points('pts0','rgb0') # draw['wld'].size(2).color('blu').points('pts1') # ,'rgb1') # draw['wld'].size(2).color('gre').points('pts2') # ,'rgb2') # draw['wld'].size(2).color('yel').points('pts3') # ,'rgb3') draw.update(30) if __name__ == '__main__': main() # # from vidar.utils.config import Config # # from easydict import EasyDict as Edict # # from omnimvs.utils.dbhelper import loadDBConfigs # # import numpy as np # # import torch # # import torchvision # # from camviz import Draw # # from camviz.utils.cmaps import jet # # from vidar.utils.read import read_image # # # path = '/data/mmt_data9-20250730T184449Z-1-001/mmt_data9' # # # depth = '/data/mmt_data9-20250730T184449Z-1-001/mmt_data9/output/data_00001.npz' # # # image = '/data/mmt_data9-20250730T184449Z-1-001/mmt_data9/cam1/00001.png' # # # path = 'abandoned-room-20250806T034543Z-1-001/abandoned-room' # # path = 'apartment-interior-scene-20250806T170043Z-1-001/apartment-interior-scene' # # rgbs, depths = [], [] # # for i in [1,2,3,4]: # # rgb = f'{path}/cam{i}/00003.png' # # depth = f'{path}/cam{i}/depth_00003.npz' # # rgbs.append(rgb) # # depths.append(depth) # # data = Edict( # # omnimvs_sweep_min_depth=0.1, # # ) # # ocams = loadDBConfigs('config', path, data)[1] # # from vidar.utils.write import write_image # # from vidar.utils.viz import viz_depth # # from omnimvs.utils.geometry import applyTransform, inverseTransform # # xyzs = [] # # for i in range(4): # # width, height = 1024, 1024 # # xs, ys = np.meshgrid(range(width), range(height)) # # p = np.concatenate((xs.reshape((1, -1)), ys.reshape((1, -1))), axis=0) # # tr = ocams[i].cam2rig # # p = applyTransform(tr, p) # # rays = ocams[i].pixelToRay(p) # # rays = torch.tensor(rays).view(1, 3, 1024, 1024) # # rays[torch.isnan(rays)] = 0.0 # # rgb = read_image(rgbs[i]) # # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255 # # depth = torch.tensor(np.load(depths[i])['depth'][:1]).unsqueeze(0) # # depth = depth / 1000 # # # depth = 1.0 / depth # # # rgb = torch.nn.functional.interpolate(rgb, size=(1024, 1024), mode='bilinear', align_corners=None) # # # depth = torch.nn.functional.interpolate(depth, size=(1024, 1024), mode='nearest', align_corners=None) # # rgbs[i] = rgb # # depths[i] = depth # # xyz = rays * depth # # write_image(f'rgb{i}.png', rgb[0]) # # write_image(f'depth{i}.png', viz_depth(depth[0])) # # print(i) # # # tr = ocams[i].cam2rig # # # xyz = applyTransform(tr, xyz[0].view(3, -1).numpy()) # # # xyz = torch.tensor(xyz).view(1, 3, 1024, 1024) # # # print(T.shape) # # xyzs.append(xyz) # # draw = Draw((1600, 900)) # # draw.add3Dworld('wld') # # for i in range(4): # # draw.addBufferf(f'pts{i}', xyzs[i][0]) # # draw.addBufferf(f'rgb{i}', rgbs[i][0]) # jet(rays[0,2])) # # cnt = 0 # # while draw.input(): # # if draw.RETURN: # # cnt = (cnt + 1) % 4 # # print(cnt) # # draw.halt(100) # # draw.clear() # # draw['wld'].size(1).color('red').points(f'pts{cnt}',f'rgb{cnt}') # # # draw['wld'].size(2).color('blu').points('pts1','rgb1') # # # draw['wld'].size(2).color('gre').points('pts2','rgb2') # # # draw['wld'].size(2).color('yel').points('pts3','rgb3') # # # draw['wld'].size(2).color('red').points('pts0','rgb0') # # # draw['wld'].size(2).color('blu').points('pts1') # ,'rgb1') # # # draw['wld'].size(2).color('gre').points('pts2') # ,'rgb2') # # # draw['wld'].size(2).color('yel').points('pts3') # ,'rgb3') # # draw.update(30) # # # from vidar.utils.write import write_image # # # from vidar.utils.viz import viz_depth # # # write_image('depth.png', viz_depth(depth)) # # # bbb = ocams[0].rayToPixel(aaa) # ###################### # #!/usr/bin/env python3 # from camviz import Draw # from camviz.utils.cmaps import jet # from vidar.utils.read import read_image # from scipy.spatial.transform import Rotation as R # from easydict import EasyDict as Edict # from scipy.io import loadmat # import numpy as np # import torch # import torchvision # from omnimvs.utils.common import * # from omnimvs.utils.geometry import applyTransform # from omnimvs.utils.ocam import OcamModel # from omnimvs.dataset import makeSphericalRays # import open3d as o3d # import glob # import yaml # import cv2 # import sys # import os # from PIL import Image # import numpy as np # from vidar.datasets.utils.misc import invert_pose # from tqdm import tqdm # sys.path.append('/home/yukinorikurahashi/ml_ws/omnimvs') # # PATH_CONFIG = 'images/w1024_h1024_fov220/config.yaml' # # PARENT_DIR = 'images/w1024_h1024_fov220/abandoned-room' # BASE_DIR = 'images/w1024_h1024_fov220/' # BASE_DIR = 'abandoned-room-20250807T024606Z-1-001/' # PROJECT_NAME = 'abandoned-room' # INDEX = 1 # OUTPUT_DIR = 'output' # def read_image(path): # img = Image.open(path) # if img.mode != 'RGB': # img = img.convert('RGB') # return img # def get_rgb(path): # rgb = read_image(path) # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 # return rgb # def get_depth(path, i): # depth = torch.tensor(np.load(path)['depth']) # depth[torch.abs(depth) > 1e6] = 0.0 # return depth # def get_intrinsics(cfg): # params = [] # for key in ['poly', 'inv_poly', 'center', 'affine', 'max_fov']: # val = cfg[key] # if isinstance(val, list): # params.extend(val) # else: # params.append(val) # params = torch.tensor(params).squeeze() # return np.squeeze(params.numpy()) # def get_pose(cfg): # cam2rig = np.array(cfg['pose']).reshape((6, 1)) # vec = torch.tensor(cam2rig).unsqueeze(0)[..., 0] # rot = torch.tensor(R.from_rotvec(vec[:, :3]).as_matrix()) # trans = vec[:, 3:].unsqueeze(-1) # T = torch.cat([rot, trans], -1) # T = torch.cat([T, torch.tensor([[[0.0, 0.0, 0.0, 1.0]]])], 1).float() # T = T[0].numpy() # return invert_pose(T) # def main(): # src = '/data/omni/20250813_bedroom-realistic-20250814T181046Z-1-001' # dst = '/data/processed/omnicam' # from glob import glob # folders1 = glob(f'{src}/*') # for f1 in folders1: # cfg = f'{f1}/config.yaml' # cfg = yaml.safe_load(open(cfg)) # cfg = Edict(cfg) # cfg = cfg.cameras # for i in range(len(cfg)): # files = glob(f'{f1}/cam{i+1}/*.png') # files = [f.split('/')[-1][:-4] for f in files] # files = [f for f in files if not f.startswith('depth_')] # for n in tqdm(files): # n = int(n) # cfg_i = cfg[i] # dst_i = f'{dst}/{f1.replace("/", "__")}' # rgb = get_rgb(f'{f1}/cam{i+1}/{n:05d}.png') # depth = get_depth(f'{f1}/cam{i+1}/depth_{n:05d}.npz', i) # from vidar.utils.write import write_image, write_npz # write_image(f'{dst_i}/rgb/cam{i}/{n:05d}.jpg', rgb[0]) # write_npz(f'{dst_i}/depth/cam{i}/{n:05d}.npz', dict(data=depth)) # intrinsics = get_intrinsics(cfg_i) # pose = get_pose(cfg_i) # write_npz(f'{dst_i}/lowdim/cam{i}/{n:05d}.npz', # dict(shape=(rgb.shape[-2], rgb.shape[-1]), intrinsics=intrinsics, pose=pose)) # import sys # sys.exit() # parent_dir = os.path.join(BASE_DIR, PROJECT_NAME) # config_path = os.path.join(parent_dir, 'config.yaml') # config_yaml = yaml.safe_load(open(config_path)) # conf = Edict(config_yaml) # camera_config = conf.cameras # for i in range(4): # cfg = camera_config[i] # intrinsics = get_intrinsics(cfg) # pose = get_pose(cfg) # np.savez_compressed(f'lowdim{i}.npz', intrinsics=params, pose=pose) # ##################################################### # from vidar.geometry.camera import Camera # cams = [] # for i in range(4): # lowdim = np.load(f'lowdim{i}.npz', allow_pickle=True) # intrinsics = torch.tensor(lowdim['intrinsics']).unsqueeze(0) # pose = torch.tensor(lowdim['pose']).unsqueeze(0) # cam = Camera(K=intrinsics, hw=(1024, 1024), Tcw=pose, geometry='omni') # cams.append(cam) # ocams = [] # xyzs = [] # rgbs = [] # for i in range(4): # ocam = OcamModel() # ocam.setConfig(camera_config[i]) # ocams.append(ocam) # cam_dir = os.path.join(parent_dir, f'cam{i+1}') # path_img_cam = os.path.join(cam_dir, f'{INDEX:05d}.png') # rgb = read_image(path_img_cam) # rgb = torchvision.transforms.functional.pil_to_tensor(rgb).unsqueeze(0) / 255.0 # width , height = rgb.shape[-1], rgb.shape[-2] # path_depth = os.path.join(cam_dir, f'depth_{INDEX:05d}.npz') # depth = torch.tensor(np.load(path_depth)['depth'][i]) # depth = depth.view(1, -1) # max_depth = 1e9 # depth[torch.abs(depth) > max_depth] = 0.0 # # xs, ys = np.meshgrid(range(width), range(height)) # # p = np.concatenate((xs.reshape((1, -1)), ys.reshape((1, -1))), axis=0) # # print('rays1a', p.shape, p.mean()) # # rays = ocams[i].pixelToRay(p) # # print('rays1b', rays.shape, rays.mean()) # # rays = torch.tensor(rays).float() # # print('rays1c', rays.shape, rays.mean()) # # rays[torch.isnan(rays)] = 0.0 # # print('rays1d', rays.shape, rays.mean()) # # rays = torch.nn.functional.normalize(rays, dim=0, p=2) # # print('rays1e', rays.shape, rays.mean()) # # rays = rays.float() # # print('rays1f', rays.shape, rays.mean()) # # xyz = rays * depth # # xyz = applyTransform(ocam.cam2rig, xyz) # # xyz = xyz.view(1, 3, 1024, 1024).float() # xyz2 = cams[i].reconstruct_depth_map(depth.view(1, 1, 1024, 1024), to_world=True) # xyzs.append(xyz2) # # xyzs.append(xyz) # rgbs.append(rgb) # draw = Draw((1600, 900)) # draw.add3Dworld('wld') # for i in range(4): # draw.addBufferf(f'pts{i}', xyzs[i][0]) # draw.addBufferf(f'rgb{i}', rgbs[i][0]) # cnt = 0 # while draw.input(): # if draw.RETURN: # cnt = (cnt + 1) % 4 # print(cnt) # draw.halt(100) # draw.clear() # # draw['wld'].size(1).color('red').points(f'pts{cnt}',f'rgb{cnt}') # draw['wld'].size(2).color('blu').points('pts0','rgb0') # draw['wld'].size(2).color('blu').points('pts1','rgb1') # draw['wld'].size(2).color('gre').points('pts2','rgb2') # draw['wld'].size(2).color('yel').points('pts3','rgb3') # # draw['wld'].size(2).color('red').points('pts0','rgb0') # # draw['wld'].size(2).color('blu').points('pts1') # ,'rgb1') # # draw['wld'].size(2).color('gre').points('pts2') # ,'rgb2') # # draw['wld'].size(2).color('yel').points('pts3') # ,'rgb3') # draw.update(30) # if __name__ == '__main__': # main()