
    fQ                     z   d dl Z d dlmZ d dlZd dlZd dlZd dlmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlZd dlmZ d d	lmZ d d
lmZ d dlZd dlZd dlZd dlZd dlmZ d dlZd dlZd dlZdZddZddZ d Z!d Z"ddZ#d Z$d Z%d Z&d Z'd Z( G d de)      Z*d Z+d Z,d Z-ddZ. G d d      Z/y)     N)
functional)glob)defaultdict)imread)Dataset)Path)tqdm)loadmat)common   c                     t        j                  |       d d d d d df   }t        j                  |      }t	        |      }|(t        j                  |||ft
        j                        }|dz  }|dz  }|S )N   interpolation      ?       @)imageior   skimageimg_as_float32square_crop_imgcv2resizeINTER_NEARESTpath
sidelengthimgs      >/home/cameronsmith/repos/flowcams/data/realestate10k_dataio.pyload_rgbr       ss    
..
q!RaRx
(C

 
 
%C
#
Cjjz:6cFWFWX3JC2ICJ    c                 t   t        j                  | t         j                        j                  t        j
                        }|(t        j                  |||ft         j                        }|dz  }t        |j                        dk(  r#|d d d d d df   }|j                  ddd      }|S |d d d d d f   }|S )Nr   g-C6?r   r      r   )r   r   IMREAD_UNCHANGEDastypenpfloat32r   r   lenshape	transposer   s      r   
load_depthr*   -   s    
**T3//
0
7
7


CCjjz:6cFWFWX4KC
399~!Q(mmmAq!$ J $1*oJr    c                    t        |       j                         j                         }t        |      dk(  rgt	        j
                  dt        j                        }t        d      D ]$  }|d   j                  d      |   ||dz  |dz  f<   & |j                         S d |d d D        D cg c]  }|d   |d   |d	   |d
   g }}t	        j                  |      j                  t        j                        j                         S c c}w )Nr      r-   dtype   r    r-   c              3   >   K   | ]  }|j                  d         ywr1   Nsplit.0xs     r   	<genexpr>zload_pose.<locals>.<genexpr>E        3TQAGGCL3T   r"   r   )openread
splitlinesr'   r%   zerosr&   ranger5   squeezeasarrayr$   )filenamelinesposeir8   s        r   	load_poserG   =   s    N!,,.E
5zQxxbjj1r 	9A"'(.."5a"8DaQ	9||~3T%PRQR)3TUa!A$!adAaD)UUzz% ''

3;;== Vs   +Dc                     | d   }||   d   }|j                         }t        j                  |      }t        j                  |      d d d d d df   }t
        j                  |      }t        |      }|dz  }|dz  }|S )Nrgb.r   r   r   )tostringioBytesIOr   r   r   r   r   )instance_dskeyrgb_dsrawsfr   s          r   load_numpy_hdf5rS   I   s    F
+c
CA


1A
..
Aq"1"H
%C

 
 
%C
#
C3JC2ICJr    c                 T   | d   }||   d   }|j                         }t        j                  |      }t        j                  |      d d d d d df   }t
        j                  |      }t        |      }|(t        j                  |||ft        j                        }|dz  }|dz  }|S )NrI   .r   r   r   r   )rJ   rK   rL   r   r   r   r   r   r   r   
INTER_AREA)rM   rN   r   rO   rP   rQ   rR   r   s           r   load_rgb_hdf5rV   Z   s    F
+c
CA


1A
..
Aq"1"H
%C

 
 
%C
#
Cjjz:6cnnU3JC2ICJr    c                 :   | d   }||   d   }t        |      }|j                  d      }|j                         }t        |      dk(  rgt	        j
                  dt        j                        }t        d      D ]$  }|d   j                  d	      |   ||d
z  |d
z  f<   & |j                         S d |d d
 D        D 	cg c]  }	|	d   |	d   |	d   |	d   g }}	t	        j                  |      j                  t        j                        j                         S c c}	w )NrE   .asciir   r,   r.   r0   r   r1   r-   c              3   >   K   | ]  }|j                  d         ywr3   r4   r6   s     r   r9   z!load_pose_hdf5.<locals>.<genexpr>}   r:   r;   r"   r   )	bytearraydecoder>   r'   r%   r?   r&   r@   r5   rA   rB   r$   )
rM   rN   pose_dsrP   barQ   rD   rE   rF   r8   s
             r   load_pose_hdf5r^   n   s
   &!G
#,s
C	3B
		'ALLNE
5zQxxbjj1r 	9A"'(.."5a"8DaQ	9 ||~3T%PRQR)3TUa!A$!adAaD)UUzz% ''

3;;== Vs   =Dc                 n    t         j                  j                  |       st        j                  |        y y N)osr   existsmakedirs)r   s    r   
cond_mkdirrd      s#    77>>$
D  r    c                     t        j                  | j                  d d       }t        j                  | j                  d d       dz  }| |d   |dz  z
  |d   |dz  z   |d   |dz  z
  |d   |dz  z   f   } | S )Nr"   r   r   )r%   aminr(   array)r   min_dimcenter_coords      r   r   r      s    ggciim$G88CIIbqM*a/L
l1o1,\!_w!|-KK
q/GqL
(a7a<)G
GH ICJr    c           	          g }dD ]:  }|j                  t        t        j                  j	                  | |                   < |S )N)z*.pngz*.jpgz*.JPEGz*.JPG)extendr   ra   r   join)r   imgsexts      r   	glob_imgsro      s;    D4 3DdC0123Kr    c                    t         j                  j                  dd      dk  r5| d d d d dd d f   } t        j                  g dg dg dg dg      }||z  }t         j                  j                  dd      dk  r0t         j                  j	                  dd	      }| || d d d d f   } nd}t         j                  j                  dd      dk  r0t         j                  j	                  dd	      }| d d || d d f   } nd}| j
                  \  }}}t        j                  | d
      } d|z  }	d|z  }
|d   |	z  |d<   |d   |
z  |d<   | ||fS )Nr   r   r   )rq   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r          rv   rv   )r   r   )r   r   )r%   randomuniformrg   randintr(   r   r   )rI   
intrinsicsc2w_mattf_flippypxHW_xscaleyscales              r   augmentr      sM    
yyA$!TrT1*o((M<|TUG# 
yyA$YYq"%"bS&!Q,	yyA$YYq"%!RVQ,iiGAq!
**S*
%C1WF1WF!$'&0Jt!$'&0Jt
G##r    c                       e Zd Zd Zy)Camerac                 R   |dd \  }}}}t        j                  |d|dgd||dgg dg dg      | _        t        j                  |dd        j                  dd      }t        j                  d      }||d dd d f<   || _        t         j                  j                  |      | _        y )	Nr      r   rr   rs      r   r-   )	r%   rg   rz   reshapeeyew2c_matlinalginvr{   )selfentryfxfycxcyr   w2c_mat_4x4s           r   __init__zCamera.__init__   s    qBB((RBN%&BN$0$0$2 3 ((59%--a3ffQi$BQBE"yy}}[1r    N)__name__
__module____qualname__r    r    r   r   r      s    
2r    r   c                 Z    | j                         } | dxx   |z  cc<   | dxx   |z  cc<   | S )Nr   r   )copy)rz   hws      r   unnormalize_intrinsicsr      s0    "JqMQMqMQMr    c                     t        | d      }i }t        |      D ]M  \  }}|dk(  r|j                         D cg c]  }t        |       }}t	        |d         }t        |      ||<   O |S c c}w )Nrr   )r<   	enumerater5   floatintr   )filerR   
cam_paramsrF   liner8   r   ids           r   parse_pose_filer      sw    T3AJQ< '46#'::<0aq00q]
2'  1s   A,c                     | d d d df   }t        j                  |      }||k(  d d df   }| |   d   }t        |      }|S )Nr   r   )r%   aroundr   )rE   timestep	timestepsmask
pose_entrycameras         r   
parse_poser      sP    QUI		)$I!1a4(DdAJJFMr    c           	         t        | j                  d            }|d   }t        j                  |      }t	        |      }t        |j                               }|D cg c]  }t        |j                  d      d         ! }	}t        j                  |	      }
t        j                  |      |
   }t        j                  |	      |
   }	|t        | j                        dz   z  }t        |      }d\  }}|t        ||      z  }|t        ||      z  }i }i }t        d|j                  d         }g }g }g }t!        d|      D ]  }|||      }|	|   }t#        ||   j$                  ||      }|d   |z  |d<   |d	   |z  |d	<   |j'                  t        j(                        d
z  dz
  }|j+                  |       |j+                  ||   j,                         |j+                  |        g }g }g }|dk(  rdg}n"|dk(  rdt        t/        |      dz
  d      g}nJ |D ]  }|||      }|	|   }t#        ||   j$                  ||      }|d   |z  |d<   |d	   |z  |d	<   |j'                  t        j(                        d
z  dz
  }|j+                  |       |j+                  ||   j,                         |j+                  |        t1        j2                  |      d    j5                         t1        j2                  |      d    j5                         t1        j2                  |      d    j5                         |j7                  dd      d   j9                  d|dz
  dd      d}t1        j2                  |      d    j5                         t1        j2                  |      d    j5                         t1        j2                  |      d    j5                         d}||dS c c}w )N*.npzr   .z.txt)rv   i     r   r   r"   r   r"        _@r"   rq   )NN)rI   	cam2worldrz   uv)rI   r   rz   )querycontext)sortedr   r%   loadr   listkeysr   r5   argsortrg   strnamer   minr(   r@   r   rz   r$   r&   appendr{   r'   torchTensorr   viewexpand) 
scene_pathall_pose_dirr   views	npz_filesnpz_filedata	rgb_filesrgb_file
timestamps
sorted_idscamera_filer   r   r   r   r   r   r   render_framequery_intrinsics	query_c2w
query_rgbsrF   rI   r   rz   context_intrinsicscontext_c2wcontext_rgbs
render_idsctxts                                    r   get_camera_poser      s   zw/0I|H778D%LTYY[!I>GH(#hnnS)!,-HJHJ'J#J/I*%j1J#joo"6"?@K -J DAqQ]FQ]F EGsIOOA./LIJ1l# 9Q< a= ,Jx,@,K,KQPQR
%d+f4
4%d+f4
4jj$u,q0
+H-556# KLzS
	!S^a/56
 !9Q< a= ,Jx,@,K,KQPQR
%d+f4
4%d+f4
4jj$u,q0!!*-:h/778C ! LL,T288:,,y1$7==? <<(89$?EEG772q>*-44Qq8H"bQSE <<-d399;k248>>@,,'9:4@FFHJD t,,U Is   $Oc                   (    e Zd Z	 	 	 ddZd ZddZy)RealEstate10kNc                 >   t        |      t        g       k(  r|d   n|| _        t        | j                  d       || _        |5t        j
                  j                  t        j                  d   |rdnd      }|5t        j
                  j                  t        j                  d   |rdnd      }t        d       || _        || _	        || _
        || _        || _        t        |      }t        |      | _        |
| _        t#        |j%                  d            | _        t)        t+        | j&                  d   j%                  d	                  }|rt-        | j&                        d | | _        t/        j0                  |      }t-        |j3                               d   }||   }|j4                  d d
 \  }}d\  }}||c| _        | _        |	| _        || _        |t?        ||      z  }|t?        ||      z  }t?        ||      }|| _         || _!        d| _        d| _        t        d| d| d       | j<                  rVtE        jF                  tE        jH                  d| j                        tE        jH                  d| j                              \  }}nAtE        jF                  tE        jH                  d|      tE        jH                  d|            \  }}tE        jJ                  |jM                         |jM                         gd      jO                  ddd
      | _(        | jP                  d    jO                  dddd
      jO                  dd
dd      | _(        | jP                  jS                  dd
      | _(        t-        t        |      j%                  d            | _*        y )Nr   n_skipRE10K_IMG_ROOTtesttrainRE10K_POSE_ROOTzLoading RealEstate10k...z*/r   r"   )rv     rv   r   zResolution is z, r   rq   )dimr   r   )+typer   printvalra   r   rl   environnum_ctxt_viewsnum_query_viewsquery_sparsity	dual_viewimslr   r
   all_poselpipsr   r   
all_scenesr   nextr   r%   r   r   r(   r   r   r   square_cropr   r   r   r   meshgridarangestackr   permuter   r   scene_path_list)r   img_root	pose_rootr   r   r   r   max_num_scenesr   r   r   r   r   r   
all_im_dirdummy_img_pathr   rN   imr   r   r   r   r   rF   js                             r   r   zRealEstate10k.__init__9  s    #'v,R"8VAYfdkk(#RZZ@P5Q\_RXel(mX"'',,rzzBS7T_bU[ho*pi(),.,"	(^
	*
 !67T$//!"4"9"9'"BCD"4??3O^DDOww~&499;"#Yxx|11A& SAYSAY!Qi qcA3a()>>%,,q$))"<ell1dii>XYDAq>>%,,q!"4ell1a6HIDAq++qwwy!'')4"=EEaAN
 ''$-''2q!4<<Q1aH''//"a(#DN$7$7$=>r    c                 ,    t        | j                        S r`   )r'   r   )r   s    r   __len__zRealEstate10k.__len__  s    4??##r    c           	      v    t         rnd j                  |n|   }t        |j                  d            }|j                  } fd}| j
                  vr |       S  j
                  |   }t        |      dk(  rt        d        |       S |d   }	 t        j                  |      }	t        |	j                               }
d}t        |
      dk  rt        d        |       S |
D cg c]  }t        |j                  d	      d         ! }}t        j                  |      }t        j                  |
      |   }
t        j                  |      |   }|t        |      k(  j!                         sJ t        |
      }d}|d
z
  }t        j"                  ||      }d
} j$                  rdnd}g } j&                  }t        j                  |      }d}|d
z
  | j(                  z  z
  }||k  r>t        | j(                  d
z   z        }|d
z
  | j(                  z  z
  }t        d|||        j$                  rd}n!t        j*                  j-                  ||      }t/         j(                        D cg c]
  }|||z  z    }}g }g }g }g } |D ]  }!|
|!   }|	|   }"|"j0                  d   dk(  r,t3        j4                  |" j6                   j8                  f      }" j:                  rt=        |"      }"t?        |||!         }#tA        |#jB                   j8                   j6                        }$ j:                  r*|$d    jD                  z  |$d<   |$d    jF                  z  |$d<    jH                  r tI        |"|$|#jJ                        \  }"}$|#_%        |"jM                  t        jN                        dz  d
z
  }"|"j0                  d d }%|"jQ                  d      }"d}& jR                  }'| jU                  |'       |jU                  |"       |jU                  |$       |jU                  |#jJ                          tW        jX                  t        jZ                  | d            j]                         } g }(g })g }*|D ]`  }!|
|!   }|	|   }"|"j0                  d   dk(  r,t3        j4                  |" j6                   j8                  f      }" j:                  rt=        |"      }"t?        |||!         }#tA        |#jB                   j8                   j6                        }$ j:                  r*|$d    jD                  z  |$d<   |$d    jF                  z  |$d<    jH                  r tI        |"|$|#jJ                        \  }"}$|#_%        |"jM                  t        jN                        dz  d
z
  }"|(jU                  |"       |)jU                  |$       |*jU                  |#jJ                         c t        jZ                  |      }t        jZ                  |      }tW        j^                  t        jZ                  |            j]                         }|  j`                  d
z
  z  } |d d d dd df   dz  |d d d dd df<   tW        j^                  |      d d d dd df   j]                         }+tW        j^                  |      jc                  ddd
      je                  dd      },|,|+|d}-|,jg                  d      |,jg                  d      }/}.|.|/z  }0|0|-d<   ti        jj                  |-d
d
dddgddg       S #  t        d        |       cY S xY wc c}w c c}w )!Nr   r   c                      j                   r dkD  r dz
     S  dz      S j                  t        j                  dt	        j
                        dz
              S )N   r   r   )r   __getitem__rw   ry   r'   r   )idxr   s   r   get_anotherz.RealEstate10k.__getitem__.<locals>.get_another  sU    xxS#XCE::CE::##FNN1c$//6JQ6N$OPPr    znpz get anotherznpz load error get anotherr      z<20 rgbs error get anotherr   r      \   z	high<low )lowhighih  r   r   r   r"   )rq   r   g        )axisrq   rv   r   ru   )rI   rz   c2w	org_ratiog      @   )hires_factorbudgetlow_reshi_res)9not_ofr   r   r   r   r   r'   r   r%   r   r   r   r   r5   r   rg   allr   r   r   r   rw   ry   r@   r(   r   r   r   r   r   r   r   r   rz   r   r   r   r{   r$   r&   r   r   r   r   r   r   r   
from_numpyr   r   	unflattensizer   make_sampleFinterpolateflatten)5r   r	  scene_queryr   r   r   r
  rE   r   r   r   window_sizer   r   r   
num_frames
left_boundright_boundcandidate_idsnframenframe_viewid_featsr   id_featr  r  base_irF   	id_renderr   r   r   uvsr   rI   	cam_paramrz   img_size
mask_lpipsr   	ctxt_rgbsctxt_intrinsicsctxt_c2wKsrm   trgtr   r   r  
imgs_large	imgs_flatmodel_inputgts5   ``                                                   r   r  zRealEstate10k.__getitem__  s   c__K,?S[Q
:??734		Q t}}$[]&:}}T"y>Q#$= Q<	!778$D
 %	y>R./=  CLLhc(..-a01L
LZZ
+
HHY'
3	XXj)*5
fZ005577^
 
 1n		*k: !XXc2  {{((8$!|F4#7#7773;d&:&:1&<=>Fa<t';'; ;;D+j5 88AVYY&&3T&:f /4D4H4H.IJVAfH_J	J 
	 "	0B }Hx.Cyy|s"jjtvvtvv&67%c*
 #4B8I/	0D0DdffdffUJ#-d#3dkk#A
4 #-d#3dkk#A
4 ||5<S*iN_N_5`2Z!2**RZZ(5014Cyy!}H++g&CJBJJrNc"##J/Y../E"	0J ll288Ca01779	 	/B }Hx.C
 yy|s"jjtvvtvv&67%c*"4B8I/	0D0DdffdffUJ#-d#3dkk#A
4 #-d#3dkk#A
4 ||5<S*iN_N_5`2Z!2**RZZ(5014CS!"":.OOI--.9	/D XXj)
88$45$$RXXi%89??A	 499Q;%5a3B3h%?%C2A2crc"./"1"RaR8>>@
+33Aa:DDR	R 

 IIbM$))B-!A#	#[!!$qa~WZ[^V_hklogpqqA	!./= " MF Ks   \ *$\11\6\.)NNr"   r"   Nrv   NTFFFF   r`   )r   r   r   r   r  r  r   r    r   r   r   8  s    04ORvxE?~$xr    r   r`   )r   )0rw   torch.nnr   r  ra   r   numpyr%   r   jsoncollectionsr   os.pathr   ospr   r   torch.utils.datar   pathlibr   r   r	   scipy.ior
   r   r   	functoolsshutilrK   r  r   r*   rG   rS   rV   r^   rd   r   ro   r   objectr   r   r   r   r   r   r   r    r   <module>rH     s     $ 	     #   $  
     
    	  	 	>"(>&
#$J2V 2	R-h[ [r    