+
    nDj$                        ^ RI t ^ RIt^ RIt^ RIt^ RIt ^ RIt^ RIt^ RI	H
t
 ^ RIt^ RIHt ^ RIHtHtHtHtHtHt ^ RIHt ^ RIHt  ^ RIHt ^ RIHtH t H!t!H"t"  ^ RI#t$^ RI%H&t' ]PP                  P8                  ]PP                  PR                  .t*R t+]PP                  PF                  R 4       t,]PP                  PF                  R 4       t-]PP                  PF                  R	 4       t.]PP                  PF                  R
 4       t/]PP                  Pa                  ]1]23RR7      ]PP                  PF                  R 4       4       t3]PP                  PF                  R 4       t4]PP                  PF                  R 4       t5]PP                  PF                  R 4       t6]PP                  PF                  R 4       t7]PP                  PF                  R 4       t8]PP                  PF                  ]PP                  Ps                  RRN.RN..]Pt                  ! R4      ^8  ]Pt                  ! RR4      ^8  ]Pt                  ! RR4      Pw                  ]Px                  ! 4       4      ^8  34      ]PP                  Ps                  RRO4      R 4       4       4       t=]PP                  PF                  R 4       t>]PP                  P~                  R 4       t@]PP                  P~                  R 4       tA]PP                  PF                  R 4       tB]PP                  PF                  ]PP                  P~                  R 4       4       tCR tDR  tER! tF]PP                  PF                  R" 4       tG]PP                  PF                  R# 4       tH]PP                  PF                  R$ 4       tI]PP                  P
                  R% 4       tJ]PP                  P
                  R& 4       tK]PP                  P
                  R' 4       tLRPR( ltMR) tN]PP                  P
                  ]PP                  Ps                  R*R+R,.4      R- 4       4       tO]PP                  P
                  R. 4       tP]PP                  P
                  R/ 4       tQ]PP                  P
                  ]PP                  Ps                  R*R+R,.4      R0 4       4       tRR1 tSR2 tTRQR3 ltURRR4 ltV]PP                  PF                  R5 4       tW]PP                  PF                  R6 4       tX]PP                  PF                  R7 4       tY]PP                  PF                  R8 4       tZ]PP                  PF                  R9 4       t[]PP                  PF                  ]PP                  P~                  R: 4       4       t\]PP                  PF                  ]PP                  P~                  ]PP                  P                  ]P                  R;8H  R<R=7      R> 4       4       4       t_]PP                  PF                  ]PP                  P~                  R? 4       4       t`]PP                  PF                  R@ 4       taRSRA ltb]PP                  PF                  RB 4       tc]PP                  PF                  RC 4       tdRD teRE tfRF tgRG thRH tiRI tjRJ tk]PP                  Ps                  RKRT4      RL 4       tl]PP                  PF                  RM 4       tmR#   ] d    Rt ELi ; i  ] d    Rt ELqi ; i  ] d	    R;t$t' ELxi ; i)U    N)FileSelector
FileSystemLocalFileSystemPyFileSystemSubTreeFileSystemFSSpecHandler)util)guid)_read_table_test_dataframe_test_table_write_tablec                    \         P                  ! R . RO/4      pV R,          pVP                  4        VR,          p\        P                  ! V\        V4      4       \        P                  ! V\        4       R7      pVP                  V4      '       g   Q h\        P                  ! R\        P                  ! V 4      R7      pVP                  V4      '       g   Q hR# )adata_dirdata.parquet
filesystemzdata_dir/data.parquetN         )patablemkdirpqwrite_tablestr
read_tabler   equalsr	   _filesystem_uri)tempdirr   	directorypathresults   &    \/data/cameron/venvs/s3viz/lib/python3.14/site-packages/pyarrow/tests/parquet/test_dataset.pytest_filesystem_urir'   ;   s    HHc9%&E*$IOO~%DNN5#d)$ ]]*,F== ]]D,@,@,IKF==    c                 0    \        4       p\        W4       R # N)r   _partition_test_for_filesystem)r"   locals   & r&   test_read_partitioned_directoryr-   N   s    E"52r(   c                     \        4       pT p\        W4       \        P                  ! V4      pVP	                  R .R7      pVP
                  R .8X  g   Q hR# )valuescolumnsN)r   r+   r   ParquetDatasetreadcolumn_names)r"   r,   	base_pathdatasetr%   s   &    r&   'test_read_partitioned_columns_selectionr7   T   sQ     EI"54	*G\\8*\-F8*,,,r(   c                    \        4       pT p^ ^.p. ROpRR.pRV.RV.RV..p\        P                  ! R\        P                  ! VRR7      P                  ^4      R\        P                  ! \        P                  ! \        P                  ! V\        R7      ^4      ^4      R\        P                  ! \        P                  ! \        P                  ! VR	R7      ^4      ^4      R
\        P                  ! ^4      /4      p\        WWg4       \        P                  ! W!. ROR7      pVP                  4       p	V	P                  4       P                  RR7      p
^ V
R,          P                  9  g   Q hRV
R,          P                  9  g   Q hRV
R,          P                  9  g   Q h. RORR..p\        P                  ! W!VR7      pVP                  4       p	V	P                  4       P                  RR7      p
V
R,          ^8H  V
R,          R8g  ,          V
R,          R8H  ,          p\        P                  ! V
R,          4      ^ 8H  V
R,          R8H  ,          pVP!                  4       ^ 8  g   Q hVP!                  4       ^ 8  g   Q hV
P"                  ^ ,          VP!                  4       VP!                  4       ,           8X  g   Q hR..R..3 F<  p\        P                  ! W!VR7      pVP                  4       P$                  ^ 8X  d   K<  Q h	  R# )r   bTFintegerstringbooleani4dtypeboolr/   Truer   filtersdropFalseNr   r9   c))r:   =r   )r;   !=r9   )r<   ==rA   )r:   rI   r   )r<   rK   rF   )r;   rK   s   1 a)r;   rK   z1 a)r   pd	DataFramenparrayrepeattileobjectarange_generate_partition_directoriesr   r2   r3   	to_pandasreset_indexr/   sumshapenum_rows)r"   r,   r5   integer_keysstring_keysboolean_keyspartition_specdfr6   r   	result_dfrC   df_filter_1df_filter_2s   &             r&   test_filters_equivalencyrb   a   s   EIq6L!K%=L	L!	;	L!N 
288L5<<R@"''"''"((;f"EqI1M277277288L#GKQO"))B-	 
B $EnI ,G
 LLNE"..D.9IIi(/////i)00000	),33333	

 
89G W6GLLNE!--4-8I Y'1,X#%'Y6)+K 88Ii01Q6Y7*,K??q   ??q   ??1+//"3koo6G"GHHH/01./02##:||~&&!+++	2r(   c           
         \        4       pT p. ROpRV..p^p\        P                  ! R\        P                  ! V4      R\        P
                  ! VRR7      /RR.R7      p\        WWF4       \        P                  ! W!RR.R7      pVP                  4       pVP                  4       P                  RR7      P                  RR	7      p	\        \        V	R,          P                  4       U
u. uF  qNK  	  pp
V^^.8X  g   Q hR
# u up
i )r   integersindexr=   r>   r0   rB   byTrD   Nr   r   r   r      )rd   <ri   )rd   >r   r   rL   rM   rN   rS   rO   rT   r   r2   r3   rU   sort_valuesrV   mapintr/   r"   r,   r5   rZ   r]   Nr^   r6   r   r_   xresult_lists   &           r&   %test_filters_cutoff_exclusive_integerrt      s   EI"L	\"N 	
A	1BHH\6 $
&B
 $EnI  
G LLNE""{g{."{{-  "#y'<'C'CDED1DKE1a&    Fs   )
Dz5Loss of type information in creation of categoricals.)raisesreasonc           
      \   \        4       pT p\        P                  ! R ^^	4      \        P                  ! R ^^
4      \        P                  ! R ^^4      \        P                  ! R ^^4      \        P                  ! R ^^4      .pRV..p^p\        P                  ! R\
        P                  ! V4      R\
        P                  ! VRR7      /RR.R7      p\        WWF4       \        P                  ! W!RR.R7      pVP                  4       pVP                  4       P                  RR7      P                  RR	7      p	\        P                  ! \
        P                  ! \        P                  ! R ^^4      .RR7      \
        P                  ! VRR7      R
7      p
V	R,          P                   V
8X  g   Q hR# )i  datesre   
datetime64r>   r0   rB   rf   TrD   
categoriesN)rx   rj   z
2018-04-12)rx   rk   z
2018-04-10)r   datetimedaterL   rM   rN   rS   rO   rT   r   r2   r3   rU   rm   rV   Categoricalr/   )r"   r,   r5   	date_keysr]   rq   r^   r6   r   r_   expecteds   &          r&   &test_filters_cutoff_exclusive_datetimer      sq    EI 	dAq!dAr"dAr"dAr"dAr"I 
)N 	
A	1)<8 !
#B
 $EnI((
G LLNE""{g{."{{-  ~~
(--a,-\B88I\:<H W$$000r(   c           
      b   V R ,          p\         P                  ! R\         P                  ! R^
RR7      R\        ^
4      /4      P	                  VRR7       \
        P                  ! VRR\        P                  ! R	^^4      3.R
7      pVP                  R4      P                  4       . RO8X  g   Q hR# )ztimestamps.parquetrx   z
2020-01-01D)periodsfreqidT)use_deprecated_int96_timestamps<=i  rC   Nrh   )
rL   rM   
date_rangerange
to_parquetr   r   r|   column	to_pylist)r"   r$   r   s   &  r&   test_filters_inclusive_datetimer      s     ))DLL|RcBeBi  z$z=MM$	$))$156) E <<'')_<<<r(   c           
      "   \        4       pT p. ROpRV..p^p\        P                  ! R\        P                  ! V4      R\        P
                  ! VRR7      /RR.R7      p\        WWF4       \        P                  ! W!RR.R7      pVP                  4       pVP                  4       P                  RR7      P                  RR	7      p	\        \        V	R,          P                  4       U
u. uF  p
\        V
4      NK  	  pp
V^^.8X  g   Q hR
# u up
i )r   rd   re   r=   r>   r0   rB   rf   TrD   Nrh   )rd   r   r   )rd   z>=r   rl   rp   s   &           r&   test_filters_inclusive_integerr     s   EI"L	\"N 	
A	1BHH\6 $
&B
 $EnI!!
G LLNE"++)+4+(  $'sIj,A,H,H#IJ#Ia3q6#IKJ1a&    Ks   )Dc                    \        4       pT p^ ^.p. ROpRR.pRV.RV.RV..p\        P                  ! R\        P                  ! VR	R
7      P                  ^4      R\        P                  ! \        P                  ! \        P                  ! V\        R
7      ^4      ^4      R\        P                  ! \        P                  ! \        P                  ! VRR
7      ^4      ^4      R\        P                  ! ^4      /4      p\        WWg4       \        P                  ! W!R.R7      pVP                  4       p	V	P                  4       P                  RR7      p
RV
R,          P                  9   g   Q hRV
R,          P                  9   g   Q hRV
R,          P                  9  g   Q h\        P                  ! W!RR^.3RRRR03.R7      pVP                  4       p	V	P                  4       P                  RR7      p
^ V
R,          P                  9  g   Q hRV
R,          P                  9  g   Q hRV
R,          P                  9  g   Q hR# )r   r   r9   rH   TFr:   r;   r<   r=   r>   r@   r/   inrB   rD   znot inrF   NrG   )r;   r   ab)r;   r   )r   r9   )r   rL   rM   rN   rO   rP   rQ   rR   rS   rT   r   r2   r3   rU   rV   r/   )r"   r,   r5   rZ   r[   r\   r]   r^   r6   r   r_   s   &          r&   test_filters_inclusive_setr   .  s
   EIq6L!K%=L	L!	;	L!N 
288L5<<R@"''"''"((;f"EqI1M277277288L#GKQO"))B-	 
B $EnI'(G LLNE"..D.9I)H%,,,,,)H%,,,,,i)00000TA3')EXy13G
 LLNE"..D.9IIi(/////i)00000	),33333r(   c           
         \        4       pT p. R
OpRV..p^p\        P                  ! R\        P                  ! V4      R\        P
                  ! VRR7      /RR.R7      p\        WWF4       \        P                  ! \        4      ;_uu_ 4        \        P                  ! VVR.R7       RRR4       \        P                  ! \        4      ;_uu_ 4        \        P                  ! VVR.R7       RRR4       \        P                  ! VVRR\        4       3.R7      pVP                  4       P                  ^ 8X  g   Q h\        P                  ! VVRR	^03.R7      p\        P                  ! \         4      ;_uu_ 4        VP                  4       P                  ^ 8X  g   Q h RRR4       R#   + '       g   i     EL	; i  + '       g   i     L; i  + '       g   i     R# ; i)r   rd   re   r=   r>   r0   r   rB   NrJ   rh   )rd   r   r   )rd   z=<r   )r   rL   rM   rN   rS   rO   rT   pytestru   	TypeErrorr   r2   
ValueErrorsetr3   rY   NotImplementedError)r"   r,   r5   rZ   r]   rq   r^   r6   s   &       r&   test_filters_invalid_pred_opr   ]  s   EI"L	\"N 	
A	1BHH\6 $
&B
 $EnI	y	!	!
)%*#8";	= 
"
 
z	"	"
)%*#8";	= 
# 	+0*4dCE)B(EGG <<>""a'''	+0*4dQC)@(ACG 
*	+	+||~&&!+++ 
,	+' 
"	!	!
 
#	" 
,	+	+s$   F)F=<"G)F:	=G	G!	c           
         \        4       pT p. R
OpRV..p^p\        P                  ! R\        P                  ! V4      R\        P
                  ! VRR7      /RR.R7      p\        WWF4       Rp\        P                  ! \        VR7      ;_uu_ 4        \        P                  ! W!R.R7      P                  4        R	R	R	4       R	#   + '       g   i     R	# ; i)r   rd   re   r=   r>   r0   z1No match for FieldRef.Name\(non_existent_column\)matchrB   Nrh   )non_existent_columnrj   r   )r   rL   rM   rN   rS   rO   rT   r   ru   r   r   r2   r3   )r"   r,   r5   rZ   r]   rq   r^   msgs   &       r&   test_filters_invalid_columnr     s     EI"L!<01N	A	1BHH\6 $
&B
 $EnI
>C	z	-	-
)#B"E	GGKtv 
.	-	-	-s   (CC	rC   rd   nestedr   r9   read_methodc                    \        \        V4      p\        4       pT p. R
OpRV..p\        V4      p\        P
                  ! R\        P                  ! V4      R\        P                  ! VRR7      R\        P                  ! \        V4       U	u. uF  p	RV	R\        V	4      /NK  	  up	4      /4      p
\        WEWz4       \        WAR7      pV! V3/ VB pVP                  ^8X  g   Q hR	# u up	i )r   rd   re   r=   r>   r   r   r9   rB   Nrh   )getattrr   r   lenrL   rM   rN   rS   rO   r   r   rT   dictrY   )r"   rC   r   r3   r,   r5   rZ   r]   rq   ir^   kwargsr   s   &&&          r&   test_filters_read_tabler     s     2{#DEI"L	\"N 	LA	1BHH\6"((58D8aS!S#a&18DE 
B $EnIU4F%f%E>>Q Es   C'c           
      t   \        4       pT pR R.pRV..p^p\        P                  ! R\        P                  ! V4      R\        P
                  ! VRR7      /RR.R7      p\        WWF4       \        P                  ! V4      pVP                  4       pVP                  R4      P                  4       V8X  g   Q hR# )2019_22019_3	year_weekre   rR   r>   r0   N)r   rL   rM   rN   rS   rO   rT   r   r2   r3   r   r   )	r"   r,   r5   r[   r]   rq   r^   r6   r%   s	   &        r&   $test_partition_keys_with_underscoresr     s     EIX&K	k"N 	
A	1RXXk: %
'B
 $EnI	*G\\^F==%//1[@@@r(   c                     V w  rVR ,           p\         P                  ! R. RO/4      p\        W2VR7       \        W!R7      pVP	                  V4      '       g   Q hR# z/test.parquetr   r   Nr   r   r   r   r   r    )s3_example_s3fsfsr$   r   r%   s   &    r&   test_read_s3fsr     sO    HB/!DHHc9%&E,-F==r(   c                     V w  rVR ,           p\         P                  ! R. RO/4      p\        WCVR7       \        W!R7      pVP	                  V4      '       g   Q hR# r   r   )r   r   r#   r$   r   r%   s   &     r&   test_read_directory_s3fsr     sO    #MB&DHHc9%&E,2F==r(   c                     \        V R ,          4      p\        P                  ! R. RO/4      p\        W!4       \        P
                  ! V.4      P                  4       pVP                  V4      '       g   Q hR# )r   r   Nr   )r   r   r   r   r   r2   r3   r    )r"   	data_pathr   r%   s   &   r&   test_read_single_file_listr     s\    Gn,-IHHc9%&E"	{+002F==r(   c                 $    V w  r\        W4       R # r*   )r+   r   r   r$   s   &  r&   $test_read_partitioned_directory_s3fsr     s     HB"2,r(   c                    ^ ^.p. ROpRV.RV..p^p\         P                  ! R\        P                  ! V4      R\        P                  ! VRR7      P                  ^4      R\        P                  ! \        P                  ! \        P                  ! V\        R7      ^4      ^4      R\        P                  P                  V4      /. ROR7      p\        WWF4       \        P                  ! WR7      pVP                  4       pVP                  4       P                  RR	7      P!                  R
R7      p	VP                  RR	7      P!                  R
R7      P#                  V	P$                  R7      p
V
R,          P'                  R4      V
R&   V
R,          P'                  R4      V
R&   V	P$                  . RO8H  P)                  4       '       g   Q h\*        P,                  ! W4       R# )r   foobarre   r=   r>   r/   r0   r   rf   TrD   categoryNrG   )re   r   r   r/   )re   r/   r   r   )rL   rM   rN   rS   rO   rP   rQ   rR   randomrandnrT   r   r2   r3   rU   rm   rV   reindexr1   astypealltmassert_frame_equal)r   r5   foo_keysbar_keysr]   rq   r^   r6   r   r_   expected_dfs   &&         r&   r+   r+      s   1vHH		N 	A	1rxx-44R8rwwrwwrxx?CQG"))//!$	
 1
2B $B>F	9GLLNE"++)+4+(  >>W>-KTK*GI$5$5G6  %U+22:>K$U+22:>K!BBGGIIII)1r(   c           	         a aaaaa \        S \        4      '       g   \        \        S 4      4      o \	        S4      o\        S R \        S RR4      4      oVVVV VV3R loS! V^ . 4       R# )pathsepsep/c                   < SV,          w  r4V EF  pW#V3.,           pSP                  \        V 4      V R V 2.4      pSP                  V4       VS^,
          8X  Ed   ^ RIHp SP                  V\        4       .4      p	\        SV4      p
\        P                  P                  V
4      pSP                  V	4      ;_uu_ 4       p\        W4       RRR4       SP                  V	4      P                  VP                  8w  g   Q hSP                  V	4      P                  VP                  8X  g   Q hSP                  VR.4      pSP                  V4      ;_uu_ 4       p RRR4       EKZ  S! Wq^,           V4       SP                  VR.4      pSP                  V4      ;_uu_ 4       p RRR4       EK  	  R#   + '       g   i     L; i  + '       g   i     EK  ; i  + '       g   i     EK  ; i)rI   )FileTypeN_SUCCESS)joinr   
create_dir
pyarrow.fsr   r
   _filter_partitionr   Tablefrom_pandasopen_output_streamr   get_file_infotypeNotFoundFile)base_dirlevel	part_keysnamer/   valuethis_part_keys	level_dirr   	file_pathfiltered_df
part_tableffile_successDEPTH_visit_levelr^   r   r]   r   s   &&&           r&   r   5_generate_partition_directories.<locals>._visit_level2  s   %e,E&-8NH&%!& I MM)$	!/#LL)TV)<=	/NCXX11+>
**955 / 6''	2778;L;LLLL''	2778==HHH&||Y
,CD**<88A 98 Y	>B&||Y
,CD**<88A 987   65 988
 988s$   F6G	%G6G	GG0N)
isinstancer   r   r   r   r   )r   r   r]   r^   r   r   r   s   f&ff@@@r&   rT   rT   '  s[     b*%%-+,Eb)WR%<=G @ 1b!r(   c                 j   \         P                  ! \        V 4      \        R 7      p. pV Fn  w  rEVP	                  V4       \        V\        P                  \        P                  34      '       d   \        P                  ! V4      pW V,          V8H  ,          pKp  	  W,          P                  V^R7      # )r>   )axis)rN   onesr   r@   appendr   r|   r}   rL   	TimestamprE   )r^   r   	predicateto_dropr   r   s   &&    r&   r   r   U  s    Bt,IG t ehmmX->->?@@LL'EX&&	 ! =gA..r(   c                 l   V R ,          pVP                  4        \        P                  P                  \        P
                  ! R. RO/4      4      p\        P                  ! W!R,          4       V R,          pVP                  4        \        P                  P                  \        P
                  ! R. RO/4      4      p\        P                  ! WCR,          4       \        P                  ! V R..R7      pVP                  R4      P                  \        P                  ! . RO.4      4      '       g   Q hR# )	zA=0Br   zA=1r   Nr   rG   )ArK   r   )r   r   r   r   rL   rM   r   r   r   r   r    chunked_array)r"   dir1table1dir2table2r   s   &     r&   "test_filter_before_validate_schemar   e  s     U?DJJLXX!!",,Y/?"@AFNN6.01U?DJJLXX!!",,_/E"FGFNN6.01 MM'^,<+=>E<<##B$4$4i[$ABBBBr(   c                    ^
p^pV \        4       ,          pVP                  4        . p. p\        V4       F  p\        W&R7      pVR,          P	                  \
        P                  4      VR&   W6 R2,          p\        P                  P                  V4      p	\        W4       VP                  V	4       VP                  V4       K  	  VR,          P                  4        RR lp
V
! V4      p\        P                  ! V4      pVP                  V4      '       g   Q h^ ^^VP                  ^,
          .pV Uu. uF  qkP!                  V4      P"                  NK  	  pp\$        P&                  ! W>R7      p\        P                  P)                  V Uu. uF  qkP+                  V4      NK  	  upVVP,                  P.                  R	7      pVP                  V4      '       g   Q h\$        P&                  ! VRR
7       \        VXR7      P0                  R,          pV \        4        R2,          p\        P                  P                  V4      p\        VV4       R# u upi u upi )
   seeduint32.parquetz_SUCCESS.crcNTc                 T    \         P                  ! V 3/ VB pVP                  WR 7      # ))r1   use_threads)r   r2   r3   )pathsr1   r  r   r6   s   &&&, r&   read_multiple_files5test_read_multiple_files.<locals>.read_multiple_files  s(    ##E4V4||G|EEr(   r0   )namesmetadata)r  )NT):NNN:Nri   N)r
   r   r   r   r   rN   int64r   r   r   r   r   touchconcat_tablesr    num_columnsfieldr   r   r   from_arraysr   schemar  iloc)r"   nfilessizedirpath	test_datar  r   r^   r$   r   r  r%   r   to_read	col_namesout	bad_applebad_apple_pathts   &                  r&   test_read_multiple_filesr  {  s   FDGMMOIE6]T* (|**288483h'$$R(U!T  ~$$&F !'F	*H=="""" !Q**Q./G/67w!a%%wI7
--
3Cxx##w$Gw!]]1%5w$G*3-3]]-C-C $ EH ::h MM't,  1-2259I$&22N
Y'AN# 8$Gs   6#IIc                    ^p^pV \        4       ,          pVP                  4        . p. p. p\        V4       F  p\        W'R7      p\        P
                  ! Wr,          V^,           V,          4      Vn        RVP                  n        W7 R2,          p	\        P                  P                  V4      p
\        W4       VP                  V
4       VP                  V4       VP                  V	4       K  	  \        P                  ! V4      pRR.pVP                  VR7      P!                  4       p\"        P$                  ! V Uu. uF  qV,          NK  	  up4      p\&        P(                  ! W4       VP                  \+        V4      R7      P!                  4       pVP,                  VP,                  8X  g   Q h\&        P(                  ! VP/                  VP0                  R7      V4       R# u upi )   r  re   r  uint8stringsr0   N)r
   r   r   r   rN   rS   re   r   r   r   r   r   r   r   r2   read_pandasrU   rL   concatr   r   r   rX   r   r1   )r"   r  r  r  r  framesr  r   r^   r$   r   r6   r1   r%   rr   r   s   &               r&   test_dataset_read_pandasr&    s   FDGMMOIFE6]T*99QXA~63h'$$R(U!bT  (G	"G   1;;=Fyyf5fG**f56H&+   W 6@@BF<<8>>)))&..1A1A.BHM 6s   G(c                    V \        4       ,          pVP                  4        \        ^
^ R7      pVR,          p\        W#RR7       \        P
                  ! VRR7      pVP                  4       P                  V4      '       g   Q hR# )r   r  	0.parquet2.6versionT)
memory_mapN)r
   r   r   r   r   r2   r3   r    )r"   r  r   r$   r6   s   &    r&   test_dataset_memory_mapr-    sk     GMMO#E[ De,D"G<<>  ''''r(   c                    V \        4       ,          pVP                  4        \        ^
^ R7      pVR,          p\        W#RR7       \        P
                  ! \        4      ;_uu_ 4        \        P                  ! VRR7       RRR4       R FA  p\        P                  ! WR7      pVP                  4       P                  V4      '       d   KA  Q h	  R#   + '       g   i     LY; i)	r   r  r(  r)  r*  )buffer_sizeNi)   i   )r
   r   r   r   r   ru   r   r   r2   r3   r    )r"   r  r   r$   r/  r6   s   &     r&   #test_dataset_enable_buffered_streamr1    s    GMMO#E[ De,	z	"	"
	& 
# ###.||~$$U++++ #	 
#	"s   'CC!	c                 |   V \        4       ,          pVP                  4        \        ^
^ R7      pVR,          p\        W#RR7       R Fq  p\        P
                  ! WR7      pVP                  4       P                  V4      '       g   Q h\        P                  ! WR7      pVP                  V4      '       d   Kq  Q h	  R# )r   r  r(  r)  r*  )
pre_bufferN)TF)	r
   r   r   r   r   r2   r3   r    r   )r"   r  r   r$   r3  r6   actuals   &      r&   test_dataset_enable_pre_bufferr5    s    GMMO#E[ De,#
##,||~$$U++++w>}}U#### $r(   c                     . p. p\        V4       FE  p\        W%R 7      pW R2,          pVP                  \        Wg4      4       VP                  V4       KG  	  V# )r  r  )r   r   r   r   )r5   r  
file_nrowsr  r  r   r   r$   s   &&&     r&   _make_example_multifile_datasetr8    sX    IE6]J/S>)e23T  Lr(   c                     V Uu. uF  p\        VP                  4       4      NK  	  pp\        V4      \        V P                  4      8X  g   Q hR # u upi r*   )r   as_posixr   files)r6   r  r$   s   && r&   _assert_dataset_pathsr<  (  sA    .34edS!eE4u:W]]++++ 5s   "A
dir_prefix_.c                     V \        4       ,          pVP                  4        \        V^
^R7      pW! R2,          P                  4        \        P                  ! V4      p\        WC4       R# )r   r  r7  stagingNr
   r   r8  r   r2   r<  r"   r=  r  r  r6   s   &&   r&   test_ignore_private_directoriesrE  -  sZ     GMMO+GB78:E W%%,,.(G')r(   c                    V \        4       ,          pVP                  4        \        V^
^R7      pVR,          P                  R4      ;_uu_ 4       pVP	                  R4       RRR4       VR,          P                  R4      ;_uu_ 4       pVP	                  R4       RRR4       \
        P                  ! V4      p\        WB4       R#   + '       g   i     Lm; i  + '       g   i     LF; i)r   rA  z	.DS_Storewbs	   gibberishNz.privater
   r   r8  openwriter   r2   r<  r"   r  r  r   r6   s   &    r&   test_ignore_hidden_files_dotrL  >  s    GMMO+GB78:E K
	%	%d	+	+q	 
, J
	$	$T	*	*a	 
+ (G') 
,	+ 
+	*   C
CC	C*	c                    V \        4       ,          pVP                  4        \        V^
^R7      pVR,          P                  R4      ;_uu_ 4       pVP	                  R4       RRR4       VR,          P                  R4      ;_uu_ 4       pVP	                  R4       RRR4       \
        P                  ! V4      p\        WB4       R#   + '       g   i     Lm; i  + '       g   i     LF; i)r   rA  _committed_123rG  s   abcdN_started_321rH  rK  s   &    r&   #test_ignore_hidden_files_underscorerQ  Q  s    GMMO+GB78:E $
$	*	*4	0	0A	 
1 N
"	(	(	.	.!	 
/ (G') 
1	0 
/	.rM  c                     W R 2,          \        4       ,          pVP                  RR7       \        V^
^R7      p\        P                  ! V4      p\        WC4       \        P                  ! V4      p\        WC4       R# )dataTparentsrA  NrC  rD  s   &&   r&   /test_ignore_no_private_directories_in_base_pathrV  d  sm    
 ,d++df4GMM$M+GB78:E &G') (G')r(   c           	      "   R .^,          R.^,          ,           p\         P                  ! \         P                  ! \        \	        V4      4      4      \         P                  ! V4      P                  4       .RR.R7      p\        P                  ! V\        V 4      R.R7       V R,          pVP                  4        \        P                  ! V\        V4      R.R7       \        P                  ! V R.R7      pVP                  V4      '       g   Q hR	# )
xxxyyyre   _partr
  partition_cols_private_duplicate_private)ignore_prefixesN)r   r   rO   r   r   dictionary_encoder   write_to_datasetr   r   r   r    )r"   partr   private_duplicater3   s   &    r&   test_ignore_custom_prefixesre  w  s    7Q;%1$DHH
s4y!"
((* w!E
 s7|WIF"66s#45(/y2 ==*/D ;;ur(   c                     V R ,          pVP                  4        \        P                  ! V4      pVP                  4       pVP                  ^ 8X  g   Q hVP
                  ^ 8X  g   Q hR# )r6   N)r   r   r2   r3   rY   r  )r"   	empty_dirr6   r%   s   &   r&   test_empty_directoryrh    sW    )#IOO	*G\\^F??a"""r(   c                 |   ^ RI p^ RIHp ^ RIHp VP
                  ! R\        R4      R\        R4      R\        \        ^
4      4      R\        P                  .^
,          R\        P                  ! R	R
RR7      P                  R4      /4      pVP                  P                  4       pRR.p	\        P                  P!                  WrRRR7      p
VP"                  ! WV	VR7       \$        P&                  P)                  \+        V 4      R4      pVeB   VP-                  VR4      ;_uu_ 4       pVP.                  ! V
P0                  V4       RRR4       MVP.                  ! V
P0                  V4       VP2                  ! V VR7      p\5        VP0                  P6                  4      pV\5        V
P0                  P6                  4      8X  g   Q hVP9                  4       pVP;                  4       pVP                  P                  4       pV	VR\=        V	4      ,          R 8X  g   Q hVV,          pV	 F  pVV,          P                  R4      VV&   K   	  V'       dE   VP?                  R4      P@                  PC                  4       pVR,          P                  V4      VR&   VPD                  ! VV4       R#   + '       g   i     ELN; i)r   Ngroup1
aaabbbbcccgroup2
eefeffgeeenumnanr}   
2017-01-01
2017-01-11datetime64[D]r>   datetime64[ns]F)r  safepreserve_indexr   _common_metadatarG  r   )#pandaspandas.testingtestingpyarrow.parquetparquetrM   listr   rN   ro  rS   r   r1   tolistr   r   r   rb  osr$   r   r   rI  write_metadatar  r2   r   r
  r3   rU   r   r  r   to_pandas_dtyper   )r5   r   r  
index_namerL   r   r   	output_dfcolspartition_byoutput_tablemetadata_pathr   r6   dataset_colsinput_tableinput_dfinput_df_colscolexpected_date_types   &&&&                r&   &_test_write_to_dataset_with_partitionsr    sO      $|$$|$tE"Ix"}		,OLSS I ##%Dh'L88''	u7< ( >L#-/ GGLLY1CDM__]D11Ql1115 21 	,--}=	+57G w~~++,L3|22889999,,.K$$&H $$++-M=c,.?)?)@AAAA~H"3..z:	#  #\\&166FFH%f-445GH	&)X.= 211s   )J**J;	c                    ^ RI p^ RIHp VP                  ! R\	        R4      R\	        R4      R\	        \        ^
4      4      R\        P                  ! RR	R
R7      P                  R4      /4      pVP                  P                  4       p\        P                  P                  V4      pVf   \        4       pM*\        V\         4      '       g   \#        \%        V4      4      p^p\        V4       F  pVP&                  ! W`VR7       K  	  \)        \+        V 4      RRR7      p	VP-                  V	4      p
V
 Uu. uF'  qP.                  P1                  R4      '       g   K%  VNK)  	  pp\3        V4      V8X  g   Q hVP4                  ! WR7      P7                  4       pVP9                  4       pVP;                  4       pW,          p\<        P>                  ! WN4       R# u upi )r   Nrj  rk  rl  rm  rn  r}   rp  rq  rr  r>   rs  r   FT)allow_not_found	recursiver  ) rx  r{  r|  rM   r}  r   rN   rS   r   r1   r~  r   r   r   r   r   r   r   r   rb  r   r   r   r$   endswithr   r2   r3   rU   drop_duplicatesr   r   )r5   r   rL   r   r  r  r  nr   selectorinfosinfooutput_filesr  r  s   &&             r&   $_test_write_to_dataset_no_partitionsr    s     $|$$|$tE"I		,OLSS	 I ##%D88''	2L$&

J//!-
";<
 	
A1X
L'1	3  C	NE&*,H $$X.E%*MUTii.@.@.LDDULM|!!! ##
df  $$&H'')H~H). Ns   5"GGc                 .    \        \        V 4      4       R # r*   r  r   r"   s   &r&   %test_write_to_dataset_with_partitionsr    s    *3w<8r(   c                    \         P                  ! \         P                  ! R \         P                  ! 4       R7      \         P                  ! R\         P                  ! 4       R7      \         P                  ! R\         P                  ! 4       R7      \         P                  ! R\         P
                  ! 4       R7      \         P                  ! R\         P                  ! RR7      R7      .4      p\        \        V 4      VR7       R	# )
rj  )r   rl  rn  ro  r}   us)unitr  N)	r   r  r  r;   r  int32	timestampr  r   )r"   r  s   & r&   0test_write_to_dataset_with_partitions_and_schemar    s    YY		<		<RXXZ8RXXZ8bll.EF	H IF
 +GV%r(   c                 2    \        \        V 4      R R7       R# )r  )r  Nr  r  s   &r&   4test_write_to_dataset_with_partitions_and_index_namer    s    *G/r(   c                 .    \        \        V 4      4       R # r*   )r  r   r  s   &r&   #test_write_to_dataset_no_partitionsr    s    (W6r(   c                 N    \        V R ,          4       \        V R,          4       R# )test1test2N)r  r  r  s   &r&   test_write_to_dataset_pathlibr    s    *7W+<=(7):;r(   c                 `   Vw  r#\         P                  ! \        R R7      ;_uu_ 4        \        V R,          VR7       RRR4       \         P                  ! \        R R7      ;_uu_ 4        \	        V R,          VR7       RRR4       R#   + '       g   i     LS; i  + '       g   i     R# ; i)z"path-like objects are only allowedr   r  r   Nr  )r   ru   r   r  r  )r"   r   r   r>  s   &&  r&   &test_write_to_dataset_pathlib_nonlocalr  #  s{     EB	y(L	M	M.g"	. 
N 
y(L	M	M,g"	. 
N	M	 
N	M 
N	M	Ms   B	*B	B	B-	win32z,test fails because of unsupported characters)rv   c                 &    V w  r\        W!R 7       R# r   N)r  r   s   &  r&   *test_write_to_dataset_with_partitions_s3fsr  2  s     HB*r(   c                 &    V w  r\        W!R 7       R# r  )r  r   s   &  r&   (test_write_to_dataset_no_partitions_s3fsr  >  s     HB(r(   c                 ,   \         P                  ! R . RO/4      p\        P                  P	                  V4      p\        V 4      p\        P                  ! W#\        4       R7       \        P                  ! V4      pVP                  V4      '       g   Q hR# )r   r   Nr   )rL   rM   r   r   r   r   r   rb  r   r   r    )r"   r^   r   r$   r%   s   &    r&    test_write_to_dataset_filesystemr  G  sh    	sI&	'BHH  $Ew<D0AB]]4 F==r(   c                 >   V R ,          p\        4       p\        P                  ! R\        P                  ! V4      R\        P
                  P                  V4      /RR.R7      p\        P                  P                  V4      p^p\        P                  ! W%P                  4      ;_uu_ 4       p\        V4       F  pVP                  V4       K  	  RRR4       \        P                  ! V4      p	V	P                   P"                  V8X  g   Q hV R,          p
VP%                  \'        V
4      4      ;_uu_ 4       p\        P(                  ! VP                  V4       RRR4       \        P*                  ! WR7      pV#   + '       g   i     L; i  + '       g   i     L<; i)r   re   r/   r0   N	_metadatar   )r   rL   rM   rN   rS   r   r   r   r   r   r   ParquetWriterr  r   r   ParquetFiler  num_row_groupsr   r   r  r2   )r"   rq   r$   r,   r^   r   
num_groupswriterr   readerr  r   r6   s   &&           r&   _make_dataset_for_picklingr  R  s5   ^#DE	1"))//!$ "
$B HH  $EJ			$	-	-z"Au% # 
. ^^D!F??))Z777k)M		!	!#m"4	5	5
%,,* 
6 #G N 
.	- 
6	5s   ,$E96"F9F		F	c                 J   a V3R  lp\        V 4      pV! V4      '       g   Q hR# )c                 J   < V SP                  SP                  V 4      4      8H  # r*   )loadsdumps)objpickle_modules   &r&   is_pickleable*test_pickle_dataset.<locals>.is_pickleablep  s$    m))-*=*=c*BCCCr(   N)r  )r"   r  r  r6   s   &f  r&   test_pickle_datasetr  n  s&    D )1G!!!!r(   c           	      d   V R ,          p\         P                  ! R. ROR. ROR. R	O/4      p\        P                  P	                  V4      p\
        P                  ! V\        V4      RR.R7       \
        P                  ! V4      P                  4       p\
        P                  ! W1R,          4       R# )
z
ARROW-3208onetwothree)	root_pathr]  zoutput.parquetN)rw  r   g      @d     r   g333333=@)rw  r   r   r  r  r      )r   r   r   r   r   r   r   )rL   rM   r   r   r   r   rb  r   r2   r3   r   )r"   r$   r^   r   s   &   r&   test_partitioned_datasetr  w  s     \!D	0,& 
B
 HH  $ET(-u~7d#((*ENN5!112r(   c                 `   V R ,          p\         P                  ! \        ^4       Uu. uF  p\        P                  ! ^
4      NK  	  up^
,          .R.R7      p\         P                  ! \        ^4       Uu. uF  p\        P                  ! ^
4      NK  	  up^
,          .R.R7      p\
        P                  ! V\        V4      R7       \
        P                  ! V\        V4      R7       \
        P                  ! VR.R7      P                  4       pV^ ,          P                  ^ 4      P                  4       V^ ,          P                  ^ 4      P                  4       .pV^ ,          P                  ^8X  g   Q hV^ ,          P                  ^ 4      V^ ,          P                  ^4      rVP                  V^ ,          4      '       d#   VP                  V^,          4      '       g   Q hR# VP                  V^,          4      '       g   Q hVP                  V^ ,          4      '       g   Q hR# u upi u upi )zARROW-3325-datasetf0r[  )r  )read_dictionaryN)r   r   r   r	   randsr   rb  r   r2   r3   chunkra  
num_chunksr    )	r"   r$   r   t1t2r%   	ex_chunksc0c1s	   &        r&   test_dataset_read_dictionaryr    s   ))D	E!H5HqDJJrNH5:;D6	JB	E!H5HqDJJrNH5:;D6	JBc$i0c$i0tf&&*df  AQ113AQ1135I !91$$$AY__Q!3	yy1yy1&&&&yy1&&&&yy1&&&&% 65s   H&5H+c                     \         P                  ! R \         P                  ! . RO\         P                  ! 4       4      /4      p\        P
                  ! WR,          4       \        P
                  ! WR,          4       \         P                  ! R.4      p\        P                  ! V R,          VR7      p\         P                  ! R . RO/VR7      pVP                  V4      '       g   Q h\        P                  ! WR7      p\         P                  ! R . RO/VR7      pVP                  V4      '       g   Q h\        P                  ! WR7      p\         P                  ! R . RO/VR7      pVP                  4       P                  V4      '       g   Q hR# )r   zdata1.parquetzdata2.parquetr  Nr   )r   r  )r   r   r   r   r   r   )r   r   rO   r  r   r   r  r   r    r2   r3   )r"   r   r  r%   r   s   &    r&   test_read_table_schemar    s   HHc288Irxxz:;<ENN5O34NN5O34YY'(F ]]7_4VDFxxi(8H=="""" ]]72Fxx01&AH==""""w6Fxx01&AH;;=))))r(   c           
         \         P                  ! R \         P                  ! . RO\         P                  ! 4       4      R\         P                  ! . RO\         P                  ! 4       4      /4      p\
        P                  ! WR,          4       \
        P                  ! V R,          R R .R7      p\         P                  ! RR.4      pVP                  R R .8X  g   Q hVP                  V8X  g   Q hR# )r   r9   r   r0   Nr   )r   r  )
r   r   rO   r  r!  r   r   r   r  r4   )r"   r   r%   expected_schemas   &   r&   *test_read_table_duplicate_column_selectionr    s    HHc288Irxxz:288Irxxz:< =ENN5N23]]7^3c3ZHFii @AO3*,,,==O+++r(   c                 :   ^ RI Hp V R,          pVR,          R,          R,          P                  RR7       \        P                  ! R. RO/4      p\
        P                  ! V\        VR,          R,          R,          R	,          4      4       VP                  . ROR
7      p\
        P                  ! \        V4      VR7      pVP                  . RO8X  g   Q h\
        P                  ! \        V4      VR7      P                  4       pVP                  . RO8X  g   Q hR# )r   Ntest_partitioning20121001TrT  r   r   )field_names)partitioningr   )yearmonthday)r   r  r  r  )pyarrow.datasetr6   r   r   r   r   r   r   r  r   r4   r2   r3   )r"   dsr  r   rc  r%   s   &     r&   test_dataset_partitioningr    s      --I$%,,T,:HHc9%&ENNs9v%,t3nDEG ??'??@D]]IT+F"????IT++/46 "????r(   c                 0   \         P                  ! R . RO/4      p\        P                  ! WR,          4       \	        \        V 4      \        4       4      p\        P                  ! RVR7      pVP                  4       pVP                  V4      '       g   Q hR# )r   r   r?  r   Nr   )
r   r   r   r   r   r   r   r2   r3   r    )r"   r   r   r6   r%   s   &    r&   #test_parquet_dataset_new_filesystemr    sk    HHc9%&ENN5N23"3w<1BCJ
;G\\^F==r(   c                    \         P                  ! R 4      pVP                  R4      p\        P                  ! R. R	O/4      p\
        P                  ! W0R,          4       \        V 4      P                  RR4      p\
        P                  ! WBR7      pVR,           pVP                  ^ ,          P                  V8X  g   Q hR# )
fsspecfiler   r   \r   r   z/data.parquetNr   )r   importorskipr   r   r   r   r   r   replacer2   	fragmentsr$   )r"   r  r   r   r$   r6   r   s   &      r&   6test_parquet_dataset_partitions_piece_path_with_fsspecr    s       *F""6*JHHc9%&ENN5N23 w<c*D%G o%HQ$$000r(   c                 L  a \         P                  ! R . R
O/4      pV R,          p. oV3R lpRp\        P                  ! WR .VVR7       VR,          R,          VR,          R,          VR,          R,          0p\	        \        \        P                  S4      4      pWe8X  g   Q hR	# )r   r  c                 >   < SP                  V P                  4       R # r*   )r   r$   )written_filepaths_writtens   &r&   file_visitorDtest_parquet_write_to_dataset_exposed_keywords.<locals>.file_visitor  s    \../r(   zpart-{i}.parquet)r  r  basename_template1zpart-0.parquet23Nr   )r   r   r   rb  r   rn   pathlibPath)r"   r   r$   r  r   expected_pathspaths_written_setr  s   &      @r&   .test_parquet_write_to_dataset_exposed_keywordsr    s    HHc9%&E^#DM0 +3%%1*;=
 	s
%%s
%%s
%%N
 Cm<=...r(   write_dataset_kwargc                8   ^ RI Hp \        P                  ! R. RO/4      pV R,          p\        P
                  ! VP                  4      pVw  rgV\        P
                  ! \        P                  4      P                  9  g   Q hWeP                  9   g   Q h\        P                  P                  VRRR7      ;_uu_ 4       p\        P                  ! W43/ Wg/B  VP                  ^ ,          w  rpW,          V8X  g   Q h RRR4       R#   + '       g   i     R# ; i)zEVerify kwargs in pq.write_to_dataset are passed onto ds.write_datasetNr   zout.parquetwrite_datasetT)autospecr   )r  r6   r   r   inspect	signaturer  r   rb  
parametersmockpatchrR   
mock_calls)r"   r	  r  r   r$   r  keyargmock_write_dataset_name_argsr   s   &&          r&   #test_write_to_dataset_kwargs_passedr    s     !HHc9%&E]"D!!""2"23I"HC g''(;(;<GGGGG&&&&&			2		>	>!
E6C:61<<Q?f{c!!!	 
?	>	>	>s   ?>DD	c                    \         P                  ! R \         P                  ! . RO. ROR7      R. R	O/4      p\        P                  ! V4      pV R,          p\
        P                  ! W R,          R .R7       VP                  4        Uu. uF&  qDP                  4       '       g   K  VP                  NK(  	  pp\        V4      ^8X  g   Q hRV9  g   Q hR# u upi )
catrz   r  r6   r\  zcat=cN)r   r9   r   rG   r   )rL   rM   r~   r   r   r   rb  iterdiris_dirr   r   )r"   r^   r   r$   r   subdirss   &     r&   'test_write_to_dataset_category_observedr  '  s    
 
r~~o/Jy 
B HHRLEYD"E7  $||~<~!vqvv~G<w<1'!!! =s   C$C)rd   rj   r   )r   r#  )r   r   )NNNr*   )r  ))r   T)r   F)nr|   r  r  r  sysnumpyrN   ImportErrorr   unittest.mockr  pyarrowr   pyarrow.computecomputepcr   r   r   r   r   r   r   pyarrow.testsr	   pyarrow.utilr
   r{  r|  r   pyarrow.tests.parquet.commonr   r   r   r   rx  rL   ry  rz  r   markr6   
pytestmarkr'   r-   r7   rb   rt   xfailr   AssertionErrorr   r   r   r   r   r   parametrizer  castr  r   r   s3r   r   r   r   r+   rT   r   r   r  r&  r-  r1  r5  r8  r<  rE  rL  rQ  rV  re  rh  r  r  r  r  r  r  r  r  skipifplatformr  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r(   r&   <module>r4     s4  $   	  
    H H   A A kk!!6;;#6#67
 & 3 3
 	- 	- C, C,L ! !B  ~&B	   '1 '1T = =  ! !B +4 +4\ %, %,P N N. /0012((:.2((8S1A5((8S166rxxzBQF	HI (EF GI 4 A A.             -  -
$2N+"\/  C C* 5$ 5$N "N "NJ ( ( , ,$ $ $ 	,
 Sz2* 3 * * *$ * *$ Sz2* 3 *"*#:/z+/\ 9 9 % % / /
 7 7 < <
 
.  
. CLLG+I  KK  
       8 " " 3 3 '.*.
,@0 1"/0 . 1 "	"* " "](  	B   	B  NBs5   Z2 [ 
[ 2	Z?>Z?	[[[! [!