o
    iP"                  
   @   s>  d dl T d dlZd dlmZ d dlmZ d dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZdd ZG dd	 d	Zd
eeef deeedf ddf fddZd9d
eeef deedf defddZd
eeef deedf defddZdedeeef fddZd9d
eeef deedf deeedf ef fddZd
eeef deeef fddZdd Zd ee fd!d"Zd eeeedf ef  fd#d$Z d
e!eeef d%eeef fd&d'Z"G d(d) d)Z#d*ee dee fd+d,Z$d9d-ee d.e%fd/d0Z&d1d2 Z'G d3d4 d4Z(d5e!eej)f d6efd7d8Z*dS ):    )*N)Path)Numberwrapsc                       t   fdd}|S )Nc               
      sj   z | i |W S  t y4 } z dd l}td j dd |jdd td W Y d }~d S d }~ww )Nr   zException in r)endF)chaing?)	Exception	tracebackprint__name__	print_exctimesleep)argskwargser   fn +/data/cameron/moge_repo/moge/utils/tools.pywrapper   s   
z catch_exception.<locals>.wrapperr   r   r   r   r   r   catch_exception   s   
r   c                   @   s.   e Zd ZdedefddZdd Zdd Zd	S )
CallbackOnExceptioncallback	exceptionc                 C   s   || _ || _d S N)r   r   )selfr   r   r   r   r   __init__      
zCallbackOnException.__init__c                 C   s   | S r   r   r    r   r   r   	__enter__"   s   zCallbackOnException.__enter__c                 C   s   t || jr|   dS dS NTF)
isinstancer   r   r    exc_typeexc_valexc_tbr   r   r   __exit__%   s   zCallbackOnException.__exit__N)r   
__module____qualname__Callabletyper!   r$   r+   r   r   r   r   r      s    r   dreturn.c                 c   sF    |   D ]\}}t|trt|D ]}|f| V  qq|fV  qd S r   )itemsr&   dicttraverse_nested_dict_keys)r0   kvZsub_keyr   r   r   r4   +   s   

r4   keysdefaultc                 C   s(   |D ]}|  ||} | d u r | S q| S r   )get)r0   r7   r8   r5   r   r   r   get_nested_dict4   s   r:   valuec                 C   s.   |d d D ]}|  |i } q|| |d < d S )N)
setdefault)r0   r7   r;   r5   r   r   r   set_nested_dict;   s   r>   list_of_dictsc                 C   s   t  }| D ]	}|t| qt|}i }|D ]0}g }| D ]}t||}|dur2t|s2|| q|r=t|t	| nt
d}t||| q|S )zd
    Returns a dictionary with the average value of each key in the input list of dictionaries.
    Nnan)setupdater4   sortedr:   mathisnanappendsumlenfloatr>   )r?   Z_nested_dict_keysr0   resultr5   valuesr6   avgr   r   r   key_averageA   s   

rM   
parent_keyc                 C   sb   g }|du rd}|   D ] \}}||f }t|tr%|t||   q|||f qt|S )z[
    Flattens a nested dictionary into a single-level dictionary, with keys as tuples.
    Nr   )r2   r&   MutableMappingextendflatten_nested_dictrF   r3   )r0   rN   r2   r5   r6   new_keyr   r   r   rQ   U   s   

rQ   c                 C   sT   i }|   D ]!\}}|}|dd D ]}||vri ||< || }q|||d < q|S )z]
    Unflattens a single-level dictionary into a nested dictionary, with keys as tuples.
    Nr<   )r2   )r0   rJ   r5   r6   Zsub_dictk_r   r   r   unflatten_nested_dicte   s   
rT   c                    sL   dd l  t| d}| }W d    n1 sw   Y   fdd|D S )Nr   r   c                    s   g | ]}  |qS r   )loads).0linejsonr   r   
<listcomp>x   s    zread_jsonl.<locals>.<listcomp>)rY   open	readlines)filefdatar   rX   r   
read_jsonlt   s
   
r`   r_   c                 C   sV   dd l }t|d}| D ]}|||d  qW d    d S 1 s$w   Y  d S )Nr   w
)rY   r[   writedumps)r_   r]   rY   r^   itemr   r   r   write_jsonl{   s   "rf   c                 C   s@   dd l }dd | D } || }|jdd}|j|j|_|S )Nr   c                 S   s   g | ]}t |qS r   )rQ   )rV   r0   r   r   r   rZ      s    z,to_hierachical_dataframe.<locals>.<listcomp>   )axis)pandas	DataFrameZ
sort_indexZ
MultiIndexfrom_tuplescolumns)r_   pddfr   r   r   to_hierachical_dataframe   s   
ro   mappingc                 C   s   t | tr| D ]
\}}| ||} q	| S t | tr-t| D ]\}}t||| |< q| S t | trB|  D ]\}}t||| |< q6| S r   )r&   strr2   replacelist	enumeraterecursive_replacer3   )r0   rp   oldnewire   r5   r6   r   r   r   ru      s   


ru   c                   @   s   e Zd ZU i Zeeed  f ed< ddededefdd	Z	d
e
fddZdd ZedefddZedefddZeded  fddZdd ZdS )timeit_historyNTFnameverboseaveragec                 C   sB   || _ || _d | _d | _|| _|r|tjvrg tj|< d S d S d S r   )r{   r|   startr	   r}   ry   rz   )r    r{   r|   r}   r   r   r   r!      s   ztimeit.__init__funcc                    s6   dd l }| r fdd}|S  fdd}|S )Nr   c                     sL   t jp j  | i |I d H }W d    |S 1 sw   Y  |S r   ry   r{   r-   r   r   retr   r    r   r   r      s   
z timeit.__call__.<locals>.wrapperc                     sD   t jp j  | i |}W d    |S 1 sw   Y  |S r   r   r   r   r   r   r      s   
)inspectiscoroutinefunction)r    r   r   r   r   r   r   __call__   s   
ztimeit.__call__c                 C   s   t   | _| S r   )r   r~   r#   r   r   r   r$      s   
ztimeit.__enter__r1   c                 C   s0   | j d us	J d| jd usJ d| j| j  S )NzTime not yet started.zTime not yet ended.)r~   r	   r#   r   r   r   r      s   ztimeit.timec                 C   s8   | j sJ dtdd tj| j D ttj| j  S )NzAverage time not available.c                 s   s    | ]}|j V  qd S r   )r   )rV   tr   r   r   	<genexpr>   s    z&timeit.average_time.<locals>.<genexpr>)r}   rG   ry   rz   r{   rH   r#   r   r   r   average_time   s   *ztimeit.average_timec                 C   s   t j| jg S r   )ry   rz   r9   r{   r#   r   r   r   history   s   ztimeit.historyc                 C   sz   t   | _| jrtj| j |  | jr;| jr*| j}t	| jpd d|dd d S t	| jp/d d| j dd d S d S )NZItz took z.6fz seconds in average.z	 seconds.)
r   r	   r}   ry   rz   r{   rF   r|   r   r   )r    r(   r)   r*   rL   r   r   r   r+      s   
 "ztimeit.__exit__r%   )r   r,   r-   rz   Dictrq   List__annotations__boolr!   r.   r   r$   propertyrI   r   r   r   r+   r   r   r   r   ry      s   
 	ry   stringsc                    s   d t tD ]tfddD r nq
t dtdd D D ] t fddD r7 nq' fddD S )	Nr   c                 3   s$    | ]}|  d    kV  qdS )r   Nr   rV   s)r~   r   r   r   r         " z-strip_common_prefix_suffix.<locals>.<genexpr>rg   c                 s   s    | ]}t |V  qd S r   rH   r   r   r   r   r      s    c                 3   s$    | ]}|      kV  qd S r   r   r   )r	   firstr   r   r      r   c                    s$   g | ]}|t |  d   qS )rg   r   r   )r	   r~   r   r   rZ      s   $ z.strip_common_prefix_suffix.<locals>.<listcomp>)rangerH   anymin)r   r   )r	   r   r~   r   r   strip_common_prefix_suffix   s   r   inputsnum_workersc                    s~   ddl m  ddlm} ddlm} d ur#tdrtnd _n|tdr-tnd ddtf fdd	}|S )
Nr   )ThreadPoolExecutor)nullcontext)tqdm__len__)totalr   c              	      s   d>}#    tt fdd}|| |jdd W d    n1 s.w   Y  W d    d S W d    d S 1 sFw   Y  d S )N)Zmax_workersc                    s    | }   |S r   )rB   )inputr   )r   pbarr   r   _fn   s   z2multithead_execute.<locals>.decorator.<locals>._fnT)wait)refreshr   suppress_tracebackmapshutdown)r   executorr   r   r   r   r   r   r   	decorator   s   Pz%multithead_execute.<locals>.decorator)	concurrent.futuresr   
contextlibr   r   hasattrrH   r   r.   )r   r   r   r   r   r   r   r   r   multithead_execute   s   r   c                    r   )Nc               
      s8   z | i |W S  t y } z|jjj|_ d }~ww r   )r   __traceback__tb_next)r   r   r   r   r   r   r      s   z#suppress_traceback.<locals>.wrapperr   r   r   r   r   r      s   r   c                   @   s4   e Zd ZddefddZdd Zdd Zd	d
 ZdS )no_warningsignoreactionc                 K   s   || _ || _d S r   )r   filter_kwargs)r    r   r   r   r   r   r!     r"   zno_warnings.__init__c                    s   t   fdd}|S )Nc                     sP   t   t jjfi j  | i |W  d    S 1 s!w   Y  d S r   )warningscatch_warningssimplefilterr   r   )r   r   r   r    r   r   r     s   
$z%no_warnings.__call__.<locals>.wrapperr   )r    r   r   r   r   r   r     s   zno_warnings.__call__c                 C   s.   t  | _| j  t j| jfi | j d S r   )r   r   warnings_managerr$   r   r   r   r#   r   r   r   r$     s   

zno_warnings.__enter__c                 C   s   | j ||| d S r   )r   r+   r'   r   r   r   r+     s   zno_warnings.__exit__N)r   )r   r,   r-   rq   r!   r   r$   r+   r   r   r   r   r     s
    r   	file_pathmodule_namec                 C   s*   t j|| }t j|}|j| |S r   )	importlibutilspec_from_file_locationmodule_from_specloaderexec_module)r   r   specmoduler   r   r   import_file_as_module  s   r   r   )+typingr   pathlibr   numbersr   	functoolsr   r   rD   rY   osr   importlib.utilr   r   r   rq   	GeneratorTupler4   Anyr:   r>   rs   rM   rQ   rT   r`   r   r3   rf   ro   Unionru   ry   r   intr   r   r   PathLiker   r   r   r   r   <module>   s:    ,(	&8""$	: 