o
    ?߱ij                      @   s   d dl Z d dlZd dlmZ d dlZd dlmZ d dlm	Z	 ee
ddfdefdd	Zd!d
dZd"ddZd"ddZd"ddZdd Zdd Zd#ddZdd Zd$ddZdd Zd$dd ZdS )%    N)contextmanager)load)easy_iometaFinclude_buffersc                 #   s   t jjj|rt jjj fdd}d fdd	} fdd}|r,dd	 d
D }ni }z?|t jj_|r;|t jj_| D ]}tt ||tt | q?d V  W t jj_|r]t jj_| D ]
\}}tt || qad S t jj_|rzt jj_| D ]
\}}tt || q~w )Nc                    sb   | || |d ur/t | j| }| j| j}|j|d< || j|  fi || j|< d S d S )Nrequires_grad)type_parameters__dict__r   to)modulenameparam	param_clskwargs)deviceold_register_parameter T/data/cameron/vidgen/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/utils.pyregister_empty_parameter    s   
&z8init_weights_on_device.<locals>.register_empty_parameterTc                    s6   | |||d |d ur| j |  | j |< d S d S )N)
persistent)_buffersr   )r   r   bufferr   )r   old_register_bufferr   r   register_empty_buffer(   s   z5init_weights_on_device.<locals>.register_empty_bufferc                    s    fdd}|S )Nc                     s    |d< | i |S )Nr   r   )argsr   )r   fnr   r   wrapper.   s   zIinit_weights_on_device.<locals>.patch_tensor_constructor.<locals>.wrapperr   )r   r   )r   )r   r   patch_tensor_constructor-   s   z8init_weights_on_device.<locals>.patch_tensor_constructorc                 S   s   i | ]}|t t|qS r   )getattrtorch).0torch_function_namer   r   r   
<dictcomp>5   s    
z*init_weights_on_device.<locals>.<dictcomp>)emptyzerosonesfullT)	r    nnModuleregister_parameterregister_bufferkeyssetattrr   items)r   r   r   r   r   tensor_constructors_to_patchr"   old_torch_functionr   )r   r   r   r   init_weights_on_device   s>   







r2   c                 C   sN   i }t | D ]}d|v r$|dd dv r$|tt j| ||d q|S )N.)safetensorsbinckptpthpt)torch_dtype)oslistdirsplitupdateload_state_dictpathjoin)	file_pathr:   
state_dict	file_namer   r   r   load_state_dict_from_folderK   s   rE   c                 C   s&   |  drt| ||dS t| ||dS )Nz.safetensors)r:   s3_credential_path)endswith load_state_dict_from_safetensorsload_state_dict_from_bin)rB   r:   rF   r   r   r   r?   S   s
   
r?   c                 C   s8   |  dr
d|dnd }i }tj| |dd}t|}|S )Ns3://s3backendrF   byte)backend_argsfile_format)
startswithr   r   safetensors_torch_load)rB   r:   rF   rO   rC   Zbyte_streamr   r   r   rH   \   s   rH   c                 C   sd   |  dr
d|dnd }tj| |dddd}|d ur0|D ]}t|| tjr/|| |||< q|S )NrJ   rK   rL   r9   cpuF)rO   rP   map_locationweights_only)rQ   r   r   
isinstancer    Tensorr   )rB   r:   rF   rO   rC   ir   r   r   rI   f   s   
rI   c                 C   sP   g }| D ]!}t | | tjr|| |  qt | | tr%|t| | 7 }q|S N)rV   r    rW   appenddictsearch_for_embeddings)rC   
embeddingskr   r   r   r\   t   s   r\   c                 C   sl   |  D ]/\}}|  | kr3| j|jkr#t| |dk r"|  S qt|  | dk r3|  S qd S )NgMbP?)r/   numelshaper    distflatten)r   rC   r   param_r   r   r   search_parameter~   s   rd   c           	   
   C   sB  t  }t ~ | D ]s}t| | |}|d ur(td| d| d || q
|r}t| | jdkr}| | jd d dkr}| | jd d }g }tdD ]}|	t| | || || |  | qMd |vr}td| d| d |D ]}|| quq
W d    n1 sw   Y  |D ]}||vrtd	||| j qd S )
N"z": "z",   r      z": ,zCannot find)
setr    no_gradrd   printaddlenr`   rangerZ   )	Zsource_state_dictZtarget_state_dictZ	split_qkvmatched_keysr   renamelengthrX   rename_r   r   r   build_rename_dict   s4   
, rs   c                 C   st   g }t j| r tt | D ]}|tt j| ||7 }q|S t j| r8|D ]}| |r7|	|   |S q(|S rY   )
r;   r@   isdirsortedr<   search_for_filesrA   isfilerG   rZ   )folder
extensionsfilesfile	extensionr   r   r   rv      s   

rv   Tc                 C   s   g }|   D ]=\}}t|trCt|tjr1|r+dttt|j}|	|d |  |	| qt|t
rC|	|d t||d  q|  d|}|S )N_:|
with_shaperh   )r/   rV   strr    rW   rA   maplistr`   rZ   r[   %convert_state_dict_keys_to_single_strsort)rC   r   r-   keyvaluer`   keys_strr   r   r   r      s   


r   c                    s   t dd  D }i }|D ]}d|vr|n|dd }||vr$g ||< || | qg }| D ]\}} fdd|D }|| q2|S )Nc                 S   s   g | ]	}t |tr|qS r   )rV   r   r!   r   r   r   r   
<listcomp>   s    z0split_state_dict_with_prefix.<locals>.<listcomp>r3   r   c                    s   i | ]}| | qS r   r   r   rC   r   r   r#      s    z0split_state_dict_with_prefix.<locals>.<dictcomp>)ru   r=   rZ   r/   )rC   r-   Zprefix_dictr   prefixZstate_dictssub_state_dictr   r   r   split_state_dict_with_prefix   s   r   c                 C   s&   t | |d}|jdd}t| S )Nr   zUTF-8)encoding)r   encodehashlibmd5	hexdigest)rC   r   r   r   r   r   hash_state_dict_keys   s   r   rY   )NN)Fr(   )r   r;   
contextlibr   r    safetensors.torchr   rR   -cosmos_predict2._src.imaginaire.utils.easy_ior   r   boolr2   rE   r?   rH   rI   r\   rd   rs   rv   r   r   r   r   r   r   r   <module>   s&   
0

	




