o
    iC                     @   s   d Z ddlZddlmZmZmZ ddlZddlmZ G dd dej	Z
dej	dej	fdd	Zddej	dedee fddZ			ddej	dededeee  deej f
ddZddej	dedee fddZddej	defddZdS )z
LoRA (Low-Rank Adaptation) for video diffusion models.
Injects trainable low-rank matrices into attention linear layers (to_q, to_k, to_v, to_out).
    N)ListOptionalSetc                       sT   e Zd ZdZ		ddejdedef fddZd	d
 Z	de
jde
jfddZ  ZS )
LoRALinearzQWraps an nn.Linear with a low-rank update: out = linear(x) + (x @ A @ B) * scale.         ?linearrankscalec                    sh   t    || _|j}|j}t|||| _|| _t	t
| j|| _t	t
|| j| _|   d S N)super__init__r   in_featuresout_featuresminr	   r
   nn	Parametertorchzeroslora_Alora_B
_init_lora)selfr   r	   r
   r   r   	__class__ :/data/cameron/vidgen/generative-models/sgm/modules/lora.pyr      s   
zLoRALinear.__init__c                 C   s*   t jj| jtdd t j| j d S )N   )a)r   initkaiming_uniform_r   mathsqrtzeros_r   )r   r   r   r   r      s   zLoRALinear._init_loraxreturnc                 C   s,   |  |}||| jj | jj | j  }|S r   )r   r   Tr   r
   )r   r$   outr   r   r   forward#   s   
zLoRALinear.forward)r   r   )__name__
__module____qualname____doc__r   Linearintfloatr   r   r   Tensorr(   __classcell__r   r   r   r   r      s    r   modelr%   c                 C   s   t | dr| jS | S )z7Unwrap OpenAIWrapper to get the actual diffusion model.diffusion_model)hasattrr3   )r2   r   r   r   _get_inner_model)   s   
r5    moduleprefixc                 C   s   g }|   D ]p\}}|r| d| n|}dt|jv s$dtt|v rndD ]F}t||rlt||}t|tjrG|	|||| d| f q&t|tj
rlt|dkrlt|d tjrl|	|||d | d| df q&q|t|| q|S )zcRecursively find CrossAttention / MemoryEfficientCrossAttention and their to_q, to_k, to_v, to_out..CrossAttention)to_qto_kto_vto_outr   z[0])named_childrentyper)   strr4   getattr
isinstancer   r-   append
Sequentiallenextend_find_attention_linears)r7   r8   resultsnamechild	full_name	attn_namesubr   r   r   rH   0   s   

("rH   r   r   r	   r
   target_modulesc                 C   s   t | } t| }g }|D ]K\}}}}	|dur|	|vrqt|||d}
tt||tjrHt||}tj|
gt| dd R  }t	||| nt	|||
 |
|
j|
jg q|  D ]}d|_q\|D ]}d|_qd|S )z
    Inject LoRA into attention linears (to_q, to_k, to_v, to_out) and freeze the rest.
    Returns list of LoRA parameters (for optimizer).
    N)r	   r
      FT)r5   rH   r   rC   rB   r   rE   listchildrensetattrrG   r   r   
parametersrequires_grad)r2   r	   r
   rO   pairslora_paramsparent	attr_namelinear_layerrL   
lora_layerseqnew_seqpr   r   r   inject_loraB   s$   

"r_   pathc                 C   sp   t | } i }|  D ]\}}t|tr'|j || d< |j || d< q
|dur0||d< t|| dS )z[Save only LoRA parameters (lora_A, lora_B) for all LoRALinear layers. Optionally save rank..lora_A.lora_BN
_lora_rank)	r5   named_modulesrC   r   r   cpur   r   save)r2   r`   r	   staterJ   modr   r   r   save_lora_state_dicta   s   
ri   c                 C   s   t j||d}|dd}t| }tdd | D }|s(t| |d t| }|dur9t|tr6t 	|n|}nt
| j	}| D ]-\}}	t|	trq| d}
| d	}|
|v rd||
 ||	j_||v rq|| ||	j_qDdS )
zgLoad LoRA state dict into model. If model has no LoRALinear layers, inject LoRA first using saved rank.)map_locationrc   r   c                 s   s    | ]}t |tV  qd S r   )rC   r   ).0mr   r   r   	<genexpr>s   s    z'load_lora_state_dict.<locals>.<genexpr>)r	   Nra   rb   )r   loadpopr5   anymodulesr_   rC   rA   devicenextrT   rd   r   tor   datar   )r2   r`   rr   rg   r	   innerhas_loratarget_devicerJ   rh   key_akey_br   r   r   load_lora_state_dictn   s(   


r{   )r6   )r   r   Nr   )r,   r!   typingr   r   r   r   torch.nnr   Moduler   r5   rA   tuplerH   r.   r/   r   r_   ri   r{   r   r   r   r   <module>   s0    

