§
    Û×j�_  ã                   ó  — d dl mZ d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	 d dl
m	c mZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddlmZmZ dd	lmZmZ e G d
„ d¦  «        ¦   «         Z G d„ de	j         ¦  «        Z! G d„ de!¦  «        Z"dS )é    )Ú	dataclassN)Úcuda_graphs)Úops)ÚTensorÚnné   )ÚDoubleStreamBlockÚEmbedNDÚEmbedNDFlux2Ú	LastLayerÚMLPEmbedderÚ
ModulationÚSingleStreamBlockÚtimestep_embeddingÚDistilledGuidanceÚChromaModulationOutÚSigLIPMultiFeatProjModel)Ú
LinearLoraÚreplace_linear_with_lora)Úapply_radiance_headÚinject_radiance_modulesc                   ó–  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   ee         ed
<   eed<   eed<   eed<   dZeed<   dZ	eed<   dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   dZedz  ed!<   dZedz  ed"<   dZedz  ed#<   dZeed$<   dZeed%<   dS )&Ú
FluxParamsÚin_channelsÚout_channelsÚ
vec_in_dimÚcontext_in_dimÚhidden_sizeÚ	mlp_ratioÚ	num_headsÚdepthÚdepth_single_blocksÚaxes_dimÚthetaÚqkv_biasÚguidance_embedFÚchromaÚesoÚradianceé   Úradiance_patch_sizeé@   Úradiance_hidden_sizeé   Úradiance_mlp_ratioÚradiance_depthé   Úradiance_max_freqsr   Úradiance_tile_sizeÚconvÚradiance_final_head_typeNÚsingle_linear1_mlp_ratioÚsingle_mlp_hidden_ratioÚdouble_mlp_ratioÚdouble_linear1_mlp_ratioÚflux2Úpiflow)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__ÚfloatÚlistÚboolr'   r(   r)   r+   r-   r/   r0   r2   r3   r5   Ústrr6   r7   r8   r9   r:   r;   © ó    ú9C:\Users\johnn\OneDrive\Documents\AI\models\flux\model.pyr   r      s½  € € € € € € àÐÐÑØÐÐÑØ€O€O�OØÐÐÑØÐÐÑØÐÐÑØ€N€N�NØ€J€J�JØÐÐÑØ�3ŒiÐÐÑØ€J€J�JØ€N€N�NØÐÐÑØ€FˆDÐÐÑØ€CˆÐÐÑØ€HˆdÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø "Ð˜#Ð"Ð"Ñ"ØÐ˜ÐÐÑØ€N�CÐÐÑØÐ˜ÐÐÑØÐ˜ÐÐÑØ$*Ð˜cÐ*Ð*Ñ*Ø-1Ð˜e d™lÐ1Ð1Ñ1Ø,0Ð˜U T™\Ð0Ð0Ñ0Ø%)Ð�e˜d‘lÐ)Ð)Ñ)Ø-1Ð˜e d™lÐ1Ð1Ñ1Ø€Eˆ4ÐÐÑØ€FˆDÐÐÑÐÐrF   r   c                   ó²   ‡ — e Zd ZdZddœdej        dedefd„Zde	fˆ fd	„Z
d
efd„Zd
edededefd„Zd„ Z	 	 	 	 	 	 	 ddededededz  dedz  defd„Zˆ xZS )ÚFluxz;
    Transformer model for flow matching on sequences.
    r   ©ÚidxÚtensorÚ
block_typerK   c                óx  — |dk    r"|d d …dd…d d …f         |d d …dd …d d …f         fS | j         j        }| j         j        }d|z  }|dk    rt          j        ||¦  «        S |dz  }|dv rC|d|z  z  }|dk    r|d	|z  z  }t          j        ||¦  «        t          j        ||dz   ¦  «        fS t          d
¦  «        ‚)NÚfinaléþÿÿÿéÿÿÿÿé   Úsingleé   >   Ú
double_imgÚ
double_txtrV   é   zBad block_type)Úparamsr"   r!   r   Úfrom_offsetÚ
ValueError)ÚselfrL   rM   rK   Úsingle_block_countÚdouble_block_countÚoffsets          rG   Úget_modulationszFlux.get_modulations?   s  € ð ˜Ò Ð Ø˜1˜1˜1˜b ˜e Q Q Q˜;Ô'¨°°°°2°3°3¸¸¸°	Ô):Ð;Ð;Ø!œ[Ô<ÐØ!œ[Ô.ÐØ�S‘ˆØ˜Ò!Ð!Ý&Ô2°6¸6ÑBÔBÐBà�!‰ˆØÐ5Ð5Ð5à�aÐ,Ñ,Ñ,ˆFØ˜\Ò)Ð)à˜!Ð0Ñ0Ñ0�å#Ô/°¸Ñ?Ô?Ý#Ô/°¸À¹
ÑCÔCðð õ Ð)Ñ*Ô*Ð*rF   rX   c           
      ó|	  •‡ ‡— t          ¦   «                              ¦   «          ‰‰ _        ‰j        ‰ _        ‰j        ‰ _        ‰j        ‰ _        t          ‰dd¦  «        ‰ _        t          ‰dd¦  «        ‰ _        t          ‰dd¦  «        ‰ _	        ‰j
        ‰j        z  dk    rt          d‰j
        › d‰j        › �¦  «        ‚‰j
        ‰j        z  }t          ‰j        ¦  «        |k    rt          d‰j        › d	|› �¦  «        ‚‰j
        ‰ _
        ‰j        ‰ _        ‰ j        r"t          |‰j        ‰j        ¬
¦  «        ‰ _        n!t%          |‰j        ‰j        ¬
¦  «        ‰ _        ‰ j        rt'          j        ¦   «         n&t'          j        ‰ j        ‰ j
        ‰ j         ¬¦  «        ‰ _        ‰j        rt1          d‰ j
        ‰ j         ¬¦  «        nd ‰ _        t'          j        ‰j        ‰ j
        ‰ j         ¬¦  «        ‰ _        ‰ j        rTt9          ‰ j
        dd¬¦  «        ‰ _        t9          ‰ j
        dd¬¦  «        ‰ _        t9          ‰ j
        dd¬¦  «        ‰ _        ‰ j        rtA          dddd¬¦  «        ‰ _!        nct1          d‰ j
        ‰ j         ¬¦  «        ‰ _"        ‰ j        s"t1          ‰j#        ‰ j
        ‰ j         ¬¦  «        nt'          j        ¦   «         ‰ _$        t'          j%        ˆˆ fd„tM          ‰j'        ¦  «        D ¦   «         ¦  «        ‰ _(        t'          j%        ˆˆ fd„tM          ‰j)        ¦  «        D ¦   «         ¦  «        ‰ _*        ‰ j        rtW          ‰ ‰¦  «         d ‰ _,        n6t[          ‰ j
        d‰ j        ‰ j        d‰ j         ‰ j         ¬¦  «        ‰ _,        ‰ j	        �rt'          j        ‰ j
        dd¬¦  «        ‰ _.        t'          j        ‰ j
        dd¬¦  «        ‰ _/        t'          j0        t'          j        ¦   «         t'          j        ‰ j
        dd¬¦  «        t'          j1        ¦   «         t'          j        ddd¬¦  «        ¦  «        ‰ _2        ‰ j.        j3        ‰ j        z  ‰ _4        ‰ j/        j3        ‰ j4        z  ‰ _5        tm          to          j8        ‰ j5        ¦  «        ¦  «        ‰ _9        d S d S )Nr:   Fr)   r;   r   zHidden size z  must be divisible by num_heads zGot z but expected positional dim )Údimr$   r#   )Úbiasé   )Úin_dimÚ
hidden_dimrb   T)Údoublerb   r,   i   i   é   )rd   re   Úout_dimÚn_layersc                 óâ   •— g | ]k}t          ‰j        ‰j        ‰j        t	          ‰d d¦  «        t	          ‰dd¦  «        ‰j         ‰j         ‰j         ‰j        ‰j        p‰j        ¬¦
  «
        ‘ŒlS )r8   Nr9   )r   r8   r9   Úmod_biasÚmlp_biasÚ	proj_biasr%   Úshared_modulation)r	   r   r    r   ÚgetattrÚis_flux2r%   r'   ©Ú.0Ú_rX   r[   s     €€rG   ú
<listcomp>z!Flux.__init__.<locals>.<listcomp>�   s•   ø€ ð ð ð ð õ "ØÔ$Ø”NØ$Ô.Ý%,¨VÐ5GÈÑ%NÔ%NÝ-4°VÐ=WÐY]Ñ-^Ô-^Ø!%¤Ð.Ø!%¤Ð.Ø"&¤-Ð/Ø#œ_Ø(,¬Ð(D°t´}ðñ ô ðð ð rF   c                 óÊ   •— g | ]_}t          ‰j        ‰j        ‰j        ‰j        p‰j        t          ‰d d¦  «        t          ‰dd¦  «        d‰j         ‰j         ¬¦	  «	        ‘Œ`S )r6   Nr7   )r   rn   r6   r7   Úqk_scaleÚlinear_biasÚmodulation_bias)r   r   r    r   r'   rp   ro   rq   s     €€rG   rt   z!Flux.__init__.<locals>.<listcomp>¡   sŒ   ø€ ð ð ð ð õ "ØÔ$Ø”NØ$Ô.Ø&*¤kÐ&B°T´]Ý-4°VÐ=WÐY]Ñ-^Ô-^Ý,3°FÐ<UÐW[Ñ,\Ô,\Ø!Ø$(¤MÐ 1Ø(,¬Ð$5ð
ñ 
ô 
ðð ð rF   r   )Úchroma_modulationÚ
use_linearrw   rx   i   é    ):ÚsuperÚ__init__rX   r   r   r'   ro   rp   r)   r;   r   r    rZ   Úsumr#   r   r$   Úpe_embedderr
   r   ÚIdentityÚLinearÚimg_inr&   r   Úguidance_inr   Útxt_inr   Údouble_stream_modulation_imgÚdouble_stream_modulation_txtÚsingle_stream_modulationr   Údistilled_guidance_layerÚtime_inr   Ú	vector_inÚ
ModuleListÚranger!   Údouble_blocksr"   Úsingle_blocksr   Úfinal_layerr   Úproj_out_meansÚproj_out_logweightsÚ
SequentialÚSiLUÚproj_out_logstdsÚout_featuresÚnum_gaussiansÚlogweights_channelsr?   ÚmathÚsqrtÚpiflow_patch_size)r[   rX   Úpe_dimÚ	__class__s   `` €rG   r}   zFlux.__init__Z   së  øøø€ Ý‰Œ×ÒÑÔÐàˆŒØ!Ô-ˆÔØ"Ô/ˆÔØ”mˆŒÝ ¨°Ñ7Ô7ˆŒÝ ¨
°EÑ:Ô:ˆŒÝ˜f h°Ñ6Ô6ˆŒØÔ Ô 0Ñ0°AÒ5Ð5ÝØe˜vÔ1ÐeÐeÐSYÔScÐeÐeñô ð ð Ô# vÔ'7Ñ7ˆÝˆvŒÑÔ 6Ò)Ð)ÝÐZ F¤OÐZÐZÐRXÐZÐZÑ[Ô[Ð[Ø!Ô-ˆÔØÔ)ˆŒàŒ=ð 	aÝ+°¸f¼lÐU[ÔUdÐeÑeÔeˆDÔÐå&¨6¸¼ÐPVÔP_Ð`Ñ`Ô`ˆDÔð Œ}ðW�BŒK‰MŒMˆMå”˜4Ô+¨TÔ-=ÈÌÐDUÐVÑVÔVð 	Œð Ô$ð�K˜s¨tÔ/?È$Ì-ÐFWÐXÑXÔXÐXàð 	Ôõ
 ”i Ô 5°tÔ7GÐRVÔR_ÐN_Ð`Ñ`Ô`ˆŒØŒ=ð 	cÝ0:¸4Ô;KÐTXÐ_dÐ0eÑ0eÔ0eˆDÔ-Ý0:¸4Ô;KÐTXÐ_dÐ0eÑ0eÔ0eˆDÔ-Ý,6°tÔ7GÐPUÐ\aÐ,bÑ,bÔ,bˆDÔ)ØŒ;ð 		NÝ,=Ø!Ø#'Ø $Ø!"ð	-ñ -ô -ˆDÔ)Ð)õ '¨c¸dÔ>NÐY]ÔYfÐUfÐgÑgÔgˆDŒLØmqÔmzð  N�[¨Ô):¸DÔ<LÐW[ÔWdÐSdÐeÑeÔeÐeõ  ACô  ALñ  ANô  ANˆDŒNåœ]ðð ð ð ð õ ˜vœ|Ñ,Ô,ðñ ô ñ
ô 
ˆÔõ$  œ]ðð ð ð ð õ ˜vÔ9Ñ:Ô:ðñ ô ñ
ô 
ˆÔð" Œ=ð 	Ý# D¨&Ñ1Ô1Ð1Ø#ˆDÔÐå(ØÔ ØØÔ!Ø"&¤+ØØ $¤Ð-Ø$(¤MÐ 1ð ñ  ô  ˆDÔð Œ;ñ 	Nå"$¤)¨DÔ,<¸dÈÐ"NÑ"NÔ"NˆDÔÝ')¤y°Ô1AÀ2ÈDÐ'QÑ'QÔ'QˆDÔ$Ý$&¤MÝ”‘”Ý”	˜$Ô*¨D°tÐ<Ñ<Ô<Ý”‘	”	Ý”	˜$ ¨Ð-Ñ-Ô-ñ	%ô %ˆDÔ!ð "&Ô!4Ô!AÀTÔEVÑ!VˆDÔØ'+Ô'?Ô'LÐPTÔPbÑ'bˆDÔ$Ý%(­¬°4Ô3KÑ)LÔ)LÑ%MÔ%MˆDÔ"Ð"Ð"ð	Nð 	NrF   Útokensc           	      óÐ  — | j         }|                     |¦  «        }| j        r0|\  }}|                     d¦  «        }|                     d¦  «        }n-|                     |¦  «                             dd¬¦  «        \  }}|gd }}t          j        ||d d …d d d …f         |d d …d d d …f         dd¬¦  «        }|j        €t          d¦  «        ‚|                     |¦  «        }	||	fS )Nr   rT   ©ra   FT©Úin_placeÚ	fused_addz(Final layer projection is not available.)
r�   Ú
norm_finalr'   ÚsqueezeÚadaLN_modulationÚchunkr   Úscale_shiftÚlinearÚRuntimeError)
r[   r�   Úvecr�   ÚnormedÚshiftÚscaleÚnormed_listÚ	modulatedÚbase_tokenss
             rG   Ú_apply_final_layerzFlux._apply_final_layerÍ   s  € ØÔ&ˆØ×'Ò'¨Ñ/Ô/ˆØŒ;ð 	MØ‰LˆE�5Ø—M’M !Ñ$Ô$ˆEØ—M’M !Ñ$Ô$ˆEˆEà&×7Ò7¸Ñ<Ô<×BÒBÀ1È!ÐBÑLÔL‰LˆE�5Ø%˜h¨�VˆÝ”O K°°q°q°q¸$ÀÀÀ°zÔ1BÀEÈ!È!È!ÈTÐSTÐSTÐSTÈ*ÔDUÐ`eÐquÐvÑvÔvˆ	ØÔÐ%ÝÐIÑJÔJÐJØ!×(Ò(¨Ñ3Ô3ˆØ˜+Ð%Ð%rF   rª   Úimg_idsÚimg_lenc           	      óÜ  — |€t          d¦  «        ‚| j        }|                     |¦  «        }|                     |¦  «                             dd¬¦  «        \  }}|gd }}	t          j        |	|d d …d d d …f         |d d …d d d …f         dd¬¦  «        }
| j        j        j	        }|
 
                    |¦  «        }
|                      |
¦  «        }|                      |
¦  «        }|                      |                     ¦   «          
                    | j        d         j        j	        ¦  «        ¦  «        }|d d …d |…f         }t          |d	                              ¦   «                              ¦   «         dz   ¦  «        }t          |d
                              ¦   «                              ¦   «         dz   ¦  «        }||z  |k    rt          d¦  «        ‚| j        }||z  | j        k    rt          d¦  «        ‚|j        d         }| j        }| j        }|||z  z  }|                     |||||||¦  «                             ddddddd¦  «                             |||||z  ||z  ¦  «        }|                     ||||d||¦  «                             ddddddd¦  «                             ||d||z  ||z  ¦  «                             d¬¦  «        }|                     |dddd¦  «        }|
|||dœfS )Nz.pi-Flow requires image ids to reshape outputs.rT   r   rŸ   FTr    rQ   ©.r   ©.rT   z0pi-Flow token length does not match latent grid.z%pi-Flow logweights channels mismatch.r   rR   r.   rg   rW   )ÚmeansÚ
logweightsÚlogstds)r©   r�   r£   r¥   r¦   r   r§   r�   ÚweightÚdtypeÚtor‘   r”   Údetachr?   ÚmaxÚitemrš   r—   Úshaper–   r   ÚviewÚpermuteÚreshapeÚlog_softmax)r[   r�   rª   r²   r³   r�   r«   r¬   r­   r®   r¯   Ú
proj_dtyper·   r¸   r¹   Úbase_img_idsÚh_lenÚw_lenÚ
patch_sizeÚbszÚkÚcÚ
c_unpackeds                          rG   Ú_apply_piflow_final_layerzFlux._apply_piflow_final_layerÝ   s  € Øˆ?ÝÐOÑPÔPÐPØÔ&ˆØ×'Ò'¨Ñ/Ô/ˆØ"×3Ò3°CÑ8Ô8×>Ò>¸qÀaÐ>ÑHÔH‰ˆˆuØ%˜h¨�VˆÝ”O K°°q°q°q¸$ÀÀÀ°zÔ1BÀEÈ!È!È!ÈTÐSTÐSTÐSTÈ*ÔDUÐ`eÐquÐvÑvÔvˆ	àÔ(Ô/Ô5ˆ
Ø—L’L Ñ,Ô,ˆ	Ø×#Ò# IÑ.Ô.ˆØ×-Ò-¨iÑ8Ô8ˆ
Ø×'Ò'¨¯
ª
©¬¯ª¸Ô8MÈbÔ8QÔ8XÔ8^Ñ(_Ô(_Ñ`Ô`ˆà˜q˜q˜q ( 7 (˜{Ô+ˆÝ�L Ô(×,Ò,Ñ.Ô.×3Ò3Ñ5Ô5¸Ñ9Ñ:Ô:ˆÝ�L Ô(×,Ò,Ñ.Ô.×3Ò3Ñ5Ô5¸Ñ9Ñ:Ô:ˆØ�5‰=˜GÒ#Ð#ÝÐQÑRÔRÐRàÔ+ˆ
Ø˜
Ñ" dÔ&>Ò>Ð>ÝÐFÑGÔGÐGàŒk˜!ŒnˆØÔˆØÔˆØ˜:¨
Ñ2Ñ3ˆ
à—
’
Ø�˜˜q *¨j¸*ñ
ô 
ç
Š'Øˆq�!�Q˜˜1˜añ
ô 
ç
Š'Ø��J ¨
Ñ 2°E¸JÑ4Fñ
ô 
ð	 	ð  —_’_Ø�˜˜q ! Z°ñ
ô 
ç
Š'Øˆq�!�Q˜˜1˜añ
ô 
ç
Š'Ø��A�u˜zÑ)¨5°:Ñ+=ñ
ô 
ç
Š+˜!ˆ+Ñ
Ô
ð 	ð —/’/ # q¨!¨Q°Ñ2Ô2ˆØ E¸ÐPWÐXÐXÐXÐXrF   c                 ó  — i }t          |¦  «        dk    r|S d„ }d}d}|                     ¦   «         D ]}d|v rd} nd|v sd|v rd} nŒt          t          |¦  «        ¦  «        }|�r"i }t	          d¦  «         g d	¢}	d
„ |	D ¦   «         }
d„ |	D ¦   «         }d„ |	D ¦   «         }|                     ¦   «         D ]Ò\  }}|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }|                     dd¦  «        }t          |
||¦  «        D ]2\  }}}|                     ||¦  «        }|                     ||¦  «        }Œ3|                     dd¦  «        }|                     dd¦  «        }|||<   ŒÓ�n÷|�ròg d¢}g d¢}g d¢}g d¢}g d¢}g d¢}|                     ¦   «         D �]Ã\  }}|                     d¦  «        r|                     dd¦  «        }|                     d¦  «        rC|                     dd ¦  «        }t          ||¦  «        D ]\  }}|                     ||¦  «        }Œn­|                     d¦  «        rC|                     dd!¦  «        }t          ||¦  «        D ]\  }}|                     ||¦  «        }ŒnUt          ||¦  «        D ]\  }}|                     ||¦  «        }Œd"|v r%d|v r ||¦  «        }|                     d"d#¦  «        }|                     d$d%¦  «        }|                     d&d'¦  «        }|                     d(d)¦  «        }|                     d*d+¦  «        }|                     d,d-¦  «        }|                     d.¦  «        sd.|z   }|||<   �ŒÅn|}|S )/Nr   c                 ój   — |                       dd¬¦  «        \  }}t          j        ||gd¬¦  «        }|S )NrT   r   rŸ   )r¦   ÚtorchÚcat)rº   r¬   r­   Ú
new_weights       rG   Úswap_scale_shiftz/Flux.preprocess_loras.<locals>.swap_scale_shift  s9   € Ø!Ÿ<š<¨¨q˜<Ñ1Ô1‰LˆE�5Ýœ E¨5 >°qÐ9Ñ9Ô9ˆJØÐrF   FÚ
lora_unet_TÚsingle_transformer_blocksÚtransformer_blocksz;Converting Lora Safetensors format to Lora Diffusers format)	Úlinear1Úlinear2Ú
modulationÚimg_attnÚtxt_attnÚimg_mlpÚtxt_mlpÚimg_modÚtxt_modc                 ó   — g | ]
}d |z   dz   ‘ŒS )rs   ú.rE   ©rr   rË   s     rG   rt   z)Flux.preprocess_loras.<locals>.<listcomp>%  ó    € Ð9Ð9Ð9¨!˜˜a™ #™Ð9Ð9Ð9rF   c                 ó   — g | ]
}d |z   d z   ‘ŒS )rs   rE   rã   s     rG   rt   z)Flux.preprocess_loras.<locals>.<listcomp>&  s    € Ð:Ð:Ð:¨1˜˜q™ 3™Ð:Ð:Ð:rF   c                 ó   — g | ]
}d |z   d z   ‘ŒS )râ   rE   rã   s     rG   rt   z)Flux.preprocess_loras.<locals>.<listcomp>'  rä   rF   Úlora_unet_blocks_zdiffusion_model.blocks.Úlora_unet__blocks_Úlora_unet_single_blocks_zdiffusion_model.single_blocks.Úlora_unet_double_blocks_zdiffusion_model.double_blocks.Úlora_upÚlora_BÚ	lora_downÚlora_A)z*time_text_embed.timestep_embedder.linear_1z*time_text_embed.timestep_embedder.linear_2z&time_text_embed.text_embedder.linear_1z&time_text_embed.text_embedder.linear_2z*time_text_embed.guidance_embedder.linear_1z*time_text_embed.guidance_embedder.linear_2Ú
x_embedderÚcontext_embedderÚproj_outz.time_guidance_embed.timestep_embedder.linear_1z.time_guidance_embed.timestep_embedder.linear_2)útime_in.in_layerútime_in.out_layerzvector_in.in_layerzvector_in.out_layerzguidance_in.in_layerzguidance_in.out_layerr‚   r„   zfinal_layer.linearrò   ró   )znorm1.linearznorm1_context.linearúattn.norm_qúattn.norm_kzff.net.0.projzff.net.2zff_context.net.0.projzff_context.net.2zattn.to_out.0zattn.to_add_outzattn.to_outz	.attn.to_z.attn.add_q_proj.z.attn.add_k_proj.z.attn.add_v_proj.z.ff_context.linear_out.z.ff_context.linear_in.z.ff.linear_out.z.ff.linear_in.)zimg_mod.linztxt_mod.linzimg_attn.norm.query_normzimg_attn.norm.key_normz	img_mlp.0z	img_mlp.2z	txt_mlp.0z	txt_mlp.2úimg_attn.projztxt_attn.projrö   z
.img_attn.z.txt_attn.q.z.txt_attn.k.z.txt_attn.v.z.txt_mlp.2.z.txt_mlp.0.z.img_mlp.2.z.img_mlp.0.)
znorm.linearrô   rõ   rñ   z.attn.to_qkv_mlp_proj.z.attn.to_q.z.attn.to_k.z.attn.to_v.z
.proj_mlp.z.attn.to_out.)
zmodulation.linznorm.query_normznorm.key_normrÙ   z	.linear1.z.linear1_attn_q.z.linear1_attn_k.z.linear1_attn_v.z.linear1_mlp.z	.linear2.ztransformer.Ú rŽ   r�   znorm_out.linearzfinal_layer.adaLN_modulation.1z$double_stream_modulation_img.linear.z!double_stream_modulation_img.lin.z$double_stream_modulation_txt.linear.z!double_stream_modulation_txt.lin.z single_stream_modulation.linear.zsingle_stream_modulation.lin.z.lora.down.weightz.lora_down.weightz.lora.up.weightz.lora_up.weightzdiffusion_model.)	ÚlenÚkeysÚnextÚiterÚprintÚitemsÚreplaceÚzipÚ
startswith)r[   Ú
model_typeÚsdÚnew_sdrÔ   Ú	lora_unetÚ	diffusersrË   Ú	first_keyÚ	repl_listÚsrc_listÚ	src_list2Útgt_listÚvÚsÚs2ÚtÚroot_srcÚroot_tgtÚ
double_srcÚ
double_tgtÚ
single_srcÚ
single_tgtÚsrcÚtgts                            rG   Úpreprocess_loraszFlux.preprocess_loras  sˆ  € ØˆÝˆr‰7Œ7�aŠ<ˆ< ˜ð	ð 	ð 	ð ˆ	Øˆ	Ø—’‘”ð 	ð 	ˆAØ˜qÐ Ð Ø �	Ø�Ø,°Ð1Ð1Ð5IÈQÐ5NÐ5NØ �	Ø�ð 6Oõ �˜R™œ‘>”>ˆ	Øñ S	ØˆFÝÐOÑPÔPÐPð Að  Að  AˆIØ9Ð9¨yÐ9Ñ9Ô9ˆHØ:Ð:°	Ð:Ñ:Ô:ˆIØ9Ð9¨yÐ9Ñ9Ô9ˆHà—x’x‘z”zð ð ‘��!Ø—I’IÐ1Ð2KÑLÔL�Ø—I’IÐ2Ð3LÑMÔM�Ø—I’IÐ8Ð9YÑZÔZ�Ø—I’IÐ8Ð9YÑZÔZ�å" 8¨Y¸ÑAÔAð (ð (‘G�A�b˜!ØŸ	š	 ! A™œ�AØŸ	š	 " Q™œ�A�Aà—I’I˜i¨Ñ1Ô1�Ø—I’I˜k¨(Ñ3Ô3�à��q‘	�	ñð ñ <	ðgð gð gˆHðhð hð hˆHð Eð  Eð  EˆJð Pð  Pð  PˆJð yð  yð  yˆJð Að  Að  AˆJð —x’x‘z”zð ñ ‘��!Ø—<’< Ñ/Ô/ð 6ØŸ	š	 .°"Ñ5Ô5�AØ—<’<Ð ;Ñ<Ô<ð [ØŸ	š	Ð"=¸ÑOÔO�AÝ$'¨
°JÑ$?Ô$?ð 0ð 0™˜˜SØŸIšI c¨3Ñ/Ô/˜˜ð0à—\’\Ð"6Ñ7Ô7ð [ØŸ	š	Ð"6¸ÑHÔH�AÝ$'¨
°JÑ$?Ô$?ð 0ð 0™˜˜SØŸIšI c¨3Ñ/Ô/˜˜ð0õ %(¨°(Ñ$;Ô$;ð 0ð 0™˜˜SØŸIšI c¨3Ñ/Ô/˜˜à(¨AÐ-Ð-Ø# q˜=˜=Ø 0Ð 0°Ñ 3Ô 3˜AØŸIšIÐ&7Ð9YÑZÔZ˜Ø—I’IÐDÐFiÑjÔj�Ø—I’IÐDÐFiÑjÔj�Ø—I’IÐ@ÐBaÑbÔb�Ø—I’IÐ1Ð3FÑGÔG�Ø—I’IÐ/Ð1BÑCÔC�Ø—|’|Ð$6Ñ7Ô7ð /Ø*¨QÑ.�Aà��q‘	‘	ð7ðX ˆFØˆrF   NÚimgÚ	timestepsÚguidanceÚNAGÚreturnc           
      óª  ‡ ‡‡‡‡‡.‡/— t          |¦  «        }d x}}d }‰ j        �r3‰ j        }‰d d …d ‰…d d …f         }t          |d                              ¦   «                              ¦   «         dz   ¦  «        |z  }t          |d                              ¦   «                              ¦   «         dz   ¦  «        |z  }|d d …d ‰…d d …f                              dd¦  «        }t          j        |||f||¬¦  «        }‰  	                    |¦  «         
                    d¦  «                             dd¦  «        }|}|dk    r|gn||                     ¦   «         g}n‰                      |¦  «        }|dk    r|gn||                     ¦   «         g}‰ j        �räd}t          |d¦  «                             |j        |j        ¦  «        }t%          j        dg|j        d	         z  |j        ¬
¦  «        }t          |d¦  «                             |j        |j        ¦  «        }t          t%          j        ||j        ¬
¦  «        d¦  «                             |j        |j        ¦  «        }|                     d	¦  «                             |j        d	         dd¦  «                             |j        |j        ¦  «        }t%          j        ||gd¬¦  «                             d¦  «                             d|d¦  «                             |j        ¦  «                             |j        |j        ¦  «        }t%          j        ||gd¬¦  «                             |j        |j        ¦  «        }‰                      |¦  «        }n~‰                      t          |d¦  «        ¦  «        Š/‰ j        j        r>‰ j        �7|€t=          d¦  «        ‚‰/‰                      t          |d¦  «        ¦  «        z  Š/ˆ ˆ/fd„|D ¦   «         Š.‰.}d }ˆ fd„|D ¦   «         }‰�ˆfd„|D ¦   «         }ˆfd„|D ¦   «         }ˆˆ fd„|D ¦   «         } t          |¦  «        d	k    r|d	         j        d         nd	}!‰ j        rˆ.ˆ fd„tA          |¦  «        D ¦   «         }"tC          j"        ¦   «         5  tG          ‰ j$        ¦  «        D ]´\  }#}$‰ j        r5‰  %                    |d|#¬¦  «        ‰  %                    |d|#¬¦  «        fg|z  }n	‰ j        r|"}|	d k    r |	dd dd¦  «         |
j&        rd g|z  c cd d d ¦  «         S tO          ||| |¦  «        D ](\  }}%}&Š/ |$||%‰/|&|¬¦  «        \  |d<   |%d<   d x}x}%x}&Š/Œ)Œµd„ tO          ||¦  «        D ¦   «         }‰ j        rˆ.ˆ fd„tA          |¦  «        D ¦   «         }'tG          ‰ j(        ¦  «        D ]’\  }#}$‰ j        r‰  %                    |d|#¬¦  «        g|z  }n	‰ j        r|'}|	d k    r |	dd dd¦  «         |
j&        rd g|z  c cd d d ¦  «         S tO          || |¦  «        D ]\  }}&Š/ |$|‰/|&|!|¬ ¦  «        |d<   d x}x}&Š/Œ Œ“	 d d d ¦  «         n# 1 swxY w Y   ˆfd!„tO          ||¦  «        D ¦   «         }‰ j        rd }(n$‰ j        r‰  %                    |d"¦  «        g|z  }(n‰.}(g })tA          |¦  «        D ] }#||#         }*‰ j        r+||#         }+tS          ‰ |*|+||¬#¦  «        },d x||#<   }+d x||#<   }*nO|(|#         Š/‰ j*        r‰  +                    |*‰/‰‰¦  «        \  }-},n‰  ,                    |*‰/¦  «        \  }-},d x||#<   x}*xŠ/}-|) -                    |,¦  «         Œ¡|)S )$Nrµ   r   r¶   rT   )Úoutput_sizeÚkernel_sizeÚstrideiX  r*   g        r   )Údevicer{   rŸ   rQ   rc   z:Didn't get guidance strength for guidance distilled model.c                 ó@   •— g | ]}‰‰                      |¦  «        z   ‘ŒS rE   )rŠ   )rr   Úyr[   rª   s     €€rG   rt   z Flux.forward.<locals>.<listcomp>±  s*   ø€ ÐEÐEÐE¸˜S 4§>¢>°!Ñ#4Ô#4Ñ4ÐEÐEÐErF   c                 ó:   •— g | ]}‰                      |¦  «        ‘ŒS rE   )r„   )rr   Útxtr[   s     €rG   rt   z Flux.forward.<locals>.<listcomp>µ  s%   ø€ Ð:Ð:Ð:¨�D—K’K Ñ$Ô$Ð:Ð:Ð:rF   c                 ó@   •— g | ]}t          j        ‰|fd ¬¦  «        ‘ŒS ©r   rŸ   ©rÑ   rÒ   )rr   r%  Úsiglip_embeddings     €rG   rt   z Flux.forward.<locals>.<listcomp>·  s.   ø€ ÐWÐWÐWÀs�œ	Ð#3°SÐ"9ÀÐBÑBÔBÐWÐWÐWrF   c                 ó@   •— g | ]}t          j        ‰|fd ¬¦  «        ‘ŒS r'  r(  )rr   Útxt_idÚsiglip_embedding_idss     €rG   rt   z Flux.forward.<locals>.<listcomp>¸  s/   ø€ ÐiÐiÐiÐRX�EœIÐ';¸VÐ&DÈ1ÐMÑMÔMÐiÐiÐirF   c                 óf   •— g | ]-}‰                      t          j        |‰fd ¬¦  «        ¦  «        ‘Œ.S r'  )r   rÑ   rÒ   )rr   Útxt_idsr²   r[   s     €€rG   rt   z Flux.forward.<locals>.<listcomp>º  s;   ø€ ÐfÐfÐfÈg�4×#Ò#¥E¤I¨w¸Ð.@ÀaÐ$HÑ$HÔ$HÑIÔIÐfÐfÐfrF   c                 ó|   •— g | ]8}‰                      ‰|         ¦  «        ‰                     ‰|         ¦  «        f‘Œ9S rE   )r…   r†   ©rr   ÚiÚbase_vec_listr[   s     €€rG   rt   z Flux.forward.<locals>.<listcomp>¾  sv   ø€ ð  cð  cð  cð  ST $×"CÒ"CÀMÐRSÔDTÑ"UÔ"UÐW[×WxÒWxð  zGð  HIô  zJñ  XKô  XKð  !Nð  cð  cð  crF   rU   rJ   rV   FT)r  r%  rª   Úper  .c                 óB   — g | ]\  }}t          j        ||fd ¦  «        ‘ŒS )r   r(  )rr   r%  r  s      rG   rt   z Flux.forward.<locals>.<listcomp>Î  s,   € ÐYÐYÐY±X°S¸#�œ	 3¨ *¨aÑ0Ô0ÐYÐYÐYrF   c                 óR   •— g | ]#}‰                      ‰|         ¦  «        d          ‘Œ$S )r   )r‡   r0  s     €€rG   rt   z Flux.forward.<locals>.<listcomp>Ñ  s3   ø€ Ð"iÐ"iÐ"iÐZ[ 4×#@Ò#@ÀÈqÔAQÑ#RÔ#RÐSTÔ#UÐ"iÐ"iÐ"irF   rS   )Úxrª   r3  Útxt_lenr  c                 óf   •— g | ]-\  }}|d d …|j         d         |j         d         ‰z   …df         ‘Œ.S )Nr   .)rÀ   )rr   r  r%  r³   s      €rG   rt   z Flux.forward.<locals>.<listcomp>à  sD   ø€ ÐoÐoÐoÉ8È3ÐPS�C˜˜˜˜3œ9 Qœ<¨#¬)°A¬,¸Ñ*@Ð@À#ÐEÔFÐoÐoÐorF   rO   )ÚmoduleÚ
hidden_seqÚ
base_imageÚheightÚwidth).rø   r)   rÉ   r?   r¾   r¿   Ú	transposeÚFÚfoldÚimg_in_patchÚflattenÚcloner‚   r'   r   r¼   r!  r»   rÑ   rL   rÀ   ÚarangeÚ	unsqueezeÚrepeatrÒ   rˆ   r‰   rX   r&   rƒ   rZ   rp   rŒ   r   ÚscopeÚ	enumerater�   r_   Ú
_interruptrÿ   rŽ   r   r;   rÎ   r±   Úappend)0r[   r  r²   Útxt_listÚtxt_ids_listr  Úy_listr³   r  ÚcallbackÚpipeliner)  r,  r  Úszr<  r=  Úbase_image_listrÉ   Úbase_idsr�   ÚimageÚhiddenÚimg_listÚmod_index_lengthÚdistill_timestepÚdistil_guidanceÚmodulation_indexÚtimestep_guidanceÚ	input_vecÚmod_vectorsÚvec_listÚpe_listr7  Údouble_vec_listr1  Úblockr%  r3  Úsingle_vec_listÚ
final_vecsÚout_listr:  r;  Úpred_tokensr¯   r2  rª   s0   ` `    `   ``                                 @@rG   ÚforwardzFlux.forwardw  s±	  øøøøøøø€ õ" �‰]Œ]ˆØÐˆ�ØˆØŒ=ñ 	#ØœˆJà˜q˜q˜q ( 7 (¨A¨A¨A˜~Ô.ˆHÝ˜ &Ô)×-Ò-Ñ/Ô/×4Ò4Ñ6Ô6¸Ñ:Ñ;Ô;¸jÑHˆFÝ˜ Ô(×,Ò,Ñ.Ô.×3Ò3Ñ5Ô5¸Ñ9Ñ:Ô:¸ZÑGˆEà˜˜˜˜H˜W˜H a a a˜Ô(×2Ò2°1°aÑ8Ô8ˆFÝ”FØØ# U˜OØ&Ø!ð	ñ ô ˆEð ×&Ò& uÑ-Ô-×5Ò5°aÑ8Ô8×BÒBÀ1ÀaÑHÔHˆFØˆCØ)+¨qª¨˜u˜g˜g°u¸e¿kºk¹m¼mÐ6LˆOˆOà—+’+˜cÑ"Ô"ˆCØ !šG˜G�C�5�5¨#¨s¯yªy©{¬{Ð);ˆàŒ;ñ 	%Ø"ÐÝ1°)¸RÑ@Ô@×CÒCÀCÄJÐPSÔPYÑZÔZÐÝœ b TÐ+;Ô+AÀ!Ô+DÑ%DÈSÌZÐXÑXÔXˆHÝ0°¸2Ñ>Ô>×AÒAÀ#Ä*ÈcÌiÑXÔXˆOÝ1µ%´,Ð?OÐX[ÔXbÐ2cÑ2cÔ2cÐegÑhÔh×kÒkÐloÔlvÐx{ô  yBñ   Cô   CÐØ/×9Ò9¸!Ñ<Ô<×CÒCÀCÄIÈaÄLÐRSÐUVÑWÔW×ZÒZÐ[^Ô[eÐgjÔgpÑqÔqÐÝ %¤	Ð+;¸_Ð*MÐSTÐ UÑ UÔ U× _Ò _Ð`aÑ bÔ b× iÒ iÐjkÐm}ð  @Añ  !Bô  !B÷  !Eò  !Eð  FIô  FOñ  !Pô  !P÷  !Sò  !Sð  TWô  T^ð  `cô  `iñ  !jô  !jÐÝœ	Ð#4Ð6FÐ"GÈRÐPÑPÔP×SÒSÐTWÔT^Ð`cÔ`iÑjÔjˆIØ×7Ò7¸	ÑBÔBˆKˆKà—,’,Õ1°)¸SÑAÔAÑBÔBˆCØŒ{Ô)ð L¨dÔ.>Ð.JØÐ#Ý$Ð%aÑbÔbÐbØ˜×(Ò(Õ);¸HÀcÑ)JÔ)JÑKÔKÑK�ØEÐEÐEÐEÐE¸fÐEÑEÔEˆMØ$ˆHàˆØ:Ð:Ð:Ð:°Ð:Ñ:Ô:ˆØÐ'ØWÐWÐWÐWÈhÐWÑWÔWˆHØiÐiÐiÐiÐ\hÐiÑiÔiˆLàfÐfÐfÐfÐfÐYeÐfÑfÔfˆÝ*-¨h©-¬-¸!Ò*;Ð*;�(˜1”+Ô# AÔ&Ð&ÀˆàŒ=ð 	cð cð  cð  cð  cð  cõ  X]ð  ^`ñ  Xaô  Xað  cñ  cô  cˆOåÔÑ Ô ð 	*ð 	*Ý% dÔ&8Ñ9Ô9ð /ð /‘��5Ø”;ð /Ø"&×"6Ò"6°{ÀLÐVWÐ"6Ñ"XÔ"XÐZ^×ZnÒZnÐozð  }Ið  OPÐZnñ  [Qô  [Qð  !Rð   Sð  VXñ   X�H�HØ”]ð /Ø.�HØ˜tÒ#Ð#Ø�H˜R  u¨dÑ3Ô3Ð3ØÔ&ð 'Ø ˜6 B™;Ð&Ð&ð	*ð 	*ð 	*ð 	*ñ 	*ô 	*ð 	*ð 	*õ *-¨X°xÀÈ(Ñ)SÔ)Sð /ð /Ñ%�C˜˜b #Ø).¨°3¸CÀSÈRÐUXÐ)YÑ)YÔ)YÑ&�C˜‘H˜c #™hØ*.Ð.�CÐ.˜#Ð.  S Sð/ð ZÐYÅÀXÈxÑAXÔAXÐYÑYÔYˆHàŒ}ð jØ"iÐ"iÐ"iÐ"iÐ"iÕ_dÐegÑ_hÔ_hÐ"iÑ"iÔ"i�å% dÔ&8Ñ9Ô9ð *ð *‘��5Ø”;ð /Ø#×3Ò3°KÀÈqÐ3ÑQÔQÐRÐUWÑW�H�HØ”]ð /Ø.�Hà˜tÒ#Ð#Ø�H˜R  u¨dÑ3Ô3Ð3ØÔ&ð 'Ø ˜6 B™;Ð&Ð&ð9	*ð 	*ð 	*ð 	*ñ 	*ô 	*ð 	*ð 	*õ: %(¨°'¸8Ñ$DÔ$Dð *ð *‘L�C˜˜SØ#˜e c¨s°rÀ7ÐPSÐTÑTÔT�C˜‘HØ%)Ð)�CÐ)˜"˜s˜sð*ð*ð'	*ð 	*ð 	*ñ 	*ô 	*ð 	*ð 	*ð 	*ð 	*ð 	*ð 	*øøøð 	*ð 	*ð 	*ð 	*ð@ pÐoÐoÐoÕWZÐ[cÐemÑWnÔWnÐoÑoÔoˆàŒ=ð 	'ØˆJˆJØŒ[ð 	'Ø×.Ò.¨{¸GÑDÔDÐEÈÑJˆJˆJà&ˆJØˆÝ�r‘”ð 	)ð 	)ˆAØ! !œˆJØŒ}ð BØ,¨QÔ/�
Ý1ØØ)Ø)Ø!Øðñ ô �ð 37Ð6� Ñ" ZØ+/Ð/�˜‘˜j˜jà  ”m�Ø”;ð VØ-1×-KÒ-KØ" C¨°'ñ.ô .Ñ*�I˜{˜{ð .2×-DÒ-DÀZÐQTÑ-UÔ-UÑ*�I˜{Ø=AÐA�˜‘ÐA˜jÐA¨3°Ø�OŠO˜KÑ(Ô(Ð(Ð(Øˆs    ÒBYÔC!YØ2YÙYÙY)r   NNNNNN)r<   r=   r>   Ú__doc__rÑ   r   rD   r?   r_   r   r}   r±   rÎ   r  Údictre  Ú__classcell__©rœ   s   @rG   rI   rI   ;   s‚  ø€ € € € € ðð ð TUð +ð +ð + e¤lð +Àð +ÈSð +ð +ð +ð +ð6qN˜zð qNð qNð qNð qNð qNð qNðf&¨ð &ð &ð &ð &ð /Y°ð /Y¸Vð /YÈfð /YÐ_bð /Yð /Yð /Yð /Yðbgð gð gðb Ø"&ØØØØ#ØðIð IàðIð ðIð ðIð ˜4‘-ðIð �D‰[ðIð 
ðIð Ið Ið Ið Ið Ið Ið IrF   rI   c                   óB   ‡ — e Zd Z	 	 d
dededdfˆ fd„Zdeddfd	„Zˆ xZS )ÚFluxLoraWrapperé€   ç      ð?Ú	lora_rankÚ
lora_scaler  Nc                 ól   •—  t          ¦   «         j        |i |¤Ž || _        t          | ||¬¦  «         d S )N)Úmax_rankr­   )r|   r}   rn  r   )r[   rn  ro  ÚargsÚkwargsrœ   s        €rG   r}   zFluxLoraWrapper.__init__  sR   ø€ ð 	�‰ŒÔ˜$Ð) &Ð)Ð)Ð)à"ˆŒå ØØØð	
ñ 	
ô 	
ð 	
ð 	
ð 	
rF   r­   c                 óŠ   — |                       ¦   «         D ]-}t          |t          ¦  «        r|                     |¬¦  «         Œ.d S )N)r­   )ÚmodulesÚ
isinstancer   Ú	set_scale)r[   r­   r9  s      rG   Úset_lora_scalezFluxLoraWrapper.set_lora_scale  sM   € Ø—l’l‘n”nð 	.ð 	.ˆFÝ˜&¥*Ñ-Ô-ð .Ø× Ò  uÐ Ñ-Ô-Ð-øð	.ð 	.rF   )rl  rm  )r<   r=   r>   r?   rA   r}   rx  rh  ri  s   @rG   rk  rk    sˆ   ø€ € € € € ð Øð
ð 
àð
ð ð
ð 
ð
ð 
ð 
ð 
ð 
ð 
ð". Eð .¨dð .ð .ð .ð .ð .ð .ð .ð .rF   rk  )#Údataclassesr   r˜   rÑ   Úmmgpr   Úshared.kernelsr   r   r   Útorch.nn.functionalÚ
functionalr?  Úmodules.layersr	   r
   r   r   r   r   r   r   r   r   r   Úmodules.lorar   r   r)   r   r   r   ÚModulerI   rk  rE   rF   rG   ú<module>r�     sÑ  ðØ !Ð !Ð !Ð !Ð !Ð !Ø €€€à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ?Ð >Ð >Ð >Ð >Ð >Ð >Ð >Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ Bð ðð ð ð ð ñ ô ñ „ðð>Eð Eð Eð Eð Eˆ2Œ9ñ Eô Eð EðP.ð .ð .ð .ð .�dñ .ô .ð .ð .ð .rF   