ó
    oM‡jÚ¦ ã                  ó‚  • S SK Jr  S SKrS SKrS SKJr  S SKrS SKJs  J	r
  S SKJr  SSKJr  SSKJrJrJr  SSKJrJrJr  SS	KJrJr  \R0                  " \5      r\" 5       (       a  S SKr\" 5       (       a	  S SKrS SKrOSr\" 5       (       a  \" S
S5      (       a  S SKJr  S SK J!r!  Sr"OSr"\ " S S\RF                  5      5       r$ " S S\RF                  5      r% " S S\RF                  5      r& " S S\RF                  5      r' " S S5      r( " S S5      r) " S S\RF                  5      r* " S S5      r+ " S  S!5      r, " S" S#5      r- " S$ S%5      r. " S& S'5      r/ " S( S)5      r0 " S* S+5      r1 " S, S-5      r2 " S. S/5      r3 " S0 S15      r4 " S2 S35      r5 " S4 S55      r6 " S6 S75      r7 " S8 S95      r8 " S: S;5      r9 " S< S=5      r: " S> S?5      r; " S@ SA5      r< " SB SC5      r= " SD SE5      r> " SF SG5      r? " SH SI5      r@ " SJ SK5      rA " SL SM5      rB " SN SO5      rC " SP SQ\RF                  5      rD " SR SS\RF                  5      rE " ST SU5      rF " SV SW5      rG " SX SY\RF                  5      rH " SZ S[\RF                  5      rI " S\ S]\R                  RF                  5      rJ " S^ S_\R                  RF                  5      rK " S` Sa\R                  RF                  5      rL " Sb Sc5      rM " Sd Se5      rN " Sf Sg5      rO " Sh Si5      rP " Sj Sk5      rQ " Sl Sm5      rR " Sn So5      rS " Sp Sq5      rT " Sr Ss5      rU " St Su5      rV " Sv Sw5      rW " Sx Sy5      rX " Sz S{5      rY " S| S}5      rZ " S~ S5      r[ " S€ S�5      r\ " S‚ Sƒ5      r]\+\G\,\74r^\)\:\8\F\I\J\Z4r_\)\*-  \+-  \,-  \--  \.-  \/-  \0-  \2-  \3-  \4-  \W-  \[-  \Y-  \\-  \5-  \6-  \7-  \8-  \;-  \9-  \:-  \<-  \(-  \=-  \>-  \?-  \@-  \A-  \B-  \C-  \D-  \E-  \F-  \G-  \T-  \U-  \V-  \&-  \O-  \%-  \I-  \J-  \K-  \L-  \M-  \N-  \P-  \Q-  \R-  \S-  r`g)„é    )ÚannotationsN)ÚCallable)Únné   )ÚIPAdapterMaskProcessor)Ú	deprecateÚis_torch_xla_availableÚlogging)Úis_torch_npu_availableÚis_torch_xla_versionÚis_xformers_available)Úis_torch_versionÚmaybe_allow_in_graphÚ>z2.2)Úflash_attention)Úis_spmdTFc                  óò  ^ • \ rS rSrSr                             S                                                       SU 4S jjjr  S     SS jjrSS jr S     SS jjrSS jr	SS jr
SSS	 jjr  S       SS
 jjrSS jrS S!S jjr S       S"S jjr S          S#S jjrS$S jr\R&                  " 5       S%S j5       rSrU =r$ )&Ú	Attentioné3   aö  
A cross attention layer.

Parameters:
    query_dim (`int`):
        The number of channels in the query.
    cross_attention_dim (`int`, *optional*):
        The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`.
    heads (`int`,  *optional*, defaults to 8):
        The number of heads to use for multi-head attention.
    kv_heads (`int`,  *optional*, defaults to `None`):
        The number of key and value heads to use for multi-head attention. Defaults to `heads`. If
        `kv_heads=heads`, the model will use Multi Head Attention (MHA), if `kv_heads=1` the model will use Multi
        Query Attention (MQA) otherwise GQA is used.
    dim_head (`int`,  *optional*, defaults to 64):
        The number of channels in each head.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
    bias (`bool`, *optional*, defaults to False):
        Set to `True` for the query, key, and value linear layers to contain a bias parameter.
    upcast_attention (`bool`, *optional*, defaults to False):
        Set to `True` to upcast the attention computation to `float32`.
    upcast_softmax (`bool`, *optional*, defaults to False):
        Set to `True` to upcast the softmax computation to `float32`.
    cross_attention_norm (`str`, *optional*, defaults to `None`):
        The type of normalization to use for the cross attention. Can be `None`, `layer_norm`, or `group_norm`.
    cross_attention_norm_num_groups (`int`, *optional*, defaults to 32):
        The number of groups to use for the group norm in the cross attention.
    added_kv_proj_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the added key and value projections. If `None`, no projection is used.
    norm_num_groups (`int`, *optional*, defaults to `None`):
        The number of groups to use for the group norm in the attention.
    spatial_norm_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the spatial normalization.
    out_bias (`bool`, *optional*, defaults to `True`):
        Set to `True` to use a bias in the output linear layer.
    scale_qk (`bool`, *optional*, defaults to `True`):
        Set to `True` to scale the query and key by `1 / sqrt(dim_head)`.
    only_cross_attention (`bool`, *optional*, defaults to `False`):
        Set to `True` to only use cross attention and not added_kv_proj_dim. Can only be set to `True` if
        `added_kv_proj_dim` is not `None`.
    eps (`float`, *optional*, defaults to 1e-5):
        An additional value added to the denominator in group normalization that is used for numerical stability.
    rescale_output_factor (`float`, *optional*, defaults to 1.0):
        A factor to rescale the output by dividing it with this value.
    residual_connection (`bool`, *optional*, defaults to `False`):
        Set to `True` to add the residual connection to the output.
    _from_deprecated_attn_block (`bool`, *optional*, defaults to `False`):
        Set to `True` if the attention block is loaded from a deprecated state dict.
    processor (`AttnProcessor`, *optional*, defaults to `None`):
        The attention processor to use. If `None`, defaults to `AttnProcessor2_0` if `torch 2.x` is used and
        `AttnProcessor` otherwise.
c                óÄ  >• [         T#U ]  5         SSKJnJn Jn!  Ub  UOXS-  U l        Uc  U R                  OXT-  U l        Xl        Xpl	        US LU l
        Ub  UOUU l        X€l        X�l        UU l        UU l        X`l        SU l        Ub  UOUU l        Ub  UOUU l        UU l        UU l        UU l        UU l        UU l        U R0                  (       a  US-  OSU l        Ub  UU-  OUU l        X0l        XÐl        UU l        U R8                  c  U R:                  (       a  [=        S5      eUb  [>        R@                  " XUSS9U l!        OS U l!        Ub  [E        UUS	9U l#        OS U l#        Uc  S U l$        S U l%        GOUS
:X  a7  [>        RL                  " UUUS9U l$        [>        RL                  " UUUS9U l%        OÖUS:X  a  U" USSUS9U l$        U" USSUS9U l%        O³US:X  a9  [>        RL                  " XS-  US9U l$        [>        RL                  " XT-  US9U l%        OtUS:X  a  U!" UUUS9U l$        U!" UUUS9U l%        OSUS:X  a  U!" XS-  US9U l$        U!" XT-  US9U l%        O0US:X  a  U " SSUS9U l$        U " SSUS9U l%        O[=        SU S35      eU
c  S U l'        OzU
S
:X  a&  [>        RL                  " U R                  5      U l'        ONU
S:X  a9  U R8                  b  Un"OU R                  n"[>        R@                  " U"USSS9U l'        O[=        SU
 S35      e[>        RP                  " XR                  US9U l)        U R:                  (       d_  [>        RP                  " U R                  U R                  US9U l*        [>        RP                  " U R                  U R                  US9U l+        OS U l*        S U l+        Xàl,        U R8                  bz  [>        RP                  " XÐR                  US9U l-        [>        RP                  " XÐR                  US9U l.        U R(                  b$  [>        RP                  " XÐR                  US9U l/        OS U l/        S U l-        S U l.        U R*                  (       dŽ  [>        R`                  " / 5      U l1        U Rb                  Re                  [>        RP                  " U R                  U R$                  US95        U Rb                  Re                  [>        Rf                  " U5      5        OS U l1        U R(                  bA  U R(                  (       d0  [>        RP                  " U R                  U R&                  US9U l4        OS U l4        Ub­  Ubª  US
:X  a7  [>        RL                  " UUUS9U l5        [>        RL                  " UUUS9U l6        O{US:X  a  U" USSUS9U l5        U" USSUS9U l6        OXUS:X  a  U!" UUS9U l5        U!" UUS9U l6        O9US:X  a  S U l5        U!" XT-  US9U l6        O[=        SU S35      eS U l5        S U l6        Uc:  [o        [p        S5      (       a  U R0                  (       a
  [s        5       O	[u        5       nU Rw                  U5        g )Né   )ÚFP32LayerNormÚLpNormÚRMSNormFg      à¿ç      ð?z¦`only_cross_attention` can only be set to True if `added_kv_proj_dim` is not None. Make sure to set either `only_cross_attention=False` or define `added_kv_proj_dim`.T©Únum_channelsÚ
num_groupsÚepsÚaffine)Ú
f_channelsÚzq_channelsÚ
layer_norm)r   Úelementwise_affineÚfp32_layer_norm)r$   Úbiasr   Úlayer_norm_across_heads)r   Úrms_normÚrms_norm_across_headsÚl2r   éÿÿÿÿ)ÚpÚdimr   zunknown qk_norm: z. Should be one of None, 'layer_norm', 'fp32_layer_norm', 'layer_norm_across_heads', 'rms_norm', 'rms_norm_across_heads', 'l2'.Ú
group_normçñhãˆµøä>zunknown cross_attention_norm: z.. Should be None, 'layer_norm' or 'group_norm'©r&   zC. Should be one of `None,'layer_norm','fp32_layer_norm','rms_norm'`Úscaled_dot_product_attention)<ÚsuperÚ__init__Únormalizationr   r   r   Ú	inner_dimÚinner_kv_dimÚ	query_dimÚuse_biasÚis_cross_attentionÚcross_attention_dimÚupcast_attentionÚupcast_softmaxÚrescale_output_factorÚresidual_connectionÚdropoutÚfused_projectionsÚout_dimÚout_context_dimÚcontext_pre_onlyÚpre_onlyÚ	is_causalÚ_from_deprecated_attn_blockÚscale_qkÚscaleÚheadsÚsliceable_head_dimÚadded_kv_proj_dimÚonly_cross_attentionÚ
ValueErrorr   Ú	GroupNormr.   ÚSpatialNormÚspatial_normÚnorm_qÚnorm_kÚ	LayerNormÚ
norm_crossÚLinearÚto_qÚto_kÚto_vÚadded_proj_biasÚ
add_k_projÚ
add_v_projÚ
add_q_projÚ
ModuleListÚto_outÚappendÚDropoutÚ
to_add_outÚnorm_added_qÚnorm_added_kÚhasattrÚFÚAttnProcessor2_0ÚAttnProcessorÚset_processor)$Úselfr7   r:   rI   Úkv_headsÚdim_headr?   r&   r;   r<   Úcross_attention_normÚcross_attention_norm_num_groupsÚqk_normrK   rY   Únorm_num_groupsÚspatial_norm_dimÚout_biasrG   rL   r   r=   r>   rF   Ú	processorrA   rB   rC   rD   r$   rE   r   r   r   Únorm_cross_num_channelsÚ	__class__s$                                      €Ú²C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffusers/models/attention_processor.pyr3   ÚAttention.__init__k   s  ø€ ôB 	‰ÑÔ÷ 	BÑAà$+Ñ$7™¸XÑ=MˆŒØ.6Ñ.>˜DŸNšNÀHÑDWˆÔØ"ŒØŒØ"5¸TÐ"AˆÔØ:MÑ:YÑ#6Ð_hˆÔ Ø 0ÔØ,ÔØ%:ˆÔ"Ø#6ˆÔ ØŒØ!&ˆÔØ")Ñ"5‘w¸9ˆŒØ2AÑ2M™ÐS\ˆÔØ 0ˆÔØ ˆŒØ"ˆŒð ,GˆÔ(à ˆŒØ'+§}§}�X˜t’^¸#ˆŒ
à,3Ñ,?�W Ò(ÀUˆŒ
ð #(Ôà!2ÔØ$8ˆÔ!à×!Ñ!Ñ)¨d×.G×.GÜð yóð ð Ñ&Ü Ÿlšl¸	ÐcfÐosÑtˆD�Oà"ˆDŒOàÑ'Ü +°yÐN^Ñ _ˆDÕà $ˆDÔà‰?ØˆDŒKØˆDŽKØ˜Ó$ÜŸ,š, x°SÐM_Ñ`ˆDŒKÜŸ,š, x°SÐM_Ñ`ˆD�KØÐ)Ó)Ù'¨ÀUÐQVÐ\_Ñ`ˆDŒKÙ'¨ÀUÐQVÐ\_Ñ`ˆD�KØÐ1Ó1äŸ,š, xÑ'7¸SÑAˆDŒKÜŸ,š, xÑ':ÀÑDˆD�KØ˜
Ó"Ù! (°ÐHZÑ[ˆDŒKÙ! (°ÐHZÑ[ˆD�KØÐ/Ó/á! (Ñ"2¸Ñ<ˆDŒKÙ! (Ñ"5¸3Ñ?ˆD�KØ˜‹_Ù  1¨"°#Ñ6ˆDŒKÙ  1¨"°#Ñ6ˆD�KäØ# G 9ð  -lð  móð ð  Ñ'Ø"ˆD�OØ! \Ó1Ü Ÿlšl¨4×+CÑ+CÓDˆD�OØ! \Ó1Ø×%Ñ%Ñ1ð +<Ñ'à*.×*BÑ*BÐ'ä ŸlšlØ4ÐA`ÐfjÐswñˆD�Oô Ø0Ð1EÐ0FÐFtÐuóð ô —I’I˜i¯©¸dÑCˆŒ	à×(×(äŸ	š	 $×":Ñ":¸D×<MÑ<MÐTXÑYˆDŒIÜŸ	š	 $×":Ñ":¸D×<MÑ<MÐTXÑYˆD�IàˆDŒIØˆDŒIà.ÔØ×!Ñ!Ñ-Ü ŸišiÐ(9×;LÑ;LÐSbÑcˆDŒOÜ ŸišiÐ(9×;LÑ;LÐSbÑcˆDŒOØ×$Ñ$Ñ0Ü"$§)¢)Ð,=¿~¹~ÐTcÑ"d�”øà"ˆDŒOØ"ˆDŒOØ"ˆDŒOà�}�}ÜŸ-š-¨Ó+ˆDŒKØ�K‰K×ÑœrŸyšy¨¯©¸¿¹ÈHÑUÔVØ�K‰K×ÑœrŸzšz¨'Ó2Õ3àˆDŒKà× Ñ Ñ,°T×5J×5JÜ Ÿiši¨¯©¸×8LÑ8LÐS[Ñ\ˆD�Oà"ˆDŒOàÑÐ#4Ñ#@Ø˜,Ó&Ü$&§L¢L°¸sÐWiÑ$j�Ô!Ü$&§L¢L°¸sÐWiÑ$j�Õ!ØÐ-Ó-Ù$1°(ÈuÐ[`ÐfiÑ$j�Ô!Ù$1°(ÈuÐ[`ÐfiÑ$j�Õ!Ø˜JÓ&Ù$+¨H¸#Ñ$>�Ô!Ù$+¨H¸#Ñ$>�Õ!ØÐ3Ó3ð %)�Ô!Ù$+¨HÑ,?ÀSÑ$I�Õ!ä Ø'¨ yÐ0sÐtóð ð !%ˆDÔØ $ˆDÔð Ñä&-¬aÐ1O×&PÑ&PÐUY×Ub×UbÔ Ô"ÔhuÓhwð ð 	×Ñ˜9Õ%ó    c                ój  • U(       aa  [         (       d  Se[        SS5      (       a  Se[        5       (       a  [        SS5      (       a  SeU(       a  [        U5      nOF[	        U5      nO:[        [        S5      (       a  U R                  (       a
  [        5       O	[        5       nU R                  U5        g)	a3  
Set whether to use xla flash attention from `torch_xla` or not.

Args:
    use_xla_flash_attention (`bool`):
        Whether to use pallas flash attention kernel from `torch_xla` or not.
    partition_spec (`tuple[]`, *optional*):
        Specify the partition specification if using SPMD. Otherwise None.
ztorch_xla is not availableÚ<ú2.3zEflash attention pallas kernel is supported from torch_xla version 2.3ú2.4zPflash attention pallas kernel using SPMD is supported from torch_xla version 2.4r1   N)r	   r   r   ÚXLAFluxFlashAttnProcessor2_0ÚXLAFlashAttnProcessor2_0rd   re   rG   rf   rg   rh   )ri   Úuse_xla_flash_attentionÚpartition_specÚis_fluxrr   s        ru   Úset_use_xla_flash_attentionÚ%Attention.set_use_xla_flash_attention7  s‘   € ö #ß)Ò)Ø2Ð2Ü% c¨5×1Ñ1Ø]Ð]Ü—‘Ô3°C¸×?Ñ?ØhÐhæÜ <¸^Ó L‘Iä 8¸Ó H‘Iô '.¬aÐ1O×&PÑ&PÐUY×Ub×UbÔ Ô"ÔhuÓhwð ð 	×Ñ˜9Õ%rw   c                ó¾   • U(       a  [        5       nO:[        [        S5      (       a  U R                  (       a
  [	        5       O	[        5       nU R                  U5        g)zB
Set whether to use npu flash attention from `torch_npu` or not.

r1   N)ÚAttnProcessorNPUrd   re   rG   rf   rg   rh   )ri   Úuse_npu_flash_attentionrr   s      ru   Úset_use_npu_flash_attentionÚ%Attention.set_use_npu_flash_attentionX  sI   € ö
 #Ü(Ó*‰Iô '.¬aÐ1O×&PÑ&PÐUY×Ub×UbÔ Ô"ÔhuÓhwð ð 	×Ñ˜9Õ%rw   c                ó”  • [        U S5      =(       a%    [        U R                  [        [        [
        45      n[        U S5      =(       a*    [        U R                  [        [        [        [        45      n[        U S5      =(       a%    [        U R                  [        [        [        45      n[        U S5      =(       a     [        U R                  [        [        45      nU(       Gaò  U(       a  U(       a  [        SU R                   35      e[!        5       (       d
  [#        SSS9e[$        R&                  R)                  5       (       d  [+        S5      e SnUb  Uu  p‰UR,                  tpz[$        R.                  " SS	US
9n[0        R2                  R5                  X»U5      n
U(       aÜ  [	        U R                  R8                  U R                  R:                  U R                  R<                  U R                  R>                  US9nURA                  U R                  RC                  5       5        [        U R                  S5      (       a9  URE                  U R                  RF                  RH                  RJ                  5        GOžU(       a   [L        RO                  S5        [        US9nGOwU(       Ga	  [        U R                  R<                  U R                  R>                  U R                  RP                  U R                  RR                  US9nURA                  U R                  RC                  5       5        [        U R                  S5      (       af  URE                  U R                  RT                  S   RH                  RJ                  U R                  RT                  S   RH                  RV                  S
9  GOfU(       a  [        US9nGOT[Y        US9nGOIU(       aù  [        [Z        S5      (       a  [
        O[        nU" U R                  R8                  U R                  R:                  U R                  R<                  U R                  R>                  S9nURA                  U R                  RC                  5       5        [        U R                  S5      (       a9  URE                  U R                  RF                  RH                  RJ                  5        GOIU(       Ga  [        U R                  R<                  U R                  R>                  U R                  RP                  U R                  RR                  S9nURA                  U R                  RC                  5       5        [        U R                  S5      (       af  URE                  U R                  RT                  S   RH                  RJ                  U R                  RT                  S   RH                  RV                  S
9  O:[        [Z        S5      (       a  U R\                  (       a
  [_        5       O	[a        5       nU Rc                  U5        g! [6         a  nUeSnAff = f)ax  
Set whether to use memory efficient attention from `xformers` or not.

Args:
    use_memory_efficient_attention_xformers (`bool`):
        Whether to use memory efficient attention from `xformers` or not.
    attention_op (`Callable`, *optional*):
        The attention operation to use. Defaults to `None` which uses the default attention operation from
        `xformers`.
rr   zhMemory efficient attention is currently not supported for custom diffusion for attention processor type zeRefer to https://github.com/facebookresearch/xformers for more information on how to install xformersÚxformers)Únamezvtorch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only available for GPU N)r   r   é(   Úcuda©ÚdeviceÚdtype)Útrain_kvÚtrain_q_outÚhidden_sizer:   Úattention_opÚto_k_custom_diffusionz‹Memory efficient attention with `xformers` might currently not work correctly if an attention mask is required for the attention operation.©r“   )r’   r:   Ú
num_tokensrH   r“   Úto_k_ipr   r1   )r�   r‘   r’   r:   )r’   r:   r–   rH   )2rd   Ú
isinstancerr   ÚCustomDiffusionAttnProcessorÚ$CustomDiffusionXFormersAttnProcessorÚCustomDiffusionAttnProcessor2_0ÚAttnAddedKVProcessorÚAttnAddedKVProcessor2_0ÚSlicedAttnAddedKVProcessorÚXFormersAttnAddedKVProcessorÚIPAdapterAttnProcessorÚIPAdapterAttnProcessor2_0ÚIPAdapterXFormersAttnProcessorÚJointAttnProcessor2_0ÚXFormersJointAttnProcessorÚNotImplementedErrorr   ÚModuleNotFoundErrorÚtorchrŒ   Úis_availablerM   ÚSUPPORTED_DTYPESÚrandnr‰   ÚopsÚmemory_efficient_attentionÚ	Exceptionr�   r‘   r’   r:   Úload_state_dictÚ
state_dictÚtor”   ÚweightrŽ   ÚloggerÚinfor–   rH   r—   r�   ÚXFormersAttnProcessorre   rG   rf   rg   rh   )ri   Ú'use_memory_efficient_attention_xformersr“   Úis_custom_diffusionÚis_added_kv_processorÚis_ip_adapterÚis_joint_processorr�   Úop_fwÚop_bwÚ_ÚqÚerr   Úattn_processor_classs                  ru   Ú+set_use_memory_efficient_attention_xformersÚ5Attention.set_use_memory_efficient_attention_xformersi  sã  € ô & d¨KÓ8÷ 
¼ZØ�N‰NÜ)Ô+OÔQpÐqó>
Ðô !(¨¨kÓ :÷ !
¼zØ�N‰Nä$Ü'Ü*Ü,ð	ó@
Ðô    kÓ2÷ 
´zØ�N‰NÜ#Ô%>Ô@^Ð_ó8
ˆô % T¨;Ó7÷ 
¼JØ�N‰Nä%Ü*ðó=
Ð÷ 3Þ$Ö)<Ü)Ø~ð  @D÷  @Nñ  @Nð  Oð  Póð ô )×*Ñ*Ü)ð$ð $ñð ô —Z‘Z×,Ñ,×.Ñ.Ü ð/óð ð
	à �EØ#Ñ/Ø'3™˜Ø$)×$:Ñ$:˜	˜ÜŸš J°vÀUÑK�AÜ Ÿ™×?Ñ?ÀÀaÓH�Aö #Ü@Ø!Ÿ^™^×4Ñ4Ø $§¡× :Ñ :Ø $§¡× :Ñ :Ø(,¯©×(JÑ(JØ!-ñ�	ð ×)Ñ)¨$¯.©.×*CÑ*CÓ*EÔFÜ˜4Ÿ>™>Ð+B×CÑCØ—L‘L §¡×!EÑ!E×!LÑ!L×!SÑ!SÔTùÞ&ô
 —‘ð bôô 9ÀlÑS’	ßÜ:Ø $§¡× :Ñ :Ø(,¯©×(JÑ(JØ#Ÿ~™~×8Ñ8ØŸ.™.×.Ñ.Ø!-ñ�	ð ×)Ñ)¨$¯.©.×*CÑ*CÓ*EÔFÜ˜4Ÿ>™>¨9×5Ñ5Ø—L‘LØ#Ÿ~™~×5Ñ5°aÑ8×?Ñ?×FÑFÈdÏnÉn×NdÑNdÐefÑNg×NnÑNn×NtÑNtð !ñ ùö $Ü6ÀLÑQ’	ä1¸|ÑL’	æ"ô œqÐ"@×AÑAõ 4ä5ð %ñ
 1Ø!Ÿ^™^×4Ñ4Ø $§¡× :Ñ :Ø $§¡× :Ñ :Ø(,¯©×(JÑ(Jñ	�	ð ×)Ñ)¨$¯.©.×*CÑ*CÓ*EÔFÜ˜4Ÿ>™>Ð+B×CÑCØ—L‘L §¡×!EÑ!E×!LÑ!L×!SÑ!SÔTùßÜ5Ø $§¡× :Ñ :Ø(,¯©×(JÑ(JØ#Ÿ~™~×8Ñ8ØŸ.™.×.Ñ.ñ	�	ð ×)Ñ)¨$¯.©.×*CÑ*CÓ*EÔFÜ˜4Ÿ>™>¨9×5Ñ5Ø—L‘LØ#Ÿ~™~×5Ñ5°aÑ8×?Ñ?×FÑFÈdÏnÉn×NdÑNdÐefÑNg×NnÑNn×NtÑNtð !ñ øô œqÐ"@×AÑAÀdÇmÇmô %Ô&ä&›ð ð 	×Ñ˜9Õ%øô_ !ó Ø�Gûðús   ÅAX6 Ø6
YÙ YÙYc                ó|  • Ub+  XR                   :”  a  [        SU SU R                    S35      eUb  U R                  b  [        U5      nOaUb  [	        U5      nORU R                  b  [        5       nO:[        [        S5      (       a  U R                  (       a
  [        5       O	[        5       nU R                  U5        g)z€
Set the slice size for attention computation.

Args:
    slice_size (`int`):
        The slice size for attention computation.
Nzslice_size z has to be smaller or equal to Ú.r1   )rJ   rM   rK   rž   ÚSlicedAttnProcessorrœ   rd   re   rG   rf   rg   rh   )ri   Ú
slice_sizerr   s      ru   Úset_attention_sliceÚAttention.set_attention_sliceû  s²   € ð Ñ! j×3JÑ3JÓ&JÜ˜{¨:¨,Ð6UÐVZ×VmÑVmÐUnÐnoÐpÓqÐqàÑ! d×&<Ñ&<Ñ&HÜ2°:Ó>‰IØÑ#Ü+¨JÓ7‰IØ×#Ñ#Ñ/Ü,Ó.‰Iô '.¬aÐ1O×&PÑ&PÐUY×Ub×UbÔ Ô"ÔhuÓhwð ð 	×Ñ˜9Õ%rw   c                ój  • [        U S5      (       aœ  [        U R                  [        R                  R
                  5      (       ai  [        U[        R                  R
                  5      (       d@  [        R                  SU R                   SU 35        U R                  R                  S5        Xl        g)zu
Set the attention processor to use.

Args:
    processor (`AttnProcessor`):
        The attention processor to use.
rr   z-You are removing possibly trained weights of z with N)
rd   r˜   rr   r§   r   ÚModuler²   r³   Ú_modulesÚpop)ri   rr   s     ru   rh   ÚAttention.set_processor  sx   € ô �D˜+×&Ñ&Ü˜4Ÿ>™>¬5¯8©8¯?©?×;Ñ;Ü˜y¬%¯(©(¯/©/×:Ñ:ä�K‰KÐGÈÏÉÐGWÐW]Ð^gÐ]hÐiÔjØ�M‰M×Ñ˜kÔ*à"�rw   c                ó*   • U(       d  U R                   $ g)zÿ
Get the attention processor in use.

Args:
    return_deprecated_lora (`bool`, *optional*, defaults to `False`):
        Set to `True` to return the deprecated LoRA attention processor.

Returns:
    "AttentionProcessor": The attention processor in use.
N©rr   )ri   Úreturn_deprecated_loras     ru   Úget_processorÚAttention.get_processor+  s   € ö &Ø—>‘>Ð!ð &rw   c                ó2  • [        [        R                  " U R                  R                  5      R
                  R                  5       5      nSS1nUR                  5        VVs/ s H  u  pxXu;  d  M  Xv;  d  M  UPM     n	nn[        U	5      S:”  a:  [        R                  SU	 SU R                  R                  R                   S35        UR                  5        VV
s0 s H  u  pzXu;   d  M  Xz_M     nnn
U R                  " U U4UUS.UD6$ s  snnf s  sn
nf )a   
The forward method of the `Attention` class.

Args:
    hidden_states (`torch.Tensor`):
        The hidden states of the query.
    encoder_hidden_states (`torch.Tensor`, *optional*):
        The hidden states of the encoder.
    attention_mask (`torch.Tensor`, *optional*):
        The attention mask to use. If `None`, no mask is applied.
    **cross_attention_kwargs:
        Additional keyword arguments to pass along to the cross attention.

Returns:
    `torch.Tensor`: The output of the attention layer.
Úip_adapter_masksÚip_hidden_statesr   zcross_attention_kwargs z are not expected by z and will be ignored.©Úencoder_hidden_statesÚattention_mask)ÚsetÚinspectÚ	signaturerr   Ú__call__Ú
parametersÚkeysÚitemsÚlenr²   Úwarningrt   Ú__name__)ri   Úhidden_statesrÖ   r×   Úcross_attention_kwargsÚattn_parametersÚquiet_attn_parametersÚkr¼   Úunused_kwargsÚws              ru   ÚforwardÚAttention.forward9  s"  € ô6 œg×/Ò/°·±×0GÑ0GÓH×SÑS×XÑXÓZÓ[ˆØ!3Ð5GÐ HÐà0×6Ñ6Ô8ô
Ú8‘$�!¸AÑ<T‹AÐYZÑYw�AÑ8ð 	ñ 
ô ˆ}Ó Ó!Ü�N‰NØ)¨-¨Ð8MÈdÏnÉn×NfÑNf×NoÑNoÐMpð  qFð  Gôð 4J×3OÑ3OÔ3QÔ!jÒ3Q©4¨1ÐUVÑUi£$ !¢$Ñ3QÐÑ!jà�~Š~ØØð
ð #8Ø)ñ	
ð
 %ñ
ð 	
ùó
ùó "ks   Á#DÁ2DÁ9DÃDÃ-Dc                ó¸   • U R                   nUR                  u  p4nUR                  X2-  X$U5      nUR                  SSSS5      R                  X2-  XEU-  5      nU$ )a+  
Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size // heads, seq_len, dim * heads]`. `heads`
is the number of heads initialized while constructing the `Attention` class.

Args:
    tensor (`torch.Tensor`): The tensor to reshape.

Returns:
    `torch.Tensor`: The reshaped tensor.
r   r   r   é   )rI   ÚshapeÚreshapeÚpermute)ri   ÚtensorÚ	head_sizeÚ
batch_sizeÚseq_lenr-   s         ru   Úbatch_to_head_dimÚAttention.batch_to_head_dimg  s_   € ð —J‘Jˆ	Ø#)§<¡<Ñ ˆ
˜SØ—‘ 
Ñ 7¸ÈSÓQˆØ—‘  1 a¨Ó+×3Ñ3°JÑ4KÈWÐ\eÑVeÓfˆØˆrw   c                ó  • U R                   nUR                  S:X  a  UR                  u  pEnSnOUR                  u  pGpVUR                  XEU-  X6U-  5      nUR	                  SSSS5      nUS:X  a  UR                  XC-  XW-  Xc-  5      nU$ )aØ  
Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size, seq_len, heads, dim // heads]` `heads` is
the number of heads initialized while constructing the `Attention` class.

Args:
    tensor (`torch.Tensor`): The tensor to reshape.
    out_dim (`int`, *optional*, defaults to `3`): The output dimension of the tensor. If `3`, the tensor is
        reshaped to `[batch_size * heads, seq_len, dim // heads]`.

Returns:
    `torch.Tensor`: The reshaped tensor.
rì   r   r   r   )rI   Úndimrí   rî   rï   )ri   rð   rA   rñ   rò   ró   r-   Ú	extra_dims           ru   Úhead_to_batch_dimÚAttention.head_to_batch_dimx  s‘   € ð —J‘Jˆ	Ø�;‰;˜!ÓØ'-§|¡|Ñ$ˆJ Ø‰Ià28·,±,Ñ/ˆJ 7Ø—‘ 
°iÑ,?ÀÐS\ÑL\Ó]ˆØ—‘  1 a¨Ó+ˆà�a‹<Ø—^‘^ JÑ$:¸GÑ<OÐQTÑQaÓbˆFàˆrw   c                ó$  • UR                   nU R                  (       a   UR                  5       nUR                  5       nUcV  [        R                  " UR
                  S   UR
                  S   UR
                  S   UR                   UR                  S9nSnOUnSn[        R                  " UUUR                  SS5      UU R                  S9nAU R                  (       a  UR                  5       nUR                  SS9nAUR                  U5      nU$ )a+  
Compute the attention scores.

Args:
    query (`torch.Tensor`): The query tensor.
    key (`torch.Tensor`): The key tensor.
    attention_mask (`torch.Tensor`, *optional*): The attention mask to use. If `None`, no mask is applied.

Returns:
    `torch.Tensor`: The attention probabilities/scores.
r   r   ©r�   rŽ   r+   éþÿÿÿ)ÚbetaÚalpha©r-   )r�   r;   Úfloatr§   Úemptyrí   rŽ   ÚbaddbmmÚ	transposerH   r<   Úsoftmaxr°   )	ri   ÚqueryÚkeyr×   r�   Úbaddbmm_inputrþ   Úattention_scoresÚattention_probss	            ru   Úget_attention_scoresÚAttention.get_attention_scores“  sõ   € ð —‘ˆØ× × Ø—K‘K“MˆEØ—)‘)“+ˆCàÑ!Ü!ŸKšKØ—‘˜A‘ §¡¨A¡°·	±	¸!±ÀEÇKÁKÐX]×XdÑXdñˆMð ‰Dà*ˆMØˆDä Ÿ=š=ØØØ�M‰M˜"˜bÓ!ØØ—*‘*ñ
Ðð à××Ø/×5Ñ5Ó7Ðà*×2Ñ2°rÐ2Ð:ˆØà)×,Ñ,¨UÓ3ˆàÐrw   c                ó^  • U R                   nUc  U$ UR                  S   nXb:w  a‘  UR                  R                  S:X  a_  UR                  S   UR                  S   U4n[        R
                  " XqR                  UR                  S9n[        R                  " X/SS9nO[        R                  " USU4SS	9nUS
:X  a8  UR                  S   X5-  :  a!  UR                  USUR                  S   U-  S9nU$ US:X  a2  UR                  S5      nUR                  USUR                  S   U-  S9nU$ )a)  
Prepare the attention mask for the attention computation.

Args:
    attention_mask (`torch.Tensor`):
        The attention mask to prepare.
    target_length (`int`):
        The target length of the attention mask. This is the length of the attention mask after padding.
    batch_size (`int`):
        The batch size, which is used to repeat the attention mask.
    out_dim (`int`, *optional*, defaults to `3`):
        The output dimension of the attention mask. Can be either `3` or `4`.

Returns:
    `torch.Tensor`: The prepared attention mask.
r+   Úmpsr   r   rü   r   r   ç        )Úvaluerì   ©r-   Úoutput_sizeé   )rI   rí   rŽ   Útyper§   Úzerosr�   Úcatre   ÚpadÚrepeat_interleaveÚ	unsqueeze)	ri   r×   Útarget_lengthrò   rA   rñ   Úcurrent_lengthÚpadding_shapeÚpaddings	            ru   Úprepare_attention_maskÚ Attention.prepare_attention_maskÂ  sQ  € ð& —J‘Jˆ	ØÑ!Ø!Ð!à,×2Ñ2°2Ñ6ˆØÓ*Ø×$Ñ$×)Ñ)¨UÓ2ð "0×!5Ñ!5°aÑ!8¸.×:NÑ:NÈqÑ:QÐS`Ð a�ÜŸ+š+ m×;OÑ;OÐXf×XmÑXmÑn�Ü!&§¢¨NÐ+DÈ!Ñ!L‘ô "#§¢ ~¸¸=Ð7IÐQTÑ!U�à�a‹<Ø×#Ñ# AÑ&¨Ñ)?Ó?Ø!/×!AÑ!AØ 1°.×2FÑ2FÀqÑ2IÈIÑ2Uð "Bð "�ð Ðð ˜‹\Ø+×5Ñ5°aÓ8ˆNØ+×=Ñ=Ø˜q¨n×.BÑ.BÀ1Ñ.EÈ	Ñ.Qð >ð ˆNð Ðrw   c                óf  • U R                   c   S5       e[        U R                   [        R                  5      (       a  U R                  U5      nU$ [        U R                   [        R                  5      (       a7  UR                  SS5      nU R                  U5      nUR                  SS5      nU$  e)a  
Normalize the encoder hidden states. Requires `self.norm_cross` to be specified when constructing the
`Attention` class.

Args:
    encoder_hidden_states (`torch.Tensor`): Hidden states of the encoder.

Returns:
    `torch.Tensor`: The normalized encoder hidden states.
zGself.norm_cross must be defined to call self.norm_encoder_hidden_statesr   r   )rT   r˜   r   rS   rN   r  )ri   rÖ   s     ru   Únorm_encoder_hidden_statesÚ$Attention.norm_encoder_hidden_statesõ  s¥   € ð �‰Ñ*ÐuÐ,uÓuÐ*ä�d—o‘o¤r§|¡|×4Ñ4Ø$(§O¡OÐ4IÓ$JÐ!ð %Ð$ô ˜Ÿ™¬¯©×6Ñ6ð %:×$CÑ$CÀAÀqÓ$IÐ!Ø$(§O¡OÐ4IÓ$JÐ!Ø$9×$CÑ$CÀAÀqÓ$IÐ!ð %Ð$ð �5rw   c                óÖ	  • U R                   R                  R                  R                  nU R                   R                  R                  R                  nU R
                  (       Gd†  [        R                  " U R                   R                  R                  U R                  R                  R                  U R                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R                  X#S9U l        U R                  R                  R                  U5        U R                  (       a˜  [        R                  " U R                   R                   R                  U R                  R                   R                  U R                  R                   R                  /5      nU R                  R                   R                  U5        GOF[        R                  " U R                  R                  R                  U R                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R                  X#S9U l        U R"                  R                  R                  U5        U R                  (       ay  [        R                  " U R                  R                   R                  U R                  R                   R                  /5      nU R"                  R                   R                  U5        [%        U SS 5      Gb¢  [%        U SS 5      Gb“  [%        U SS 5      Gb„  [        R                  " U R&                  R                  R                  U R(                  R                  R                  U R*                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R,                  X#S9U l        U R.                  R                  R                  U5        U R,                  (       a˜  [        R                  " U R&                  R                   R                  U R(                  R                   R                  U R*                  R                   R                  /5      nU R.                  R                   R                  U5        Xl        g )Nr   r   )r&   rŽ   r�   r\   rZ   r[   )rV   r±   ÚdatarŽ   r�   r9   r§   r  rW   rX   rí   r   rU   r8   Úto_qkvÚcopy_r&   Úto_kvÚgetattrr\   rZ   r[   rY   Úto_added_qkvr@   )ri   ÚfuserŽ   r�   Úconcatenated_weightsÚin_featuresÚout_featuresÚconcatenated_biass           ru   Úfuse_projectionsÚAttention.fuse_projections  sg  € à—‘×!Ñ!×&Ñ&×-Ñ-ˆØ—	‘	× Ñ ×%Ñ%×+Ñ+ˆà×&×&Ð&ä#(§9¢9¨d¯i©i×.>Ñ.>×.CÑ.CÀTÇYÁY×EUÑEU×EZÑEZÐ\`×\eÑ\e×\lÑ\l×\qÑ\qÐ-rÓ#sÐ Ø.×4Ñ4°QÑ7ˆKØ/×5Ñ5°aÑ8ˆLô Ÿ)š) KÀDÇMÁMÐZ`ÑnˆDŒKØ�K‰K×Ñ×$Ñ$Ð%9Ô:Ø�}�}Ü$)§I¢I¨t¯y©y¯~©~×/BÑ/BÀDÇIÁIÇNÁN×DWÑDWÐY]×YbÑYb×YgÑYg×YlÑYlÐ.mÓ$nÐ!Ø—‘× Ñ ×&Ñ&Ð'8Ô9ùô $)§9¢9¨d¯i©i×.>Ñ.>×.CÑ.CÀTÇYÁY×EUÑEU×EZÑEZÐ-[Ó#\Ð Ø.×4Ñ4°QÑ7ˆKØ/×5Ñ5°aÑ8ˆLäŸš ;À4Ç=Á=ÐY_ÑmˆDŒJØ�J‰J×Ñ×#Ñ#Ð$8Ô9Ø�}�}Ü$)§I¢I¨t¯y©y¯~©~×/BÑ/BÀDÇIÁIÇNÁN×DWÑDWÐ.XÓ$YÐ!Ø—
‘
—‘×%Ñ%Ð&7Ô8ô �D˜,¨Ó-Ò9Ü˜˜l¨DÓ1Ò=Ü˜˜l¨DÓ1Ò=ä#(§9¢9Ø—‘×'Ñ'×,Ñ,¨d¯o©o×.DÑ.D×.IÑ.IÈ4Ï?É?×KaÑKa×KfÑKfÐgó$Ð ð /×4Ñ4°QÑ7ˆKØ/×5Ñ5°aÑ8ˆLä "§	¢	Ø°×0DÑ0DÈVñ!ˆDÔð ×Ñ×$Ñ$×*Ñ*Ð+?Ô@Ø×#×#Ü$)§I¢IØ—_‘_×)Ñ)×.Ñ.°·±×0DÑ0D×0IÑ0IÈ4Ï?É?×K_ÑK_×KdÑKdÐeó%Ð!ð ×!Ñ!×&Ñ&×,Ñ,Ð->Ô?à!%Õrw   ),rF   rZ   r\   r[   rK   rY   rC   r:   r?   r@   r.   rI   r5   r6   rE   r9   rc   rb   rT   rR   rQ   rL   rB   rA   rD   rr   r7   r=   r>   rH   rG   rJ   rP   ra   r)  rW   r'  r^   rV   r%  rX   r;   r<   r8   )Né   Né@   r  FFFNé    NNTNNTTFr/   r   FFNNNNFTF)8r7   Úintr:   ú
int | NonerI   r4  rj   r5  rk   r4  r?   r  r&   Úboolr;   r6  r<   r6  rl   ú
str | Nonerm   r4  rn   r7  rK   r5  rY   zbool | Nonero   r5  rp   r5  rq   r6  rG   r6  rL   r6  r   r  r=   r  r>   r6  rF   r6  rr   z'AttnProcessor' | NonerA   r4  rB   r4  r$   r6  rE   r6  )NF)r~   r6  r   útuple[str | None, ...] | NoneÚreturnÚNone)r…   r6  r9  r:  ©N)rµ   r6  r“   úCallable | Noner9  r:  )rÅ   r4  r9  r:  )rr   z'AttnProcessor'r9  r:  )F)rÏ   r6  r9  z'AttentionProcessor'©NN)râ   útorch.TensorrÖ   útorch.Tensor | Noner×   r?  r9  r>  )rð   r>  r9  r>  )rì   )rð   r>  rA   r4  r9  r>  )r  r>  r  r>  r×   r?  r9  r>  )
r×   r>  r  r4  rò   r4  rA   r4  r9  r>  )rÖ   r>  r9  r>  )T)rá   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r3   r�   r†   rÀ   rÆ   rh   rÐ   ré   rô   rù   r  r  r!  r§   Úno_gradr/  Ú__static_attributes__Ú__classcell__©rt   s   @ru   r   r   3   sÇ  ø† ñ4ðr +/ØØ#ØØØØ!&Ø$Ø+/Ø/1Ø"Ø(,Ø'+Ø&*Ø'+ØØØ%*ØØ'*Ø$)Ø,1Ø,0ØØ#ØØØ#'Øð?J&àðJ&ð (ðJ&ð ð	J&ð
 ðJ&ð ðJ&ð ðJ&ð ðJ&ð ðJ&ð ðJ&ð )ðJ&ð *-ðJ&ð ðJ&ð &ðJ&ð %ðJ&ð  $ð!J&ð" %ð#J&ð$ ð%J&ð& ð'J&ð( #ð)J&ð* ð+J&ð,  %ð-J&ð. "ð/J&ð0 &*ð1J&ð2 *ð3J&ð4 ð5J&ð6 ð7J&ð< !ð=J&ð> ÷?J&ð J&ð^ 9=Øð	&à!%ð&ð 6ð&ð
 
õ&ôB&ð$ ^bðP&Ø7;ðP&ØKZðP&à	õP&ôd&ô8#ö("ð" 6:Ø.2ð	,
à#ð,
ð  3ð,
ð ,ð	,
ð 
õ,
ô\ö"ð8 ]að-Ø!ð-Ø(4ð-ØFYð-à	õ-ð` abð1Ø*ð1Ø;>ð1ØLOð1ØZ]ð1à	õ1ôf%ð: ‡]‚]ƒ_ó2&ó ö2&rw   r   c                  óF   ^ • \ rS rSr        SU 4S jjrSS jrSrU =r$ )Ú!SanaMultiscaleAttentionProjectioniH  c           
     ó°   >• [         TU ]  5         SU-  n[        R                  " UUUUS-  USS9U l        [        R                  " XDSSSSU-  SS9U l        g )Nrì   r   F)r  Úgroupsr&   r   r   )rK  r&   )r2   r3   r   ÚConv2dÚproj_inÚproj_out)ri   Úin_channelsÚnum_attention_headsÚkernel_sizeÚchannelsrt   s        €ru   r3   Ú*SanaMultiscaleAttentionProjection.__init__I  sc   ø€ ô 	‰ÑÔà�{‘?ˆÜ—y’yØØØØ 1Ñ$ØØñ
ˆŒô Ÿ	š	 (°a¸¸AÀaÐJ]ÑF]ÐdiÑjˆ�rw   c                óJ   • U R                  U5      nU R                  U5      nU$ r;  ©rM  rN  ©ri   râ   s     ru   ré   Ú)SanaMultiscaleAttentionProjection.forward\  s$   € ØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐrw   rU  )rO  r4  rP  r4  rQ  r4  r9  r:  ©râ   r>  r9  r>  ©rá   r@  rA  rB  r3   ré   rE  rF  rG  s   @ru   rI  rI  H  s=   ø† ðkàðkð !ðkð ð	kð
 
÷k÷&ò rw   rI  c                  ó‚   ^ • \ rS rSrSr       S                 S	U 4S jjjrS
S jrS
S jrSS jrSr	U =r
$ )ÚSanaMultiscaleLinearAttentionib  z(Lightweight multi-scale linear attentionc
                óx  >• [         TU ]  5         SSKJn
  X€l        X@l        X`l        X�l        Uc  [        X-  U-  5      OUnX4-  n[        R                  " XSS9U l        [        R                  " XSS9U l        [        R                  " XSS9U l        [        R                  " 5       U l        U H(  nU R                  R!                  [#        X³U5      5        M*     [        R$                  " 5       U l        [        R                  " US[)        U5      -   -  USS9U l        U
" XbS9U l        [/        5       U l        g )Nr   )Úget_normalizationFr0   )Únum_features)r2   r3   r4   r]  r   Úattention_head_dimÚ	norm_typer>   r4  r   rU   rV   rW   rX   r]   Úto_qkv_multiscaler_   rI  ÚReLUÚnonlinearityrß   r^   Únorm_outÚSanaMultiscaleAttnProcessor2_0rr   )ri   rO  Úout_channelsrP  r_  Úmultr`  Úkernel_sizesr   r>   r]  r5   rQ  rt   s                €ru   r3   Ú&SanaMultiscaleLinearAttention.__init__e  s  ø€ ô 	‰ÑÔõ 	5àŒØ"4ÔØ"ŒØ#6Ô ð >QÑ=XŒC�Ñ1°DÑ8Ô9Ð^qð 	ð (Ñ<ˆ	ä—I’I˜k¸5ÑAˆŒ	Ü—I’I˜k¸5ÑAˆŒ	Ü—I’I˜k¸5ÑAˆŒ	ä!#§¢£ˆÔÛ'ˆKØ×"Ñ"×)Ñ)Ü1°)ÐR]Ó^öñ (ô
 ŸGšG›IˆÔÜ—i’i 	¨Q´°\Ó1BÑ-BÑ CÀ\ÐX]Ñ^ˆŒÙ)¨)ÑOˆŒä7Ó9ˆ�rw   c                ó:  • [         R                  " USSSS9n[        R                  " X2R	                  SS5      5      n[        R                  " XA5      nUR                  [        R                  S9nUS S 2S S 2S S24   US S 2S S 2SS 24   U R                  -   -  nU$ )N©r   r   r   r   Úconstantr   ©Úmoder  r+   rý   ©r�   )re   r  r§   Úmatmulr  r°   Úfloat32r   ©ri   r  r  r  Úscoresrâ   s         ru   Úapply_linear_attentionÚ4SanaMultiscaleLinearAttention.apply_linear_attention�  s‰   € Ü—’�e˜\°
À!ÑDˆÜ—’˜e§]¡]°2°rÓ%:Ó;ˆÜŸš VÓ3ˆà%×(Ñ(¬u¯}©}Ð(Ð=ˆØ%¢aª¨C¨R¨C iÑ0°MÂ!ÂQÈÉÀ)Ñ4LÈtÏxÉxÑ4WÑXˆØÐrw   c                ó8  • [         R                  " UR                  SS5      U5      nUR                  [         R                  S9nU[         R
                  " USSS9U R                  -   -  n[         R                  " X4R                  UR                  5      5      nU$ )Nr+   rý   ro  r   T)r-   Úkeepdim)r§   rp  r  r°   rq  Úsumr   r�   rr  s         ru   Úapply_quadratic_attentionÚ7SanaMultiscaleLinearAttention.apply_quadratic_attention™  sr   € Ü—’˜cŸm™m¨B°Ó3°UÓ;ˆØ—‘¤§¡�Ð/ˆØœ5Ÿ9š9 V°¸DÑAÀDÇHÁHÑLÑMˆÜŸš U¯I©I°e·k±kÓ,BÓCˆØÐrw   c                ó$   • U R                  X5      $ r;  rÎ   rV  s     ru   ré   Ú%SanaMultiscaleLinearAttention.forward   s   € Ø�~‰~˜dÓ2Ð2rw   )r_  r   rc  rd  r`  rr   r>   rW   r^   rV   ra  rX   )Nr1  r   Ú
batch_norm)é   çVçž¯Ò<F)rO  r4  rf  r4  rP  r5  r_  r4  rg  r  r`  Ústrrh  ztuple[int, ...]r   r  r>   r6  )r  r>  r  r>  r  r>  r9  r>  rX  )rá   r@  rA  rB  rC  r3   rt  ry  ré   rE  rF  rG  s   @ru   r[  r[  b  s”   ø† Ù3ð +/Ø"#ØØ%Ø(,ØØ$)ð):àð):ð ð):ð (ð	):ð
  ð):ð ð):ð ð):ð &ð):ð ð):ð "÷):ð ):ôVô÷3ò 3rw   r[  c                  ó’   ^ • \ rS rSr          S                         SU 4S jjjr  S     SS jjrSrU =r$ )	ÚMochiAttentioni¤  c                óŠ  >• [         TU ]  5         SSKJn  U	b  U	OXT-  U l        U	b  U	OUU l        U
(       a  U
OUU l        XÀl        U	b  X•-  OUU l        U" X]S5      U l	        U" X]S5      U l
        U" X]S5      U l        U" X]S5      U l        [        R                  " XR                  US9U l        [        R                  " XR                  US9U l        [        R                  " XR                  US9U l        [        R                  " X R                  US9U l        [        R                  " X R                  US9U l        U R                  b$  [        R                  " X R                  US9U l        [        R*                  " / 5      U l        U R,                  R/                  [        R                  " U R                  U R
                  US95        U R,                  R/                  [        R0                  " U5      5        U R                  (       d/  [        R                  " U R                  U R                  US9U l        X0l        g )Nr   )ÚMochiRMSNormTr0   )r2   r3   r4   r„  r5   rA   rB   rC   rI   rQ   rR   rb   rc   r   rU   rV   rW   rX   rZ   r[   r\   r]   r^   r_   r`   ra   rr   )ri   r7   rK   rr   rI   rk   r?   r&   rY   rA   rB   rq   rC   r   r„  rt   s                  €ru   r3   ÚMochiAttention.__init__¥  s¢  ø€ ô  	‰ÑÔÝ/à$+Ñ$7™¸XÑ=MˆŒØ")Ñ"5‘w¸9ˆŒÞ2A™ÀyˆÔØ 0Ôà,3Ñ,?�WÒ(ÀUˆŒ
á" 8°$Ó7ˆŒÙ" 8°$Ó7ˆŒÙ(¨¸Ó=ˆÔÙ(¨¸Ó=ˆÔä—I’I˜i¯©¸dÑCˆŒ	Ü—I’I˜i¯©¸dÑCˆŒ	Ü—I’I˜i¯©¸dÑCˆŒ	äŸ)š)Ð$5·~±~ÈOÑ\ˆŒÜŸ)š)Ð$5·~±~ÈOÑ\ˆŒØ× Ñ Ñ,Ü ŸišiÐ(9¿>¹>ÐP_Ñ`ˆDŒOä—m’m BÓ'ˆŒØ�‰×Ñœ2Ÿ9š9 T§^¡^°T·\±\ÈÑQÔRØ�‰×Ñœ2Ÿ:š: gÓ.Ô/à×$×$Ü Ÿiši¨¯©¸×8LÑ8LÐS[Ñ\ˆDŒOà"�rw   c                ó0   • U R                   " U U4UUS.UD6$ )NrÕ   rÎ   )ri   râ   rÖ   r×   Úkwargss        ru   ré   ÚMochiAttention.forwardÖ  s2   € ð �~Š~ØØð
ð #8Ø)ñ	
ð
 ñ
ð 	
rw   )rZ   r\   r[   rC   rI   r5   rc   rb   rR   rQ   rB   rA   rr   ra   rW   r^   rV   rX   )
r1  r2  r  FTNNTFr/   )r7   r4  rK   r4  rr   z'MochiAttnProcessor2_0'rI   r4  rk   r4  r?   r  r&   r6  rY   r6  rA   r5  rB   r5  rq   r6  rC   r6  r   r  r=  )râ   r>  rÖ   r?  r×   r?  rY  rG  s   @ru   r‚  r‚  ¤  sÜ   ø† ð ØØØØ $Ø"Ø&*ØØ!&Øð/#àð/#ð ð/#ð +ð	/#ð
 ð/#ð ð/#ð ð/#ð ð/#ð ð/#ð ð/#ð $ð/#ð ð/#ð ð/#ð ÷/#ð /#ðh 6:Ø.2ð	
à#ð
ð  3ð
ð ,÷	
ó 
rw   r‚  c                  óD   • \ rS rSrSrS r S           SS jjrSrg)	ÚMochiAttnProcessor2_0iæ  z"Attention processor used in Mochi.c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   zUMochiAttnProcessor2_0 requires PyTorch 2.0. To use it, please upgrade PyTorch to 2.0.©rd   re   ÚImportError©ri   s    ru   r3   ÚMochiAttnProcessor2_0.__init__é  ó!   € Ü”qÐ8×9Ñ9ÜÐuÓvÐvð :rw   Nc           	     ó:  • UR                  U5      nUR                  U5      nUR                  U5      nUR                  SUR                  S45      nUR                  SUR                  S45      nUR                  SUR                  S45      nUR
                  b  UR                  U5      nUR                  b  UR                  U5      nUR                  U5      n	UR                  U5      n
UR                  U5      nU	R                  SUR                  S45      n	U
R                  SUR                  S45      n
UR                  SUR                  S45      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
Ub  S nU" U/UQ76 nU" U/UQ76 nUR                  SS5      UR                  SS5      UR                  SS5      p‡nU	R                  SS5      U
R                  SS5      UR                  SS5      pºn	UR                  S5      nU	R                  S5      nXÞ-   nUR                  u  nnnn/ n[        U5       GH&  nUU   S S S 24   n[         R"                  " UR%                  5       SS9R%                  5       nU	UUS-   2S S 2US S 24   nU
UUS-   2S S 2US S 24   nUUUS-   2S S 2US S 24   n[         R&                  " UUUS-    U/SS9n[         R&                  " UUUS-    U/SS9n[         R&                  " UUUS-    U/SS9n[(        R*                  " UUUSSS	9nUR                  S5      n[(        R,                  " US
S
S
UU-
  45      nUR/                  U5        GM)     [         R&                  " US
S9nUR                  SS5      R%                  SS5      nUR1                  XÞ4SS9u  p#UR2                  S
   " U5      nUR2                  S   " U5      n[5        US5      (       a  UR7                  U5      nX#4$ )Nr   r+   c                ó4  • U SSS S24   R                  5       nU SSS S24   R                  5       nX1-  XB-  -
  R                  U R                  5      nX2-  XA-  -   R                  U R                  5      n[        R                  " XV/SS9R                  S5      $ )N.r   r   r   r+   r   rý   )r  r°   r�   r§   ÚstackÚflatten)ÚxÚ	freqs_cosÚ	freqs_sinÚx_evenÚx_oddÚcosÚsins          ru   Úapply_rotary_embÚ8MochiAttnProcessor2_0.__call__.<locals>.apply_rotary_emb  s–   € Ø˜3   1 ˜9™×+Ñ+Ó-�Ø˜#˜q˜t !˜t˜)™×*Ñ*Ó,�àÑ)¨EÑ,=Ñ=×AÑAÀ!Ç'Á'ÓJ�ØÑ)¨EÑ,=Ñ=×AÑAÀ!Ç'Á'ÓJ�ä—{’{ C :°2Ñ6×>Ñ>¸rÓBÐBrw   r   F)Úas_tupler   r  ©Ú	dropout_prE   r   rì   ra   )rV   rW   rX   Ú	unflattenrI   rQ   rR   r\   rZ   r[   rb   rc   r  Úsizerí   Úranger§   Únonzeror”  r  re   r1   r  r_   Úsplit_with_sizesr^   rd   ra   ) ri   Úattnrâ   rÖ   r×   Úimage_rotary_embr  r  r  Úencoder_queryÚencoder_keyÚencoder_valuerœ  Úsequence_lengthÚencoder_sequence_lengthÚtotal_lengthrò   rI   r¼   r-   Úattn_outputsÚidxÚmaskÚvalid_prompt_token_indicesÚvalid_encoder_queryÚvalid_encoder_keyÚvalid_encoder_valueÚvalid_queryÚ	valid_keyÚvalid_valueÚattn_outputÚvalid_sequence_lengths                                    ru   rÛ   ÚMochiAttnProcessor2_0.__call__í  s,  € ð —	‘	˜-Ó(ˆØ�i‰i˜Ó&ˆØ—	‘	˜-Ó(ˆà—‘  D§J¡J°Ð#3Ó4ˆØ�m‰m˜A §
¡
¨BÐ/Ó0ˆØ—‘  D§J¡J°Ð#3Ó4ˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCàŸ™Ð(=Ó>ˆØ—o‘oÐ&;Ó<ˆØŸ™Ð(=Ó>ˆà%×/Ñ/°°D·J±JÀÐ3CÓDˆØ!×+Ñ+¨A°·
±
¸BÐ/?Ó@ˆØ%×/Ñ/°°D·J±JÀÐ3CÓDˆà×ÑÑ(Ø ×-Ñ-¨mÓ<ˆMØ×ÑÑ(Ø×+Ñ+¨KÓ8ˆKàÑ'òCñ % UÐ>Ð-=Ò>ˆEÙ" 3Ð:Ð)9Ò:ˆCà!ŸO™O¨A¨qÓ1°3·=±=ÀÀAÓ3FÈÏÉÐXYÐ[\ÓH]�Eˆà×#Ñ# A qÓ)Ø×!Ñ! ! QÓ'Ø×#Ñ# A qÓ)ð %2ˆð  Ÿ*™* Q›-ˆØ"/×"4Ñ"4°QÓ"7ÐØ&Ñ@ˆà$)§K¡KÑ!ˆ
�E˜1˜cØˆÜ˜×$ˆCØ! #Ñ& tªQ wÑ/ˆDÜ).¯ª°t·|±|³~ÐPUÑ)V×)^Ñ)^Ó)`Ð&à"/°°c¸A±g°ºqÐB\Ò^_Ð0_Ñ"`ÐØ +¨C°#¸±'¨Mº1Ð>XÒZ[Ð,[Ñ \ÐØ"/°°c¸A±g°ºqÐB\Ò^_Ð0_Ñ"`ÐäŸ)š) U¨3°°q±Ð%9Ð;NÐ$OÐUVÑWˆKÜŸ	š	 3 s¨S°1©WÐ#5Ð7HÐ"IÈqÑQˆIÜŸ)š) U¨3°°q±Ð%9Ð;NÐ$OÐUVÑWˆKä×8Ò8Ø˜Y¨¸sÈeñˆKð %0×$4Ñ$4°QÓ$7Ð!ÜŸ%š% ¨a°°A°|ÐF[Ñ7[Ð-\Ó]ˆKØ×Ñ ×,ñ# %ô& Ÿ	š	 ,°AÑ6ˆØ%×/Ñ/°°1Ó5×=Ñ=¸aÀÓCˆà/<×/MÑ/MØÐ6¸Að 0Nð 0
Ñ,ˆð
 Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆä�4˜×&Ñ&Ø$(§O¡OÐ4IÓ$JÐ!àÐ3Ð3rw   © r;  )r¦  z'MochiAttention'râ   r>  rÖ   r>  r×   r>  r§  r?  r9  r>  ©rá   r@  rA  rB  rC  r3   rÛ   rE  r»  rw   ru   rŠ  rŠ  æ  s\   † Ù,òwð 15ð_4àð_4ð $ð_4ð  ,ð	_4ð
 %ð_4ð .ð_4ð 
÷_4ð _4rw   rŠ  c                  óB   • \ rS rSrSr   S           SS jjrSrg)rg   iO  zB
Default processor for performing attention-related computations.
Nc                óŽ  • [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  p¼pÞUR                  X¼XÞ-  5      R                  SS5      nUc  UR
                  OUR
                  u  p¿nUR                  XOU5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR!                  UUU5      n["        R$                  " UU5      nUR'                  U5      nUR(                  S   " U5      nUR(                  S   " U5      nU
S:X  a$  UR                  SS	5      R+                  UWWW5      nUR,                  (       a  X)-   nX!R.                  -  nU$ )
Nr   rH   úðThe `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.ú1.0.0r  r   r   r+   rý   )rß   Úgetr   rP   r÷   rí   Úviewr  r  r.   rV   rT   r!  rW   rX   rù   r  r§   Úbmmrô   r^   rî   r>   r=   )ri   r¦  râ   rÖ   r×   ÚtembÚargsr‡  Údeprecation_messageÚresidualÚ
input_ndimrò   ÚchannelÚheightÚwidthr«  r¼   r  r  r  r
  s                        ru   rÛ   ÚAttnProcessor.__call__T  s3  € ô ˆt‹9�q‹=˜FŸJ™J w°Ó5ÑAð #UÐÜ�g˜wÐ(;Ô<à ˆà×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð ×4Ñ4°^ÐV`Óaˆà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà×&Ñ& uÓ-ˆØ×$Ñ$ SÓ)ˆØ×&Ñ& uÓ-ˆà×3Ñ3°E¸3ÀÓOˆÜŸ	š	 /°5Ó9ˆØ×.Ñ.¨}Ó=ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  ©NNN©r¦  r   râ   r>  rÖ   r?  r×   r?  rÄ  r?  r9  r>  ©rá   r@  rA  rB  rC  rÛ   rE  r»  rw   ru   rg   rg   O  s^   † ñð 6:Ø.2Ø$(ð@àð@ð $ð@ð  3ð	@ð
 ,ð@ð "ð@ð 
÷@ð @rw   rg   c                  óz   ^ • \ rS rSrSr      S           SU 4S jjjr  S         S	S jjrSrU =r$ )
r™   i—  a  
Processor for implementing attention for the Custom Diffusion method.

Args:
    train_kv (`bool`, defaults to `True`):
        Whether to newly train the key and value matrices corresponding to the text features.
    train_q_out (`bool`, defaults to `True`):
        Whether to newly train query matrices corresponding to the latent image features.
    hidden_size (`int`, *optional*, defaults to `None`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`, *optional*, defaults to `None`):
        The number of channels in the `encoder_hidden_states`.
    out_bias (`bool`, defaults to `True`):
        Whether to include the bias parameter in `train_q_out`.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
c                óL  >• [         TU ]  5         Xl        X l        X0l        X@l        U R                  (       aH  [        R                  " U=(       d    UUSS9U l        [        R                  " U=(       d    UUSS9U l	        U R                  (       a“  [        R                  " X3SS9U l
        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g ©NFr0   ©r2   r3   r�   r‘   r’   r:   r   rU   r”   Úto_v_custom_diffusionÚto_q_custom_diffusionr]   Úto_out_custom_diffusionr_   r`   ©ri   r�   r‘   r’   r:   rq   r?   rt   s          €ru   r3   Ú%CustomDiffusionAttnProcessor.__init__ª  óÚ   ø€ ô 	‰ÑÔØ ŒØ&Ôà&ÔØ#6Ô ð �=�=Ü)+¯ªÐ3F×3UÈ+ÐWbÐinÑ)oˆDÔ&Ü)+¯ªÐ3F×3UÈ+ÐWbÐinÑ)oˆDÔ&Ø××Ü)+¯ª°;ÐRWÑ)XˆDÔ&Ü+-¯=ª=¸Ó+<ˆDÔ(Ø×(Ñ(×/Ñ/´·	²	¸+ÐYaÑ0bÔcØ×(Ñ(×/Ñ/´·
²
¸7Ó0CÕDð	 rw   c                ó^  • UR                   u  pVnUR                  XFU5      nU R                  (       a?  U R                  U5      R	                  UR
                  R                  R                  5      nO>UR                  UR	                  UR
                  R                  R                  5      5      nUc  Sn	UnO$Sn	UR                  (       a  UR                  U5      nU R                  (       aÛ  U R                  UR	                  U R                  R                  R                  5      5      n
U R                  UR	                  U R                  R                  R                  5      5      nU
R	                  UR
                  R                  R                  5      n
UR	                  UR
                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   XÊ-  SU-
  U
R#                  5       -  -   n
XË-  SU-
  UR#                  5       -  -   nUR%                  U5      nUR%                  U
5      n
UR%                  U5      nUR'                  XŠU5      n[        R(                  " XÛ5      nUR+                  U5      nU R                  (       a,  U R,                  S   " U5      nU R,                  S   " U5      nU$ UR.                  S   " U5      nUR.                  S   " U5      nU$ )NFTr   r  r   )rí   r  r‘   rÕ  r°   rV   r±   r�   rT   r!  r�   r”   rÔ  rW   rX   r§   Ú	ones_likeÚdetachrù   r  rÃ  rô   rÖ  r^   )ri   r¦  râ   rÖ   r×   rò   r«  r¼   r  Ú	crossattnr  r  rÜ  r
  s                 ru   rÛ   Ú%CustomDiffusionAttnProcessor.__call__Ä  s¦  € ð *7×)<Ñ)<Ñ&ˆ
 QØ×4Ñ4°^ÐV`ÓaˆØ××Ø×.Ñ.¨}Ó=×@Ñ@ÀÇÁ×AQÑAQ×AWÑAWÓX‰Eà—I‘I˜m×.Ñ.¨t¯y©y×/?Ñ/?×/EÑ/EÓFÓGˆEà Ñ(ØˆIØ$1Ñ!àˆIØ��Ø(,×(GÑ(GÐH]Ó(^Ð%à�=�=Ø×,Ñ,Ð-B×-EÑ-EÀd×F`ÑF`×FgÑFg×FmÑFmÓ-nÓoˆCØ×.Ñ.Ð/D×/GÑ/GÈ×HbÑHb×HiÑHi×HoÑHoÓ/pÓqˆEØ—&‘&˜Ÿ™×)Ñ)×/Ñ/Ó0ˆCØ—H‘H˜TŸY™Y×-Ñ-×3Ñ3Ó4‰Eà—)‘)Ð1Ó2ˆCØ—I‘IÐ3Ó4ˆEæÜ—_’_ SÓ)ˆFØ%¢a¨¨!¨ªQ hÑ/°#Ñ5ˆF’1�b�q�bš!�8ÑØ‘, ! f¡*°·
±
³Ñ!<Ñ<ˆCØ‘N a¨&¡j°E·L±L³NÑ%BÑBˆEà×&Ñ& uÓ-ˆØ×$Ñ$ SÓ)ˆØ×&Ñ& uÓ-ˆà×3Ñ3°EÀÓOˆÜŸ	š	 /Ó9ˆØ×.Ñ.¨}Ó=ˆà××à ×8Ñ8¸Ò;¸MÓJˆMà ×8Ñ8¸Ò;¸MÓJˆMð Ðð	 !ŸK™K¨šN¨=Ó9ˆMà ŸK™K¨šN¨=Ó9ˆMàÐrw   ©r:   r’   r”   rÖ  rÕ  rÔ  r�   r‘   ©TTNNTr  ©r�   r6  r‘   r6  r’   r5  r:   r5  rq   r6  r?   r  r=  ©
r¦  r   râ   r>  rÖ   r?  r×   r?  r9  r>  ©	rá   r@  rA  rB  rC  r3   rÛ   rE  rF  rG  s   @ru   r™   r™   —  sª   ø† ñð( Ø Ø"&Ø*.ØØðEàðEð ðEð  ð	Eð
 (ðEð ðEð ÷Eð Eð< 6:Ø.2ð8àð8ð $ð8ð  3ð	8ð
 ,ð8ð 
÷8ó 8rw   r™   c                  ó<   • \ rS rSrSr  S         SS jjrSrg)rœ   iÿ  z{
Processor for performing attention-related computations with extra learnable key and value matrices for the text
encoder.
Nc                óÚ  • [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pšnUR                  XJU	5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  (       dr  UR                  U5      nUR!                  U5      nUR                  U5      nUR                  U5      n["        R$                  " Xß/SS9n["        R$                  " UU/SS9nOUnUnUR'                  XÏU5      n["        R(                  " UU5      nUR+                  U5      nUR,                  S   " U5      nUR,                  S   " U5      nUR                  SS	5      R/                  UR                  5      nX(-   nU$ )
Nr   rH   r¿  rÀ  r   r+   r   r   rý   )rß   rÁ  r   rÂ  rí   r  r  rT   r!  r.   rV   rù   rZ   r[   rL   rW   rX   r§   r  r  rÃ  rô   r^   rî   )ri   r¦  râ   rÖ   r×   rÅ  r‡  rÆ  rÇ  rò   r«  r¼   r  Úencoder_hidden_states_key_projÚ encoder_hidden_states_value_projr  r  r
  s                     ru   rÛ   ÚAttnAddedKVProcessor.__call__  sI  € ô ˆt‹9�q‹=˜FŸJ™J w°Ó5ÑAð #UÐÜ�g˜wÐ(;Ô<à ˆà%×*Ñ*¨=×+>Ñ+>¸qÑ+AÀ=×CVÑCVÐWXÑCYÐ[]Ó^×hÑhÐijÐlmÓnˆØ)6×)<Ñ)<Ñ&ˆ
 Qà×4Ñ4°^ÐV`Óaˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!àŸ™¨×(?Ñ(?ÀÀ1Ó(EÓF×PÑPÐQRÐTUÓVˆà—	‘	˜-Ó(ˆØ×&Ñ& uÓ-ˆà)-¯©Ð9NÓ)OÐ&Ø+/¯?©?Ð;PÓ+QÐ(Ø)-×)?Ñ)?Ð@^Ó)_Ð&Ø+/×+AÑ+AÐBbÓ+cÐ(à×(×(Ø—)‘)˜MÓ*ˆCØ—I‘I˜mÓ,ˆEØ×(Ñ(¨Ó-ˆCØ×*Ñ*¨5Ó1ˆEÜ—)’)Ð;ÐAÀqÑIˆCÜ—I’IÐ?ÀÐGÈQÑO‰Eà0ˆCØ4ˆEà×3Ñ3°EÀÓOˆÜŸ	š	 /°5Ó9ˆØ×.Ñ.¨}Ó=ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà%×/Ñ/°°BÓ7×?Ñ?ÀÇÁÓOˆØ%Ñ0ˆàÐrw   r»  r=  râ  rÏ  r»  rw   ru   rœ   rœ   ÿ  sI   † ñð 6:Ø.2ð:àð:ð $ð:ð  3ð	:ð
 ,ð:ð 
÷:ð :rw   rœ   c                  óB   • \ rS rSrSrS r  S         SS jjrSrg)	r�   iB  z«
Processor for performing scaled dot-product attention (enabled by default if you're using PyTorch 2.0), with extra
learnable key and value matrices for the text encoder.
c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   zWAttnAddedKVProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú AttnAddedKVProcessor2_0.__init__H  ó%   € Ü”qÐ8×9Ñ9ÜØióð ð :rw   Nc           	     óê  • [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pšnUR                  XJU	SS	9nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  USS	9nUR                  U5      nUR                  U5      nUR                  USS	9nUR                  USS	9nUR                  (       dp  UR                  U5      nUR!                  U5      nUR                  USS	9nUR                  USS	9n["        R$                  " Xß/SS
9n["        R$                  " UU/SS
9nOUnUn[&        R(                  " XÏUUSSS9nUR                  SS5      R+                  U	SUR                  S   5      nUR,                  S   " U5      nUR,                  S   " U5      nUR                  SS5      R+                  UR                  5      nX(-   nU$ )Nr   rH   r¿  rÀ  r   r+   r   r  )rA   r   r  F©Ú	attn_maskr   rE   rý   )rß   rÁ  r   rÂ  rí   r  r  rT   r!  r.   rV   rù   rZ   r[   rL   rW   rX   r§   r  re   r1   rî   r^   )ri   r¦  râ   rÖ   r×   rÅ  r‡  rÆ  rÇ  rò   r«  r¼   r  ræ  rç  r  r  s                    ru   rÛ   Ú AttnAddedKVProcessor2_0.__call__N  sƒ  € ô ˆt‹9�q‹=˜FŸJ™J w°Ó5ÑAð #UÐÜ�g˜wÐ(;Ô<à ˆà%×*Ñ*¨=×+>Ñ+>¸qÑ+AÀ=×CVÑCVÐWXÑCYÐ[]Ó^×hÑhÐijÐlmÓnˆØ)6×)<Ñ)<Ñ&ˆ
 Qà×4Ñ4°^ÐV`ÐjkÐ4Ðlˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!àŸ™¨×(?Ñ(?ÀÀ1Ó(EÓF×PÑPÐQRÐTUÓVˆà—	‘	˜-Ó(ˆØ×&Ñ& u°aÐ&Ð8ˆà)-¯©Ð9NÓ)OÐ&Ø+/¯?©?Ð;PÓ+QÐ(Ø)-×)?Ñ)?Ð@^ÐhiÐ)?Ð)jÐ&Ø+/×+AÑ+AÐBbÐlmÐ+AÐ+nÐ(à×(×(Ø—)‘)˜MÓ*ˆCØ—I‘I˜mÓ,ˆEØ×(Ñ(¨°aÐ(Ð8ˆCØ×*Ñ*¨5¸!Ð*Ð<ˆEÜ—)’)Ð;ÐAÀqÑIˆCÜ—I’IÐ?ÀÐGÈQÑO‰Eà0ˆCØ4ˆEô ×6Ò6Ø˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈhÏnÉnÐ]^ÑN_Ó`ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà%×/Ñ/°°BÓ7×?Ñ?ÀÇÁÓOˆØ%Ñ0ˆàÐrw   r»  r=  râ  r¼  r»  rw   ru   r�   r�   B  sN   † ñò
ð 6:Ø.2ð=àð=ð $ð=ð  3ð	=ð
 ,ð=ð 
÷=ð =rw   r�   c                  óB   • \ rS rSrSrS r  S         SS jjrSrg)	r£   iŽ  úYAttention processor used typically in processing the SD3-like self-attention projections.c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   zUJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   ÚJointAttnProcessor2_0.__init__‘  r�  rw   Nc                ó   • UnUR                   S   nUR                  U5      n	UR                  U5      n
UR                  U5      nU
R                   S   nXÁR                  -  nU	R                  USUR                  U5      R                  SS5      n	U
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
UGb<  UR                  U5      nUR                  U5      nUR                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " Xž/SS9n	[        R                  " X¯/SS9n
[        R                  " UU/SS9n[         R"                  " XšUSSS9nUR                  SS5      R%                  USUR                  U-  5      nUR'                  U	R(                  5      nUbQ  US S 2S UR                   S   24   US S 2UR                   S   S 24   p2UR*                  (       d  UR-                  U5      nUR.                  S   " U5      nUR.                  S   " U5      nUb  X#4$ U$ )	Nr   r+   r   r   r   r  FrŸ  )rí   rV   rW   rX   rI   rÂ  r  rQ   rR   r\   rZ   r[   rb   rc   r§   r  re   r1   rî   r°   r�   rC   ra   r^   )ri   r¦  râ   rÖ   r×   rÅ  r‡  rÇ  rò   r  r  r  r5   Úhead_dimÚ encoder_hidden_states_query_projræ  rç  s                    ru   rÛ   ÚJointAttnProcessor2_0.__call__•  s%  € ð !ˆà"×(Ñ(¨Ñ+ˆ
ð —	‘	˜-Ó(ˆØ�i‰i˜Ó&ˆØ—	‘	˜-Ó(ˆà—I‘I˜b‘Mˆ	Ø§
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð !Ò,Ø/3¯©Ð?TÓ/UÐ,Ø-1¯_©_Ð=RÓ-SÐ*Ø/3¯©Ð?TÓ/UÐ,à/O×/TÑ/TØ˜B §
¡
¨Hó0ç‰i˜˜1‹oð -ð .L×-PÑ-PØ˜B §
¡
¨Hó.ç‰i˜˜1‹oð +ð 0P×/TÑ/TØ˜B §
¡
¨Hó0ç‰i˜˜1‹oð -ð × Ñ Ñ,Ø37×3DÑ3DÐEeÓ3fÐ0Ø× Ñ Ñ,Ø15×1BÑ1BÐCaÓ1bÐ.ä—I’I˜uÐGÈQÑOˆEÜ—)’)˜SÐAÀqÑIˆCÜ—I’I˜uÐ&FÐGÈQÑOˆEä×6Ò6°uÀ5ÐTWÐchÑiˆØ%×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆà Ñ,ð šaÐ!4 8§>¡>°!Ñ#4Ð!4Ð4Ñ5Øša §¡°Ñ!2Ñ!4Ð4Ñ5ð 1ð ×(×(Ø(,¯©Ð8MÓ(NÐ%ð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà Ñ,Ø Ð7Ð7à Ð rw   r»  r=  ©
r¦  r   râ   útorch.FloatTensorrÖ   rú  r×   útorch.FloatTensor | Noner9  rú  r¼  r»  rw   ru   r£   r£   Ž  sT   † Ùcòwð 48Ø37ðL!àðL!ð )ðL!ð  1ð	L!ð
 1ðL!ð 
÷L!ð L!rw   r£   c                  óB   • \ rS rSrSrS r  S         SS jjrSrg)	ÚPAGJointAttnProcessor2_0iä  rò  c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   zXPAGJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú!PAGJointAttnProcessor2_0.__init__ç  s%   € Ü”qÐ8×9Ñ9ÜØjóð ð :rw   Nc           	     óT  • UnUR                   nUS:X  a3  UR                  u  pxpšUR                  XxXš-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  pxpšUR                  XxXš-  5      R                  SS5      nUR                  S   nUR	                  S5      u  pÞUR	                  S5      u  nnUR                  S   nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " UU/SS9n[        R                  " UU/SS9n[        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      n[        R                  " UUUSSS	9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   pýUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUR                  S   nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " UU/SS9n[        R                  " UU/SS9n[        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR-                  S5      n[        R.                  " UU4UR0                  UR$                  S9n [3        S5      U S U2S U24'   U S U2S U24   R5                  S5        U R7                  S5      R7                  S5      n [        R                  " UUUU SSS9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      n[        R                  " XÞ/5      n[        R                  " UU/5      nX#4$ )Nr  r   r   r   r   r+   r  FrŸ  rý   r�   ú-infrî  )r÷   rí   rÂ  r  ÚchunkrV   rW   rX   r\   rZ   r[   r§   r  rI   re   r1   rî   r°   r�   r^   rC   ra   r¢  r  rŽ   r  Úfill_diagonal_r  )!ri   r¦  râ   rÖ   r×   rÇ  rÈ  rò   rÉ  rÊ  rË  Úcontext_input_ndimÚidentity_block_sizeÚhidden_states_orgÚhidden_states_ptbÚencoder_hidden_states_orgÚencoder_hidden_states_ptbÚ	query_orgÚkey_orgÚ	value_orgÚ$encoder_hidden_states_org_query_projÚ"encoder_hidden_states_org_key_projÚ$encoder_hidden_states_org_value_projr5   rö  Ú	query_ptbÚkey_ptbÚ	value_ptbÚ$encoder_hidden_states_ptb_query_projÚ"encoder_hidden_states_ptb_key_projÚ$encoder_hidden_states_ptb_value_projró   Ú	full_masks!                                    ru   rÛ   Ú!PAGJointAttnProcessor2_0.__call__í  sÂ  € ð !ˆà"×'Ñ'ˆ
Ø˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMØ2×7Ñ7ÐØ Ó"Ø1F×1LÑ1LÑ.ˆJ Ø$9×$>Ñ$>¸zÐTZÑTbÓ$c×$mÑ$mÐnoÐqrÓ$sÐ!ð ,×1Ñ1°!Ñ4Ðð 0=×/BÑ/BÀ1Ó/EÑ,ÐØ?T×?ZÑ?ZÐ[\Ó?]Ñ<Ð!Ð#<ð /×4Ñ4°QÑ7ˆ
ð —I‘IÐ/Ó0ˆ	Ø—)‘)Ð-Ó.ˆØ—I‘IÐ/Ó0ˆ	ð 04¯©Ð?XÓ/YÐ,Ø-1¯_©_Ð=VÓ-WÐ*Ø/3¯©Ð?XÓ/YÐ,ô —I’I˜yÐ*NÐOÐUVÑWˆ	Ü—)’)˜WÐ&HÐIÈqÑQˆÜ—I’I˜yÐ*NÐOÐUVÑWˆ	à—M‘M "Ñ%ˆ	Ø §
¡
Ñ*ˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	Ø—,‘,˜z¨2¨t¯z©z¸8ÓD×NÑNÈqÐRSÓTˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	ä×:Ò:Ø�w 	°SÀEñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±ÓAÐð šaÐ!4 8§>¡>°!Ñ#4Ð!4Ð4Ñ5Øša §¡°Ñ!2Ñ!4Ð4Ñ5ð 5ð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=ÐØ×$×$Ø(,¯©Ð8QÓ(RÐ%à˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐØ Ó"Ø(A×(KÑ(KÈBÐPRÓ(S×([Ñ([Ø˜G V¨Uó)Ð%ð /×4Ñ4°QÑ7ˆ
ð —I‘IÐ/Ó0ˆ	Ø—)‘)Ð-Ó.ˆØ—I‘IÐ/Ó0ˆ	ð 04¯©Ð?XÓ/YÐ,Ø-1¯_©_Ð=VÓ-WÐ*Ø/3¯©Ð?XÓ/YÐ,ô —I’I˜yÐ*NÐOÐUVÑWˆ	Ü—)’)˜WÐ&HÐIÈqÑQˆÜ—I’I˜yÐ*NÐOÐUVÑWˆ	à—M‘M "Ñ%ˆ	Ø §
¡
Ñ*ˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	Ø—,‘,˜z¨2¨t¯z©z¸8ÓD×NÑNÈqÐRSÓTˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	ð —.‘. Ó#ˆÜ—K’K ¨'Ð 2¸9×;KÑ;KÐS\×SbÑSbÑcˆ	ô AFÀfÃˆ	Ð&Ð&Ð&Ð(<Ð)<Ð(<Ð<Ñ=ð 	Ð&Ð&Ð&Ð(<Ð)<Ð(<Ð<Ñ=×LÑLÈQÔOð ×'Ñ'¨Ó*×4Ñ4°QÓ7ˆ	ä×:Ò:Ø�w 	°YÈ#ÐY^ñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±ÓAÐð šaÐ!4 8§>¡>°!Ñ#4Ð!4Ð4Ñ5Øša §¡°Ñ!2Ñ!4Ð4Ñ5ð 5Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=ÐØ×$×$Ø(,¯©Ð8QÓ(RÐ%à˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐØ Ó"Ø(A×(KÑ(KÈBÐPRÓ(S×([Ñ([Ø˜G V¨Uó)Ð%ô
 Ÿ	š	Ð#4Ð"HÓIˆÜ %§	¢	Ð+DÐF_Ð*`Ó aÐàÐ3Ð3rw   r»  r=  rù  r¼  r»  rw   ru   rý  rý  ä  sS   † Ùcòð 48Ø37ðP4àðP4ð )ðP4ð  1ð	P4ð
 1ðP4ð 
÷P4ð P4rw   rý  c                  óB   • \ rS rSrSrS r  S         SS jjrSrg)	ÚPAGCFGJointAttnProcessor2_0i€  rò  c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   z[PAGCFGJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú$PAGCFGJointAttnProcessor2_0.__init__ƒ  ó%   € Ü”qÐ8×9Ñ9ÜØmóð ð :rw   Nc           	     ó¾  • UnUR                   nUS:X  a3  UR                  u  pšp¼UR                  XšX¼-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  pšp¼UR                  XšX¼-  5      R                  SS5      nUR                  S   nUR	                  S5      u  nnn[
        R                  " UU/5      nUR	                  S5      u  nnn[
        R                  " UU/5      nUR                  S   n	UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[
        R                  " UU/SS9n[
        R                  " UU/SS9n[
        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      n[        R                  " UUUSS	S
9nUR                  SS5      R!                  U	SUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUR                  S   n	UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n UR                  U5      n!UR                  U5      n"[
        R                  " UU /SS9n[
        R                  " UU!/SS9n[
        R                  " UU"/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR-                  S5      n#[
        R.                  " U#U#4UR0                  UR$                  S9n$[3        S5      U$S U2S U24'   U$S U2S U24   R5                  S5        U$R7                  S5      R7                  S5      n$[        R                  " UUUU$SS	S9nUR                  SS5      R!                  U	SUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      n[
        R                  " UU/5      n[
        R                  " UU/5      nX#4$ )Nr  r   r   rì   r   r   r+   r  FrŸ  rý   r�   r  rî  )r÷   rí   rÂ  r  r  r§   r  rV   rW   rX   r\   rZ   r[   rI   re   r1   rî   r°   r�   r^   rC   ra   r¢  r  rŽ   r  r  r  )%ri   r¦  râ   rÖ   r×   rÅ  r‡  rÇ  rÈ  rò   rÉ  rÊ  rË  r  r  Úhidden_states_uncondr  r  Úencoder_hidden_states_uncondr  r	  r
  r  r  r  r  r  r5   rö  r  r  r  r  r  r  ró   r  s%                                        ru   rÛ   Ú$PAGCFGJointAttnProcessor2_0.__call__‰  s  € ð !ˆà"×'Ñ'ˆ
Ø˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMØ2×7Ñ7ÐØ Ó"Ø1F×1LÑ1LÑ.ˆJ Ø$9×$>Ñ$>¸zÐTZÑTbÓ$c×$mÑ$mÐnoÐqrÓ$sÐ!à+×1Ñ1Øñ
Ðð
 FS×EXÑEXÐYZÓE[ÑBÐÐ/Ð1BÜ!ŸIšIÐ';Ð=NÐ&OÓPÐð "×'Ñ'¨Ó*ñ		
Ø(Ø%Ø%ä$)§I¢IÐ/KÐMfÐ.gÓ$hÐ!ð /×4Ñ4°QÑ7ˆ
ð —I‘IÐ/Ó0ˆ	Ø—)‘)Ð-Ó.ˆØ—I‘IÐ/Ó0ˆ	ð 04¯©Ð?XÓ/YÐ,Ø-1¯_©_Ð=VÓ-WÐ*Ø/3¯©Ð?XÓ/YÐ,ô —I’I˜yÐ*NÐOÐUVÑWˆ	Ü—)’)˜WÐ&HÐIÈqÑQˆÜ—I’I˜yÐ*NÐOÐUVÑWˆ	à—M‘M "Ñ%ˆ	Ø §
¡
Ñ*ˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	Ø—,‘,˜z¨2¨t¯z©z¸8ÓD×NÑNÈqÐRSÓTˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	ä×:Ò:Ø�w 	°SÀEñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±ÓAÐð šaÐ!4 8§>¡>°!Ñ#4Ð!4Ð4Ñ5Øša §¡°Ñ!2Ñ!4Ð4Ñ5ð 5Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=ÐØ×$×$Ø(,¯©Ð8QÓ(RÐ%à˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐØ Ó"Ø(A×(KÑ(KÈBÐPRÓ(S×([Ñ([Ø˜G V¨Uó)Ð%ð /×4Ñ4°QÑ7ˆ
ð —I‘IÐ/Ó0ˆ	Ø—)‘)Ð-Ó.ˆØ—I‘IÐ/Ó0ˆ	ð 04¯©Ð?XÓ/YÐ,Ø-1¯_©_Ð=VÓ-WÐ*Ø/3¯©Ð?XÓ/YÐ,ô —I’I˜yÐ*NÐOÐUVÑWˆ	Ü—)’)˜WÐ&HÐIÈqÑQˆÜ—I’I˜yÐ*NÐOÐUVÑWˆ	à—M‘M "Ñ%ˆ	Ø §
¡
Ñ*ˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	Ø—,‘,˜z¨2¨t¯z©z¸8ÓD×NÑNÈqÐRSÓTˆØ—N‘N :¨r°4·:±:¸xÓH×RÑRÐSTÐVWÓXˆ	ð —.‘. Ó#ˆÜ—K’K ¨'Ð 2¸9×;KÑ;KÐS\×SbÑSbÑcˆ	ô AFÀfÃˆ	Ð&Ð&Ð&Ð(<Ð)<Ð(<Ð<Ñ=ð 	Ð&Ð&Ð&Ð(<Ð)<Ð(<Ð<Ñ=×LÑLÈQÔOð ×'Ñ'¨Ó*×4Ñ4°QÓ7ˆ	ä×:Ò:Ø�w 	°YÈ#ÐY^ñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±ÓAÐð šaÐ!4 8§>¡>°!Ñ#4Ð!4Ð4Ñ5Øša §¡°Ñ!2Ñ!4Ð4Ñ5ð 5Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=ÐØ×$×$Ø(,¯©Ð8QÓ(RÐ%à˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐØ Ó"Ø(A×(KÑ(KÈBÐPRÓ(S×([Ñ([Ø˜G V¨Uó)Ð%ô
 Ÿ	š	Ð#4Ð6GÐ"HÓIˆÜ %§	¢	Ð+DÐF_Ð*`Ó aÐàÐ3Ð3rw   r»  r=  rù  r¼  r»  rw   ru   r  r  €  sS   † Ùcòð 48Ø37ðY4àðY4ð )ðY4ð  1ð	Y4ð
 1ðY4ð 
÷Y4ð Y4rw   r  c                  óB   • \ rS rSrSrS r  S         SS jjrSrg)	ÚFusedJointAttnProcessor2_0i%  rò  c                óD   • [        [        S5      (       d  [        S5      eg ©Nr1   zPAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú#FusedJointAttnProcessor2_0.__init__(  ó!   € Ü”qÐ8×9Ñ9ÜÐpÓqÐqð :rw   Nc                ón  • UnUR                   nUS:X  a3  UR                  u  pšp¼UR                  XšX¼-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  pšp¼UR                  XšX¼-  5      R                  SS5      nUR                  S   n	UR	                  U5      nUR                  S   S-  n[
        R                  " XïSS9u  nnnUR                  U5      nUR                  S   S-  n[
        R                  " UUSS9u  nnn[
        R                  " UU/SS9n[
        R                  " UU/SS9n[
        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      n[        R                  " UUUSS	S
9nUR                  SS5      R                  U	SUR                  U-  5      nUR                  UR                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   p2UR                  S   " U5      nUR                  S   " U5      nUR                   (       d  UR#                  U5      nUS:X  a$  UR                  SS5      R                  U	W
WW5      nUS:X  a$  UR                  SS5      R                  U	W
WW5      nX#4$ )Nr  r   r   r   r+   rì   r   r  FrŸ  rý   )r÷   rí   rÂ  r  r%  r§   Úsplitr)  r  rI   re   r1   rî   r°   r�   r^   rC   ra   )ri   r¦  râ   rÖ   r×   rÅ  r‡  rÇ  rÈ  rò   rÉ  rÊ  rË  r  ÚqkvÚ
split_sizer  r  r  Úencoder_qkvr÷  ræ  rç  r5   rö  s                            ru   rÛ   Ú#FusedJointAttnProcessor2_0.__call__,  s1  € ð !ˆà"×'Ñ'ˆ
Ø˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMØ2×7Ñ7ÐØ Ó"Ø1F×1LÑ1LÑ.ˆJ Ø$9×$>Ñ$>¸zÐTZÑTbÓ$c×$mÑ$mÐnoÐqrÓ$sÐ!à*×0Ñ0°Ñ3ˆ
ð �k‰k˜-Ó(ˆØ—Y‘Y˜r‘] aÑ'ˆ
Ü!ŸKšK¨¸RÑ@Ñˆˆs�Eð ×'Ñ'Ð(=Ó>ˆØ ×&Ñ& rÑ*¨aÑ/ˆ
ô
 �KŠK˜ Z°RÑ8ñ		
Ø,Ø*Ø,ô —	’	˜5Ð"BÐCÈÑKˆÜ�iŠi˜Ð<Ð=À1ÑEˆÜ—	’	˜5Ð"BÐCÈÑKˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆä×6Ò6°u¸cÀ5ÐTWÐchÑiˆØ%×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð š!Ð0˜xŸ~™~¨aÑ0Ð0Ð0Ñ1Øš!˜XŸ^™^¨AÑ.Ñ0Ð0Ñ1ð -ð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆØ×$×$Ø$(§O¡OÐ4IÓ$JÐ!à˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMØ Ó"Ø$9×$CÑ$CÀBÈÓ$K×$SÑ$SÐT^Ð`gÐioÐqvÓ$wÐ!àÐ3Ð3rw   r»  r=  rù  r¼  r»  rw   ru   r"  r"  %  sT   † Ùcòrð 48Ø37ðE4àðE4ð )ðE4ð  1ð	E4ð
 1ðE4ð 
÷E4ð E4rw   r"  c                  óJ   • \ rS rSrSrSSS jjr  S	         S
S jjrSrg)r¤   it  á•  
Processor for implementing memory efficient attention using xFormers.

Args:
    attention_op (`Callable`, *optional*, defaults to `None`):
        The base
        [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
        use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
        operator.
Nc                ó   • Xl         g r;  r•   ©ri   r“   s     ru   r3   Ú#XFormersJointAttnProcessor.__init__€  ó   € Ø(Õrw   c           	     óŽ  • UnUR                  U5      nUR                  U5      n	UR                  U5      n
UR                  U5      R	                  5       nUR                  U	5      R	                  5       n	UR                  U
5      R	                  5       n
UR
                  b  UR                  U5      nUR                  b  UR                  U	5      n	UGb  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      R	                  5       nUR                  U5      R	                  5       nUR                  U5      R	                  5       nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " X‹/SS9n[        R                  " Xœ/SS9n	[        R                  " X­/SS9n
[        R                  R!                  X‰X¤U R"                  UR$                  S9nUR'                  UR(                  5      nUR+                  U5      nUbQ  US S 2S UR,                  S   24   US S 2UR,                  S   S 24   p2UR.                  (       d  UR1                  U5      nUR2                  S   " U5      nUR2                  S   " U5      nUb  X#4$ U$ )Nr   r   ©Ú	attn_biasÚoprH   r   )rV   rW   rX   rù   Ú
contiguousrQ   rR   r\   rZ   r[   rb   rc   r§   r  r‰   r«   r¬   r“   rH   r°   r�   rô   rí   rC   ra   r^   )ri   r¦  râ   rÖ   r×   rÅ  r‡  rÇ  r  r  r  r÷  ræ  rç  s                 ru   rÛ   Ú#XFormersJointAttnProcessor.__call__ƒ  sœ  € ð !ˆð —	‘	˜-Ó(ˆØ�i‰i˜Ó&ˆØ—	‘	˜-Ó(ˆà×&Ñ& uÓ-×8Ñ8Ó:ˆØ×$Ñ$ SÓ)×4Ñ4Ó6ˆØ×&Ñ& uÓ-×8Ñ8Ó:ˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð !Ò,Ø/3¯©Ð?TÓ/UÐ,Ø-1¯_©_Ð=RÓ-SÐ*Ø/3¯©Ð?TÓ/UÐ,à/3×/EÑ/EÐFfÓ/g×/rÑ/rÓ/tÐ,Ø-1×-CÑ-CÐDbÓ-c×-nÑ-nÓ-pÐ*Ø/3×/EÑ/EÐFfÓ/g×/rÑ/rÓ/tÐ,à× Ñ Ñ,Ø37×3DÑ3DÐEeÓ3fÐ0Ø× Ñ Ñ,Ø15×1BÑ1BÐCaÓ1bÐ.ä—I’I˜uÐGÈQÑOˆEÜ—)’)˜SÐAÀqÑIˆCÜ—I’I˜uÐGÈQÑOˆEä Ÿ™×?Ñ?Ø˜¸D×<MÑ<MÐUY×U_ÑU_ð @ð 
ˆð &×(Ñ(¨¯©Ó5ˆØ×.Ñ.¨}Ó=ˆà Ñ,ð šaÐ!4 8§>¡>°!Ñ#4Ð!4Ð4Ñ5Øša §¡°Ñ!2Ñ!4Ð4Ñ5ð 1ð ×(×(Ø(,¯©Ð8MÓ(NÐ%ð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà Ñ,Ø Ð7Ð7à Ð rw   r•   r;  ©r“   r<  r=  rù  r¼  r»  rw   ru   r¤   r¤   t  sU   † ñ	ö)ð 48Ø37ðC!àðC!ð )ðC!ð  1ð	C!ð
 1ðC!ð 
÷C!ð C!rw   r¤   c                  óN   • \ rS rSrSrS r    S             SS jjrSrg)	ÚAllegroAttnProcessor2_0iÉ  zá
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the Allegro model. It applies a normalization layer and rotary embedding on the query and key vector.
c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   zWAllegroAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú AllegroAttnProcessor2_0.__init__Ï  rì  rw   Nc           	     ó4  • UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  pšp¼UR                  XšX¼-  5      R	                  SS5      nUc  UR                  OUR                  u  p�nUb<  UR                  XMU	5      nUR                  X‘R                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R	                  SS5      nUR                  U	SUR                  U5      R	                  SS5      nUR                  U	SUR                  U5      R	                  SS5      nUb6  UR                  (       d%  SSKJn  U" XöS   US   5      nU" UUS   US   5      n[         R"                  " UUUUSSS	9nUR	                  SS5      R%                  U	SUR                  U-  5      nUR'                  UR(                  5      nUR*                  S   " U5      nUR*                  S   " U5      nUS:X  a$  UR	                  SS
5      R%                  U	W
WW5      nUR,                  (       a  X'-   nX!R.                  -  nU$ )Nr  r   r   r+   )Úapply_rotary_emb_allegror   r  Frî  rý   )rP   r÷   rí   rÂ  r  r  rI   r.   rV   rT   r!  rW   rX   r9   Ú
embeddingsr?  re   r1   rî   r°   r�   r^   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  r§  rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  r5   rö  r?  s                        ru   rÛ   Ú AllegroAttnProcessor2_0.__call__Õ  sû  € ð !ˆà×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆð Ñ'°×0G×0GÝ<á,¨UÀQÑ4GÐIYÐZ[ÑI\Ó]ˆEÙ*¨3Ð0@ÀÑ0CÐEUÐVWÑEXÓYˆCô ×6Ò6Ø�3˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  ©NNNN©r¦  r   râ   r>  rÖ   r?  r×   r?  rÄ  r?  r§  r?  r9  r>  r¼  r»  rw   ru   r;  r;  É  sq   † ñò
ð 6:Ø.2Ø$(Ø04ðOàðOð $ðOð  3ð	Oð
 ,ðOð "ðOð .ðOð 
÷Oð Orw   r;  c                  ó<   • \ rS rSrSrS r S       SS jjrSrg)	ÚAuraFlowAttnProcessor2_0i'  z;Attention processor used typically in processing Aura Flow.c                óh   • [        [        S5      (       d  [        SS5      (       a  [        S5      eg g )Nr1   ry   ú2.1z£AuraFlowAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to at least 2.1 or above as we use `scale` in `F.scaled_dot_product_attention()`. ©rd   re   r   r�  rŽ  s    ru   r3   Ú!AuraFlowAttnProcessor2_0.__init__*  s:   € Ü”qÐ8×9Ñ9Ô>NÈsÐTY×>ZÑ>ZÜð vóð ð ?[Ð9rw   Nc           	     óD  • UR                   S   nUR                  U5      nUR                  U5      nUR                  U5      n	Ub3  UR	                  U5      n
UR                  U5      nUR                  U5      nUR                   S   nXÑR                  -  nUR                  USUR                  U5      nUR                  USUR                  U5      nU	R                  USUR                  U5      n	UR                  b  UR                  U5      nUR                  b  UR                  U5      nUbØ  W
R                  USUR                  U5      n
WR                  USUR                  U5      nWR                  USUR                  U5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      n[        R                  " X§/SS9n[        R                  " X¸/SS9n[        R                  " XÉ/SS9n	UR                  SS5      nUR                  SS5      nU	R                  SS5      n	[         R"                  " XxU	SUR$                  SS9nUR                  SS5      R'                  USUR                  U-  5      nUR)                  UR*                  5      nUb/  US S 2UR                   S   S 24   US S 2S UR                   S   24   p2UR,                  S   " U5      nUR,                  S   " U5      nUb  UR/                  U5      nUb  X#4$ U$ )	Nr   r+   r   r   r   r  F©r   rH   rE   )rí   rV   rW   rX   r\   rZ   r[   rI   rÂ  rQ   rR   rb   rc   r§   r  r  re   r1   rH   rî   r°   r�   r^   ra   )ri   r¦  râ   rÖ   rÅ  r‡  rò   r  r  r  r÷  ræ  rç  r5   rö  s                  ru   rÛ   Ú!AuraFlowAttnProcessor2_0.__call__0  s  € ð #×(Ñ(¨Ñ+ˆ
ð —	‘	˜-Ó(ˆØ�i‰i˜Ó&ˆØ—	‘	˜-Ó(ˆð !Ñ,Ø/3¯©Ð?TÓ/UÐ,Ø-1¯_©_Ð=RÓ-SÐ*Ø/3¯©Ð?TÓ/UÐ,ð —I‘I˜b‘Mˆ	Ø§
¡
Ñ*ˆØ—
‘
˜: r¨4¯:©:°xÓ@ˆØ�h‰h�z 2 t§z¡z°8Ó<ˆØ—
‘
˜: r¨4¯:©:°xÓ@ˆð �;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð !Ñ,Ø/O×/TÑ/TØ˜B §
¡
¨Hó0Ð,ð .L×-PÑ-PÐQ[Ð]_Ðae×akÑakÐmuÓ-vÐ*Ø/O×/TÑ/TØ˜B §
¡
¨Hó0Ð,ð × Ñ Ñ,Ø37×3DÑ3DÐEeÓ3fÐ0Ø× Ñ Ñ,Ø15×1BÑ1BÐCaÓ1bÐ.ä—I’IÐ?ÐGÈQÑOˆEÜ—)’)Ð;ÐAÀqÑIˆCÜ—I’IÐ?ÐGÈQÑOˆEà—‘  1Ó%ˆØ�m‰m˜A˜qÓ!ˆØ—‘  1Ó%ˆô ×6Ò6Ø˜¨°D·J±JÈ%ñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð !Ñ,àšaÐ!6×!<Ñ!<¸QÑ!?Ñ!AÐAÑBØšaÐ!AÐ#8×#>Ñ#>¸qÑ#AÐ!AÐAÑBð 1ð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆØ Ñ,Ø$(§O¡OÐ4IÓ$JÐ!à Ñ,Ø Ð7Ð7à Ð rw   r»  r;  ©r¦  r   râ   rú  rÖ   rú  r9  rú  r¼  r»  rw   ru   rE  rE  '  sE   † ÙEòð 48ð	Q!àðQ!ð )ðQ!ð  1ð	Q!ð 
÷Q!ð Q!rw   rE  c                  ó<   • \ rS rSrSrS r S       SS jjrSrg)	ÚFusedAuraFlowAttnProcessor2_0i„  zRAttention processor used typically in processing Aura Flow with fused projections.c                óh   • [        [        S5      (       d  [        SS5      (       a  [        S5      eg g )Nr1   ry   rG  z¨FusedAuraFlowAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to at least 2.1 or above as we use `scale` in `F.scaled_dot_product_attention()`. rH  rŽ  s    ru   r3   Ú&FusedAuraFlowAttnProcessor2_0.__init__‡  s:   € Ü”qÐ8×9Ñ9Ô>NÈsÐTY×>ZÑ>ZÜð {óð ð ?[Ð9rw   Nc           	     óh  • UR                   S   nUR                  U5      nUR                   S   S-  n[        R                  " XxSS9u  pšnUb<  UR	                  U5      nUR                   S   S-  n[        R                  " XÈSS9u  nnnU
R                   S   nUUR
                  -  nU	R                  USUR
                  U5      n	U
R                  USUR
                  U5      n
UR                  USUR
                  U5      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
UbØ  WR                  USUR
                  U5      nWR                  USUR
                  U5      nWR                  USUR
                  U5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " XÙ/SS9n	[        R                  " Xê/SS9n
[        R                  " Xû/SS9nU	R                  SS5      n	U
R                  SS5      n
UR                  SS5      n[        R                  " XšUSUR                  SS	9nUR                  SS5      R!                  USUR
                  U-  5      nUR#                  U	R$                  5      nUb/  US S 2UR                   S   S 24   US S 2S UR                   S   24   p2UR&                  S   " U5      nUR&                  S   " U5      nUb  UR)                  U5      nUb  X#4$ U$ )
Nr   r+   rì   r   r   r   r  FrK  )rí   r%  r§   r(  r)  rI   rÂ  rQ   rR   rb   rc   r  r  re   r1   rH   rî   r°   r�   r^   ra   )ri   r¦  râ   rÖ   rÅ  r‡  rò   r)  r*  r  r  r  r+  r÷  ræ  rç  r5   rö  s                     ru   rÛ   Ú&FusedAuraFlowAttnProcessor2_0.__call__�  s;  € ð #×(Ñ(¨Ñ+ˆ
ð �k‰k˜-Ó(ˆØ—Y‘Y˜r‘] aÑ'ˆ
Ü!ŸKšK¨¸RÑ@Ñˆ�Eð !Ñ,Ø×+Ñ+Ð,AÓBˆKØ$×*Ñ*¨2Ñ.°!Ñ3ˆJô
 —’˜K¸Ñ<ñ	Ø0Ø.Ø0ð —I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆØ—
‘
˜: r¨4¯:©:°xÓ@ˆØ�h‰h�z 2 t§z¡z°8Ó<ˆØ—
‘
˜: r¨4¯:©:°xÓ@ˆð �;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð !Ñ,Ø/O×/TÑ/TØ˜B §
¡
¨Hó0Ð,ð .L×-PÑ-PÐQ[Ð]_Ðae×akÑakÐmuÓ-vÐ*Ø/O×/TÑ/TØ˜B §
¡
¨Hó0Ð,ð × Ñ Ñ,Ø37×3DÑ3DÐEeÓ3fÐ0Ø× Ñ Ñ,Ø15×1BÑ1BÐCaÓ1bÐ.ä—I’IÐ?ÐGÈQÑOˆEÜ—)’)Ð;ÐAÀqÑIˆCÜ—I’IÐ?ÐGÈQÑOˆEà—‘  1Ó%ˆØ�m‰m˜A˜qÓ!ˆØ—‘  1Ó%ˆô ×6Ò6Ø˜¨°D·J±JÈ%ñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð !Ñ,àšaÐ!6×!<Ñ!<¸QÑ!?Ñ!AÐAÑBØšaÐ!AÐ#8×#>Ñ#>¸qÑ#AÐ!AÐAÑBð 1ð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆØ Ñ,Ø$(§O¡OÐ4IÓ$JÐ!à Ñ,Ø Ð7Ð7à Ð rw   r»  r;  rM  r¼  r»  rw   ru   rO  rO  „  sE   † Ù\òð 48ð	U!àðU!ð )ðU!ð  1ð	U!ð 
÷U!ð U!rw   rO  c                  óF   • \ rS rSrSrS r  S           SS jjrSrg)	ÚCogVideoXAttnProcessor2_0iå  ú¶
Processor for implementing scaled dot-product attention for the CogVideoX model. It applies a rotary embedding on
query and key vectors, but does not include spatial normalization.
c                óD   • [        [        S5      (       d  [        S5      eg ©Nr1   zVCogVideoXAttnProcessor requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú"CogVideoXAttnProcessor2_0.__init__ë  ó!   € Ü”qÐ8×9Ñ9ÜÐvÓwÐwð :rw   Nc           	     ó  • UR                  S5      n[        R                  " X2/SS9nUR                  u  pxn	Ub<  UR	                  XHU5      nUR                  XqR                  SUR                  S   5      nUR                  U5      n
UR                  U5      nUR                  U5      nUR                  S   nXÑR                  -  nU
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      nUbY  SSKJn  U" U
S S 2S S 2US 24   U5      U
S S 2S S 2US 24'   UR                  (       d!  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   [         R"                  " X«XÄSSS9nUR                  SS5      R%                  USUR                  U-  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUR)                  XbR                  S5      U-
  /SS9u  p2X#4$ )
Nr   r   r+   r   ©rœ  r  Frî  r   )r¢  r§   r  rí   r  rÂ  rI   rV   rW   rX   r  rQ   rR   r@  rœ  r9   re   r1   rî   r^   r(  )ri   r¦  râ   rÖ   r×   r§  Útext_seq_lengthrò   r«  r¼   r  r  r  r5   rö  rœ  s                   ru   rÛ   Ú"CogVideoXAttnProcessor2_0.__call__ï  sk  € ð 0×4Ñ4°QÓ7ˆäŸ	š	Ð#8Ð"HÈaÑPˆà)6×)<Ñ)<Ñ&ˆ
 QàÑ%Ø!×8Ñ8¸ÐZdÓeˆNØ+×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà—	‘	˜-Ó(ˆØ�i‰i˜Ó&ˆØ—	‘	˜-Ó(ˆà—I‘I˜b‘Mˆ	Ø§
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð Ñ'Ý4á,<¸UÂ1ÂaÈÑIYÐCYÑ=ZÐ\lÓ,mˆE’!’Q˜Ñ(Ð(Ñ)Ø×*×*Ù.>¸sÂ1ÂaÈÑIYÐCYÑ?ZÐ\lÓ.m�’A’q˜/Ñ*Ð*Ñ+ä×6Ò6Ø˜À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà/<×/BÑ/BØ×0Ñ0°Ó3°oÑEÐFÈAð 0Cð 0
Ñ,Ðð Ð3Ð3rw   r»  r=  ©r¦  r   râ   r>  rÖ   r>  r×   r?  r§  r?  r9  r>  r¼  r»  rw   ru   rU  rU  å  sY   † ñò
xð /3Ø04ð84àð84ð $ð84ð  ,ð	84ð
 ,ð84ð .ð84ð 
÷84ð 84rw   rU  c                  óF   • \ rS rSrSrS r  S           SS jjrSrg)	ÚFusedCogVideoXAttnProcessor2_0i*	  rV  c                óD   • [        [        S5      (       d  [        S5      eg rX  rŒ  rŽ  s    ru   r3   Ú'FusedCogVideoXAttnProcessor2_0.__init__0	  rZ  rw   Nc           	     ó0  • UR                  S5      n[        R                  " X2/SS9nUc  UR                  OUR                  u  pxn	Ub<  UR	                  XHU5      nUR                  XqR                  SUR                  S   5      nUR                  U5      n
U
R                  S   S-  n[        R                  " X«SS9u  pÍnUR                  S   nXñR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      nUbY  SSKJn  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   UR                  (       d!  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   [        R                   " XÍXäSSS	9nUR                  SS5      R#                  USUR                  U-  5      nUR$                  S
   " U5      nUR$                  S   " U5      nUR                  XbR                  S5      U-
  /SS9u  p2X#4$ )Nr   r   r+   rì   r   r\  r  Frî  r   )r¢  r§   r  rí   r  rÂ  rI   r%  r(  r  rQ   rR   r@  rœ  r9   re   r1   rî   r^   )ri   r¦  râ   rÖ   r×   r§  r]  rò   r«  r¼   r)  r*  r  r  r  r5   rö  rœ  s                     ru   rÛ   Ú'FusedCogVideoXAttnProcessor2_0.__call__4	  sŠ  € ð 0×4Ñ4°QÓ7ˆäŸ	š	Ð#8Ð"HÈaÑPˆð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNØ+×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�k‰k˜-Ó(ˆØ—Y‘Y˜r‘] aÑ'ˆ
Ü!ŸKšK¨¸RÑ@Ñˆ�Eà—I‘I˜b‘Mˆ	Ø§
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð Ñ'Ý4á,<¸UÂ1ÂaÈÑIYÐCYÑ=ZÐ\lÓ,mˆE’!’Q˜Ñ(Ð(Ñ)Ø×*×*Ù.>¸sÂ1ÂaÈÑIYÐCYÑ?ZÐ\lÓ.m�’A’q˜/Ñ*Ð*Ñ+ä×6Ò6Ø˜À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà/<×/BÑ/BØ×0Ñ0°Ó3°oÑEÐFÈAð 0Cð 0
Ñ,Ðð Ð3Ð3rw   r»  r=  r_  r¼  r»  rw   ru   ra  ra  *	  sY   † ñò
xð /3Ø04ð:4àð:4ð $ð:4ð  ,ð	:4ð
 ,ð:4ð .ð:4ð 
÷:4ð :4rw   ra  c                  óJ   • \ rS rSrSrSSS jjr  S	         S
S jjrSrg)rŸ   iq	  r.  Nc                ó   • Xl         g r;  r•   r0  s     ru   r3   Ú%XFormersAttnAddedKVProcessor.__init__}	  r2  rw   c           	     óÂ  • UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pgnUR                  XGU5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      n	UR                  U	5      n	UR                  U5      n
UR                  U5      nUR                  U
5      n
UR                  U5      nUR                  (       dq  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " X¬/SS9n[        R                  " X½/SS9nOU
nUn[         R"                  R%                  XœXÔU R&                  UR(                  S9nUR+                  U	R,                  5      nUR/                  U5      nUR0                  S   " U5      nUR0                  S   " U5      nUR                  SS5      R3                  UR                  5      nX%-   nU$ )Nr   r   r+   r   r   r4  rý   )rÂ  rí   r  r  rT   r!  r.   rV   rù   rZ   r[   rL   rW   rX   r§   r  r‰   r«   r¬   r“   rH   r°   r�   rô   r^   rî   )ri   r¦  râ   rÖ   r×   rÇ  rò   r«  r¼   r  ræ  rç  r  r  s                 ru   rÛ   Ú%XFormersAttnAddedKVProcessor.__call__€	  s:  € ð !ˆØ%×*Ñ*¨=×+>Ñ+>¸qÑ+AÀ=×CVÑCVÐWXÑCYÐ[]Ó^×hÑhÐijÐlmÓnˆØ)6×)<Ñ)<Ñ&ˆ
 Qà×4Ñ4°^ÐV`Óaˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!àŸ™¨×(?Ñ(?ÀÀ1Ó(EÓF×PÑPÐQRÐTUÓVˆà—	‘	˜-Ó(ˆØ×&Ñ& uÓ-ˆà)-¯©Ð9NÓ)OÐ&Ø+/¯?©?Ð;PÓ+QÐ(Ø)-×)?Ñ)?Ð@^Ó)_Ð&Ø+/×+AÑ+AÐBbÓ+cÐ(à×(×(Ø—)‘)˜MÓ*ˆCØ—I‘I˜mÓ,ˆEØ×(Ñ(¨Ó-ˆCØ×*Ñ*¨5Ó1ˆEÜ—)’)Ð;ÐAÀqÑIˆCÜ—I’IÐ?ÐGÈQÑO‰Eà0ˆCØ4ˆEä Ÿ™×?Ñ?Ø˜¸D×<MÑ<MÐUY×U_ÑU_ð @ð 
ˆð &×(Ñ(¨¯©Ó5ˆØ×.Ñ.¨}Ó=ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà%×/Ñ/°°BÓ7×?Ñ?ÀÇÁÓOˆØ%Ñ0ˆàÐrw   r•   r;  r9  r=  râ  r¼  r»  rw   ru   rŸ   rŸ   q	  sN   † ñ	ö)ð 6:Ø.2ð5àð5ð $ð5ð  3ð	5ð
 ,ð5ð 
÷5ð 5rw   rŸ   c                  óP   • \ rS rSrSrSSS jjr   S	           S
S jjrSrg)r´   i¸	  r.  Nc                ó   • Xl         g r;  r•   r0  s     ru   r3   ÚXFormersAttnProcessor.__init__Ä	  r2  rw   c           	     óz  • [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  p¼pÞUR                  X¼XÞ-  5      R                  SS5      nUc  UR
                  OUR
                  u  p¿nUR                  XOU5      nUb#  UR
                  u  nnnUR                  SUS5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR!                  U5      R#                  5       nUR!                  U5      R#                  5       nUR!                  U5      R#                  5       n[$        R&                  R)                  UUUX@R*                  UR,                  S	9nUR/                  UR0                  5      nUR3                  U5      nUR4                  S   " U5      nUR4                  S   " U5      nU
S:X  a$  UR                  SS
5      R7                  UWWW5      nUR8                  (       a  X)-   nX!R:                  -  nU$ )Nr   rH   r¿  rÀ  r  r   r   r+   r4  rý   )rß   rÁ  r   rP   r÷   rí   rÂ  r  r  Úexpandr.   rV   rT   r!  rW   rX   rù   r7  r‰   r«   r¬   r“   rH   r°   r�   rô   r^   rî   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  rÅ  r‡  rÆ  rÇ  rÈ  rò   rÉ  rÊ  rË  Ú
key_tokensr¼   Úquery_tokensr  r  r  s                        ru   rÛ   ÚXFormersAttnProcessor.__call__Ç	  s   € ô ˆt‹9�q‹=˜FŸJ™J w°Ó5ÑAð #UÐÜ�g˜wÐ(;Ô<à ˆà×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	"ˆ
 ð ×4Ñ4°^ÐQ[Ó\ˆØÑ%ð "/×!4Ñ!4ÑˆAˆ|˜QØ+×2Ñ2°2°|ÀRÓHˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà×&Ñ& uÓ-×8Ñ8Ó:ˆØ×$Ñ$ SÓ)×4Ñ4Ó6ˆØ×&Ñ& uÓ-×8Ñ8Ó:ˆä Ÿ™×?Ñ?Ø�3˜¨×<MÑ<MÐUY×U_ÑU_ð @ð 
ˆð &×(Ñ(¨¯©Ó5ˆØ×.Ñ.¨}Ó=ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r•   r;  r9  rÍ  rÎ  r¼  r»  rw   ru   r´   r´   ¸	  sc   † ñ	ö)ð 6:Ø.2Ø$(ðLàðLð $ðLð  3ð	Lð
 ,ðLð "ðLð 
÷Lð Lrw   r´   c                  óH   • \ rS rSrSrS r   S           SS jjrSrg)	r„   i
  zõ
Processor for implementing flash attention using torch_npu. Torch_npu supports only fp16 and bf16 data types. If
fp32 is used, F.scaled_dot_product_attention will be used for computation, but the acceleration effect on NPU is
not significant.

c                ó8   • [        5       (       d  [        S5      eg )NzTAttnProcessorNPU requires torch_npu extensions and is supported only on npu devices.)r   r�  rŽ  s    ru   r3   ÚAttnProcessorNPU.__init__
  s   € Ü%×'Ñ'ÜÐtÓuÐuð (rw   Nc                ó  • [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  p¼pÞUR                  X¼XÞ-  5      R                  SS5      nUc  UR
                  OUR
                  u  p¿nUb°  UR                  XOU5      nUR                  X±R                  SUR
                  S   5      nUR                  SSUR
                  S   S5      nUR                  [        R                  :X  a%  [        R                  " UR                  5       5      nOUR                  5       nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR!                  U5      nUc  UnO"UR"                  (       a  UR%                  U5      nUR'                  U5      nUR)                  U5      nUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  [        R*                  [        R,                  4;   aR  [.        R0                  " UUUUR                  S	S US
[2        R4                  " UR
                  S   5      -  SSS
SSS9S   nO[6        R8                  " UUUUSSS9nUR                  SS5      R;                  USUR                  U-  5      nUR=                  UR                  5      nUR>                  S   " U5      nUR>                  S   " U5      nU
S:X  a$  UR                  SS5      R;                  UWWW5      nUR@                  (       a  X)-   nX!RB                  -  nU$ )Nr   rH   r¿  rÀ  r  r   r   r+   ÚBNSDr   i   F)	Úinput_layoutÚpseÚ
atten_maskrH   Úpre_tockensÚnext_tockensÚ	keep_probÚsyncÚinner_preciser  rî  rý   )"rß   rÁ  r   rP   r÷   rí   rÂ  r  r  rI   Úrepeatr�   r§   r6  Úlogical_notr.   rV   rT   r!  rW   rX   Úfloat16Úbfloat16Ú	torch_npuÚnpu_fusion_attentionÚmathÚsqrtre   r1   rî   r°   r^   r>   r=   ©ri   r¦  râ   rÖ   r×   rÄ  rÅ  r‡  rÆ  rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  r5   rö  s                         ru   rÛ   ÚAttnProcessorNPU.__call__"
  s·  € ô ˆt‹9�q‹=˜FŸJ™J w°Ó5ÑAð #UÐÜ�g˜wÐ(;Ô<à ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNØ+×2Ñ2°1°a¸×9LÑ9LÈQÑ9OÐQRÓSˆNØ×#Ñ#¤u§z¡zÓ1Ü!&×!2Ò!2°>×3FÑ3FÓ3HÓ!I‘à!/×!4Ñ!4Ó!6�à�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆð �;‰;œ5Ÿ=™=¬%¯.©.Ð9Ó9Ü%×:Ò:ØØØØ—
‘
Ø#ØØ)ØœDŸIšI e§k¡k°"¡oÓ6Ñ6Ø!Ø"ØØØñð ñ‰Mô" ×:Ò:Ø�s˜E¨^ÀsÐV[ñˆMð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rÍ  rÎ  r¼  r»  rw   ru   r„   r„   
  sd   † ñòvð 6:Ø.2Ø$(ðcàðcð $ðcð  3ð	cð
 ,ðcð "ðcð 
÷cð crw   r„   c                  óH   • \ rS rSrSrS r   S           SS jjrSrg)	rf   iˆ
  úk
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
c                óD   • [        [        S5      (       d  [        S5      eg r$  rŒ  rŽ  s    ru   r3   ÚAttnProcessor2_0.__init__�
  r&  rw   Nc           	     óœ  • [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  p¼pÞUR                  X¼XÞ-  5      R                  SS5      nUc  UR
                  OUR
                  u  p¿nUb<  UR                  XOU5      nUR                  X±R                  SUR
                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                   b  UR!                  U5      nUR"                  b  UR#                  U5      n[$        R&                  " UUUUS	S
S9nUR                  SS5      R)                  USUR                  U-  5      nUR+                  UR,                  5      nUR.                  S   " U5      nUR.                  S   " U5      nU
S:X  a$  UR                  SS5      R)                  UWWW5      nUR0                  (       a  X)-   nX!R2                  -  nU$ )Nr   rH   r¿  rÀ  r  r   r   r+   r  Frî  rý   )rß   rÁ  r   rP   r÷   rí   rÂ  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   re   r1   rî   r°   r�   r^   r>   r=   rˆ  s                         ru   rÛ   ÚAttnProcessor2_0.__call__‘
  s  € ô ˆt‹9�q‹=˜FŸJ™J w°Ó5ÑAð #UÐÜ�g˜wÐ(;Ô<à ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCô ×6Ò6Ø�3˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rÍ  rÎ  r¼  r»  rw   ru   rf   rf   ˆ
  sd   † ñòrð 6:Ø.2Ø$(ðRàðRð $ðRð  3ð	Rð
 ,ðRð "ðRð 
÷Rð Rrw   rf   c                  óP   • \ rS rSrSrSSS jjr   S	           S
S jjrSrg)r}   iæ
  úr
Processor for implementing scaled dot-product attention with pallas flash attention kernel if using `torch_xla`.
Nc                óÞ   • [        [        S5      (       d  [        S5      e[        SS5      (       a  [        S5      e[	        5       (       a  [        SS5      (       a  [        S5      eXl        g )Nr1   zXXLAFlashAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.ry   rz   ú6XLA flash attention requires torch_xla version >= 2.3.r{   úDSPMD support for XLA flash attention needs torch_xla version >= 2.4.)rd   re   r�  r   r   r   )ri   r   s     ru   r3   Ú!XLAFlashAttnProcessor2_0.__init__ë
  sa   € Ü”qÐ8×9Ñ9ÜØjóð ô    U×+Ñ+ÜÐVÓWÐWÜ�9‰9Ô-¨c°5×9Ñ9ÜÐdÓeÐeØ,Õrw   c           	     ó  • UnUR                   b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  p«pÍUR                  X«XÍ-  5      R	                  SS5      nUc  UR                  OUR                  u  p®nUb<  UR                  XNU
5      nUR                  X¡R                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R	                  SS5      nUR                  U
SUR                  U5      R	                  SS5      nUR                  U
SUR                  U5      R	                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        S UUU4 5       5      (       aÂ  Ubn  UR                  U
SSUR                  S   5      nUR!                  5       R#                  US:H  [!        S5      5      R#                  US:H  [!        S5      5      nUU-   nU[$        R&                  " UR                  S	   5      -  n[)        5       (       a  U R*                  OS n[-        UUUS
US9nO.[.        R1                  S5        [2        R4                  " UUUUSS
S9nUR	                  SS5      R7                  U
SUR                  U-  5      nUR9                  UR:                  5      nUR<                  S   " U5      nUR<                  S   " U5      nU	S:X  a$  UR	                  SS5      R7                  U
WWW5      nUR>                  (       a  X(-   nX!R@                  -  nU$ )Nr  r   r   r+   c              3  óD   #   • U  H  oR                   S    S:¬  v •  M     g7f)r   i   N)rí   )Ú.0rð   s     ru   Ú	<genexpr>Ú4XLAFlashAttnProcessor2_0.__call__.<locals>.<genexpr>0  s   é € ÐIÒ5H¨6�|‰|˜A‰ $Ö&Ò5Hùs   ‚ r   r  r  rì   F)Úcausalr   z[Unable to use the flash attention pallas kernel API call due to QKV sequence length < 4096.rî  rý   )!rP   r÷   rí   rÂ  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   Úallr  Úmasked_fillr†  r‡  r   r   r   r²   rà   re   r1   rî   r°   r�   r^   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  rÅ  r‡  rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  r5   rö  r   s                         ru   rÛ   Ú!XLAFlashAttnProcessor2_0.__call__ö
  sÀ  € ð !ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCô ÑI°e¸SÀ%Ñ5HÓI×IÑIØÑ)Ø!/×!4Ñ!4°ZÀÀAÀ~×G[ÑG[Ð\^ÑG_Ó!`�ð #×(Ñ(Ó*ß ‘[ °1Ñ!4´e¸F³mÓDß ‘[ °1Ñ!4´e¸C³jÓAð ð ˜NÑ*�Ø”T—Y’Y˜uŸ{™{¨1™~Ó.Ñ.ˆEÜ4;·I±I˜T×0Ò0À4ˆNÜ+¨E°3¸ÀeÐ\jÑk‰Mä�N‰NØmôô ×:Ò:Ø�s˜E¨^ÀsÐV[ñˆMð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   )r   r;  )r   r8  rÍ  rÎ  r¼  r»  rw   ru   r}   r}   æ
  sc   † ñö	-ð 6:Ø.2Ø$(ðaàðað $ðað  3ð	að
 ,ðað "ðað 
÷að arw   r}   c                  óB   • \ rS rSrSrS r  S         SS jjrSrg)	ÚMochiVaeAttnProcessor2_0iZ  z(
Attention processor used in Mochi VAE.
c                óD   • [        [        S5      (       d  [        S5      eg r$  rŒ  rŽ  s    ru   r3   Ú!MochiVaeAttnProcessor2_0.__init___  r&  rw   Nc           	     ó^  • UnUR                   S   S:H  nUc  UR                   OUR                   u  pxn	Ub<  UR                  XHU5      nUR                  XqR                  SUR                   S   5      nU(       a`  UR	                  U5      nUR
                  S   " U5      nUR
                  S   " U5      nUR                  (       a  X%-   nX!R                  -  nU$ UR                  U5      n
Uc  UnUR                  U5      nUR	                  U5      nUR                   S   nXÑR                  -  nU
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      n[        R                  " X«XÄSUR                  S9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  U
R$                  5      nUR
                  S   " U5      nUR
                  S   " U5      nUR                  (       a  X%-   nX!R                  -  nU$ )Nr   r+   r   r   r  rî  )rí   r  rÂ  rI   rX   r^   r>   r=   rV   rW   r  rQ   rR   re   r1   rE   rî   r°   r�   )ri   r¦  râ   rÖ   r×   rÇ  Úis_single_framerò   r«  r¼   r  r  r  r5   rö  s                  ru   rÛ   Ú!MochiVaeAttnProcessor2_0.__call__c  s{  € ð !ˆØ'×-Ñ-¨aÑ0°AÑ5ˆð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNæØ ŸI™I mÓ4ˆMð !ŸK™K¨šN¨=Ó9ˆMà ŸK™K¨šN¨=Ó9ˆMà×'×'Ø -Ñ 8�à)×,FÑ,FÑFˆMØ Ð à—	‘	˜-Ó(ˆà Ñ(Ø$1Ð!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø§
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCô ×6Ò6Ø˜À3ÐRV×R`ÑR`ñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  r=  râ  r¼  r»  rw   ru   r   r   Z  sV   † ñòrð 6:Ø.2ðIàðIð $ðIð  3ð	Ið
 ,ðIð 
÷Ið Irw   r   c                  ó^   • \ rS rSrSrS r      SS jr   S	           S
S jjrSrg)ÚStableAudioAttnProcessor2_0i¯  zä
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the Stable Audio model. It applies rotary embedding on query and key vector, and allows MHA, GQA or MQA.
c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   z[StableAudioAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú$StableAudioAttnProcessor2_0.__init__µ  r  rw   c                ó”   • SSK Jn  US   R                  S   nUSS U24   USUS 24   peU" XRSSS9n[        R                  " Xv4SS	9nU$ )
Nr   r\  r   r+   .Trý   ©Úuse_realÚuse_real_unbind_dimr   )r@  rœ  rí   r§   r  )	ri   r•  Ú	freqs_cisrœ  Úrot_dimÚx_to_rotateÚx_unrotatedÚ	x_rotatedÚouts	            ru   Úapply_partial_rotary_embÚ4StableAudioAttnProcessor2_0.apply_partial_rotary_emb»  sa   € õ
 	1à˜A‘,×$Ñ$ RÑ(ˆØ#$ S¨(¨7¨( ]Ñ#3°Q°s¸G¹H°}Ñ5E�[á$ [ÀdÐ`bÑcˆ	ä�iŠi˜Ð0°bÑ9ˆØˆ
rw   Nc           	     ód  • SSK Jn  UnUR                  nUS:X  a3  UR                  u  pšp¼UR	                  XšX¼-  5      R                  SS5      nUc  UR                  OUR                  u  p�nUb<  UR                  XMU	5      nUR	                  X‘R                  SUR                  S   5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   UR                  -  nUR                  S   U-  nUR	                  U	SUR                  U5      R                  SS5      nUR	                  U	SUU5      R                  SS5      nUR	                  U	SUU5      R                  SS5      nUUR                  :w  a]  UR                  U-  n[        R                  " UUSUR                  S   U-  S9n[        R                  " UUSUR                  S   U-  S9nUR                  b  UR                  U5      nUR                   b  UR!                  U5      nUbÿ  UR"                  nUR"                  nUR%                  [        R&                  5      nUR%                  [        R&                  5      nUS   R                  S   nUSS U24   USUS 24   nnU" UUS	S
S9n[        R(                  " UU4SS9nUR*                  (       d2  USS U24   USUS 24   nnU" UUS	S
S9n[        R(                  " UU4SS9nUR%                  U5      nUR%                  U5      n[,        R.                  " UUUUSSS9nUR                  SS5      R1                  U	SUR                  U-  5      nUR%                  UR"                  5      nUR2                  S   " U5      nUR2                  S   " U5      nUS:X  a$  UR                  SS
5      R1                  U	W
WW5      nUR4                  (       a  X'-   nX!R6                  -  nU$ )Nr   r\  r  r   r+   r  r   .Trý   r«  r   r  Frî  )r@  rœ  r÷   rí   rÂ  r  r  rI   rV   rT   r!  rW   rX   r§   r  rQ   rR   r�   r°   rq  r  r9   re   r1   rî   r^   r>   r=   )ri   r¦  râ   rÖ   r×   Ú
rotary_embrœ  rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  rö  rj   Úheads_per_kv_headÚquery_dtypeÚ	key_dtyper¯  Úquery_to_rotateÚquery_unrotatedÚquery_rotatedÚkey_to_rotateÚkey_unrotatedÚkey_rotateds                                 ru   rÛ   Ú$StableAudioAttnProcessor2_0.__call__Ê  s  € õ 	1à ˆà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—;‘;˜r‘? d§j¡jÑ0ˆØ—9‘9˜R‘= HÑ,ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 x°Ó:×DÑDÀQÈÓJˆØ—
‘
˜: r¨8°XÓ>×HÑHÈÈAÓNˆà�t—z‘zÓ!à $§
¡
¨hÑ 6ÐÜ×)Ò)¨#Ð/@ÀaÐUX×U^ÑU^Ð_`ÑUaÐduÑUuÑvˆCÜ×+Ò+ØÐ(¨a¸U¿[¹[È¹^ÐN_Ñ=_ñˆEð �;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð Ñ!ØŸ+™+ˆKØŸ	™	ˆIØ—H‘HœUŸ]™]Ó+ˆEØ—&‘&œŸ™Ó'ˆCà  ‘m×)Ñ)¨"Ñ-ˆGØ/4°S¸(¸7¸(°]Ñ/CÀUÈ3ÐPWÑPXÈ=ÑEY˜_ˆOÙ,¨_¸jÐSWÐmoÑpˆMä—I’I˜}¨oÐ>ÀBÑGˆEà×*×*Ø/2°3¸¸¸°=Ñ/AÀ3ÀsÈGÉHÀ}ÑCU˜}�Ù.¨}¸jÐSWÐmoÑp�ä—i’i ¨mÐ <À"ÑE�à—H‘H˜[Ó)ˆEØ—&‘&˜Ó#ˆCô ×6Ò6Ø�3˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  )r•  r>  r®  ztuple[torch.Tensor]r9  r>  rÍ  )r¦  r   râ   r>  rÖ   r?  r×   r?  r·  r?  r9  r>  )	rá   r@  rA  rB  rC  r3   r´  rÛ   rE  r»  rw   ru   r§  r§  ¯  s„   † ñò
ðàðð 'ðð 
ô	ð& 6:Ø.2Ø*.ðgàðgð $ðgð  3ð	gð
 ,ðgð (ðgð 
÷gð grw   r§  c                  óN   • \ rS rSrSrS r    S             SS jjrSrg)	ÚHunyuanAttnProcessor2_0i4  zâ
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the HunyuanDiT model. It applies a s normalization layer and rotary embedding on query and key vector.
c                óD   • [        [        S5      (       d  [        S5      eg r$  rŒ  rŽ  s    ru   r3   Ú HunyuanAttnProcessor2_0.__init__:  r&  rw   Nc           	     ó’  • SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  p«pÍUR                  X«XÍ-  5      R                  SS5      nUc  UR                  OUR                  u  p®nUb<  UR                  XNU
5      nUR                  X¡R                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                   b  UR!                  U5      nUb#  U" UU5      nUR"                  (       d	  U" UU5      n[$        R&                  " UUUUSSS9nUR                  SS5      R)                  U
SUR                  U-  5      nUR+                  UR,                  5      nUR.                  S	   " U5      nUR.                  S   " U5      nU	S:X  a$  UR                  SS
5      R)                  U
WWW5      nUR0                  (       a  X(-   nX!R2                  -  nU$ ©Nr   r\  r  r   r+   r  Frî  r   rý   )r@  rœ  rP   r÷   rí   rÂ  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   r9   re   r1   rî   r°   r�   r^   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  r§  rœ  rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  r5   rö  s                        ru   rÛ   Ú HunyuanAttnProcessor2_0.__call__>  s  € õ 	1à ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð Ñ'Ù$ UÐ,<Ó=ˆEØ×*×*Ù& sÐ,<Ó=�ô ×6Ò6Ø�3˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rB  rC  r¼  r»  rw   ru   rÃ  rÃ  4  sr   † ñò
rð 6:Ø.2Ø$(Ø04ðUàðUð $ðUð  3ð	Uð
 ,ðUð "ðUð .ðUð 
÷Uð Urw   rÃ  c                  óN   • \ rS rSrSrS r    S             SS jjrSrg)	ÚFusedHunyuanAttnProcessor2_0i–  zÿ
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0) with fused
projection layers. This is used in the HunyuanDiT model. It applies a s normalization layer and rotary embedding on
query and key vector.
c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   z\FusedHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú%FusedHunyuanAttnProcessor2_0.__init__�  s%   € Ü”qÐ8×9Ñ9ÜØnóð ð :rw   Nc           	     ó<  • SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  p«pÍUR                  X«XÍ-  5      R                  SS5      nUc  UR                  OUR                  u  p®nUb<  UR                  XNU
5      nUR                  X¡R                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUc>  UR                  U5      nUR                  S   S-  n[        R                  " UUSS9u  nnnOoUR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR                  S   S-  n[        R                  " UUSS9u  nnUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR"                  b  UR#                  U5      nUR$                  b  UR%                  U5      nUb#  U" UU5      nUR&                  (       d	  U" UU5      n[(        R*                  " UUUUSS	S
9nUR                  SS5      R-                  U
SUR                  U-  5      nUR/                  UR0                  5      nUR2                  S   " U5      nUR2                  S   " U5      nU	S:X  a$  UR                  SS5      R-                  U
WWW5      nUR4                  (       a  X(-   nX!R6                  -  nU$ )Nr   r\  r  r   r+   rì   r   r  Frî  r   rý   )r@  rœ  rP   r÷   rí   rÂ  r  r  rI   r.   r%  r§   r(  rT   r!  rV   r'  rQ   rR   r9   re   r1   rî   r°   r�   r^   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  r§  rœ  rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r)  r*  r  r  r  Úkvr5   rö  s                           ru   rÛ   Ú%FusedHunyuanAttnProcessor2_0.__call__£  s_  € õ 	1à ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà Ñ(Ø—+‘+˜mÓ,ˆCØŸ™ 2™¨!Ñ+ˆJÜ %§¢¨C°ÀÑ DÑˆE�3™à��Ø(,×(GÑ(GÐH]Ó(^Ð%Ø—I‘I˜mÓ,ˆEà—‘Ð1Ó2ˆBØŸ™ "™¨Ñ*ˆJÜŸš R¨¸Ñ<‰JˆC�à—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð Ñ'Ù$ UÐ,<Ó=ˆEØ×*×*Ù& sÐ,<Ó=�ô ×6Ò6Ø�3˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rB  rC  r¼  r»  rw   ru   rÊ  rÊ  –  sq   † ñòð 6:Ø.2Ø$(Ø04ðWàðWð $ðWð  3ð	Wð
 ,ðWð "ðWð .ðWð 
÷Wð Wrw   rÊ  c                  óN   • \ rS rSrSrS r    S             SS jjrSrg)	ÚPAGHunyuanAttnProcessor2_0iý  áO  
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the HunyuanDiT model. It applies a normalization layer and rotary embedding on query and key vector. This
variant of the processor employs [Pertubed Attention Guidance](https://huggingface.co/papers/2403.17377).
c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   zZPAGHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú#PAGHunyuanAttnProcessor2_0.__init__  s%   € Ü”qÐ8×9Ñ9ÜØlóð ð :rw   Nc           	     óf  • SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  p«pÍUR                  X«XÍ-  5      R                  SS5      nUR                  S5      u  pïUc  UR                  OUR                  u  n
nnUb=  UR                  UUU
5      nUR                  X¡R                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                   b  UR!                  U5      nUR"                  b  UR#                  U5      nUb#  U" UU5      nUR$                  (       d	  U" UU5      n[&        R(                  " UUUUSSS9nUR                  SS5      R+                  U
SUR                  U-  5      nUR-                  UR.                  5      nUR0                  S	   " U5      nUR0                  S   " U5      nU	S:X  a$  UR                  SS
5      R+                  U
WWW5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR-                  UR.                  5      nUR0                  S	   " U5      nUR0                  S   " U5      nU	S:X  a$  UR                  SS
5      R+                  U
WWW5      n[2        R4                  " Xï/5      nUR6                  (       a  X(-   nX!R8                  -  nU$ rÇ  )r@  rœ  rP   r÷   rí   rÂ  r  r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   r9   re   r1   rî   r°   r�   r^   r§   r  r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  r§  rœ  rÇ  rÈ  rò   rÉ  rÊ  rË  r  r  r«  r¼   r  r  r  r5   rö  s                          ru   rÛ   Ú#PAGHunyuanAttnProcessor2_0.__call__
  s  € õ 	1à ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð 0=×/BÑ/BÀ1Ó/EÑ,Ðð (=Ñ'DÐ×#Ò#ÐJ_×JeÑJeñ 	'ˆ
�O Qð Ñ%Ø!×8Ñ8¸ÈÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà—	‘	Ð+Ó,ˆà Ñ(Ø$5Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð Ñ'Ù$ UÐ,<Ó=ˆEØ×*×*Ù& sÐ,<Ó=�ô ×:Ò:Ø�3˜¨À3ÐRWñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐð �?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà ŸI™IÐ&7Ó8ÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐô Ÿ	š	Ð#4Ð"HÓIˆà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rB  rC  r¼  r»  rw   ru   rÑ  rÑ  ý  sq   † ñòð 6:Ø.2Ø$(Ø04ðkàðkð $ðkð  3ð	kð
 ,ðkð "ðkð .ðkð 
÷kð krw   rÑ  c                  óN   • \ rS rSrSrS r    S             SS jjrSrg)	ÚPAGCFGHunyuanAttnProcessor2_0ix  rÒ  c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   z]PAGCFGHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú&PAGCFGHunyuanAttnProcessor2_0.__init__  s%   € Ü”qÐ8×9Ñ9ÜØoóð ð :rw   Nc           	     ó˜  • SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  p«pÍUR                  X«XÍ-  5      R                  SS5      nUR                  S5      u  pïn[        R                  " Xï/5      nUc  UR                  OUR                  u  n
nnUb=  UR                  UUU
5      nUR                  X¡R                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR!                  U5      nUR#                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR$                  b  UR%                  U5      nUR&                  b  UR'                  U5      nUb#  U" UU5      nUR(                  (       d	  U" UU5      n[*        R,                  " UUUUSSS	9nUR                  SS5      R/                  U
SUR                  U-  5      nUR1                  UR2                  5      nUR4                  S
   " U5      nUR4                  S   " U5      nU	S:X  a$  UR                  SS5      R/                  U
WWW5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR#                  U5      nUR1                  UR2                  5      nUR4                  S
   " U5      nUR4                  S   " U5      nU	S:X  a$  UR                  SS5      R/                  U
WWW5      n[        R                  " UU/5      nUR6                  (       a  X(-   nX!R8                  -  nU$ )Nr   r\  r  r   rì   r+   r  Frî  r   rý   )r@  rœ  rP   r÷   rí   rÂ  r  r  r§   r  r  rI   r.   rV   rT   r!  rW   rX   rQ   rR   r9   re   r1   rî   r°   r�   r^   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  r§  rœ  rÇ  rÈ  rò   rÉ  rÊ  rË  r  r  r  r«  r¼   r  r  r  r5   rö  s                           ru   rÛ   Ú&PAGCFGHunyuanAttnProcessor2_0.__call__…  s"  € õ 	1à ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð FS×EXÑEXÐYZÓE[ÑBÐÐ1BÜ!ŸIšIÐ';Ð&OÓPÐð (=Ñ'DÐ×#Ò#ÐJ_×JeÑJeñ 	'ˆ
�O Qð Ñ%Ø!×8Ñ8¸ÈÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà—	‘	Ð+Ó,ˆà Ñ(Ø$5Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð Ñ'Ù$ UÐ,<Ó=ˆEØ×*×*Ù& sÐ,<Ó=�ô ×:Ò:Ø�3˜¨À3ÐRWñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐð �?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà ŸI™IÐ&7Ó8ÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐô Ÿ	š	Ð#4Ð6GÐ"HÓIˆà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rB  rC  r¼  r»  rw   ru   rØ  rØ  x  sq   † ñòð 6:Ø.2Ø$(Ø04ðlàðlð $ðlð  3ð	lð
 ,ðlð "ðlð .ðlð 
÷lð lrw   rØ  c                  óR   • \ rS rSrSrS r    S               SS jjrSrg)	ÚLuminaAttnProcessor2_0iô  zå
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the LuminaNextDiT model. It applies a s normalization layer and rotary embedding on query and key vector.
c                óD   • [        [        S5      (       d  [        S5      eg r$  rŒ  rŽ  s    ru   r3   ÚLuminaAttnProcessor2_0.__init__ú  r&  rw   Nc                ó  • SSK Jn  UR                  n	U	S:X  a3  UR                  u  p«pÍUR	                  X«XÍ-  5      R                  SS5      nUR                  u  p®nUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUR                  S   nUUR                  -  nUR                  nUU-  nUR                  b  UR                  U5      nUR                  b  UR                  U5      nUR	                  U
SUR                  U5      nUR	                  U
SUU5      nUR	                  U
SUU5      nUb  U" UUSS9nUb  U" UUSS9nUR                  U5      UR                  U5      nnUc  S nOGUb8  [        R                  " [        R                   " Xç5      5      UR"                  -  nOUR"                  nUR                  U-  nUS:¼  ah  UR%                  S5      R'                  SSSUS5      R)                  SS5      nUR%                  S5      R'                  SSSUS5      R)                  SS5      nUR+                  5       R	                  U
SSS5      nUR-                  SUR                  US5      nUR                  SS5      nUR                  SS5      nUR                  SS5      n[.        R0                  " UUUUUS	9nUR                  SS5      R                  U5      nU$ )
Nr   r\  r  r   r+   F)r¬  rì   )rï  rH   )r@  rœ  r÷   rí   rÂ  r  rV   rW   rX   rI   r�   rQ   rR   r°   r†  r‡  ÚlogrH   r  r€  r”  r6  ro  re   r1   )ri   r¦  râ   rÖ   r×   Úquery_rotary_embÚkey_rotary_embÚbase_sequence_lengthrœ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  r7   r5   rö  r�   rj   Úsoftmax_scaleÚn_reps                             ru   rÛ   ÚLuminaAttnProcessor2_0.__call__þ  sÝ  € õ 	1à"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMà)6×)<Ñ)<Ñ&ˆ
 Qð —	‘	˜-Ó(ˆØ�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—K‘K ‘Oˆ	Ø—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆØ—‘ˆð  Ñ(ˆð �;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCà—
‘
˜: r¨4¯:©:°xÓ@ˆà�h‰h�z 2 x°Ó:ˆØ—
‘
˜: r¨8°XÓ>ˆð Ñ'Ù$ UÐ,<ÀuÑMˆEØÑ%Ù" 3¨ÀÑGˆCà—X‘X˜e“_ c§f¡f¨U£mˆsˆð Ñ!Ø ‰Mà#Ñ/Ü $§	¢	¬$¯(ª(°?Ó*YÓ ZÐ]a×]gÑ]gÑ g‘à $§
¡
�ð —
‘
˜hÑ&ˆØ�A‹:Ø—-‘- Ó"×)Ñ)¨!¨Q°°5¸!Ó<×DÑDÀQÈÓJˆCØ—O‘O AÓ&×-Ñ-¨a°°A°u¸aÓ@×HÑHÈÈAÓNˆEð (×,Ñ,Ó.×3Ñ3°JÀÀ1ÀbÓIˆØ'×.Ñ.¨r°4·:±:¸ÐPRÓSˆà—‘  1Ó%ˆØ�m‰m˜A˜qÓ!ˆØ—‘  1Ó%ˆô ×6Ò6Ø�3˜¨¸}ñ
ˆð &×/Ñ/°°1Ó5×8Ñ8¸Ó?ˆàÐrw   r»  rB  )r¦  r   râ   r>  rÖ   r>  r×   r?  rã  r?  rä  r?  rå  r5  r9  r>  r¼  r»  rw   ru   rÞ  rÞ  ô  s}   † ñò
rð /3Ø04Ø.2Ø+/ðSàðSð $ðSð  ,ð	Sð
 ,ðSð .ðSð ,ðSð )ðSð 
÷Sð Srw   rÞ  c                  óH   • \ rS rSrSrS r   S           SS jjrSrg)	ÚFusedAttnProcessor2_0iT  u‹  
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). It uses
fused projection layers. For self-attention modules, all projection matrices (i.e., query, key, value) are fused.
For cross-attention modules, key and value projection matrices are fused.

> [!WARNING] > This API is currently ðŸ§ª experimental in nature and can change in future.
c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   z`FusedAttnProcessor2_0 requires at least PyTorch 2.0, to use it. Please upgrade PyTorch to > 2.0.rŒ  rŽ  s    ru   r3   ÚFusedAttnProcessor2_0.__init__]  s%   € Ü”qÐ8×9Ñ9ÜØróð ð :rw   Nc           	     óF  • [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  p¼pÞUR                  X¼XÞ-  5      R                  SS5      nUc  UR
                  OUR
                  u  p¿nUb<  UR                  XOU5      nUR                  X±R                  SUR
                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUc>  UR                  U5      nUR
                  S   S	-  n[        R                  " UUSS
9u  nnnOoUR                  (       a  UR                  U5      nUR!                  U5      nUR#                  U5      nUR
                  S   S-  n[        R                  " UUSS
9u  nnUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR$                  b  UR%                  U5      nUR&                  b  UR'                  U5      n[(        R*                  " UUUUSSS9nUR                  SS5      R-                  USUR                  U-  5      nUR/                  UR0                  5      nUR2                  S   " U5      nUR2                  S   " U5      nU
S:X  a$  UR                  SS5      R-                  UWWW5      nUR4                  (       a  X)-   nX!R6                  -  nU$ )Nr   rH   r¿  rÀ  r  r   r   r+   rì   r   r  Frî  rý   )rß   rÁ  r   rP   r÷   rí   rÂ  r  r  rI   r.   r%  r§   r(  rT   r!  rV   r'  rQ   rR   re   r1   rî   r°   r�   r^   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  rÅ  r‡  rÆ  rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r)  r*  r  r  r  rÎ  r5   rö  s                            ru   rÛ   ÚFusedAttnProcessor2_0.__call__c  sb  € ô ˆt‹9�q‹=˜FŸJ™J w°Ó5ÑAð #UÐÜ�g˜wÐ(;Ô<à ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð Ñ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà Ñ(Ø—+‘+˜mÓ,ˆCØŸ™ 2™¨!Ñ+ˆJÜ %§¢¨C°ÀÑ DÑˆE�3™à��Ø(,×(GÑ(GÐH]Ó(^Ð%Ø—I‘I˜mÓ,ˆEà—‘Ð1Ó2ˆBØŸ™ "™¨Ñ*ˆJÜŸš R¨¸Ñ<‰JˆC�à—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCô ×6Ò6Ø�3˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rÍ  rÎ  r¼  r»  rw   ru   rê  rê  T  sc   † ñòð 6:Ø.2Ø$(ðTàðTð $ðTð  3ð	Tð
 ,ðTð "ðTð 
÷Tð Trw   rê  c                  ó€   ^ • \ rS rSrSr       S             SU 4S jjjr  S         S	S jjrSrU =r$ )
rš   iº  a.  
Processor for implementing memory efficient attention using xFormers for the Custom Diffusion method.

Args:
train_kv (`bool`, defaults to `True`):
    Whether to newly train the key and value matrices corresponding to the text features.
train_q_out (`bool`, defaults to `True`):
    Whether to newly train query matrices corresponding to the latent image features.
hidden_size (`int`, *optional*, defaults to `None`):
    The hidden size of the attention layer.
cross_attention_dim (`int`, *optional*, defaults to `None`):
    The number of channels in the `encoder_hidden_states`.
out_bias (`bool`, defaults to `True`):
    Whether to include the bias parameter in `train_q_out`.
dropout (`float`, *optional*, defaults to 0.0):
    The dropout probability to use.
attention_op (`Callable`, *optional*, defaults to `None`):
    The base
    [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to use
    as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best operator.
c                óX  >• [         TU ]  5         Xl        X l        X0l        X@l        Xpl        U R                  (       aH  [        R                  " U=(       d    UUSS9U l	        [        R                  " U=(       d    UUSS9U l
        U R                  (       a“  [        R                  " X3SS9U l        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g rÒ  )r2   r3   r�   r‘   r’   r:   r“   r   rU   r”   rÔ  rÕ  r]   rÖ  r_   r`   )	ri   r�   r‘   r’   r:   rq   r?   r“   rt   s	           €ru   r3   Ú-CustomDiffusionXFormersAttnProcessor.__init__Ñ  sà   ø€ ô 	‰ÑÔØ ŒØ&Ôà&ÔØ#6Ô Ø(Ôð �=�=Ü)+¯ªÐ3F×3UÈ+ÐWbÐinÑ)oˆDÔ&Ü)+¯ªÐ3F×3UÈ+ÐWbÐinÑ)oˆDÔ&Ø××Ü)+¯ª°;ÐRWÑ)XˆDÔ&Ü+-¯=ª=¸Ó+<ˆDÔ(Ø×(Ñ(×/Ñ/´·	²	¸+ÐYaÑ0bÔcØ×(Ñ(×/Ñ/´·
²
¸7Ó0CÕDð	 rw   c           	     ó  • Uc  UR                   OUR                   u  pVnUR                  XFU5      nU R                  (       a?  U R                  U5      R	                  UR
                  R                  R                  5      nO>UR                  UR	                  UR
                  R                  R                  5      5      nUc  Sn	UnO$Sn	UR                  (       a  UR                  U5      nU R                  (       aÛ  U R                  UR	                  U R                  R                  R                  5      5      n
U R                  UR	                  U R                  R                  R                  5      5      nU
R	                  UR
                  R                  R                  5      n
UR	                  UR
                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   XÊ-  SU-
  U
R#                  5       -  -   n
XË-  SU-
  UR#                  5       -  -   nUR%                  U5      R'                  5       nUR%                  U
5      R'                  5       n
UR%                  U5      R'                  5       n[(        R*                  R-                  XŠX´U R.                  UR0                  S9nUR	                  UR                  5      nUR3                  U5      nU R                  (       a,  U R4                  S   " U5      nU R4                  S   " U5      nU$ UR6                  S   " U5      nUR6                  S   " U5      nU$ )NFTr   r  r4  r   )rí   r  r‘   rÕ  r°   rV   r±   r�   rT   r!  r�   r”   rÔ  rW   rX   r§   rÛ  rÜ  rù   r7  r‰   r«   r¬   r“   rH   rô   rÖ  r^   )ri   r¦  râ   rÖ   r×   rò   r«  r¼   r  rÝ  r  r  rÜ  s                ru   rÛ   Ú-CustomDiffusionXFormersAttnProcessor.__call__í  sù  € ð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð ×4Ñ4°^ÐV`Óaˆà××Ø×.Ñ.¨}Ó=×@Ñ@ÀÇÁ×AQÑAQ×AWÑAWÓX‰Eà—I‘I˜m×.Ñ.¨t¯y©y×/?Ñ/?×/EÑ/EÓFÓGˆEà Ñ(ØˆIØ$1Ñ!àˆIØ��Ø(,×(GÑ(GÐH]Ó(^Ð%à�=�=Ø×,Ñ,Ð-B×-EÑ-EÀd×F`ÑF`×FgÑFg×FmÑFmÓ-nÓoˆCØ×.Ñ.Ð/D×/GÑ/GÈ×HbÑHb×HiÑHi×HoÑHoÓ/pÓqˆEØ—&‘&˜Ÿ™×)Ñ)×/Ñ/Ó0ˆCØ—H‘H˜TŸY™Y×-Ñ-×3Ñ3Ó4‰Eà—)‘)Ð1Ó2ˆCØ—I‘IÐ3Ó4ˆEæÜ—_’_ SÓ)ˆFØ%¢a¨¨!¨ªQ hÑ/°#Ñ5ˆF’1�b�q�bš!�8ÑØ‘, ! f¡*°·
±
³Ñ!<Ñ<ˆCØ‘N a¨&¡j°E·L±L³NÑ%BÑBˆEà×&Ñ& uÓ-×8Ñ8Ó:ˆØ×$Ñ$ SÓ)×4Ñ4Ó6ˆØ×&Ñ& uÓ-×8Ñ8Ó:ˆä Ÿ™×?Ñ?Ø˜¸D×<MÑ<MÐUY×U_ÑU_ð @ð 
ˆð &×(Ñ(¨¯©Ó5ˆØ×.Ñ.¨}Ó=ˆà××à ×8Ñ8¸Ò;¸MÓJˆMà ×8Ñ8¸Ò;¸MÓJˆMð Ðð	 !ŸK™K¨šN¨=Ó9ˆMà ŸK™K¨šN¨=Ó9ˆMàÐrw   )	r“   r:   r’   r”   rÖ  rÕ  rÔ  r�   r‘   )TFNNTr  N)r�   r6  r‘   r6  r’   r5  r:   r5  rq   r6  r?   r  r“   r<  r=  râ  rã  rG  s   @ru   rš   rš   º  s¹   ø† ñð0 Ø!Ø"&Ø*.ØØØ(,ðEàðEð ðEð  ð	Eð
 (ðEð ðEð ðEð &÷Eð Eð@ 6:Ø.2ð>àð>ð $ð>ð  3ð	>ð
 ,ð>ð 
÷>ó >rw   rš   c                  óz   ^ • \ rS rSrSr      S           SU 4S jjjr  S         S	S jjrSrU =r$ )
r›   i.  uS  
Processor for implementing attention for the Custom Diffusion method using PyTorch 2.0â€™s memory-efficient scaled
dot-product attention.

Args:
    train_kv (`bool`, defaults to `True`):
        Whether to newly train the key and value matrices corresponding to the text features.
    train_q_out (`bool`, defaults to `True`):
        Whether to newly train query matrices corresponding to the latent image features.
    hidden_size (`int`, *optional*, defaults to `None`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`, *optional*, defaults to `None`):
        The number of channels in the `encoder_hidden_states`.
    out_bias (`bool`, defaults to `True`):
        Whether to include the bias parameter in `train_q_out`.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
c                óL  >• [         TU ]  5         Xl        X l        X0l        X@l        U R                  (       aH  [        R                  " U=(       d    UUSS9U l        [        R                  " U=(       d    UUSS9U l	        U R                  (       a“  [        R                  " X3SS9U l
        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g rÒ  rÓ  r×  s          €ru   r3   Ú(CustomDiffusionAttnProcessor2_0.__init__B  rÙ  rw   c           	     óä  • UR                   u  pVnUR                  XFU5      nU R                  (       a  U R                  U5      nOUR	                  U5      nUc  Sn	UnO$Sn	UR
                  (       a  UR                  U5      nU R                  (       aÛ  U R                  UR                  U R                  R                  R                  5      5      n
U R                  UR                  U R                  R                  R                  5      5      nU
R                  UR                  R                  R                  5      n
UR                  UR                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   XÊ-  SU-
  U
R#                  5       -  -   n
XË-  SU-
  UR#                  5       -  -   nUR                   S   nXÑR$                  -  nUR'                  USUR$                  U5      R)                  SS5      nU
R'                  USUR$                  U5      R)                  SS5      n
UR'                  USUR$                  U5      R)                  SS5      n[*        R,                  " XŠX´SSS9nUR)                  SS5      R/                  USUR$                  U-  5      nUR                  UR                  5      nU R                  (       a,  U R0                  S   " U5      nU R0                  S   " U5      nU$ UR2                  S   " U5      nUR2                  S   " U5      nU$ )	NFTr   r  r+   r   rî  r   )rí   r  r‘   rÕ  rV   rT   r!  r�   r”   r°   r±   r�   rÔ  rW   rX   r§   rÛ  rÜ  rI   rÂ  r  re   r1   rî   rÖ  r^   )ri   r¦  râ   rÖ   r×   rò   r«  r¼   r  rÝ  r  r  rÜ  r5   rö  s                  ru   rÛ   Ú(CustomDiffusionAttnProcessor2_0.__call__\  sú  € ð *7×)<Ñ)<Ñ&ˆ
 QØ×4Ñ4°^ÐV`ÓaˆØ××Ø×.Ñ.¨}Ó=‰Eà—I‘I˜mÓ,ˆEà Ñ(ØˆIØ$1Ñ!àˆIØ��Ø(,×(GÑ(GÐH]Ó(^Ð%à�=�=Ø×,Ñ,Ð-B×-EÑ-EÀd×F`ÑF`×FgÑFg×FmÑFmÓ-nÓoˆCØ×.Ñ.Ð/D×/GÑ/GÈ×HbÑHb×HiÑHi×HoÑHoÓ/pÓqˆEØ—&‘&˜Ÿ™×)Ñ)×/Ñ/Ó0ˆCØ—H‘H˜TŸY™Y×-Ñ-×3Ñ3Ó4‰Eð —)‘)Ð1Ó2ˆCØ—I‘IÐ3Ó4ˆEæÜ—_’_ SÓ)ˆFØ%¢a¨¨!¨ªQ hÑ/°#Ñ5ˆF’1�b�q�bš!�8ÑØ‘, ! f¡*°·
±
³Ñ!<Ñ<ˆCØ‘N a¨&¡j°E·L±L³NÑ%BÑBˆEà!×'Ñ'¨Ñ+ˆ	à§
¡
Ñ*ˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆô ×6Ò6Ø˜À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆà××à ×8Ñ8¸Ò;¸MÓJˆMà ×8Ñ8¸Ò;¸MÓJˆMð Ðð	 !ŸK™K¨šN¨=Ó9ˆMà ŸK™K¨šN¨=Ó9ˆMàÐrw   rß  rà  rá  r=  râ  rã  rG  s   @ru   r›   r›   .  s±   ø† ñð* Ø Ø"&Ø*.ØØðEàðEð ðEð  ð	Eð
 (ðEð ðEð ÷Eð Eð< 6:Ø.2ðAàðAð $ðAð  3ð	Að
 ,ðAð 
÷Aó Arw   r›   c                  óF   • \ rS rSrSrSS jr  S         S	S jjrSrg)
rÄ   i   a  
Processor for implementing sliced attention.

Args:
    slice_size (`int`, *optional*):
        The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and
        `attention_head_dim` must be a multiple of the `slice_size`.
c                ó   • Xl         g r;  ©rÅ   ©ri   rÅ   s     ru   r3   ÚSlicedAttnProcessor.__init__ª  ó   € Ø$�rw   Nc                ód  • UnUR                   nUS:X  a3  UR                  u  pxpšUR                  XxXš-  5      R                  SS5      nUc  UR                  OUR                  u  p{nUR	                  XKU5      nUR
                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  S   nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  u  nnn[        R                  " UUXáR                  -  4UR                  UR                   S9n[#        US-
  U R$                  -  S-   5       Hj  nUU R$                  -  nUS-   U R$                  -  nUUU nUUU nUb  UUU OS nUR'                  UUU5      n[        R(                  " UUUU 5      nUUUU& Ml     UR+                  U5      nUR,                  S   " U5      nUR,                  S   " U5      nUS:X  a$  UR                  SS5      R/                  UWW	W
5      nUR0                  (       a  X%-   nX!R2                  -  nU$ )Nr  r   r   r+   r�   r   rý   )r÷   rí   rÂ  r  r  r.   rV   rù   rT   r!  rW   rX   r§   r  rI   rŽ   r�   r£  rÅ   r  rÃ  rô   r^   rî   r>   r=   )ri   r¦  râ   rÖ   r×   rÇ  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r-   r  r  Úbatch_size_attentionrq  ÚiÚ	start_idxÚend_idxÚquery_sliceÚ	key_sliceÚattn_mask_sliceÚ
attn_slices                             ru   rÛ   ÚSlicedAttnProcessor.__call__­  s¹  € ð !ˆà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
 Qð ×4Ñ4°^ÐV`Óaˆà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆØ�k‰k˜"‰oˆØ×&Ñ& uÓ-ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆØ×$Ñ$ SÓ)ˆØ×&Ñ& uÓ-ˆà05·±Ñ-Ð˜l¨AÜŸšØ! <°¿
¹
Ñ1BÐCÈEÏLÉLÐ`e×`kÑ`kñ
ˆô Ð,¨qÑ0°T·_±_ÑDÀqÑHÖIˆAØ˜DŸO™OÑ+ˆIØ˜1‘u §¡Ñ/ˆGà 	¨'Ð2ˆKØ˜I gÐ.ˆIØCQÑC]˜n¨Y°wÑ?ÐcgˆOà×2Ñ2°;À	È?Ó[ˆJäŸš :¨u°Y¸wÐ/GÓHˆJà/9ˆM˜) GÒ,ñ Jð ×.Ñ.¨}Ó=ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   rû  )rÅ   r4  r=  râ  r¼  r»  rw   ru   rÄ   rÄ      sU   † ñô%ð 6:Ø.2ðGàðGð $ðGð  3ð	Gð
 ,ðGð 
÷Gð Grw   rÄ   c                  óH   • \ rS rSrSrS r   S           SS jjrSrg)	rž   i÷  aP  
Processor for implementing sliced attention with extra learnable key and value matrices for the text encoder.

Args:
    slice_size (`int`, *optional*):
        The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and
        `attention_head_dim` must be a multiple of the `slice_size`.
c                ó   • Xl         g r;  rû  rü  s     ru   r3   Ú#SlicedAttnAddedKVProcessor.__init__  rþ  rw   Nc                ó(  • UnUR                   b  UR                  X%5      nUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pxn	UR	                  XHU5      nUc  UnO"UR
                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      n
U
R                  S   nUR                  U
5      n
UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  (       dq  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                   " XÎ/SS9n[        R                   " Xß/SS9nOUnUnU
R                  u  nnn	[        R"                  " UUX±R$                  -  4U
R&                  U
R(                  S9n[+        US-
  U R,                  -  S-   5       Hj  nUU R,                  -  nUS-   U R,                  -  nU
UU nUUU nUb  UUU OS nUR/                  UUU5      n[        R0                  " UUUU 5      nUUUU& Ml     UR3                  U5      nUR4                  S   " U5      nUR4                  S   " U5      nUR                  SS5      R7                  UR                  5      nX&-   nU$ )Nr   r   r+   r   r   r�   rý   )rP   rÂ  rí   r  r  rT   r!  r.   rV   rù   rZ   r[   rL   rW   rX   r§   r  r  rI   rŽ   r�   r£  rÅ   r  rÃ  rô   r^   rî   )ri   r¦  râ   rÖ   r×   rÄ  rÇ  rò   r«  r¼   r  r-   ræ  rç  r  r  r   rq  r  r  r  r  r  r  r  s                            ru   rÛ   Ú#SlicedAttnAddedKVProcessor.__call__  s  € ð !ˆà×ÑÑ(Ø ×-Ñ-¨mÓBˆMà%×*Ñ*¨=×+>Ñ+>¸qÑ+AÀ=×CVÑCVÐWXÑCYÐ[]Ó^×hÑhÐijÐlmÓnˆà)6×)<Ñ)<Ñ&ˆ
 Qà×4Ñ4°^ÐV`Óaˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!àŸ™¨×(?Ñ(?ÀÀ1Ó(EÓF×PÑPÐQRÐTUÓVˆà—	‘	˜-Ó(ˆØ�k‰k˜"‰oˆØ×&Ñ& uÓ-ˆà)-¯©Ð9NÓ)OÐ&Ø+/¯?©?Ð;PÓ+QÐ(à)-×)?Ñ)?Ð@^Ó)_Ð&Ø+/×+AÑ+AÐBbÓ+cÐ(à×(×(Ø—)‘)˜MÓ*ˆCØ—I‘I˜mÓ,ˆEØ×(Ñ(¨Ó-ˆCØ×*Ñ*¨5Ó1ˆEÜ—)’)Ð;ÐAÀqÑIˆCÜ—I’IÐ?ÐGÈQÑO‰Eà0ˆCØ4ˆEà05·±Ñ-Ð˜l¨AÜŸšØ! <°¿
¹
Ñ1BÐCÈEÏLÉLÐ`e×`kÑ`kñ
ˆô Ð,¨qÑ0°T·_±_ÑDÀqÑHÖIˆAØ˜DŸO™OÑ+ˆIØ˜1‘u §¡Ñ/ˆGà 	¨'Ð2ˆKØ˜I gÐ.ˆIØCQÑC]˜n¨Y°wÑ?ÐcgˆOà×2Ñ2°;À	È?Ó[ˆJäŸš :¨u°Y¸wÐ/GÓHˆJà/9ˆM˜) GÒ,ñ Jð ×.Ñ.¨}Ó=ˆð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà%×/Ñ/°°BÓ7×?Ñ?ÀÇÁÓOˆØ%Ñ0ˆàÐrw   rû  rÍ  )r¦  z'Attention'râ   r>  rÖ   r?  r×   r?  rÄ  r?  r9  r>  r¼  r»  rw   ru   rž   rž   ÷  sc   † ñò%ð 6:Ø.2Ø$(ðLàðLð $ðLð  3ð	Lð
 ,ðLð "ðLð 
÷Lð Lrw   rž   c                  óB   ^ • \ rS rSrSr    SU 4S jjrSS jrSrU =r$ )rO   iS  aU  
Spatially conditioned normalization as defined in https://huggingface.co/papers/2209.09002.

Args:
    f_channels (`int`):
        The number of channels for input to group normalization layer, and output of the spatial norm layer.
    zq_channels (`int`):
        The number of channels for the quantized vector as described in the paper.
c                óÊ   >• [         TU ]  5         [        R                  " USSSS9U l        [        R
                  " X!SSSS9U l        [        R
                  " X!SSSS9U l        g )Nr3  ç�íµ ÷Æ°>Tr   r   r   )rQ  Ústrider  )r2   r3   r   rN   Ú
norm_layerrL  Úconv_yÚconv_b)ri   r!   r"   rt   s      €ru   r3   ÚSpatialNorm.__init__^  sV   ø€ ô
 	‰ÑÔÜŸ,š,°JÈ2ÐSWÐ`dÑeˆŒÜ—i’i ÀQÈqÐZ[Ñ\ˆŒÜ—i’i ÀQÈqÐZ[Ñ\ˆ�rw   c                óº   • UR                   SS  n[        R                  " X#SS9nU R                  U5      nX@R	                  U5      -  U R                  U5      -   nU$ )Nrý   Únearest)r¢  rn  )rí   re   Úinterpolater  r  r  )ri   ÚfÚzqÚf_sizeÚnorm_fÚnew_fs         ru   ré   ÚSpatialNorm.forwardh  sQ   € Ø—‘˜˜�ˆÜ�]Š]˜2°Ñ;ˆØ—‘ Ó#ˆØŸ™ R›Ñ(¨4¯;©;°r«?Ñ:ˆØˆrw   )r  r  r  )r!   r4  r"   r4  )r  r>  r  r>  r9  r>  )	rá   r@  rA  rB  rC  r3   ré   rE  rF  rG  s   @ru   rO   rO   S  s,   ø† ñð]àð]ð ÷]÷ò rw   rO   c                  ób   ^ • \ rS rSrSrSU 4S jjr     S             SS jjrSrU =r$ )	r    ip  a»  
Attention processor for Multiple IP-Adapters.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or list[`float`], defaults to 1.0):
        the weight scale of image prompt.
c                ót  >• [         TU ]  5         Xl        X l        [	        U[
        [        45      (       d  U/nX0l        [	        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l
        [        R                  " [        [        U5      5       Vs/ s H  n[        R                  " X!SS9PM     sn5      U l        [        R                  " [        [        U5      5       Vs/ s H  n[        R                  " X!SS9PM     sn5      U l        g s  snf s  snf ©NúJ`scale` should be a list of integers with the same length as `num_tokens`.Fr0   )r2   r3   r’   r:   r˜   ÚtupleÚlistr–   rß   rM   rH   r   r]   r£  rU   r—   Úto_v_ip©ri   r’   r:   r–   rH   r¼   rt   s         €ru   r3   ÚIPAdapterAttnProcessor.__init__  sû   ø€ Ü‰ÑÔà&ÔØ#6Ô ä˜*¤u¬d m×4Ñ4Ø$˜ˆJØ$Œä˜%¤×&Ñ&Ø�Gœc *›oÑ-ˆEÜˆu‹:œ˜Z›Ó(ÜÐiÓjÐjØŒ
ä—}’}ÜNSÔTWÐXbÓTcÔNdÓeÒNdÈŒR�YŠYÐ*¸eÔDÑNdÑeó
ˆŒô —}’}ÜNSÔTWÐXbÓTcÔNdÓeÒNdÈŒR�YŠYÐ*¸eÔDÑNdÑeó
ˆ�ùò fùò fs   Â3D0ÄD5c           
     ó  • UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /p“UR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pÞnnUR                  XÞUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUR                  UUU5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR!                  U5      nUR!                  U5      nUR!                  U5      nUR#                  UUU5      n[$        R&                  " UU5      nUR)                  U5      nUGbµ  [        U[*        5      (       d  [+        UR-                  S5      5      n[/        U5      [/        U R0                  5      s=:X  a  [/        W	5      :X  d<  O  [3        S
[/        U5       S[/        U R0                  5       S[/        W	5       S35      e[5        [7        XpR0                  U	5      5       Hô  u  nu  nnnUc  M  [        U[$        R8                  5      (       a  UR                  S:w  a  [3        S5      eUR                  S   UR                  S   :w  a.  [3        SUR                  S    SUR                  S    SU 35      e[        U[*        5      (       d  M®  [/        U5      UR                  S   :X  a  MÌ  [3        SUR                  S    S[/        U5       SU 35      e   OS /[/        U R0                  5      -  n[7        W	U R0                  U R:                  U R<                  U5       GH  u  nnnnnSn[        U[*        5      (       a  [?        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGbD  [        U[*        5      (       d  U/UR                  S   -  nUR                  S   n[A        U5       Hû  nU" US S 2US S 2S S 24   5      n U" US S 2US S 2S S 24   5      n!UR!                  U 5      n UR!                  U!5      n!UR#                  UU S 5      n"[$        R&                  " U"U!5      n#UR)                  U#5      n#[B        RD                  " US S 2US S 2S S 24   UU#R                  S   U#R                  S	   5      n$U$RG                  URH                  URJ                  S9n$X&U   U#U$-  -  -   nMý     GM”  U" U5      n U" U5      n!UR!                  U 5      n UR!                  U!5      n!UR#                  UU S 5      n"[$        R&                  " U"U!5      nUR)                  U5      nX&U-  -   nGM     URL                  S   " U5      nURL                  S   " U5      nUS:X  a$  UR                  SS5      RO                  UWWW5      nURP                  (       a  X(-   nX!RR                  -  nU$ )NúÜYou have passed a tensor as `encoder_hidden_states`. This is deprecated and will be removed in a future release. Please make sure to update your script to pass `encoder_hidden_states` as a tuple to suppress this warning.ú!encoder_hidden_states not a tuplerÀ  F©Ústandard_warnr   r   r  r   ú"Length of ip_adapter_masks array (ú)) must match length of self.scale array (ú") and number of ip_hidden_states (Ú)ú³Each element of the ip_adapter_masks array should be a tensor with shape [1, num_images_for_ip_adapter, height, width]. Please use `IPAdapterMaskProcessor` to preprocess your maskúNumber of masks (ú&) does not match number of ip images (ú) at index ú#) does not match number of scales (c              3  ó*   #   • U  H	  oS :H  v •  M     g7f©r   Nr»  ©r˜  Úss     ru   r™  Ú2IPAdapterAttnProcessor.__call__.<locals>.<genexpr>ü  ó   é € Ð-¢u !˜A–v¢uùó   ‚Trü   r+   rý   )*r˜   r#  r   rí   r–   rP   r÷   rÂ  r  r  r.   rV   rT   r!  rW   rX   rù   r  r§   rÃ  rô   r$  r  rß   rH   rM   Ú	enumerateÚzipÚTensorr—   r%  rœ  r£  r   Ú
downsampler°   r�   rŽ   r^   rî   r>   r=   )%ri   r¦  râ   rÖ   r×   rÄ  rH   rÓ   rÇ  rÔ   rÆ  Úend_posrÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  r
  Úindexr°  Úip_stateÚcurrent_ip_hidden_statesr—   r%  ÚskipÚcurrent_num_imagesr  Úip_keyÚip_valueÚip_attention_probsÚ_current_ip_hidden_statesÚmask_downsamples%                                        ru   rÛ   ÚIPAdapterAttnProcessor.__call__–  s•  € ð !ˆð !Ñ,ÜÐ/´×7Ñ7Ø:OÑ7Ð%Ð'7ðCð $ô Ð=¸wÐH[ÐkpÒqØ/×5Ñ5°aÑ8¸4¿?¹?È1Ñ;MÑM�à)ª!¨X¨g¨X²q¨.Ñ9Ø*ª1¨g©hº¨>Ñ:Ð;ð (8ð
 ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ ¨Ø)×.Ñ.¨zÀFÈUÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
�O Qð ×4Ñ4°^À_ÐV`Óaˆà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà×&Ñ& uÓ-ˆØ×$Ñ$ SÓ)ˆØ×&Ñ& uÓ-ˆà×3Ñ3°E¸3ÀÓOˆÜŸ	š	 /°5Ó9ˆØ×.Ñ.¨}Ó=ˆàÒ'ÜÐ.´×5Ñ5ä#'Ð(8×(BÑ(BÀ1Ó(EÓ#FÐ ÜÐ(Ó)¬S°·±«_ÕUÄÐDTÓ@UÕUÜ Ø8¼Ð=MÓ9NÐ8Oð P3Ü36°t·z±z³?Ð2Cð DÜÐ,Ó-Ð.¨að1óð ô 7@ÄÐDT×V`ÑV`ÐbrÓ@sÖ6tÑ2�EÑ2˜D %¨Ø‘|Ù Ü% d¬E¯L©L×9Ñ9¸T¿Y¹YÈ!»^Ü(ð[óð ð
 —z‘z !‘}¨¯©°qÑ(9Ó9Ü(Ø/°·
±
¸1±¨ð ?4Ø4<·N±NÀ1Ñ4EÐ3FÀkÐRWÐQXðZóð ô " %¬×.Ó.´s¸5³zÀTÇZÁZÐPQÁ]Õ7RÜ(Ø/°·
±
¸1±¨ð ?1Ü14°U³°¸KÈÀwðPóð ò 7uð( !%˜v¬¨D¯J©J«Ñ7Ðô HKØ˜dŸj™j¨$¯,©,¸¿¹ÐFV÷H
ÑCÐ$ e¨W°g¸tð ˆDÜ˜%¤×&Ñ&ÜÑ-¡uÓ-×-Ñ-Ø�DøØ˜!“Ø�ß�4ØÒ#Ü% e¬T×2Ñ2Ø!& ¨$¯*©*°Q©-Ñ 7˜à)-¯©°A©Ð&Ü"Ð#5Ö6˜Ù!(Ð)AÂ!ÀQÊÊ1À*Ñ)MÓ!N˜Ù#*Ð+CÂAÀqÊ!ÊQÀJÑ+OÓ#P˜à!%×!7Ñ!7¸Ó!?˜Ø#'×#9Ñ#9¸(Ó#C˜à-1×-FÑ-FÀuÈfÐVZÓ-[Ð*Ü49·I²IÐ>PÐRZÓ4[Ð1Ø48×4JÑ4JÐKdÓ4eÐ1ä*@×*KÒ*KØ ¢ A¢qª! Ñ,Ø&Ø5×;Ñ;¸AÑ>Ø5×;Ñ;¸AÑ>ó	+˜ð +:×*<Ñ*<À5Ç;Á;ÐW\×WcÑWcÐ*<Ð*d˜à(5¸a¹ÐD]Ð`oÑDoÑ8pÑ(pšô) 7ñ, %Ð%=Ó>�FÙ&Ð'?Ó@�Hà!×3Ñ3°FÓ;�FØ#×5Ñ5°hÓ?�Hà)-×)BÑ)BÀ5È&ÐRVÓ)WÐ&Ü/4¯yªyÐ9KÈXÓ/VÐ,Ø/3×/EÑ/EÐF^Ó/_Ð,à$1Ð<TÑ4TÑ$T“Mñ_H
ðd Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   ©r:   r’   r–   rH   r—   r%  ©N©r  r   ©NNNr   N©r¦  r   râ   r>  rÖ   r?  r×   r?  rÄ  r?  rH   r  rÓ   r?  rã  rG  s   @ru   r    r    p  su   ø† ñ÷
ð6 6:Ø.2Ø$(ØØ04ð_àð_ð $ð_ð  3ð	_ð
 ,ð_ð "ð_ð ð_ð .÷_ó _rw   r    c                  ób   ^ • \ rS rSrSrSU 4S jjr     S             SS jjrSrU =r$ )	r¡   i8  aÁ  
Attention processor for IP-Adapter for PyTorch 2.0.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or `list[float]`, defaults to 1.0):
        the weight scale of image prompt.
c                óâ  >• [         TU ]  5         [        [        S5      (       d"  [	        U R
                  R                   S35      eXl        X l        [        U[        [        45      (       d  U/nX0l        [        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l        [         R"                  " [%        [        U5      5       Vs/ s H  n[         R&                  " X!SS9PM     sn5      U l        [         R"                  " [%        [        U5      5       Vs/ s H  n[         R&                  " X!SS9PM     sn5      U l        g s  snf s  snf )Nr1   z@ requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r"  Fr0   )r2   r3   rd   re   r�  rt   rá   r’   r:   r˜   r#  r$  r–   rß   rM   rH   r   r]   r£  rU   r—   r%  r&  s         €ru   r3   Ú"IPAdapterAttnProcessor2_0.__init__G  s.  ø€ Ü‰ÑÔä”qÐ8×9Ñ9ÜØ—>‘>×*Ñ*Ð+Ð+kÐlóð ð 'ÔØ#6Ô ä˜*¤u¬d m×4Ñ4Ø$˜ˆJØ$Œä˜%¤×&Ñ&Ø�Gœc *›oÑ-ˆEÜˆu‹:œ˜Z›Ó(ÜÐiÓjÐjØŒ
ä—}’}ÜNSÔTWÐXbÓTcÔNdÓeÒNdÈŒR�YŠYÐ*¸eÔDÑNdÑeó
ˆŒô —}’}ÜNSÔTWÐXbÓTcÔNdÓeÒNdÈŒR�YŠYÐ*¸eÔDÑNdÑeó
ˆ�ùò fùò fs   Ã*E'Ä;E,c                ó>  • UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /p“UR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pÞnnUR                  XÞUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUb=  UR                  UUU5      nUR                  XÑR                  S
UR                  S
   5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR                  S
   nUUR                  -  nUR                  US
UR                  U5      R                  SS	5      nUR                  US
UR                  U5      R                  SS	5      nUR                  US
UR                  U5      R                  SS	5      n["        R$                  " UUUUSSS9nUR                  SS	5      R'                  US
UR                  U-  5      nUR)                  UR*                  5      nUGbµ  [        U[,        5      (       d  [-        UR/                  S5      5      n[1        U5      [1        U R2                  5      s=:X  a  [1        W	5      :X  d<  O  [5        S[1        U5       S[1        U R2                  5       S[1        W	5       S35      e[7        [9        XpR2                  U	5      5       Hô  u  nu  nnnUc  M  [        U[:        R<                  5      (       a  UR                  S:w  a  [5        S5      eUR                  S   UR                  S   :w  a.  [5        SUR                  S    SUR                  S    SU 35      e[        U[,        5      (       d  M®  [1        U5      UR                  S   :X  a  MÌ  [5        SUR                  S    S[1        U5       SU 35      e   OS /[1        U R2                  5      -  n[9        W	U R2                  U R>                  U R@                  U5       GHÐ  u  nnnnnSn[        U[,        5      (       a  [C        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGb©  [        U[,        5      (       d  U/UR                  S   -  nUR                  S   n[E        U5       GH_  n U" US S 2U S S 2S S 24   5      n!U" US S 2U S S 2S S 24   5      n"U!R                  US
UR                  U5      R                  SS	5      n!U"R                  US
UR                  U5      R                  SS	5      n"["        R$                  " UU!U"S SSS9n#U#R                  SS	5      R'                  US
UR                  U-  5      n#U#R)                  UR*                  5      n#[F        RH                  " US S 2U S S 2S S 24   UU#R                  S   U#R                  S	   5      n$U$R)                  UR*                  URJ                  S9n$X&U    U#U$-  -  -   nGMb     GMù  U" U5      n!U" U5      n"U!R                  US
UR                  U5      R                  SS	5      n!U"R                  US
UR                  U5      R                  SS	5      n"["        R$                  " UU!U"S SSS9nUR                  SS	5      R'                  US
UR                  U-  5      nUR)                  UR*                  5      nX&U-  -   nGMÓ     URL                  S   " U5      nURL                  S   " U5      nUS:X  a$  UR                  S
S5      R'                  UWWW5      nURN                  (       a  X(-   nX!RP                  -  nU$ )Nr)  r*  rÀ  Fr+  r   r   r  r   r+   r  rî  r-  r.  r/  r0  r1  r2  r3  r4  r5  c              3  ó*   #   • U  H	  oS :H  v •  M     g7fr7  r»  r8  s     ru   r™  Ú5IPAdapterAttnProcessor2_0.__call__.<locals>.<genexpr>×  r;  r<  Trü   rý   ))r˜   r#  r   rí   r–   rP   r÷   rÂ  r  r  rI   r.   rV   rT   r!  rW   rX   re   r1   rî   r°   r�   r$  r  rß   rH   rM   r=  r>  r§   r?  r—   r%  rœ  r£  r   r@  rŽ   r^   r>   r=   )%ri   r¦  râ   rÖ   r×   rÄ  rH   rÓ   rÇ  rÔ   rÆ  rA  rÈ  rò   rÉ  rÊ  rË  r«  r¼   r  r  r  r5   rö  rB  r°  rC  rD  r—   r%  rE  rF  r  rG  rH  rJ  rK  s%                                        ru   rÛ   Ú"IPAdapterAttnProcessor2_0.__call__c  s
  € ð !ˆð !Ñ,ÜÐ/´×7Ñ7Ø:OÑ7Ð%Ð'7ðCð $ô Ð=¸wÐH[ÐkpÒqØ/×5Ñ5°aÑ8¸4¿?¹?È1Ñ;MÑM�à)ª!¨X¨g¨X²q¨.Ñ9Ø*ª1¨g©hº¨>Ñ:Ð;ð (8ð
 ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ ¨Ø)×.Ñ.¨zÀFÈUÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
�O Qð Ñ%Ø!×8Ñ8¸ÈÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆô ×6Ò6Ø�3˜¨À3ÐRWñ
ˆð &×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆàÒ'ÜÐ.´×5Ñ5ä#'Ð(8×(BÑ(BÀ1Ó(EÓ#FÐ ÜÐ(Ó)¬S°·±«_ÕUÄÐDTÓ@UÕUÜ Ø8¼Ð=MÓ9NÐ8Oð P3Ü36°t·z±z³?Ð2Cð DÜÐ,Ó-Ð.¨að1óð ô 7@ÄÐDT×V`ÑV`ÐbrÓ@sÖ6tÑ2�EÑ2˜D %¨Ø‘|Ù Ü% d¬E¯L©L×9Ñ9¸T¿Y¹YÈ!»^Ü(ð[óð ð
 —z‘z !‘}¨¯©°qÑ(9Ó9Ü(Ø/°·
±
¸1±¨ð ?4Ø4<·N±NÀ1Ñ4EÐ3FÀkÐRWÐQXðZóð ô " %¬×.Ó.´s¸5³zÀTÇZÁZÐPQÁ]Õ7RÜ(Ø/°·
±
¸1±¨ð ?1Ü14°U³°¸KÈÀwðPóð ò 7uð( !%˜v¬¨D¯J©J«Ñ7Ðô HKØ˜dŸj™j¨$¯,©,¸¿¹ÐFV÷H
ÑCÐ$ e¨W°g¸tð ˆDÜ˜%¤×&Ñ&ÜÑ-¡uÓ-×-Ñ-Ø�DøØ˜!“Ø�ß�4ØÒ#Ü% e¬T×2Ñ2Ø!& ¨$¯*©*°Q©-Ñ 7˜à)-¯©°A©Ð&Ü"Ð#5×6˜Ù!(Ð)AÂ!ÀQÊÊ1À*Ñ)MÓ!N˜Ù#*Ð+CÂAÀqÊ!ÊQÀJÑ+OÓ#P˜à!'§¡¨Z¸¸T¿Z¹ZÈÓ!R×!\Ñ!\Ð]^Ð`aÓ!b˜Ø#+§=¡=°¸RÀÇÁÈXÓ#V×#`Ñ#`ÐabÐdeÓ#f˜ô 56×4RÒ4RØ! 6¨8¸tÈsÐ^cñ5Ð1ð 5N×4WÑ4WÐXYÐ[\Ó4]×4eÑ4eØ&¨¨D¯J©J¸Ñ,Aó5Ð1ð 5N×4PÑ4PÐQV×Q\ÑQ\Ó4]Ð1ä*@×*KÒ*KØ ¢ A¢qª! Ñ,Ø&Ø5×;Ñ;¸AÑ>Ø5×;Ñ;¸AÑ>ó	+˜ð +:×*<Ñ*<À5Ç;Á;ÐW\×WcÑWcÐ*<Ð*d˜Ø(5¸a¹ÐD]Ð`oÑDoÑ8pÑ(p›ô5 7ñ8 %Ð%=Ó>�FÙ&Ð'?Ó@�Hà#Ÿ[™[¨°R¸¿¹ÀXÓN×XÑXÐYZÐ\]Ó^�FØ'Ÿ}™}¨Z¸¸T¿Z¹ZÈÓR×\Ñ\Ð]^Ð`aÓb�Hô 01×/MÒ/MØ˜v x¸4È3ÐZ_ñ0Ð,ð 0H×/QÑ/QÐRSÐUVÓ/W×/_Ñ/_Ø" B¨¯
©
°XÑ(=ó0Ð,ð 0H×/JÑ/JÈ5Ï;É;Ó/WÐ,à$1Ð<TÑ4TÑ$T“MñyH
ð~ Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   rM  rN  rP  rQ  rã  rG  s   @ru   r¡   r¡   8  sv   ø† ñ÷
ð@ 6:Ø.2Ø$(ØØ04ðzàðzð $ðzð  3ð	zð
 ,ðzð "ðzð ðzð .÷zó zrw   r¡   c                  óp   ^ • \ rS rSrSr    S SU 4S jjjr     S             S	S jjrSrU =r$ )
r¢   i   a  
Attention processor for IP-Adapter using xFormers.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `tuple[int]` or `list[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or `list[float]`, defaults to 1.0):
        the weight scale of image prompt.
    attention_op (`Callable`, *optional*, defaults to `None`):
        The base
        [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
        use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
        operator.
c                ó¨  >• [         TU ]  5         Xl        X l        XPl        [        U[        [        45      (       d  U/nX0l        [        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l        [        R                  " [        [        U5      5       Vs/ s H"  n[        R                  " U=(       d    UUSS9PM$     sn5      U l        [        R                  " [        [        U5      5       Vs/ s H"  n[        R                  " U=(       d    UUSS9PM$     sn5      U l        g s  snf s  snf r!  )r2   r3   r’   r:   r“   r˜   r#  r$  r–   rß   rM   rH   r   r]   r£  rU   r—   r%  )ri   r’   r:   r–   rH   r“   r¼   rt   s          €ru   r3   Ú'IPAdapterXFormersAttnProcessor.__init__4  s  ø€ ô 	‰ÑÔà&ÔØ#6Ô Ø(Ôä˜*¤u¬d m×4Ñ4Ø$˜ˆJØ$Œä˜%¤×&Ñ&Ø�Gœc *›oÑ-ˆEÜˆu‹:œ˜Z›Ó(ÜÐiÓjÐjØŒ
ä—}’}Ü]bÔcfÐgqÓcrÔ]sÓtÒ]sÐXYŒR�YŠYÐ*×9¨k¸;ÈUÔSÑ]sÑtó
ˆŒô —}’}Ü]bÔcfÐgqÓcrÔ]sÓtÒ]sÐXYŒR�YŠYÐ*×9¨k¸;ÈUÔSÑ]sÑtó
ˆ�ùò uùò us   Â9)E
Ä)Ec           
     ó  • UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /p“UR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pÞnnUR                  XÞUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUb6  UR                  UUU5      nUR                  u  nnnUR                  S
US
5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR#                  U5      R%                  5       nUR#                  U5      R%                  5       nUR#                  U5      R%                  5       n[&        R(                  R+                  UUUX@R,                  S9nUR/                  UR0                  5      nUR3                  U5      nW	(       Gaš  UGbµ  [        U[4        5      (       d  [5        UR7                  S5      5      n[9        U5      [9        U R:                  5      s=:X  a  [9        U	5      :X  d<  O  [=        S[9        U5       S[9        U R:                  5       S[9        U	5       S35      e[?        [A        XpR:                  U	5      5       Hô  u  nu  nnnUc  M  [        U[B        RD                  5      (       a  UR                  S:w  a  [=        S5      eUR                  S   UR                  S   :w  a.  [=        SUR                  S    SUR                  S    SU 35      e[        U[4        5      (       d  M®  [9        U5      UR                  S   :X  a  MÌ  [=        SUR                  S    S[9        U5       SU 35      e   OS /[9        U R:                  5      -  n[A        X�R:                  U RF                  U RH                  U5       GH—  u  nnnnnSn[        U[4        5      (       a  [K        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGbœ  UR/                  [B        RL                  5      n[        U[4        5      (       d  U/UR                  S   -  nUR                  S   n[O        U5       GH3  nU" US S 2US S 2S S 24   5      n U" US S 2US S 2S S 24   5      n!UR#                  U 5      R%                  5       n UR#                  U!5      R%                  5       n![&        R(                  R+                  UU U!U R,                  S9n"U"R/                  UR0                  5      n"UR3                  U"5      n"[P        RR                  " US S 2US S 2S S 24   UU"R                  S   U"R                  S	   5      n#U#R/                  UR0                  URT                  S9n#X&U   U"U#-  -  -   nGM6     GMì  U" U5      n U" U5      n!UR#                  U 5      R%                  5       n UR#                  U!5      R%                  5       n![&        R(                  R+                  UU U!U R,                  S9nUR/                  UR0                  5      nUR3                  U5      nX&U-  -   nGMš     URV                  S   " U5      nURV                  S   " U5      nUS:X  a$  UR                  S
S5      RY                  UWWW5      nURZ                  (       a  X(-   nX!R\                  -  nU$ )Nr)  r*  rÀ  Fr+  r   r   r  r   r+   )r5  r6  r-  r.  r/  r0  r1  r2  r3  r4  r5  c              3  ó*   #   • U  H	  oS :H  v •  M     g7fr7  r»  r8  s     ru   r™  Ú:IPAdapterXFormersAttnProcessor.__call__.<locals>.<genexpr>È  s   é € Ð1ª5 a ž6ª5ùr<  T)r6  rü   rý   )/r˜   r#  r   rí   r–   rP   r÷   rÂ  r  r  ro  r.   rV   rT   r!  rW   rX   rù   r7  r‰   r«   r¬   r“   r°   r�   rô   r$  r  rß   rH   rM   r=  r>  r§   r?  r—   r%  rœ  r‚  r£  r   r@  rŽ   r^   rî   r>   r=   )$ri   r¦  râ   rÖ   r×   rÄ  rH   rÓ   rÇ  rÔ   rÆ  rA  rÈ  rò   rÉ  rÊ  rË  r«  r¼   rq  r  r  r  rB  r°  rC  rD  r—   r%  rE  rF  r  rG  rH  rJ  rK  s$                                       ru   rÛ   Ú'IPAdapterXFormersAttnProcessor.__call__S  sk  € ð !ˆð !Ñ,ÜÐ/´×7Ñ7Ø:OÑ7Ð%Ð'7ðCð $ô Ð=¸wÐH[ÐkpÒqØ/×5Ñ5°aÑ8¸4¿?¹?È1Ñ;MÑM�à)ª!¨X¨g¨X²q¨.Ñ9Ø*ª1¨g©hº¨>Ñ:Ð;ð (8ð
 ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
à˜‹?Ø1>×1DÑ1DÑ.ˆJ ¨Ø)×.Ñ.¨zÀFÈUÁNÓS×]Ñ]Ð^_ÐabÓcˆMð $9Ñ#@ˆM×ÒÐF[×FaÑFañ 	'ˆ
�O Qð Ñ%Ø!×8Ñ8¸ÈÐZdÓeˆNð "/×!4Ñ!4ÑˆAˆ|˜QØ+×2Ñ2°2°|ÀRÓHˆNà�?‰?Ñ&Ø ŸO™O¨M×,CÑ,CÀAÀqÓ,IÓJ×TÑTÐUVÐXYÓZˆMà—	‘	˜-Ó(ˆà Ñ(Ø$1Ñ!Ø�_�_Ø$(×$CÑ$CÐDYÓ$ZÐ!à�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà×&Ñ& uÓ-×8Ñ8Ó:ˆØ×$Ñ$ SÓ)×4Ñ4Ó6ˆØ×&Ñ& uÓ-×8Ñ8Ó:ˆä Ÿ™×?Ñ?Ø�3˜¨×<MÑ<Mð @ð 
ˆð &×(Ñ(¨¯©Ó5ˆØ×.Ñ.¨}Ó=ˆçØÒ+Ü!Ð"2´D×9Ñ9ä'+Ð,<×,FÑ,FÀqÓ,IÓ'JÐ$ÜÐ,Ó-´°T·Z±Z³ÕYÄCÐHXÓDYÕYÜ$Ø<¼SÐAQÓ=RÐ<Sð T7Ü7:¸4¿:¹:³Ð6Gð HÜÐ 0Ó1Ð2°!ð5óð ô ;DÜÐ,¯j©jÐ:JÓKö;Ñ6˜Ñ6  e¨Xð  ™<Ù$Ü)¨$´·±×=Ñ=ÀÇÁÈaÃÜ",ð!_ó#ð ð
  Ÿ:™: a™=¨H¯N©N¸1Ñ,=Ó=Ü",Ø"3°D·J±J¸q±M°?ð C8Ø8@¿¹ÀqÑ8IÐ7JÈ+ÐV[ÐU\ð!^ó#ð ô & e¬T×2Ó2¼3¸u»:ÈÏÉÐTUÉÕ;VÜ",Ø"3°D·J±J¸q±M°?ð C5Ü58¸³Z°LÀÈEÈ7ð!Tó#ð ò#;ð, %) 6¬C°·
±
«OÑ#;Ð ô LOØ §*¡*¨d¯l©l¸D¿L¹LÐJZ÷LÑGÐ(¨%°¸'À4ð �Ü˜e¤T×*Ñ*ÜÑ1©5Ó1×1Ñ1Ø#˜øØ˜a“ZØ�Dß�tØÒ'Ø#Ÿw™w¤u§}¡}Ó5˜Ü)¨%´×6Ñ6Ø%* G¨d¯j©j¸©mÑ$;˜Eà-1¯Z©Z¸©]Ð*Ü!&Ð'9×!:˜AÙ%,Ð-EÂaÈÊAÊqÀjÑ-QÓ%R˜FÙ'.Ð/GÊÈ1ÊaÒQRÈ
Ñ/SÓ'T˜Hà%)×%;Ñ%;¸FÓ%C×%NÑ%NÓ%P˜FØ'+×'=Ñ'=¸hÓ'G×'RÑ'RÓ'T˜Hä8@¿¹×8_Ñ8_Ø % v¨x¸D×<MÑ<Mð 9`ð 9Ð5ð 9R×8TÑ8TÐUZ×U`ÑU`Ó8aÐ5Ø8<×8NÑ8NÐOhÓ8iÐ5ä.D×.OÒ.OØ $¢Q¨ª1ªa ZÑ 0Ø *Ø 9× ?Ñ ?ÀÑ BØ 9× ?Ñ ?ÀÑ Bó	/˜Oð />×.@Ñ.@ÀuÇ{Á{Ð[`×[gÑ[gÐ.@Ð.h˜OØ,9À!¹HÐHaÐdsÑHsÑ<tÑ,t›Mô+ ";ñ. ")Ð)AÓ!B˜Ù#*Ð+CÓ#D˜à!%×!7Ñ!7¸Ó!?×!JÑ!JÓ!L˜Ø#'×#9Ñ#9¸(Ó#C×#NÑ#NÓ#P˜ä3;·<±<×3ZÑ3ZØ! 6¨8¸×8IÑ8Ið 4[ð 4Ð0ð 4L×3NÑ3NÈuÏ{É{Ó3[Ð0Ø37×3IÑ3IÐJbÓ3cÐ0à(5Ð@XÑ8XÑ(X›ñgLðl Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà˜‹?Ø)×3Ñ3°B¸Ó;×CÑCÀJÐPWÐY_ÐafÓgˆMà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   )r“   r:   r’   r–   rH   r—   r%  )NrO  r   Nr9  rP  )r¦  r   râ   rú  rÖ   rû  r×   rû  rÄ  rû  rH   r  rÓ   rû  rã  rG  s   @ru   r¢   r¢      s“   ø† ñð, !ØØØ(,ð
ð &÷
ð 
ðF ;?Ø37Ø)-ØØ59ðràðrð )ðrð  8ð	rð
 1ðrð 'ðrð ðrð 3÷ró rrw   r¢   c                  óz   ^ • \ rS rSrSr  S         SU 4S jjjr    S             S	S jjrSrU =r$ )
Ú!SD3IPAdapterJointAttnProcessor2_0i  a"  
Attention processor for IP-Adapter used typically in processing the SD3-like self-attention projections, with
additional image-based information and timestep embeddings.

Args:
    hidden_size (`int`):
        The number of hidden channels.
    ip_hidden_states_dim (`int`):
        The image feature dimension.
    head_dim (`int`):
        The number of head channels.
    timesteps_emb_dim (`int`, defaults to 1280):
        The number of input channels for timestep embedding.
    scale (`float`, defaults to 0.5):
        IP-Adapter scale.
c                ó  >• [         TU ]  5         SSKJnJn  U" XBS-  SSS9U l        [        R                  " X!SS9U l        [        R                  " X!SS9U l	        U" US5      U l
        U" US5      U l        U" US5      U l        XPl        g )Nr   )ÚAdaLayerNormr   r   r  )Ú
output_dimÚnorm_epsÚ	chunk_dimFr0   )r2   r3   r4   rc  r   Únorm_ipr   rU   r—   r%  rQ   rR   Ú	norm_ip_krH   )	ri   r’   Úip_hidden_states_dimrö  Útimesteps_emb_dimrH   rc  r   rt   s	           €ru   r3   Ú*SD3IPAdapterJointAttnProcessor2_0.__init__  sƒ   ø€ ô 	‰ÑÔ÷ 	9á#Ð$5ÐYZÑBZÐeiÐuvÑwˆŒÜ—y’yÐ!5ÈÑOˆŒÜ—y’yÐ!5ÈÑOˆŒÙ˜h¨Ó-ˆŒÙ˜h¨Ó-ˆŒÙ  ¨4Ó0ˆŒØ�
rw   c                óœ	  • UnUR                   S   nUR                  U5      n	UR                  U5      n
UR                  U5      nU
R                   S   nXÁR                  -  nU	R                  USUR                  U5      R                  SS5      n	U
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nU	nU
nUnUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
UGb>  UR                  U5      nUR                  U5      nUR                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " U	U/SS9n	[        R                  " U
U/SS9n
[        R                  " UU/SS9n[         R"                  " XšUSSS	9nUR                  SS5      R%                  USUR                  U-  5      nUR'                  U	R(                  5      nUbQ  USS2SUR                   S   24   USS2UR                   S   S24   p2UR*                  (       d  UR-                  U5      nU R.                  S:w  Gae  UGba  U R1                  XVS
9nU R3                  U5      nU R5                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nU R                  U5      n	U R                  U5      nU R7                  U5      n[        R                  " UU/SS9n
[        R                  " UU/SS9n[         R"                  " XšUSSS	9nUR                  SS5      R                  USUR                  U-  5      nUR'                  U	R(                  5      nX%U R.                  -  -   nUR8                  S   " U5      nUR8                  S   " U5      nUb  X#4$ U$ )aÀ  
Perform the attention computation, integrating image features (if provided) and timestep embeddings.

If `ip_hidden_states` is `None`, this is equivalent to using JointAttnProcessor2_0.

Args:
    attn (`Attention`):
        Attention instance.
    hidden_states (`torch.FloatTensor`):
        Input `hidden_states`.
    encoder_hidden_states (`torch.FloatTensor`, *optional*):
        The encoder hidden states.
    attention_mask (`torch.FloatTensor`, *optional*):
        Attention mask.
    ip_hidden_states (`torch.FloatTensor`, *optional*):
        Image embeddings.
    temb (`torch.FloatTensor`, *optional*):
        Timestep embeddings.

Returns:
    `torch.FloatTensor`: Output hidden states.
r   r+   r   r   Nr   r  FrŸ  )rÄ  )rí   rV   rW   rX   rI   rÂ  r  rQ   rR   r\   rZ   r[   rb   rc   r§   r  re   r1   rî   r°   r�   rC   ra   rH   rg  r—   r%  rh  r^   )ri   r¦  râ   rÖ   r×   rÔ   rÄ  rÇ  rò   r  r  r  r5   rö  Ú	img_queryÚimg_keyÚ	img_valuer÷  ræ  rç  Únorm_ip_hidden_statesrG  rH  s                          ru   rÛ   Ú*SD3IPAdapterJointAttnProcessor2_0.__call__/  s–  € ð> !ˆà"×(Ñ(¨Ñ+ˆ
ð —	‘	˜-Ó(ˆØ�i‰i˜Ó&ˆØ—	‘	˜-Ó(ˆà—I‘I˜b‘Mˆ	Ø§
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØˆ	ØˆØˆ	à�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCð !Ò,Ø/3¯©Ð?TÓ/UÐ,Ø-1¯_©_Ð=RÓ-SÐ*Ø/3¯©Ð?TÓ/UÐ,à/O×/TÑ/TØ˜B §
¡
¨Hó0ç‰i˜˜1‹oð -ð .L×-PÑ-PØ˜B §
¡
¨Hó.ç‰i˜˜1‹oð +ð 0P×/TÑ/TØ˜B §
¡
¨Hó0ç‰i˜˜1‹oð -ð × Ñ Ñ,Ø37×3DÑ3DÐEeÓ3fÐ0Ø× Ñ Ñ,Ø15×1BÑ1BÐCaÓ1bÐ.ä—I’I˜uÐ&FÐGÈQÑOˆEÜ—)’)˜SÐ"@ÐAÀqÑIˆCÜ—I’I˜uÐ&FÐGÈQÑOˆEä×6Ò6°uÀ5ÐTWÐchÑiˆØ%×/Ñ/°°1Ó5×=Ñ=¸jÈ"ÈdÏjÉjÐ[cÑNcÓdˆØ%×(Ñ(¨¯©Ó5ˆà Ñ,ð šaÐ!4 8§>¡>°!Ñ#4Ð!4Ð4Ñ5Øša §¡°Ñ!2Ñ!4Ð4Ñ5ð 1ð ×(×(Ø(,¯©Ð8MÓ(NÐ%ð �:‰:˜Œ?Ð/Ò;à$(§L¡LÐ1A LÐ$MÐ!ð —\‘\Ð"7Ó8ˆFØ—|‘|Ð$9Ó:ˆHð —[‘[ ¨R°·±¸XÓF×PÑPÐQRÐTUÓVˆFØ—}‘} Z°°T·Z±ZÀÓJ×TÑTÐUVÐXYÓZˆHð —K‘K 	Ó*ˆEØ—k‘k 'Ó*ˆGØ—^‘^ FÓ+ˆFô —)’)˜W fÐ-°1Ñ5ˆCÜ—I’I˜y¨(Ð3¸Ñ;ˆEä ×=Ò=¸eÈ%Ð[^ÐjoÑpÐØ/×9Ñ9¸!¸QÓ?×DÑDÀZÐQSÐUY×U_ÑU_ÐbjÑUjÓkÐØ/×2Ñ2°5·;±;Ó?Ðà)¸t¿z¹zÑ,IÑIˆMð Ÿ™ Aš }Ó5ˆàŸ™ Aš }Ó5ˆà Ñ,Ø Ð7Ð7à Ð rw   )rg  rh  rR   rQ   rH   r—   r%  )i   g      à?)
r’   r4  ri  r4  rö  r4  rj  r4  rH   r  rB  )r¦  r   râ   rú  rÖ   rú  r×   rû  rÔ   rú  rÄ  rú  r9  rú  rã  rG  s   @ru   ra  ra    s¯   ø† ñð, "&Øðàðð "ðð ð	ð
 ðð ÷ð ð2 48Ø37Ø.2Ø"&ðA!àðA!ð )ðA!ð  1ð	A!ð
 1ðA!ð ,ðA!ð  ðA!ð 
÷A!ó A!rw   ra  c                  óH   • \ rS rSrSrS r   S           SS jjrSrg)	ÚPAGIdentitySelfAttnProcessor2_0i³  ú­
Processor for implementing PAG using scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
PAG reference: https://huggingface.co/papers/2403.17377
c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   z_PAGIdentitySelfAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú(PAGIdentitySelfAttnProcessor2_0.__init__¹  s%   € Ü”qÐ8×9Ñ9ÜØqóð ð :rw   Nc           	     óB  • UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  p‰p«UR                  X‰X«-  5      R	                  SS5      nUR                  S5      u  pÍUR                  u  pŽnUb<  UR                  XNU5      nUR                  X�R                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      n[        R                  " UUUUSSS9nUR	                  SS5      R                  USUR                  U-  5      nUR                  UR                   5      nUR"                  S   " U5      nUR"                  S   " U5      nUS:X  a$  UR	                  SS	5      R                  UW	W
W5      nUR                  u  pŽnUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  UR                   5      nUR"                  S   " U5      nUR"                  S   " U5      nUS:X  a$  UR	                  SS	5      R                  UW	W
W5      n[$        R&                  " XÍ/5      nUR(                  (       a  X&-   nX!R*                  -  nU$ )
Nr  r   r   r+   r  Frî  r   rý   )rP   r÷   rí   rÂ  r  r  r  rI   r.   rV   rW   rX   re   r1   rî   r°   r�   r^   r§   r  r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  rÇ  rÈ  rò   rÉ  rÊ  rË  r  r  r«  r¼   r  r  r  r5   rö  s                        ru   rÛ   Ú(PAGIdentitySelfAttnProcessor2_0.__call__¿  s|  € ð !ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
Ø˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð 0=×/BÑ/BÀ1Ó/EÑ,Ðð *;×)@Ñ)@Ñ&ˆ
 QàÑ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà—	‘	Ð+Ó,ˆØ�i‰iÐ)Ó*ˆØ—	‘	Ð+Ó,ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆØ�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆô ×:Ò:Ø�3˜¨À3ÐRWñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐð *;×)@Ñ)@Ñ&ˆ
 Qà�?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà ŸI™IÐ&7Ó8ÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐô Ÿ	š	Ð#4Ð"HÓIˆà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rÍ  ©r¦  r   râ   rú  rÖ   rû  r×   rû  rÄ  rû  r9  r>  r¼  r»  rw   ru   rs  rs  ³  sc   † ñò
ð ;?Ø37Ø)-ðTàðTð )ðTð  8ð	Tð
 1ðTð 'ðTð 
÷Tð Trw   rs  c                  óH   • \ rS rSrSrS r   S           SS jjrSrg)	Ú"PAGCFGIdentitySelfAttnProcessor2_0i  rt  c                óD   • [        [        S5      (       d  [        S5      eg )Nr1   zbPAGCFGIdentitySelfAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.rŒ  rŽ  s    ru   r3   Ú+PAGCFGIdentitySelfAttnProcessor2_0.__init__  s%   € Ü”qÐ8×9Ñ9ÜØtóð ð :rw   Nc           	     óv  • UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  p‰p«UR                  X‰X«-  5      R	                  SS5      nUR                  S5      u  pÍn[        R                  " XÍ/5      nUR                  u  p�nUb<  UR                  XOU5      nUR                  X�R                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      n[        R                  " UUUUSSS9nUR	                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUS:X  a$  UR	                  SS
5      R!                  UW	W
W5      nUR                  u  p�nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUnUR#                  UR$                  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUS:X  a$  UR	                  SS
5      R!                  UW	W
W5      n[        R                  " XÞ/5      nUR(                  (       a  X&-   nX!R*                  -  nU$ )Nr  r   r   rì   r+   r  Frî  r   rý   )rP   r÷   rí   rÂ  r  r  r§   r  r  rI   r.   rV   rW   rX   re   r1   rî   r°   r�   r^   r>   r=   )ri   r¦  râ   rÖ   r×   rÄ  rÇ  rÈ  rò   rÉ  rÊ  rË  r  r  r  r«  r¼   r  r  r  r5   rö  s                         ru   rÛ   Ú+PAGCFGIdentitySelfAttnProcessor2_0.__call__"  sš  € ð !ˆØ×ÑÑ(Ø ×-Ñ-¨mÓBˆMà"×'Ñ'ˆ
Ø˜‹?Ø1>×1DÑ1DÑ.ˆJ Ø)×.Ñ.¨zÀFÁNÓS×]Ñ]Ð^_ÐabÓcˆMð FS×EXÑEXÐYZÓE[ÑBÐÐ1BÜ!ŸIšIÐ';Ð&OÓPÐð *;×)@Ñ)@Ñ&ˆ
 QàÑ%Ø!×8Ñ8¸ÐZdÓeˆNð ,×0Ñ0°¿Z¹ZÈÈ^×MaÑMaÐbdÑMeÓfˆNà�?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà—	‘	Ð+Ó,ˆØ�i‰iÐ)Ó*ˆØ—	‘	Ð+Ó,ˆà—I‘I˜b‘Mˆ	Ø §
¡
Ñ*ˆà—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆà�h‰h�z 2 t§z¡z°8Ó<×FÑFÀqÈ!ÓLˆØ—
‘
˜: r¨4¯:©:°xÓ@×JÑJÈ1ÈaÓPˆô ×:Ò:Ø�3˜¨À3ÐRWñ
Ðð .×7Ñ7¸¸1Ó=×EÑEÀjÐRTÐVZ×V`ÑV`ÐckÑVkÓlÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐð *;×)@Ñ)@Ñ&ˆ
 Qà�?‰?Ñ&Ø $§¡Ð0A×0KÑ0KÈAÈqÓ0QÓ R× \Ñ \Ð]^Ð`aÓ bÐà—	‘	Ð+Ó,ˆØ!ÐØ-×0Ñ0°·±Ó=Ðð !ŸK™K¨šNÐ+<Ó=Ðà ŸK™K¨šNÐ+<Ó=Ðà˜‹?Ø 1× ;Ñ ;¸BÀÓ C× KÑ KÈJÐX_ÐagÐinÓ oÐô Ÿ	š	Ð#4Ð"HÓIˆà×#×#Ø)Ñ4ˆMà%×(BÑ(BÑBˆàÐrw   r»  rÍ  ry  r¼  r»  rw   ru   r{  r{    sc   † ñò
ð ;?Ø37Ø)-ðXàðXð )ðXð  8ð	Xð
 1ðXð 'ðXð 
÷Xð Xrw   r{  c                  ó"   • \ rS rSrSrSS jrSrg)re  i}  z<
Processor for implementing multiscale quadratic attention.
c                óÜ  • UR                   SS  u  p4X4-  UR                  :”  a  SnOSnUn[        UR                  5       5      u  pxp4UR                  n	UR                  SS5      nUR                  U5      n
UR                  U5      nUR                  U5      n[        R                  " X«U/SS9nUR                  SS5      nU/nUR                   H  nUR                  U" U5      5        M     [        R                  " USS9nU(       a  UR                  [        R                  S9nUR                  USSUR                  -  X4-  5      nUR!                  SS	S9u  p«nUR#                  U
5      n
UR#                  U5      nU(       a"  UR%                  X«U5      nUR                  U	S9nOUR'                  X«U5      n[        R                  " X'SX445      nUR)                  UR                  SS5      5      R                  SS5      nUR*                  S
:X  a2  UR-                  UR                  SS5      5      R                  SS5      nOUR-                  U5      nUR.                  (       a  X&-   nU$ )Nrý   TFr   r+   rì   r   ro  r   r(   )rí   r_  r$  r¢  r�   ÚmovedimrV   rW   rX   r§   r  ra  r_   r°   rq  rî   r  rc  rt  ry  r^   r`  rd  r>   )ri   r¦  râ   rÊ  rË  Úuse_linear_attentionrÇ  rò   r¼   Úoriginal_dtyper  r  r  Úmulti_scale_qkvÚblocks                  ru   rÛ   Ú'SanaMultiscaleAttnProcessor2_0.__call__‚  sD  € Ø%×+Ñ+¨B¨CÐ0‰ˆØ‰>˜D×3Ñ3Ó3Ø#'Ñ à#(Ð à ˆä'+¨M×,>Ñ,>Ó,@Ó'AÑ$ˆ
�vØ&×,Ñ,ˆà%×-Ñ-¨a°Ó4ˆØ—	‘	˜-Ó(ˆØ�i‰i˜Ó&ˆØ—	‘	˜-Ó(ˆÜŸ	š	 5¨uÐ"5¸1Ñ=ˆØ%×-Ñ-¨b°!Ó4ˆà(˜/ˆØ×+Ô+ˆEØ×"Ñ"¡5¨Ó#7Ö8ñ ,ô Ÿ	š	 /°qÑ9ˆæà)×,Ñ,´5·=±=Ð,ÐAˆMà%×-Ñ-¨j¸"¸aÀ$×BYÑBYÑ>YÐ[aÑ[iÓjˆà)×/Ñ/°°qÐ/Ð9Ñˆ�EØ×!Ñ! %Ó(ˆØ×Ñ Ó$ˆæØ ×7Ñ7¸ÀEÓJˆMØ)×,Ñ,°>Ð,ÐB‰Mà ×:Ñ:¸5ÀuÓMˆMäŸš mÀ"ÀfÐ5TÓUˆØŸ™ M×$9Ñ$9¸!¸RÓ$@ÓA×IÑIÈ"ÈaÓPˆà�>‰>˜ZÓ'Ø ŸM™M¨-×*?Ñ*?ÀÀ2Ó*FÓG×OÑOÐPRÐTUÓV‰Mà ŸM™M¨-Ó8ˆMà×#×#Ø)Ñ4ˆMàÐrw   r»  N)r¦  r[  râ   r>  r9  r>  rÏ  r»  rw   ru   re  re  }  s   † ñ÷4rw   re  c                  ó   • \ rS rSrSrS rSrg)ÚLoRAAttnProcessori¹  z1
Processor for implementing attention with LoRA.
c                ó   • g r;  r»  rŽ  s    ru   r3   ÚLoRAAttnProcessor.__init__¾  ó   € Ørw   r»  N©rá   r@  rA  rB  rC  r3   rE  r»  rw   ru   r‰  r‰  ¹  ó   † ñõrw   r‰  c                  ó   • \ rS rSrSrS rSrg)ÚLoRAAttnProcessor2_0iÂ  zb
Processor for implementing attention with LoRA (enabled by default if you're using PyTorch 2.0).
c                ó   • g r;  r»  rŽ  s    ru   r3   ÚLoRAAttnProcessor2_0.__init__Ç  rŒ  rw   r»  Nr�  r»  rw   ru   r�  r�  Â  rŽ  rw   r�  c                  ó   • \ rS rSrSrS rSrg)ÚLoRAXFormersAttnProcessoriË  z@
Processor for implementing attention with LoRA using xFormers.
c                ó   • g r;  r»  rŽ  s    ru   r3   Ú"LoRAXFormersAttnProcessor.__init__Ð  rŒ  rw   r»  Nr�  r»  rw   ru   r”  r”  Ë  rŽ  rw   r”  c                  ó   • \ rS rSrSrS rSrg)ÚLoRAAttnAddedKVProcessoriÔ  zr
Processor for implementing attention with LoRA with extra learnable key and value matrices for the text encoder.
c                ó   • g r;  r»  rŽ  s    ru   r3   Ú!LoRAAttnAddedKVProcessor.__init__Ù  rŒ  rw   r»  Nr�  r»  rw   ru   r˜  r˜  Ô  rŽ  rw   r˜  c                  ó<   • \ rS rSrSr  S         SS jjrSrg)ÚSanaLinearAttnProcessor2_0iÝ  úA
Processor for implementing scaled dot-product linear attention.
Nc                ó¼  • UR                   nUc  UnUR                  U5      nUR                  U5      nUR                  U5      nUR                  b  UR	                  U5      nUR
                  b  UR                  U5      nUR                  SS5      R                  SUR                  S45      nUR                  SS5      R                  SUR                  S45      R                  SS5      nUR                  SS5      R                  SUR                  S45      n[        R                  " U5      n[        R                  " U5      nUR                  5       UR                  5       UR                  5       p‡n[        R                  " USSSS9n[        R                  " X‡5      n	[        R                  " X–5      nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR                  SS5      R                  SS5      nUR!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      nU[        R$                  :X  a  UR'                  SS5      nU$ )Nr   r   r+   rì   rk  rl  r   rm  r  r   é  ÿÿéàÿ  )r�   rV   rW   rX   rQ   rR   r  r¡  rI   re   Úrelur  r  r§   rp  r”  r°   r^   r‚  Úclip)
ri   r¦  râ   rÖ   r×   r„  r  r  r  rs  s
             ru   rÛ   Ú#SanaLinearAttnProcessor2_0.__call__â  sÿ  € ð '×,Ñ,ˆà Ñ(Ø$1Ð!à—	‘	˜-Ó(ˆØ�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà�;‰;Ñ"Ø—K‘K Ó&ˆEØ�;‰;Ñ"Ø—+‘+˜cÓ"ˆCà—‘  1Ó%×/Ñ/°°D·J±JÀÐ3CÓDˆØ�m‰m˜A˜qÓ!×+Ñ+¨A°·
±
¸BÐ/?Ó@×JÑJÈ1ÈaÓPˆØ—‘  1Ó%×/Ñ/°°D·J±JÀÐ3CÓDˆä—’�u“ˆÜ�fŠf�S‹kˆà!ŸK™K›M¨3¯9©9«;¸¿¹»�Eˆä—’�e˜\°
À#ÑFˆÜ—’˜eÓ)ˆÜŸš VÓ3ˆà%¢aª¨C¨R¨C iÑ0°MÂ!ÂQÈÉÀ)Ñ4LÈuÑ4TÑUˆØ%×-Ñ-¨a°Ó3×=Ñ=¸aÀÓCˆØ%×(Ñ(¨Ó8ˆàŸ™ Aš }Ó5ˆØŸ™ Aš }Ó5ˆàœUŸ]™]Ó*Ø)×.Ñ.¨v°uÓ=ˆMàÐrw   r»  r=  râ  rÏ  r»  rw   ru   rœ  rœ  Ý  sI   † ñð 6:Ø.2ð,àð,ð $ð,ð  3ð	,ð
 ,ð,ð 
÷,ð ,rw   rœ  c                  ó<   • \ rS rSrSr  S         SS jjrSrg)Ú PAGCFGSanaLinearAttnProcessor2_0i  r�  Nc                óR  • UR                   nUR                  S5      u  pgn[        R                  " Xg/5      nUR	                  U5      n	UR                  U5      n
UR                  U5      nU	R                  SS5      R                  SUR                  S45      n	U
R                  SS5      R                  SUR                  S45      R                  SS5      n
UR                  SS5      R                  SUR                  S45      n[        R                  " U	5      n	[        R                  " U
5      n
U	R                  5       U
R                  5       UR                  5       pºn	[        R                  " USSSS9n[        R                  " Xº5      n[        R                  " XÉ5      nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR                  SS5      R                  SS5      nUR!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      nUR                  U5      R!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      n[        R                  " Xx/5      nU[        R$                  :X  a  UR'                  SS5      nU$ )Nrì   r   r   r+   rk  rl  r   rm  r  r   rŸ  r   )r�   r  r§   r  rV   rW   rX   r  r¡  rI   re   r¡  r  r  rp  r”  r°   r^   r‚  r¢  )ri   r¦  râ   rÖ   r×   r„  r  r  r  r  r  r  rs  s                ru   rÛ   Ú)PAGCFGSanaLinearAttnProcessor2_0.__call__  sS  € ð '×,Ñ,ˆàER×EXÑEXÐYZÓE[ÑBÐÐ1BÜ!ŸIšIÐ';Ð&OÓPÐà—	‘	Ð+Ó,ˆØ�i‰iÐ)Ó*ˆØ—	‘	Ð+Ó,ˆà—‘  1Ó%×/Ñ/°°D·J±JÀÐ3CÓDˆØ�m‰m˜A˜qÓ!×+Ñ+¨A°·
±
¸BÐ/?Ó@×JÑJÈ1ÈaÓPˆØ—‘  1Ó%×/Ñ/°°D·J±JÀÐ3CÓDˆä—’�u“ˆÜ�fŠf�S‹kˆà!ŸK™K›M¨3¯9©9«;¸¿¹»�Eˆä—’�e˜\°
À#ÑFˆÜ—’˜eÓ)ˆÜ!ŸLšL¨Ó7Ðà-ªa²°C°R°C¨iÑ8Ð<MÊaÒQRÐTVÑTWÈiÑ<XÐ[`Ñ<`ÑaÐØ-×5Ñ5°a¸Ó;×EÑEÀaÈÓKÐØ-×0Ñ0°Ó@Ðà ŸK™K¨šNÐ+<Ó=ÐØ ŸK™K¨šNÐ+<Ó=Ðð !ŸI™IÐ&7Ó8×;Ñ;¸NÓKÐà ŸK™K¨šNÐ+<Ó=ÐØ ŸK™K¨šNÐ+<Ó=ÐäŸ	š	Ð#4Ð"HÓIˆàœUŸ]™]Ó*Ø)×.Ñ.¨v°uÓ=ˆMàÐrw   r»  r=  râ  rÏ  r»  rw   ru   r¥  r¥    sI   † ñð 6:Ø.2ð/àð/ð $ð/ð  3ð	/ð
 ,ð/ð 
÷/ð /rw   r¥  c                  ó<   • \ rS rSrSr  S         SS jjrSrg)Ú%PAGIdentitySanaLinearAttnProcessor2_0iH  r�  Nc                óž  • UR                   nUR                  S5      u  pgUR                  U5      nUR                  U5      n	UR	                  U5      n
UR                  SS5      R                  SUR                  S45      nU	R                  SS5      R                  SUR                  S45      R                  SS5      n	U
R                  SS5      R                  SUR                  S45      n
[        R                  " U5      n[        R                  " U	5      n	UR                  5       U	R                  5       U
R                  5       p©n[        R                  " U
SSSS9n
[        R                  " X©5      n[        R                  " X¸5      nUR                   [        R                  [        R                  4;   a  UR                  5       nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR!                  SS5      R                  SS5      nUR#                  U5      nUR$                  S
   " U5      nUR$                  S   " U5      nUR	                  U5      R#                  U5      nUR$                  S
   " U5      nUR$                  S   " U5      n[        R&                  " Xg/5      nU[        R                  :X  a  UR)                  SS5      nU$ )Nr   r   r+   rì   rk  rl  r   rm  r  r   rŸ  r   )r�   r  rV   rW   rX   r  r¡  rI   re   r¡  r  r  r§   rp  r‚  rƒ  r”  r°   r^   r  r¢  )ri   r¦  râ   rÖ   r×   r„  r  r  r  r  r  rs  s               ru   rÛ   Ú.PAGIdentitySanaLinearAttnProcessor2_0.__call__M  sf  € ð '×,Ñ,ˆà/<×/BÑ/BÀ1Ó/EÑ,Ðà—	‘	Ð+Ó,ˆØ�i‰iÐ)Ó*ˆØ—	‘	Ð+Ó,ˆà—‘  1Ó%×/Ñ/°°D·J±JÀÐ3CÓDˆØ�m‰m˜A˜qÓ!×+Ñ+¨A°·
±
¸BÐ/?Ó@×JÑJÈ1ÈaÓPˆØ—‘  1Ó%×/Ñ/°°D·J±JÀÐ3CÓDˆä—’�u“ˆÜ�fŠf�S‹kˆà!ŸK™K›M¨3¯9©9«;¸¿¹»�Eˆä—’�e˜\°
À#ÑFˆÜ—’˜eÓ)ˆÜ!ŸLšL¨Ó7Ðà×"Ñ"¤u§}¡}´e·n±nÐ&EÓEØ 1× 7Ñ 7Ó 9Ðà-ªa²°C°R°C¨iÑ8Ð<MÊaÒQRÐTVÑTWÈiÑ<XÐ[`Ñ<`ÑaÐØ-×5Ñ5°a¸Ó;×EÑEÀaÈÓKÐØ-×0Ñ0°Ó@Ðà ŸK™K¨šNÐ+<Ó=ÐØ ŸK™K¨šNÐ+<Ó=Ðð !ŸI™IÐ&7Ó8×;Ñ;¸NÓKÐà ŸK™K¨šNÐ+<Ó=ÐØ ŸK™K¨šNÐ+<Ó=ÐäŸ	š	Ð#4Ð"HÓIˆàœUŸ]™]Ó*Ø)×.Ñ.¨v°uÓ=ˆMàÐrw   r»  r=  râ  rÏ  r»  rw   ru   r©  r©  H  sI   † ñð 6:Ø.2ð1àð1ð $ð1ð  3ð	1ð
 ,ð1ð 
÷1ð 1rw   r©  c                  ó   • \ rS rSrS rSrg)ÚFluxAttnProcessor2_0i�  c                ó<   • Sn[        SSU5        SSKJn  U" U0 UD6$ )Nzq`FluxAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxAttnProcessor`r­  rÀ  r   ©ÚFluxAttnProcessor©r   Útransformers.transformer_fluxr°  ©ÚclsrÅ  r‡  rÆ  r°  s        ru   Ú__new__ÚFluxAttnProcessor2_0.__new__‚  s,   € ð RÐÜÐ(¨'Ð3FÔGåDá  $Ð1¨&Ñ1Ð1rw   r»  N©rá   r@  rA  rB  rµ  rE  r»  rw   ru   r­  r­  �  ó   † õ2rw   r­  c                  ó   • \ rS rSrSrS rSrg)ÚFluxSingleAttnProcessor2_0i‹  r‹  c                ó<   • Sn[        SSU5        SSKJn  U" U0 UD6$ )Nz|`FluxSingleAttnProcessor` is deprecated and will be removed in a future version. Please use `FluxAttnProcessorSDPA` instead.rº  rÀ  r   r¯  r±  r³  s        ru   rµ  Ú"FluxSingleAttnProcessor2_0.__new__�  s,   € ð ]ÐÜÐ.°Ð9LÔMåDá  $Ð1¨&Ñ1Ð1rw   r»  N©rá   r@  rA  rB  rC  rµ  rE  r»  rw   ru   rº  rº  ‹  s   † ñõ2rw   rº  c                  ó   • \ rS rSrS rSrg)ÚFusedFluxAttnProcessor2_0i™  c                ó<   • Sn[        SSU5        SSKJn  U" U0 UD6$ )Nzv`FusedFluxAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxAttnProcessor`r¿  rÀ  r   r¯  r±  r³  s        ru   rµ  Ú!FusedFluxAttnProcessor2_0.__new__š  s,   € ð WÐÜÐ-¨wÐ8KÔLåDá  $Ð1¨&Ñ1Ð1rw   r»  Nr·  r»  rw   ru   r¿  r¿  ™  r¸  rw   r¿  c                  ó   • \ rS rSrS rSrg)Ú"FluxIPAdapterJointAttnProcessor2_0i£  c                ó<   • Sn[        SSU5        SSKJn  U" U0 UD6$ )Nzˆ`FluxIPAdapterJointAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxIPAdapterAttnProcessor`rÃ  rÀ  r   )ÚFluxIPAdapterAttnProcessor)r   r²  rÅ  )r´  rÅ  r‡  rÆ  rÅ  s        ru   rµ  Ú*FluxIPAdapterJointAttnProcessor2_0.__new__¤  s,   € ð iÐÜÐ6¸ÐATÔUåMá)¨4Ð:°6Ñ:Ð:rw   r»  Nr·  r»  rw   ru   rÃ  rÃ  £  s   † õ;rw   rÃ  c                  ó   • \ rS rSrS rSrg)ÚFluxAttnProcessor2_0_NPUi­  c                óJ   • Sn[        SSUSS9  SSKJn  U" 5       nSUl        U$ )	Nz¢FluxAttnProcessor2_0_NPU is deprecated and will be removed in a future version. An alternative solution to use NPU Flash Attention will be provided in the future.rÈ  rÀ  Fr+  r   r¯  Ú_native_npu©r   r²  r°  Ú_attention_backend©r´  rÅ  r‡  rÆ  r°  rr   s         ru   rµ  Ú FluxAttnProcessor2_0_NPU.__new__®  s9   € ð^ð 	ô 	Ð,¨gÐ7JÐZ_Ò`åDá%Ó'ˆ	Ø'4ˆ	Ô$ØÐrw   r»  Nr·  r»  rw   ru   rÈ  rÈ  ­  ó   † õrw   rÈ  c                  ó   • \ rS rSrS rSrg)ÚFusedFluxAttnProcessor2_0_NPUi¼  c                óJ   • Sn[        SSUSS9  SSKJn  U" 5       nSUl        U$ )	Nz§FusedFluxAttnProcessor2_0_NPU is deprecated and will be removed in a future version. An alternative solution to use NPU Flash Attention will be provided in the future.rÑ  rÀ  Fr+  r   r¯  Ú
_fused_npurË  )ri   rÆ  r°  rr   s       ru   rµ  Ú%FusedFluxAttnProcessor2_0_NPU.__new__½  s9   € ð^ð 	ô 	Ð1°7Ð<OÐ_dÒeåDá%Ó'ˆ	Ø'3ˆ	Ô$ØÐrw   r»  Nr·  r»  rw   ru   rÑ  rÑ  ¼  rÏ  rw   rÑ  c                  ó   • \ rS rSrSrS rSrg)r|   iË  r‘  c                ó<  • Sn[        SSUSS9  [        SS5      (       a  [        S5      e[        5       (       a  [        SS	5      (       a  [        S
5      eSSKJn  [        U5      S:”  d  UR                  SS 5      b  Sn[        SSU5        U" U0 UD6nSUl        U$ )Nz§XLAFluxFlashAttnProcessor2_0 is deprecated and will be removed in diffusers 1.0.0. An alternative solution to using XLA Flash Attention will be provided in the future.r|   rÀ  Fr+  ry   rz   r“  r{   r”  r   r¯  r   r   zˆpartition_spec was not used in the processor implementation when it was added. Passing it is a no-op and support for it will be removed.Ú_native_xla)	r   r   r�  r   r²  r°  rß   rÁ  rÌ  rÍ  s         ru   rµ  Ú$XLAFluxFlashAttnProcessor2_0.__new__Ð  s°   € ð`ð 	ô 	Ð0°'Ð;NÐ^cÒdä  U×+Ñ+ÜÐVÓWÐWÜ�9‰9Ô-¨c°5×9Ñ9ÜÐdÓeÐeåDäˆt‹9�q‹=˜FŸJ™JÐ'7¸Ó>ÑJðAð  ô Ð&¨Ð1DÔEá% tÐ6¨vÑ6ˆ	Ø'4ˆ	Ô$ØÐrw   r»  Nr½  r»  rw   ru   r|   r|   Ë  s   † ñõrw   r|   )aÚ
__future__r   rÙ   r†  Útypingr   r§   Útorch.nn.functionalr   Ú
functionalre   Úimage_processorr   Úutilsr   r	   r
   Úutils.import_utilsr   r   r   Úutils.torch_utilsr   r   Ú
get_loggerrá   r²   r„  r‰   Úxformers.opsÚ$torch_xla.experimental.custom_kernelr   Útorch_xla.runtimer   ÚXLA_AVAILABLErÉ   r   rI  r[  r‚  rŠ  rg   r™   rœ   r�   r£   rý  r  r"  r¤   r;  rE  rO  rU  ra  rŸ   r´   r„   rf   r}   r   r§  rÃ  rÊ  rÑ  rØ  rÞ  rê  rš   r›   rÄ   rž   rO   r    r¡   r¢   ra  rs  r{  re  r‰  r�  r”  r˜  rœ  r¥  r©  r­  rº  r¿  rÃ  rÈ  rÑ  r|   ÚADDED_KV_ATTENTION_PROCESSORSÚCROSS_ATTENTION_PROCESSORSÚAttentionProcessorr»  rw   ru   Ú<module>ré     s  ðõ #ã Û Ý ã ß Ð Ý å 4ß >Ñ >ß dÑ dß Fð 
×	Ò	˜HÓ	%€á×ÑÛá×ÑÛÜà€Há×Ñá˜C ×'Ñ'ÝHÝ-Ø�Mà€Mð ôQ&�—	‘	ó Q&ó ðQ&ôh¨¯	©	ô ô4?3 B§I¡Iô ?3ôD?
�R—Y‘Yô ?
÷Df4ñ f4÷REñ EôPe 2§9¡9ô e÷P@ñ @÷FIñ I÷XS!ñ S!÷lY4ñ Y4÷xb4ñ b4÷JL4ñ L4÷^R!ñ R!÷j[ñ [÷|Z!ñ Z!÷z^!ñ ^!÷BB4ñ B4÷JD4ñ D4÷NDñ D÷N[ñ [÷|oñ o÷d[ñ [÷|qñ q÷hRñ R÷jBñ B÷J_ñ _÷Ddñ d÷Nxñ x÷vyñ y÷x]ñ ]÷@cñ côLq¨2¯9©9ô qôho b§i¡iô o÷dTñ T÷nYñ Yôx�"—)‘)ô ô:E˜RŸY™Yô EôPe §¡§¡ô eôPe U§X¡X§_¡_ô eôPh!¨¯©¯©ô h!÷V`ñ `÷Fdñ d÷N9ñ 9÷xñ ÷ñ ÷ñ ÷ñ ÷1ñ 1÷h4ñ 4÷n6ñ 6÷r2ñ 2÷2ñ 2÷2ñ 2÷;ñ ;÷ñ ÷ñ ÷ñ ð@ ØØØ ð	!Ð ð ØØØØØØ&ðÐ ð Ø"ñ#àñð ñð ñ	ð
 ñð "ñ"ð !ñ!ð ñð ñ	ð $ñ
$ð ñð ñð  ñ ð $ñ$ð  ñ ð  %ñ!%ð" #ñ##ð$ ñ%ð& ñ'ð( ñ)ð* ñ+ð, ñ-ð. ñ/ð0 "ñ1"ð2 ñ3ð4 #ñ5#ð6 !ñ7!ð8 $ñ9$ð: ñ;ð< ñ=ð> +ñ?+ð@ &ñA &ðB ñC!ðD !ñE"!ðF !ñG#!ðH 'ñI$'ðJ ,ñK%,ðL $ñM&$ðN %ñO'%ðP (ñQ((ðR ñS)ðT  ñU* ðV %ñW+%ðX (ñY,(ðZ &ñ[-&ð\ )ñ].)ð^ ñ_/ð` ña0ðb  ñc1 ðd ñe2ñ rw   