§
     ’Þj®p  ã                   ó2  — d dl Z d dlZd dlmZmZ d dlZd dlmZmZ d dl	m	Z	 d dl
mZ d dlmZmZ d dlmc mZ d dlZd dl
mZ ej        dk    o. eej        d¦  «        oej        j                             ¦   «         Zerd	nej                             d¦  «        \  ZZed
k    Zdada  e!¦   «         Z"	 d dl#Z#dZ$n	#  dZ$Y nxY we$e$oeefdv dœe$e$oed
k    oej        j%         dœe$e$oed
k    oej        j%         dœdœZ&	 d dl'm(Z( n# e)$ r dZ(Y nw xY w	 d dl*Z*dZ+n# e,$ r dZ+Y nw xY w	 d dl-Z-dZ.n# e,$ r dZ.dZ-Y nw xY w	 d dl/m0Z0 ddl1m2Z2 	 	 	 d<d„Z3n# e)$ r dxZ3Z2Y nw xY w	 d dl4m5Z5 n2# e)$ r* dZ5e$s#	  ed¦  «        Z6 e7d¦  «         n# e)$ r Y nw xY wY nw xY w	 ddl8m9Z:m;Z;m<Z< ddl1m8Z8  e;¦   «         Z=n;# e)$ r3 dxZ:Z8dZ=d„ Z<e$s%	  ed¦  «        Z6e$s e7d¦  «         n# e)$ r Y nw xY wY nw xY wd=d„Z>ej?         @                    ¦   «         	 	 	 	 d>d„¦   «         ZA	 d dl9mBZC e$s e7d¦  «         n2# e)$ r* dZCe$s#	  ed ¦  «        ZD e7d!¦  «         n# e)$ r Y nw xY wY nw xY weC€:	 d d"lCmEZC n2# e)$ r* dZCe$s#	  ed#¦  «        ZD e7d!¦  «         n# e)$ r Y nw xY wY nw xY wej?         @                    ¦   «         d$„ ¦   «         ZFd%„ ZGej?         @                    ¦   «         	 	 	 	 d?d'„¦   «         ZH e jI        d¬(¦  «        d)„ ¦   «         ZJd*„ ZKd+„ ZL e jI        d¬(¦  «        d,„ ¦   «         ZMd-„ ZNd.„ ZOd/„ ZPd=d0„ZQh d1£ZRd@d2„ZSed=d3„¦   «         ZTedeSfd4„¦   «         ZUd5„ ZVd@d6„ZWg d7¢ZXd8„ ZYej?         @                    ¦   «         d9„ ¦   «         ZZ	 	 	 	 	 	 	 	 	 	 	 dAd;„Z[dS )Bé    N)ÚcontextmanagerÚnullcontext)Ú
SDPBackendÚsdpa_kernel)Útqdm)Úversion)Úcuda_graphsÚoffloadÚdarwinÚmps)r   r   é   FT))r   é   )r   é	   )r   r   )é
   r   )é   r   )r   é   )Ú	installedÚ	supported)ÚsolÚvedaÚvsa)Úmemory_efficient_attention)Úsageattn_varlenr   )Ú
sage1_corec
                 óÖ   — |	rt          j        |d¦  «        nd }
t          | ||
€|n||
                     |j        ¦  «        z  ||||||¬¦	  «	        }|
€|nt          j        ||
d¦  «        S )N)r   )Ú	is_causalÚsm_scaler   )r   Úv_scalesr   ÚtoÚdtypeÚscale_output)ÚqÚkÚvÚcu_seqlens_qÚcu_seqlens_kvÚmax_seqlen_qÚmax_seqlen_kvr   r   Úscale_vÚscaleÚos               ú8C:\Users\johnn\OneDrive\Documents\AI\shared\attention.pyÚsageattn_varlen_wrapperr-   G   s‰   € ð 18ÐA•
Ô# A tÑ,Ô,Ð,¸TˆÝØØØ�ˆAˆA A¨¯ª°´Ñ(9Ô(9Ñ$9ØØØØØØð

ñ 

ô 

ˆð �Mˆqˆq¥zÔ'>¸qÀ%ÈÑ'KÔ'KÐKó    )Úblock_sparse_sage2_attn_cudaÚsageattentionzMSage Attention has been detected but it won't work until Triton is installed.)ÚsageattnÚis_sage2_supportedÚ&sageattn_attention_mask_support_reason)Ú
sage2_corec                  ó   — dS )NzSageAttention 2 is unavailable© )ÚargsÚkwargss     r,   ú<lambda>r9   w   s   € ÐEe€ r.   zOSage Attention 2 has been detected but it won't work until Triton is installed.c                 óh  — | j         dk    r|                      dd¦  «        } nT| j         dk    r|                      d¦  «        } n3| j         dk    r(|                      d¦  «                             d¦  «        } t          j        | ¦  «        rL|                      |¬¦  «        } |�3|                      | t          j        | j        ¦  «        j	        ¦  «        } n[| j        t          j
        k    rF|�| |z  } |                      dd	¬
¦  «        }t          j        || t          j        | ¦  «        ¦  «        } | S )zÐpay_attention's mask ((batch, q, heads, k), 3D or 2D) as SageAttention takes it: (batch, heads, q, k), in q's dtype when it is
    additive; a boolean mask lets the queries that see no key attend to all keys.é   r   é   é   r   )r    NéÿÿÿÿT)ÚdimÚkeepdim)ÚndimÚ	transposeÚ	unsqueezeÚtorchÚis_floating_pointr   Úmasked_fillÚfinfor    ÚminÚboolÚanyÚwhereÚ	ones_like)Úattention_maskÚq_dtypeÚcausal_maskÚhas_keyss       r,   Úsage_attention_maskrQ      s/  € ð Ô˜aÒÐØ'×1Ò1°!°QÑ7Ô7ˆˆØ	Ô	 Ò	!Ð	!Ø'×1Ò1°!Ñ4Ô4ˆˆØ	Ô	 Ò	!Ð	!Ø'×1Ò1°!Ñ4Ô4×>Ò>¸qÑAÔAˆÝÔ˜~Ñ.Ô.ð `Ø'×*Ò*°Ð*Ñ9Ô9ˆØÐ"Ø+×7Ò7¸¸ÅeÄkÐR`ÔRfÑFgÔFgÔFkÑlÔlˆNøØ	Ô	¥¤Ò	+Ð	+ØÐ"Ø+¨kÑ9ˆNØ!×%Ò%¨"°dÐ%Ñ;Ô;ˆÝœ X¨~½u¼È~Ñ?^Ô?^Ñ_Ô_ˆØÐr.   c                 ó¾  — |�±d }|r‘| d         j         d         | d         j         d         | d         j        }	}}t          j        ||	¬¦  «        d d …d f         }
t          j        ||	¬¦  «        d d d …f         }||
k                         dd||¦  «        }d}t          || d         j        |¦  «        }t          | d||||¬¦  «        }|                      ¦   «          |S )Nr   r   ©ÚdeviceFÚNHD)Útensor_layoutr   Ú	recycle_qÚ	attn_maskr)   )	ÚshaperT   rD   ÚarangeÚviewrQ   r    Ú	sageattn2Úclear)Úqkv_listÚattention_lengthrW   rM   Úcausalr)   rO   ÚlqÚlkrT   ÚrowÚcolr+   s                r,   Úsageattn2_wrapperre   ”   sÿ   € ð Ð!ØˆØð 	Ø% aœ[Ô.¨qÔ1°8¸A´;Ô3DÀQÔ3GÈÐRSÌÔI[�F�ˆBÝ”,˜r¨&Ð1Ñ1Ô1°!°!°!°T°'Ô:ˆCÝ”,˜r¨&Ð1Ñ1Ô1°$¸¸¸°'Ô:ˆCØ #š:×+Ò+¨A¨q°"°bÑ9Ô9ˆKØˆFÝ,¨^¸XÀa¼[Ô=NÐP[Ñ\Ô\ˆÝ�(¨%¸6ÈYÐbpð  {Bð  	Cñ  	Cô  	C€AØ‡N‚NÑÔÐà€Hr.   )Úsageattn_blackwellzRSage Attention 3 is installed but it won't be supported until Triton is installed.rf   zOSage Attention 3 has been detected but it won't work until Triton is installed.)Úsageattn3_blackwellrg   c                 ó  — | \  }}}|                      dd¦  «        }|                      dd¦  «        }|                      dd¦  «        }t          |||¦  «        }|                      dd¦  «        }|                      ¦   «          |S )Nr   r<   )rB   Ú	sageattn3r]   )r^   r_   r"   r#   r$   r+   s         r,   Úsageattn3_wrapperrj   Å   sƒ   € ð
 �F€A€aˆð 	
�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AÝ�!�Q˜ÑÔ€AØ	�Š�A�aÑÔ€AØ‡N‚NÑÔÐà€Hr.   c                 ó6  — t           j        j        rB| j        j        dk    s2| j        d         dk    s!|�| j        t           j        t           j        fvrdS t           j	         
                    | j        ¦  «        j                             dd¦  «        d         dv S )	NÚcudar>   i   Fú:r   r   )Úgfx1100Úgfx1151)rD   r   ÚhiprT   ÚtyperY   r    Úbfloat16Úfloat32rl   Úget_device_propertiesÚgcnArchNameÚsplit)ÚqueryrM   s     r,   Ú_sdpa_requires_mathrx   ô   s©   € åŒ=Ôð  ¤Ô 1°VÒ ;Ð ;¸u¼{È2¼ÐRUÒ?UÐ?UÐYgÐYoÐsxÔs~õ  HMô  HVõ  X]ô  Xeð  Gfð  tfð  tfØˆuÝŒ:×+Ò+¨E¬LÑ9Ô9ÔE×KÒKÈCÐQRÑSÔSÐTUÔVÐZpÐpÐpr.   ç        c           
      óø  — | \  }}}|                      dd¦  «        }|                      dd¦  «        }|                      dd¦  «        }|d k    r|                      dd¦  «        }t          ||¦  «        rt          t          j        ¦  «        nt          ¦   «         5  t          j        |||||||¬¦  «                              dd¦  «        }	d d d ¦  «         n# 1 swxY w Y   ~~~|                      ¦   «          |	S )Nr   r<   )rX   r   r*   Ú	dropout_p)	rB   rx   r   r   ÚMATHr   ÚFÚscaled_dot_product_attentionr]   )
r^   r_   rM   r`   Úsoftmax_scaler{   r"   r#   r$   r+   s
             r,   Úsdpa_wrapperr€   û   sj  € ð �G€A€qˆ!à	�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AØ˜ÒÐØ'×1Ò1°!°AÑ6Ô6ˆÝ)<¸QÀÑ)OÔ)OÐ	b�•Z”_Ñ	%Ô	%Ð	%ÕU`ÑUbÔUbð Yð YÝÔ*¨1¨a°¸nÐX^Ðfsð  @Ið  Jñ  Jô  J÷  Tò  Tð  UVð  WXñ  Yô  YˆðYð Yð Yñ Yô Yð Yð Yð Yð Yð Yð Yøøøð Yð Yð Yð Yà	ˆ1ˆaØ‡N‚NÑÔÐà€Hs   Â0CÃCÃC)Úmaxsizec                  ó0  — ddg} t           d k    r|                      d¦  «         t          d k    r|                      d¦  «         t          d k    r|                      d¦  «         t          d k    r7t          d¦  «                             d¦  «        r|                      d¦  «         t          d k    r7t          d¦  «                             d¦  «        r|                      d	¦  «         t          d k    r|                      d
¦  «         t          | ¦  «        S )NÚsdpaÚautoÚflashÚxformersÚsager0   Ú2Úsage2ÚradialÚsage3)
Ú
flash_attnÚappendr   r-   r\   r   Ú
startswithr/   ri   Útuple)Úrets    r,   Ú_attention_modesr‘     sÿ   € ð �6Ð
€CÝ�TÒÐØ�
Š
�7ÑÔÐÝ! TÒ)Ð)Ø�
Š
�:ÑÔÐÝ $Ò&Ð&Ø�
Š
�6ÑÔÐÝ�DÒÐ�W _Ñ5Ô5×@Ò@ÀÑEÔEÐØ�
Š
�7ÑÔÐÝ# tÒ+Ð+µ¸Ñ0HÔ0H×0SÒ0SÐTWÑ0XÔ0XÐ+Ø�
Š
�8ÑÔÐå�DÒÐØ�
Š
�7ÑÔÐå�‰:Œ:Ðr.   c                  ó8   — t          t          ¦   «         ¦  «        S ©N)Úlistr‘   r6   r.   r,   Úget_attention_modesr•   (  s   € ÝÕ Ñ"Ô"Ñ#Ô#Ð#r.   c                  ó8   — t          t          ¦   «         ¦  «        S r“   )r”   Ú_supported_attention_modesr6   r.   r,   Úget_supported_attention_modesr˜   +  s   € ÝÕ*Ñ,Ô,Ñ-Ô-Ð-r.   c                  ó¨  — t           rdS t          ¦   «         } t          j                             ¦   «         \  }}|dk     st
          sd| v r|                      d¦  «         t          rt
          s2d| v r|                      d¦  «         d| v r|                      d¦  «         |dk     st
          sd| v r|                      d¦  «         t          | ¦  «        S )N)rƒ   r„   r   r‹   r‰   rŠ   é   r‡   )	Ú_is_mpsr•   rD   rl   Úget_device_capabilityÚtriton_installedÚremoveÚsage2_supportedr�   )r�   ÚmajorÚminors      r,   r—   r—   .  sØ   € õ ð  ØÐÝ
Ñ
Ô
€CÝ”:×3Ò3Ñ5Ô5�L€Eˆ5Ø�Š
ˆ
Õ*ˆ
Ø�cˆ>ˆ>Ø�JŠJ�wÑÔÐåð !Õ"2ð !Ø�cˆ>ˆ>Ø�JŠJ�wÑÔÐØ�sˆ?ˆ?Ø�JŠJ�xÑ Ô Ð àˆq‚y€yÕ(€yØ�Sˆ=ˆ=Ø�JŠJ�vÑÔÐå�‰:Œ:Ðr.   c                  óf   — t          ¦   «         d„ t                               ¦   «         D ¦   «         z   S )Nc                 ó(   — g | ]\  }}|d          ¯|‘ŒS )r   r6   ©Ú.0ÚmodeÚstatuss      r,   ú
<listcomp>z0get_override_attention_modes.<locals>.<listcomp>G  s(   € Ð#xÐ#xÐ#x©\¨T°6ÐdjÐkvÔdwÐ#x DÐ#xÐ#xÐ#xr.   )r•   ÚATTENTION_MODE_AVAILABILITYÚitemsr6   r.   r,   Úget_override_attention_modesr«   F  s/   € ÝÑ Ô Ð#xÐ#xÕ=X×=^Ò=^Ñ=`Ô=`Ð#xÑ#xÔ#xÑxÐxr.   c                  óf   — t          ¦   «         d„ t                               ¦   «         D ¦   «         z   S )Nc                 ó(   — g | ]\  }}|d          ¯|‘ŒS )r   r6   r¤   s      r,   r¨   z:get_supported_override_attention_modes.<locals>.<listcomp>K  sE   € ð  .Cð  .Cð  .C±|°t¸VÐntð  vAô  oBð  .C¨dð  .Cð  .Cð  .Cr.   )r˜   r©   rª   r6   r.   r,   Ú&get_supported_override_attention_modesr®   J  sD   € Ý(Ñ*Ô*ð  .Cð  .CÕGb×GhÒGhÑGjÔGjð  .Cñ  .Cô  .Cñ  Cð  Cr.   c                  ó8   — dD ]} | t          ¦   «         v r| c S ŒdS )N)r‰   r‡   rƒ   rƒ   )r˜   )Úattns    r,   Úget_default_attention_moder±   N  s6   € Ø)ð ð ˆØÕ0Ñ2Ô2Ð2Ð2ØˆKˆKˆKð 3àˆ6r.   c                 ó¤   — t           st          j                             ¦   «         sd S t          j                             | ¦  «        \  }}|dz  |z   S )Nr   )r›   rD   rl   Úis_availablerœ   )rT   r    r¡   s      r,   Úget_current_cuda_architecturer´   U  sL   € Ýð •e”j×-Ò-Ñ/Ô/ð ØˆtÝ”:×3Ò3°FÑ;Ô;�L€Eˆ5Ø�2‰:˜ÑÐr.   >   r‡   r‰   r‹   rŠ   c                 ó,  — t          | pd¦  «                             ¦   «                              ¦   «         }|dk    rt          ¦   «         n|}|rt	          ¦   «         nd }|�|dk     r|t
          v rd}|t          ¦   «         vrt          d|› d�¦  «        ‚|S )Nr„   éY   rƒ   zAttention mode 'z/' is not installed or supported on this system.)ÚstrÚstripÚlowerr±   r´   Ú_SAGE_ATTENTION_MODESr˜   Ú
ValueError)Úattention_modeÚdisable_sage_pre_adar°   Úarchitectures       r,   Úresolve_attention_moder¿   _  s­   € Ýˆ~Ð' Ñ(Ô(×.Ò.Ñ0Ô0×6Ò6Ñ8Ô8€DØ+/°6ª>¨>Õ%Ñ'Ô'Ð'¸t€DØ6JÐTÕ0Ñ2Ô2Ð2ÐPT€LØÐ L°2Ò$5Ð$5¸$ÕBWÐ:WÐ:WØˆØÕ0Ñ2Ô2Ð2Ð2ÝÐa¨DÐaÐaÐaÑbÔbÐbØ€Kr.   c              #   óÆ  K  — t           j                             dt          ¦  «        }|t          u s|dk    r| pt	          ¦   «         t           j        d<   n|t           j        d<   	 d V — |t          u r"t           j                             dd ¦  «         d S |t           j        d<   d S # |t          u r!t           j                             dd ¦  «         n|t           j        d<   w xY w)NÚ
_attentionr„   )r
   Úshared_stateÚgetÚ_MISSINGr±   Úpop)Údefault_attentionÚpreviouss     r,   Úattention_shared_staterÈ   j  sê   è è € åÔ#×'Ò'¨µhÑ?Ô?€HØ•8ÐÐ˜x¨6Ò1Ð1Ø->Ð-^ÕB\ÑB^ÔB^�Ô˜\Ñ*Ð*à-5�Ô˜\Ñ*ð:Øˆˆˆà•xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8Ð8à19�GÔ  Ñ.Ð.Ð.øð •xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8à19�GÔ  Ñ.Ð9Ð9Ð9Ð9s   Á%B% Â%;C c              +   óš  K  — t           j                             dt          ¦  «        } || fi |¤Žt           j        d<   	 t           j        d         V — |t          u r"t           j                             dd ¦  «         d S |t           j        d<   d S # |t          u r!t           j                             dd ¦  «         n|t           j        d<   w xY w)NrÁ   )r
   rÂ   rÃ   rÄ   rÅ   )r¼   ÚresolverÚresolver_kwargsrÇ   s       r,   Úattention_config_shared_staterÌ   z  sÖ   è è € åÔ#×'Ò'¨µhÑ?Ô?€HØ)1¨°.Ð)TÐ)TÀOÐ)TÐ)T…GÔ˜Ñ&ð:ÝÔ" <Ô0Ð0Ð0Ð0à•xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8Ð8à19�GÔ  Ñ.Ð.Ð.øð •xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8à19�GÔ  Ñ.Ð9Ð9Ð9Ð9s   ¿B Â;C
c                  ó8   — 	 ddl m}  n# t          $ r Y dS w xY w| S )zYshared.spas_sage_attn_core when the SpargeAttn package and its kernels import, else None.r   ©Úspas_sage_attn_coreN)ÚsharedrÏ   Ú	ExceptionrÎ   s    r,   Ú_sparge_corerÒ   †  sD   € ðØ.Ð.Ð.Ð.Ð.Ð.Ð.øÝð ð ð ØˆtˆtðøøøàÐs   ‚	 ‰
–c                 óª  — |pt           j        d         }t           j                             dd¦  «        }|dk    r-t           j                             d¦  «        pt          ¦   «         }|dv rt          ¦   «         n|}|dk    r4|rt	          ¦   «         nd}|�|                     | |¦  «        nd}|€dn||fS |d	k    rVt          �,|t          j        | ¦  «        vs|rt          j        | ¦  «        sdS t          |rt          j	        nt          j
        fS |d
vst          �|rt          | ¬¦  «        �dS t          j        | |||¦  «        }|€dnt          |fS )aÜ  (core, settings) of the staged_* functions (q, k and v quantized as soon as each is computed) of the SageAttention that
    pay_attention would run without sequence lengths on this device (with this force_attention, and a mask when masked):
    shared.sage2_core for SageAttention 2, shared.sage1_core for SageAttention without a mask; None otherwise. The model def key
    sage_v_scaling (shared state _sage_v_scaling, set for each generation) scales v on their Triton paths.rÁ   Ú_sage_v_scalingFr   Ú_attention_fallback©r   ÚvdnÚspargeNr‡   )r‰   rŠ   rS   )r
   rÂ   rÃ   r±   rÒ   Ústaged_settingsr   Ú	head_dimsÚmasks_supportedÚSCALED_V_SETTINGSÚSETTINGSr\   r3   r4   )rT   Úhead_dimÚforce_attentionÚmaskedr°   r)   ÚcoreÚsettingss           r,   Úsage_stagedrã   �  sŠ  € ð
 Ð@�gÔ2°<Ô@€DÝÔ"×&Ò&Ð'8¸%Ñ@Ô@€GØˆv‚~€~ÝÔ#×'Ò'Ð(=Ñ>Ô>Ð^ÕB\ÑB^ÔB^ˆØ+/°>Ð+AÐ+AÕ%Ñ'Ô'Ð'Àt€DØˆxÒÐØ!'Ð1�|‰~Œ~ˆ~¨TˆØ=AÐ=M�4×'Ò'¨°Ñ9Ô9Ð9ÐSWˆØÐ'ˆtˆt¨d°HÐ-=Ð=Øˆv‚~€~ÝÐ µÔ1EÀfÑ1MÔ1MÐ!MÐ!MÐQWÐ!MÕ`jÔ`zð  |Bñ  aCô  aCÐ!MØ�4Ý¸7Ð[�:Ô7Ð7Í
ÔH[Ð[Ð[ØÐ&Ð&Ð&­)Ð*;¸vÐ*;ÕJpÐx~ÐJÑJÔJð  KLØˆtÝÔ)¨&°&¸'À8ÑLÔL€HØÐ#ˆ4ˆ4­*°hÐ)?Ð?r.   )r©   rÌ   rÈ   r´   r«   r®   r¿   Úpay_attentionrã   rQ   Ú	attentionc                 óì   — t           rdnd}t          j        d| z  dz   gt          j        |¬¦  «        }t	          | ¦  «        D ]0}||         }||z  |z   }|dz   |z  }||d|z  dz   <   ||d|z  dz   <   Œ1|S )Nr   rl   r<   r   ©r    rT   )r›   rD   ÚzerosÚint32Úrange)	Ú
batch_sizeÚlensÚmax_lenÚ
_cu_deviceÚ
cu_seqlensÚiÚsÚs1Ús2s	            r,   Úget_cu_seqlensrô   ´  sœ   € å!Ð-�� v€JÝ”˜a *™n¨qÑ0Ð1½¼ÈZÐXÑXÔX€Jå�:ÑÔð #ð #ˆØ�ŒGˆØ�‰[˜1‰_ˆØ�!‰e�wÑˆØ "ˆ
�1�q‘5˜1‘9ÑØ "ˆ
�1�q‘5˜1‘9ÑÐàÐr.   c                 ó’   — | j         d         }t          j        || j        ¦  «        d uo|d u p|t          j        | j        ¦  «        vS )Nr>   )rY   r   Úpadded_head_dimrT   rÚ   )r"   r   rÞ   s      r,   Ú_sage_rows_head_dimr÷   Â  sq   € ð Œw�rŒ{€HÝÔ% h°´Ñ9Ô9ÀÐEð  TÈ=Ð\`ÐK`ð  LSÐdlÕt~ô  uIð  JKô  JRñ  uSô  uSð  eSð  Tr.   ©r>   r>   c                 ó  — |�|rt          d¦  «        ‚|d k    �rˆ|d k    rt          j        d         n|}|dv r-t          j                             d¦  «        pt	          ¦   «         }|dv rt	          ¦   «         n|}|dk    rdn|}d }t
          rd}n/|dk    r)t          d k    r|
d k    r|d k    rt          | |d	¬
¦  «        }|dk    r|€t          d k    r|
d k    r	|d k    rd}n‡|dk    rKt          �D|
d k    r>|d k    r8t          | d         |¦  «        r"t          j
        | d         j        ¦  «        rd}n6d}|dk    r.t          s'|rd|› d�nd}t          j        d|› d|› d�¦  «         da|j        t           j        k    r&t$          s|                     t           j        ¦  «        }|d k    rt          j        d         n|}|dv r-t          j                             d¦  «        pt	          ¦   «         }|dv rt	          ¦   «         n|}| \  }}}|                      ¦   «          |j        }|dk    o|
d k    o|d k    o| ot          ||¦  «        }|j        t           j        k    rft$          s_|s]|                     t           j        ¦  «        }|                     t           j        ¦  «        }|                     t           j        ¦  «        }d}|                     d¦  «        |                     d¦  «        |                     d¦  «        }}}|                     |j        ¦  «        }|                     |j        ¦  «        }t/          |¦  «        }t/          |¦  «        |k    r|                     |ddd¦  «        }t/          |¦  «        |k    r|                     |ddd¦  «        }|j        j        dk    r<|                     ¦   «         |                     ¦   «         |                     ¦   «         }}}|dk    rddlm}m} ddlm}m } |dk    rd}|dk    rt          �tC          j"        ¦   «         rd}|dk    �r©|d k    �r¢|dv �r�|d k    sJ ‚|
d k    sJ ‚g }g }|d         } d}!|dd …         D ]<}"|"| k    r|!dz  }!Œ| #                    |!¦  «         | #                    | ¦  «         d}!|"} Œ=| #                    |!¦  «         | #                    |"¦  «         t/          |¦  «        dk    s|d         |j$        d         k    rãt!          j%        ||¦  «        }#t!          j%        ||¦  «        }$t!          j%        ||¦  «        }%d\  }}}d„ tM          |$|¦  «        D ¦   «         }$d„ tM          |%|¦  «        D ¦   «         }%g }&tM          |#|$|%¦  «        D ]8\  }'}(})|'|(|)g} d\  }'}(})|& #                    tO          | ||||¬ ¦  «        ¦  «         Œ9d\  }#}$}%t!          j(        |&d¬!¦  «        }&|&S n‹|
d k    s|d k    r|dv r{|dk    sJ ‚|
d k    r|
d          )                    ¦   «         n|}*|d k    r|d          )                    ¦   «         n|}+||*z
  }|d d …d |*…f         }|d d …d |+…f         }|d d …d |+…f         }|�!|d"k    rtT          stW          j,        d#¦  «         |dk    r?|s=|j$        d         d$vr.tZ          s%t          j        d%|j$        d         › d&�¦  «         da-d}|dk    r|r|d'k    �r|dk    rÑ|d k    r(t!          j.        |f|t           j/        |j        ¬(¦  «        }|
d k    r(t!          j.        |f|t           j/        |j        ¬(¦  «        }
 |j0        dg|j$        d)d …         ¢R Ž } |j0        dg|j$        d)d …         ¢R Ž } |j0        dg|j$        d)d …         ¢R Ž }tc          ||
|¦  «        },tc          |||¦  «        }-�n8|
d k    r|
d          )                    ¦   «         n|}*|d k    r|d          )                    ¦   «         n|}+|*|k    s|+|k    rSt!          j2        d|*|gt           j/        |j        ¬(¦  «        },t!          j2        d|+|gt           j/        |j        ¬(¦  «        }-nVt!          j3        dd*|z  |t           j/        |j        ¬(¦  «        },t!          j3        dd*|z  |t           j/        |j        ¬(¦  «        }-| 4                    d¦  «        }| 4                    d¦  «        }| 4                    d¦  «        }|r\d+„ |||fD ¦   «         } ~~~t          j5        | |	t          j                             d,d-¦  «        |€d ntm          ||¦  «        |¦  «        }.�nT|dk    rOto          ||||,|-||||t          j                             d,d-¦  «        ¬.¦
  «
         8                    d||f¦  «        }.�nÿ|dk    r|||g} ~~~ts          | |¦  «        }.�nß|dk    r=|||g} ~~~tu          | ||	||t          j                             d,d-¦  «        ¬/¦  «        }.�nœ|dk    r|||g} ~~~tw          | |||||¬0¦  «        }.�nw|d'k    rD|d"k    r>ty          j=        ||||,|-d d |||||¬1¦  «        d          8                    d||f¦  «        }.�n-|d'k    r7t}          j=        ||||,|-|||||||¬2¦  «         8                    d||f¦  «        }.nð|d3k    rêdd4l?m@}/ |d k    r|
d k    rtƒ          |||¦  «        }.nÆ|d k    rB|
d k    r<|/ B                    |g|z  |t‡          |¦  «        ¦  «        }0tƒ          ||||0¬5¦  «        }.n~|dk    rv|
d k    r|
d          )                    ¦   «         n|}*|d k    r|d          )                    ¦   «         n|}+|/ B                    |*||*z
  g||+dg¦  «        }0tƒ          ||||0¬5¦  «        }.nJ ‚|.                     |¦  «        }.|dk    rSt!          j(        |.t!          jD        |.j$        d         |g|.j$        d)d …         ¢R |.j        |.j        ¬(¦  «        gd¦  «        }.|.S )6Nzmpay_attention received both attention_mask and causal=True; build a combined mask once and pass causal=False.rÁ   )r   r   rÕ   rÖ   rŠ   r‰   z"MPS uses SDPA for masked attentionrU   )rV   r‡   r   rƒ   z (ú)Ú z>[WAN2GP] Attention mask is unsupported by selected attention 'ú'z!. Masked attention will use SDPA.Tr   r>   r   Úchipmunk)ÚSparseDiffMlpÚSparseDiffAttn)ÚLayerCounterÚGLOBAL_CONFIGr‹   )r‰   r‹   rƒ   )NNNc                 ó0   — g | ]\  }}|d d …d |…f         ‘ŒS r“   r6   ©r¥   ÚuÚszs      r,   r¨   z!pay_attention.<locals>.<listcomp>+  ó,   € ÐGÐGÐG¡u q¨"˜˜1˜1˜1˜c˜r˜c˜6œÐGÐGÐGr.   c                 ó0   — g | ]\  }}|d d …d |…f         ‘ŒS r“   r6   r  s      r,   r¨   z!pay_attention.<locals>.<listcomp>,  r  r.   )r{   r   r`   rß   )r?   r=   zBFlash attention 3 is not available, use flash attention 2 instead.)é@   é€   zA[WAN2GP] SageAttention does not support this attention (head_dim z): it will use SDPA.r…   rç   éþÿÿÿr<   c                 ót   — g | ]5} |j         d g|j        dd…         ¢R Ž                      |j        ¦  «        ‘Œ6S )r>   r<   N)ÚreshaperY   r[   )r¥   Úts     r,   r¨   z!pay_attention.<locals>.<listcomp>c  sF   € ÐQÐQÐQÀ!�I�A”I˜bÐ/ 1¤7¨1¨2¨2¤;Ð/Ð/Ð/×4Ò4°Q´WÑ=Ô=ÐQÐQÐQr.   rÔ   F)
r"   r#   r$   r%   r&   r'   r(   r   r   r)   )rW   rM   r`   r)   )rM   r`   r   r{   )r"   r#   r$   r%   Úcu_seqlens_kÚ	seqused_qÚ	seqused_kr'   Úmax_seqlen_kr   r`   Údeterministic)r"   r#   r$   r%   r  r'   r  r{   r   r`   Úwindow_sizer  r†   )ÚBlockDiagonalPaddedKeysMask)Ú	attn_bias)Er»   r
   rÂ   rÃ   r±   r›   r\   r3   r   r÷   rÛ   rT   Ú_MASKED_ATTENTION_SDPA_WARNEDr   Úwriter    rD   rr   Úbfloat16_supportedr   Úfloat16r]   ÚsizeÚlenÚexpandrq   Ú
contiguousÚsrc.chipmunk.modulesrþ   rÿ   Úsrc.chipmunk.utilr   r  r	   Ú	recordingr�   rY   rv   Úziprä   ÚcatÚitemÚFLASH_ATTN_3_AVAILABLEÚwarningsÚwarnÚ_HEAD_DIM_SDPA_WARNEDÚfullré   r  rô   ÚtensorrZ   Úsqueezerå   rQ   r-   Ú	unflattenrj   re   r€   Úflash_attn_interfaceÚflash_attn_varlen_funcrŒ   Úxformers.ops.fmha.attn_biasr  r   Úfrom_seqlensr”   Úempty)1r^   r{   r   r`   r  r  r   rß   rM   rW   Úq_lensÚk_lensÚrequested_attnÚsupport_reasonÚdetailr°   r"   r#   r$   Ú	out_dtypeÚ	sage_rowsÚfinal_paddingÚbra   rb   Úbatchrþ   rÿ   r   r  Úchunk_sizesÚk_sizesÚcurrent_sizeÚcurrent_countÚk_lenÚq_chunksÚk_chunksÚv_chunksr+   Úsub_qÚsub_kÚsub_vÚszqÚszkr%   r  Úxr  rX   s1                                                    r,   rä   rä   Ê  s9  € ð$ Ð! fÐ!Ýð  Iñ  Jô  Jð  	JØ˜ÒÑØ?NÐRVÒ?VÐ?V�Ô-¨lÔ;Ð;Ð\kˆØ˜_Ð,Ð,Ý$Ô1×5Ò5Ð6KÑLÔLÐlÕPjÑPlÔPlˆNØ9GÈ>Ð9YÐ9YÕ3Ñ5Ô5Ð5Ð_mˆØ$2°hÒ$>Ð$>˜˜ÀNˆØˆÝð 	sØAˆNˆNØ˜wÒ&Ð&­9¸Ò+<Ð+<ÀÈ4ÂÀÐTZÐ^bÒTbÐTbÝCÀHÈnÐlqÐrÑrÔrˆNØ˜WÒ$Ð$¨Ð)?ÅIÐQUÒDUÐDUÐZ`ÐdhÒZhÐZhÐmsÐw{Òm{Ðm{Ø%ˆOˆOØ Ò&Ð&­:Ð+AÀfÐPTÂnÀnÐY_ÐcgÒYgÐYgÕlð  AIð  JKô  ALð  N[ñ  m\ô  m\ÐYgÝÔ,¨X°a¬[Ô-?Ñ@Ô@ð Zhà$ˆOˆOà$ˆOØ Ò'Ð'Õ0MÐ'Ø3AÐIÐ/˜nÐ/Ð/Ð/Ð/Àr�Ý”
ð  WÐ\jð  Wð  WÐmsð  Wð  Wð  Wñ  Xô  Xð  XØ04Ð-ØÔ ¥E¤NÒ2Ð2Õ;MÐ2Ø+×.Ò.­u¬}Ñ=Ô=ˆNØ1@À4Ò1GÐ1G�7Ô Ô-Ð-È_€DØˆÐÐÝÔ#×'Ò'Ð(=Ñ>Ô>Ð^ÕB\ÑB^ÔB^ˆØ+/°>Ð+AÐ+AÕ%Ñ'Ô'Ð'Àt€Dà�E€A€aˆØ‡N‚NÑÔÐØ”€Ià˜’Ð} 6¨T¢>Ð}°fÀ²nÐ}ÈVÈÐ}ÕXkÐlmÐo|ÑX}ÔX}€IØ„w•%”.Ò Ð Õ);Ð ÀIÐ Ø�DŠD•”ÑÔˆØ�DŠD•”ÑÔˆØ�DŠD•”ÑÔˆØ€MØ—’�q‘	”	˜1Ÿ6š6 !™9œ9 a§f¢f¨Q¡i¤iˆ2€r€Aà	�ŠˆQŒW‰Œ€AØ	�ŠˆQŒW‰Œ€AÝ�‰FŒF€EÝ
ˆ1�v„v�‚€˜AŸHšH U¨B°°BÑ7Ô7˜Ý
ˆ1�v„v�‚€˜AŸHšH U¨B°°BÑ7Ô7˜Ø„x„}˜ÒÐ¨¯ª©¬¸¿º¹¼ÈÏÊÉÌ Q !˜qØˆzÒÐØFÐFÐFÐFÐFÐFÐFÐFØAÐAÐAÐAÐAÐAÐAÐAØˆxÒÐ˜w˜àˆw‚€�9Ð0µ[Ô5JÑ5LÔ5LÐ0ÐU\Èdàˆ1‚u�u�˜4’‘ DÐ,FÐ$FÑ$FØ Ò%Ð%Ð%Ð%à˜Š~ˆ~ˆ~ˆ~ØˆØˆØ˜a”yˆØˆØ˜A˜B˜B”Zð 	%ð 	%ˆEØ˜Ò$Ð$Ø Ñ"��à×"Ò" =Ñ1Ô1Ð1Ø—’˜|Ñ,Ô,Ð,Ø !�Ø$��Ø×Ò˜=Ñ)Ô)Ð)Ø�Š�uÑÔÐÝˆ{ÑÔ˜aÒÐ 6¨!¤9°´¸´
Ò#:Ð#:Ý”k ! [Ñ1Ô1ˆHÝ”k ! [Ñ1Ô1ˆHÝ”k ! [Ñ1Ô1ˆHØ&‰GˆAˆq�!ØGÐGµ°H¸gÑ0FÔ0FÐGÑGÔGˆHØGÐGµ°H¸gÑ0FÔ0FÐGÑGÔGˆHØˆAÝ'*¨8°X¸xÑ'HÔ'Hð Ið IÑ#��u˜eØ! 5¨%Ð0�Ø&6Ñ#��u˜eØ—’� x¸9ÐTaÐjpð  CGð  Hñ  Hô  Hñ  Iô  Ið  Ið  IØ+;Ñ(ˆH�h Ý”	˜! 1Ð%Ñ%Ô%ˆAØˆHð $;ð �DŠ.ˆ.˜F dšN˜N°Ð8RÐ0RÐ0RØ�AŠvˆvˆvˆvØ"(¨D¢. .ˆf�QŒi�nŠnÑÔÐ°bˆØ"(¨D¢. .ˆf�QŒi�nŠnÑÔÐ°bˆØ˜S™ˆØˆaˆaˆa��#�ˆgŒJˆØˆaˆaˆa��#�ˆgŒJˆØˆaˆaˆa��#�ˆgŒJˆàÐ˜w¨!š|˜|Õ4J˜|ÝŒØPñ	
ô 	
ð 	
ð ˆv‚~€~˜i€~¨A¬G°B¬K¸yÐ,HÐ,HÝ$ð 	)ÝŒJÐ|Ð[\Ô[bÐceÔ[fÐ|Ð|Ð|Ñ}Ô}Ð}Ø$(Ð!ØˆØˆV‚|€|˜I€|¨¨wª©Ø�Š6ˆ6Ø˜Š~ˆ~Ýœ Q D¨"µE´KÈÌÐQÑQÔQ�Ø˜Š~ˆ~Ýœ Q D¨"µE´KÈÌÐQÑQÔQ�Ø�”	˜"Ð,˜qœw r s sœ|Ð,Ð,Ð,ˆAØ�”	˜"Ð,˜qœw r s sœ|Ð,Ð,Ð,ˆAØ�”	˜"Ð,˜qœw r s sœ|Ð,Ð,Ð,ˆAÝ'¨¨6°2Ñ6Ô6ˆLÝ'¨¨6°2Ñ6Ô6ˆL‰Là&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ�bŠyˆy˜C 2šI˜IÝ$œ|¨Q°°R¨LÅÄÐTUÔT\Ð]Ñ]Ô]�Ý$œ|¨Q°°R¨LÅÄÐTUÔT\Ð]Ñ]Ô]��å$œ|¨A¨q°2©v°rÅÄÐUVÔU]Ð^Ñ^Ô^�Ý$œ|¨A¨q°2©v°rÅÄÐUVÔU]Ð^Ñ^Ô^�Ø—	’	˜!‘”ˆAØ—	’	˜!‘”ˆAØ—	’	˜!‘”ˆAð ð JØQÐQÈÈ1ÈaÀyÐQÑQÔQˆØˆq�!ÝÔ  ¨9µgÔ6J×6NÒ6NÐO`ÐbgÑ6hÔ6hØ)7Ð)?  ÕEXÐYgÐirÑEsÔEsð  vCñDô Dˆ‰à	ˆvŠˆÝ#ØØØØ&Ø'ØØØØ"ÝÔ(×,Ò,Ð->ÀÑFÔFð
ñ 
ô 
÷ Š)�A˜˜2�wÑ
Ô
ð 	
‰ð 
ˆwŠˆØ�a˜�7ˆØˆa�Ý˜h¨Ñ+Ô+ˆ‰Ø	ˆwŠˆØ�a˜�7ˆØˆa�Ý˜h¨°iÐP^ÐgmÕw~ô  xL÷  xPò  xPð  Qbð  diñ  xjô  xjð  kñ  kô  kˆ‰Ø	ˆvŠˆØ�q˜!�9ˆØˆq�!Ý˜ 2°nÈVÐcpð  }Fð  Gñ  Gô  Gˆ‰Ø	ˆwŠˆ˜7 aš<˜<Ý Ô7ØØØØ&Ø&ØØØØØ'ØØ'ð)ñ )ô )ð *+ô,÷ -6ªI°a¸!¸R¸Ñ,AÔ,Að 	
‰ð 
ˆwŠˆÝÔ-ØØØØ&Ø&ØØØØ'ØØ#Ø'ð)ñ )ô )÷ *3ª°1°q¸"°gÑ)>Ô)>ð 	
ˆð  
ˆzÒ	Ð	ØKÐKÐKÐKÐKÐKØ�TŠ>ˆ>˜f¨šn˜nÝ*¨1¨a°Ñ4Ô4ˆAˆAØ�tŠ^ˆ^ ¨$¢ Ø3×@Ò@À"ÀÈÁÈBÕQUÐV\ÑQ]ÔQ]Ñ_Ô_ˆIÝ*¨1¨a°¸yÐJÑJÔJˆAˆAØ�!ŠVˆVØ&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ3×@Ò@À#ÀrÈCÁxÐAQÐTVÐZ]Ð_`ÐYaÑcÔcˆIÝ*¨1¨a°¸yÐJÑJÔJˆAˆAà�5Ø	�ŠˆyÑÔ€AØ�qÒÐÝŒI�q�%œ+¨¬°¬
°MÐ'QÀAÄGÈBÈCÈCÄLÐ'QÐ'QÐZ[ÔZaÐjkÔjrÐuÑuÔuÐwÐyzÑ{Ô{ˆð €Hr.   )FNFr“   )FNFF)NFNry   )NF)ry   NFrø   FNNNFNN)\Ú	functoolsÚsysÚ
contextlibr   r   rD   Útorch.nn.attentionr   r   r   Úimportlib.metadatar   Úmmgpr	   r
   Útorch.nn.functionalÚnnÚ
functionalr}   r%  ÚplatformÚhasattrÚbackendsr   r³   r›   rl   rœ   r    r¡   r  r  r'  ÚobjectrÄ   Útritonr�   rp   r©   Úxformers.opsr   ÚImportErrorr,  r$  ÚModuleNotFoundErrorrŒ   ÚFLASH_ATTN_2_AVAILABLEr0   r   rû   r   r-   Úspas_sage_attnr/   Úsg2_versionÚprintr4   r1   r\   r2   r3   rŸ   rQ   ÚcompilerÚdisablere   rf   ri   Úsg3_versionrg   rj   rx   r€   Ú	lru_cacher‘   r•   r˜   r—   r«   r®   r±   r´   rº   r¿   rÈ   rÌ   rÒ   rã   Ú__all__rô   r÷   rä   r6   r.   r,   ú<module>rc     sl  ðð Ð Ð Ð Ø 
€
€
€
Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2à €€€Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø Ð Ð Ð Ð Ð Ø &Ð &Ð &Ð &Ð &Ð &Ø %Ð %Ð %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø €€€Ø &Ð &Ð &Ð &Ð &Ð &à
Œ,˜(Ò
"Ð
k w w¨u¬~¸uÑ'EÔ'EÐ
kÈ%Ì.ÔJ\×JiÒJiÑJkÔJk€à ÐLˆvˆv e¤j×&FÒ&FÀtÑ&LÔ&L�€€uØ˜q’jÐ Ø %Ð ØÐ Øˆ6‰8Œ8€ðØ€M€M€MØÐÐøðØÐÐÐøøøð &Ø%Ðo¨5°%¨.Ð<oÐ*oðð ð
 &Ø%ÐN¨%°1ª*ÐN¸U¼]Ô=NÐ9Nðð ð
 &Ø%ÐN¨%°1ª*ÐN¸U¼]Ô=NÐ9Nðð ðð Ð ð &Ø7Ð7Ð7Ð7Ð7Ð7Ð7øØð &ð &ð &Ø!%ÐÐÐð&øøøð#ØÐÐÐØ!ÐÐøØð #ð #ð #Ø"ÐÐÐð#øøøðØÐÐÐØ!ÐÐøØð ð ð Ø"ÐØ€J€J€Jðøøøð0Ø-Ð-Ð-Ð-Ð-Ð-ØÐÐÐÐÐð ØØðLð Lð Lð Lð Løð6 ð 0ð 0ð 0Ø+/Ð/Ð˜j˜j˜jð0øøøð	Ø;Ð;Ð;Ð;Ð;Ð;Ð;øØð ð ð Ø#'Ð Øð ð	Ø!˜' /Ñ2Ô2ˆKØˆEÐaÑbÔbÐbÐbøØð 	ð 	ð 	ØˆDð	øøøøøðøøøðØmÐmÐmÐmÐmÐmÐmÐmÐmÐmØÐÐÐÐÐØ)Ð)Ñ+Ô+€O€OøØð 	ð 	ð 	Ø!Ð!€I�
Ø€OØ-eÐ-eÐ*Øð ð	Ø!˜' /Ñ2Ô2ˆKØ#Ð} U UÐ+|Ñ%}Ô%}Ð%}øøØð 	ð 	ð 	ØˆDð	øøøøøð	øøøðð ð ð ð* „×ÒÑÔð ØØØðð ð ñ Ôðð.Ø8Ð8Ð8Ð8Ð8Ð8Øð dØˆÐbÑcÔcÐcøøØð ð ð Ø€IØð ð	Ø!˜'Ð"6Ñ7Ô7ˆKØˆEÐcÑdÔdÐdÐdøØð 	ð 	ð 	ØˆDð	øøøøøðøøøð Ðð	Ø>Ð>Ð>Ð>Ð>Ð>Ð>øØð ð ð Øˆ	Øð 	ðØ%˜gÐ&;Ñ<Ô<�Ø�ÐgÑhÔhÐhÐhøØð ð ð Ø�ðøøøøøðøøøð „×ÒÑÔðð ñ Ôðð\qð qð qð „×ÒÑÔð ØØØðð ð ñ Ôðð. €Ô˜TÐ"Ñ"Ô"ðð ñ #Ô"ðð($ð $ð $ð.ð .ð .ð €Ô˜TÐ"Ñ"Ô"ðð ñ #Ô"ðð.yð yð yðCð Cð Cðð ð ðð ð ð ð =Ð<Ð<Ð ðð ð ð ð ð:ð :ð :ñ „ð:ð Ø15Ð@Vð 	:ð 	:ð 	:ñ „ð	:ðð ð ð@ð @ð @ð @ð.ð ð €ðð ð ð „×ÒÑÔðTð Tñ ÔðTð ØØØØØØØØØØðhð hð hð hð hð hs,  Â4B; Â;CÄ D ÄDÄDÄD ÄD&Ä%D&Ä*D1 Ä1	D=Ä<D=ÅE Å	E!Å E!Å%E, Å,	FÅ6FÆFÆFÆFÆFÆFÆFÆF: Æ:G2ÇG$Ç#G2Ç$G,Ç)G2Ç+G,Ç,G2Ç1G2È!H5 È5	I$È?IÉI$ÉIÉI$ÉIÉI$É#I$É*I1 É1	J É;JÊJ ÊJÊJ ÊJÊJ ÊJ 