§
    oF§jvS  ã                   óô  — d dl Z d dlmZ d dlZd dlmZ d dlmZ d dlm	c m
Z d dlZd dlmZ e j        dk    o. eej        d¦  «        oej        j                             ¦   «         Zerdnej                             d¦  «        \  ZZedk    Zd	a e¦   «         Z	 d dlZd
Zn	#  d	ZY nxY wdeeoeefdv dœiZ	 d dlmZ n# e $ r dZY nw xY w	 d dl!Z!d
Z"n# e#$ r d	Z"Y nw xY w	 d dl$Z$d
Z%n# e#$ r d	Z%dZ$Y nw xY w	 d dl&m'Z' 	 	 d0d„Z(n# e $ r dZ(Y nw xY w	 d dl)m*Z* n2# e $ r* dZ*es#	  ed¦  «        Z+ e,d¦  «         n# e $ r Y nw xY wY nw xY w	 ddl-m.Z/m0Z0m1Z1  e0¦   «         Z2n9# e $ r1 dZ/d	Z2d„ Z1es%	  ed¦  «        Z+es e,d¦  «         n# e $ r Y nw xY wY nw xY wej3         4                    ¦   «         	 	 	 d1d„¦   «         Z5	 d dl.m6Z7 es e,d¦  «         n2# e $ r* dZ7es#	  ed¦  «        Z8 e,d¦  «         n# e $ r Y nw xY wY nw xY we7€:	 d dl7m9Z7 n2# e $ r* dZ7es#	  ed¦  «        Z8 e,d¦  «         n# e $ r Y nw xY wY nw xY wej3         4                    ¦   «         d„ ¦   «         Z:ej3         4                    ¦   «         	 	 d2d „¦   «         Z;d!„ Z<d"„ Z=d#„ Z>d$„ Z?d%„ Z@d3d&„ZAh d'£ZBd2d(„ZCed3d)„¦   «         ZDedeCfd*„¦   «         ZEg d+¢ZFd,„ ZGej3         4                    ¦   «         	 	 	 	 	 	 	 	 	 	 	 d4d/„¦   «         ZHdS )5é    N)Úcontextmanager)Úversion)ÚoffloadÚdarwinÚmps)r   r   é   FTÚsol))r   é   )r   é	   )r   r   )é
   r   )é   r   )r   é   )Ú	installedÚ	supported)Úmemory_efficient_attention©Úsageattn_varlenc	                 ó2   — t          | ||||||||¬¦	  «	        S )N)Ú	is_causalÚsm_scaler   ©	ÚqÚkÚvÚcu_seqlens_qÚcu_seqlens_kvÚmax_seqlen_qÚmax_seqlen_kvr   r   s	            ú)C:\Users\johnn\Wan2GP\shared\attention.pyÚsageattn_varlen_wrapperr    9   s6   € õ ØØØØØØØØØð

ñ 

ô 

ð 
	
ó    )Úblock_sparse_sage2_attn_cudaÚsageattentionzMSage Attention has been detected but it won't work until Triton is installed.r   )ÚsageattnÚis_sage2_supportedÚ&sageattn_attention_mask_support_reasonc                  ó   — dS )NzSageAttention 2 is unavailable© )ÚargsÚkwargss     r   ú<lambda>r+   d   s   € ÐEe€ r!   zOSage Attention 2 has been detected but it won't work until Triton is installed.c                 óî  — | \  }}}|j         }|||g} |��·|j        dk    r|                     dd¦  «        }nT|j        dk    r|                     d¦  «        }n3|j        dk    r(|                     d¦  «                             d¦  «        }d }	|r‚|j        d         |j        d         }}
t          j        |
|j        ¬¦  «        d d …d f         }t          j        ||j        ¬¦  «        d d d …f         }||k                         dd|
|¦  «        }	d}t          j	        |¦  «        rL| 
                    |¬¦  «        }|	�3|                     |	 t          j        |j         ¦  «        j        ¦  «        }n[|j         t
          j        k    rF|	�||	z  }|                     d	d
¬¦  «        }t          j        ||t          j        |¦  «        ¦  «        }t%          | d|||¬¦  «        }|                      ¦   «          |S )Né   r   é   é   r   )ÚdeviceF©ÚdtypeéÿÿÿÿT)ÚdimÚkeepdimÚNHD)Útensor_layoutr   Ú	recycle_qÚ	attn_mask)r2   ÚndimÚ	transposeÚ	unsqueezeÚshapeÚtorchÚaranger0   ÚviewÚis_floating_pointÚtoÚmasked_fillÚfinfoÚminÚboolÚanyÚwhereÚ	ones_likeÚ	sageattn2Úclear)Úqkv_listÚattention_lengthr8   Úattention_maskÚcausalr   r   r   Úq_dtypeÚcausal_maskÚlqÚlkÚrowÚcolÚhas_keysÚos                   r   Úsageattn2_wrapperrX   l   s  € ð �F€A€aˆØŒg€GØ�!�Aˆw€HØÑ!ØÔ !Ò#Ð#Ø+×5Ò5°a¸Ñ;Ô;ˆNˆNØÔ  AÒ%Ð%Ø+×5Ò5°aÑ8Ô8ˆNˆNØÔ  AÒ%Ð%Ø+×5Ò5°aÑ8Ô8×BÒBÀ1ÑEÔEˆNØˆØð 	Ø”W˜Q”Z ¤¨¤�ˆBÝ”,˜r¨!¬(Ð3Ñ3Ô3°A°A°A°t°GÔ<ˆCÝ”,˜r¨!¬(Ð3Ñ3Ô3°D¸!¸!¸!°GÔ<ˆCØ #š:×+Ò+¨A¨q°"°bÑ9Ô9ˆKØˆFÝÔ" >Ñ2Ô2ð 	dØ+×.Ò.°WÐ.Ñ=Ô=ˆNØÐ&Ø!/×!;Ò!;¸[¸LÍ%Ì+ÐVdÔVjÑJkÔJkÔJoÑ!pÔ!p�øØÔ!¥U¤ZÒ/Ð/ØÐ&Ø!/°+Ñ!=�Ø%×)Ò)¨b¸$Ð)Ñ?Ô?ˆHÝ"œ[¨°>Å5Ä?ÐSaÑCbÔCbÑcÔcˆNÝ�(¨%¸6ÈYÐbpÐqÑqÔq€AØ‡N‚NÑÔÐà€Hr!   )Úsageattn_blackwellzRSage Attention 3 is installed but it won't be supported until Triton is installed.rY   zOSage Attention 3 has been detected but it won't work until Triton is installed.)Úsageattn3_blackwellrZ   c                 ó  — | \  }}}|                      dd¦  «        }|                      dd¦  «        }|                      dd¦  «        }t          |||¦  «        }|                      dd¦  «        }|                      ¦   «          |S )Nr   r.   )r;   Ú	sageattn3rK   )rL   rM   r   r   r   rW   s         r   Úsageattn3_wrapperr]   ¬   sƒ   € ð
 �F€A€aˆð 	
�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AÝ�!�Q˜ÑÔ€AØ	�Š�A�aÑÔ€AØ‡N‚NÑÔÐà€Hr!   c                 óV  — | \  }}}|                      dd¦  «        }|                      dd¦  «        }|                      dd¦  «        }|d k    r|                      dd¦  «        }t          j        |||||¬¦  «                              dd¦  «        }~~~|                      ¦   «          |S )Nr   r.   )r9   r   )r;   ÚFÚscaled_dot_product_attentionrK   )rL   rM   rN   rO   r   r   r   rW   s           r   Úsdpa_wrapperra   Û   s²   € ð �G€A€qˆ!à	�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AØ	�Š�A�aÑÔ€AØ˜ÒÐØ'×1Ò1°!°AÑ6Ô6ˆÝ	Ô& q¨!¨Q¸.ÐTZÐ[Ñ[Ô[×eÒeÐfgÐhiÑjÔj€AØ	ˆ1ˆaØ‡N‚NÑÔÐà€Hr!   c                  ó  — ddg} t           d k    r|                      d¦  «         t          d k    r|                      d¦  «         t          d k    r|                      d¦  «         t          d k    r7t          d¦  «                             d¦  «        r|                      d¦  «         t          d k    r7t          d¦  «                             d¦  «        r|                      d	¦  «         t          d k    r|                      d
¦  «         | S )NÚsdpaÚautoÚflashÚxformersÚsager#   Ú2Úsage2ÚradialÚsage3)	Ú
flash_attnÚappendr   r    rJ   r   Ú
startswithr"   r\   )Úrets    r   Úget_attention_modesrp   ð   sö   € Ø�6Ð
€CÝ�TÒÐØ�
Š
�7ÑÔÐÝ! TÒ)Ð)Ø�
Š
�:ÑÔÐÝ $Ò&Ð&Ø�
Š
�6ÑÔÐÝ�DÒÐ�W _Ñ5Ô5×@Ò@ÀÑEÔEÐØ�
Š
�7ÑÔÐÝ# tÒ+Ð+µ¸Ñ0HÔ0H×0SÒ0SÐTWÑ0XÔ0XÐ+Ø�
Š
�8ÑÔÐå�DÒÐØ�
Š
�7ÑÔÐà€Jr!   c                  ó’  — t           rddgS t          ¦   «         } t          j                             ¦   «         \  }}|dk     st
          sd| v r|                      d¦  «         t          rt
          s2d| v r|                      d¦  «         d| v r|                      d¦  «         |dk     st
          sd| v r|                      d¦  «         | S )	Nrc   rd   r   rk   ri   rj   é   rg   )Ú_is_mpsrp   r>   ÚcudaÚget_device_capabilityÚtriton_installedÚremoveÚsage2_supported)ro   ÚmajorÚminors      r   Úget_supported_attention_modesr{     sÔ   € åð  Ø˜ÐÐÝ
Ñ
Ô
€CÝ”:×3Ò3Ñ5Ô5�L€Eˆ5Ø�Š
ˆ
Õ*ˆ
Ø�cˆ>ˆ>Ø�JŠJ�wÑÔÐåð !Õ"2ð !Ø�cˆ>ˆ>Ø�JŠJ�wÑÔÐØ�sˆ?ˆ?Ø�JŠJ�xÑ Ô Ð àˆq‚y€yÕ(€yØ�Sˆ=ˆ=Ø�JŠJ�vÑÔÐà€Jr!   c                  óf   — t          ¦   «         d„ t                               ¦   «         D ¦   «         z   S )Nc                 ó(   — g | ]\  }}|d          ¯|‘ŒS )r   r(   ©Ú.0ÚmodeÚstatuss      r   ú
<listcomp>z0get_override_attention_modes.<locals>.<listcomp>  s(   € Ð#xÐ#xÐ#x©\¨T°6ÐdjÐkvÔdwÐ#x DÐ#xÐ#xÐ#xr!   )rp   ÚATTENTION_MODE_AVAILABILITYÚitemsr(   r!   r   Úget_override_attention_modesr…     s/   € ÝÑ Ô Ð#xÐ#xÕ=X×=^Ò=^Ñ=`Ô=`Ð#xÑ#xÔ#xÑxÐxr!   c                  óf   — t          ¦   «         d„ t                               ¦   «         D ¦   «         z   S )Nc                 ó(   — g | ]\  }}|d          ¯|‘ŒS )r   r(   r~   s      r   r‚   z:get_supported_override_attention_modes.<locals>.<listcomp>  sE   € ð  .Cð  .Cð  .C±|°t¸VÐntð  vAô  oBð  .C¨dð  .Cð  .Cð  .Cr!   )r{   rƒ   r„   r(   r!   r   Ú&get_supported_override_attention_modesrˆ     sD   € Ý(Ñ*Ô*ð  .Cð  .CÕGb×GhÒGhÑGjÔGjð  .Cñ  .Cô  .Cñ  Cð  Cr!   c                  ó8   — dD ]} | t          ¦   «         v r| c S ŒdS )N)ri   rg   rc   rc   )r{   )Úattns    r   Úget_default_attention_moder‹   !  s6   € Ø)ð ð ˆØÕ0Ñ2Ô2Ð2Ð2ØˆKˆKˆKð 3àˆ6r!   c                 ó¤   — t           st          j                             ¦   «         sd S t          j                             | ¦  «        \  }}|dz  |z   S )Nr   )rs   r>   rt   Úis_availableru   )r0   ry   rz   s      r   Úget_current_cuda_architecturerŽ   (  sL   € Ýð •e”j×-Ò-Ñ/Ô/ð ØˆtÝ”:×3Ò3°FÑ;Ô;�L€Eˆ5Ø�2‰:˜ÑÐr!   >   rg   ri   rk   rj   c                 ó,  — t          | pd¦  «                             ¦   «                              ¦   «         }|dk    rt          ¦   «         n|}|rt	          ¦   «         nd }|�|dk     r|t
          v rd}|t          ¦   «         vrt          d|› d�¦  «        ‚|S )Nrd   éY   rc   zAttention mode 'z/' is not installed or supported on this system.)ÚstrÚstripÚlowerr‹   rŽ   Ú_SAGE_ATTENTION_MODESr{   Ú
ValueError)Úattention_modeÚdisable_sage_pre_adarŠ   Úarchitectures       r   Úresolve_attention_moder™   2  s­   € Ýˆ~Ð' Ñ(Ô(×.Ò.Ñ0Ô0×6Ò6Ñ8Ô8€DØ+/°6ª>¨>Õ%Ñ'Ô'Ð'¸t€DØ6JÐTÕ0Ñ2Ô2Ð2ÐPT€LØÐ L°2Ò$5Ð$5¸$ÕBWÐ:WÐ:WØˆØÕ0Ñ2Ô2Ð2Ð2ÝÐa¨DÐaÐaÐaÑbÔbÐbØ€Kr!   c              #   óÆ  K  — t           j                             dt          ¦  «        }|t          u s|dk    r| pt	          ¦   «         t           j        d<   n|t           j        d<   	 d V — |t          u r"t           j                             dd ¦  «         d S |t           j        d<   d S # |t          u r!t           j                             dd ¦  «         n|t           j        d<   w xY w)NÚ
_attentionrd   )r   Úshared_stateÚgetÚ_MISSINGr‹   Úpop)Údefault_attentionÚpreviouss     r   Úattention_shared_stater¢   =  sê   è è € åÔ#×'Ò'¨µhÑ?Ô?€HØ•8ÐÐ˜x¨6Ò1Ð1Ø->Ð-^ÕB\ÑB^ÔB^�Ô˜\Ñ*Ð*à-5�Ô˜\Ñ*ð:Øˆˆˆà•xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8Ð8à19�GÔ  Ñ.Ð.Ð.øð •xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8à19�GÔ  Ñ.Ð9Ð9Ð9Ð9s   Á%B% Â%;C c              +   óš  K  — t           j                             dt          ¦  «        } || fi |¤Žt           j        d<   	 t           j        d         V — |t          u r"t           j                             dd ¦  «         d S |t           j        d<   d S # |t          u r!t           j                             dd ¦  «         n|t           j        d<   w xY w)Nr›   )r   rœ   r�   rž   rŸ   )r–   ÚresolverÚresolver_kwargsr¡   s       r   Úattention_config_shared_stater¦   M  sÖ   è è € åÔ#×'Ò'¨µhÑ?Ô?€HØ)1¨°.Ð)TÐ)TÀOÐ)TÐ)T…GÔ˜Ñ&ð:ÝÔ" <Ô0Ð0Ð0Ð0à•xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8Ð8à19�GÔ  Ñ.Ð.Ð.øð •xÐÐÝÔ ×$Ò$ \°4Ñ8Ô8Ð8Ð8à19�GÔ  Ñ.Ð9Ð9Ð9Ð9s   ¿B Â;C
)	rƒ   r¦   r¢   rŽ   r…   rˆ   r™   Úpay_attentionÚ	attentionc                 óì   — t           rdnd}t          j        d| z  dz   gt          j        |¬¦  «        }t	          | ¦  «        D ]0}||         }||z  |z   }|dz   |z  }||d|z  dz   <   ||d|z  dz   <   Œ1|S )Nr   rt   r.   r   ©r2   r0   )rs   r>   ÚzerosÚint32Úrange)	Ú
batch_sizeÚlensÚmax_lenÚ
_cu_deviceÚ
cu_seqlensÚiÚsÚs1Ús2s	            r   Úget_cu_seqlensr·   e  sœ   € å!Ð-�� v€JÝ”˜a *™n¨qÑ0Ð1½¼ÈZÐXÑXÔX€Jå�:ÑÔð #ð #ˆØ�ŒGˆØ�‰[˜1‰_ˆØ�!‰e�wÑˆØ "ˆ
�1�q‘5˜1‘9ÑØ "ˆ
�1�q‘5˜1‘9ÑÐàÐr!   ç        ©r3   r3   c                 óŽ  — |�|rt          d¦  «        ‚|d k    �r|d k    rt          j        d         n|}|dv rt          ¦   «         n|}|dk    rdn|}d }t          rd}n/|dk    r)t
          d k    r|
d k    r|d k    rt          | |d¬¦  «        }|dk    r|€t
          d k    r|
d k    r	|d k    rd}n1d	}|d	k    r)t          s"|rd
|› d�nd}t          d|› d|› d�¦  «         da|j	        t          j        k    r&t          s|                     t          j        ¦  «        }|d k    rt          j        d         n|}|dv rt          ¦   «         n|}| \  }}}|                      ¦   «          |j	        }|j	        t          j        k    rdt          s]|                     t          j        ¦  «        }|                     t          j        ¦  «        }|                     t          j        ¦  «        }d}|                     d¦  «        |                     d¦  «        |                     d¦  «        }}}|                     |j	        ¦  «        }|                     |j	        ¦  «        }t#          |¦  «        }t#          |¦  «        |k    r|                     |ddd¦  «        }t#          |¦  «        |k    r|                     |ddd¦  «        }|j        j        dk    r<|                     ¦   «         |                     ¦   «         |                     ¦   «         }}}|dk    rddlm}m} ddlm}m} |dk    rd}|dk    �r¤|d k    �r�|dv �r˜|d k    sJ ‚|
d k    sJ ‚g }g }|d         }d} |dd …         D ]<}!|!|k    r| dz  } Œ|                     | ¦  «         |                     |¦  «         d} |!}Œ=|                     | ¦  «         |                     |!¦  «         t#          |¦  «        dk    s|d         |j        d         k    rÞt          j        ||¦  «        }"t          j        ||¦  «        }#t          j        ||¦  «        }$d\  }}}d„ t?          |#|¦  «        D ¦   «         }#d„ t?          |$|¦  «        D ¦   «         }$g }%t?          |"|#|$¦  «        D ]3\  }&}'}(|&|'|(g} d\  }&}'}(|%                     tA          | ¦  «        ¦  «         Œ4d\  }"}#}$t          j!        |%d¬¦  «        }%|%S n‹|
d k    s|d k    r|dv r{|dk    sJ ‚|
d k    r|
d          "                    ¦   «         n|})|d k    r|d          "                    ¦   «         n|}*||)z
  }|d d …d |)…f         }|d d …d |*…f         }|d d …d |*…f         }|�!|dk    rtF          stI          j%        d¦  «         |dk    s|d k    �r5|dk    rý|d k    r>t          j&        |g|z  t          j'        ¬!¦  «                             |j        d¬"¦  «        }|
d k    r>t          j&        |g|z  t          j'        ¬!¦  «                             |j        d¬"¦  «        }
 |j(        dg|j        d#d …         ¢R Ž } |j(        dg|j        d#d …         ¢R Ž } |j(        dg|j        d#d …         ¢R Ž }tS          ||
|¦  «        }+tS          |||¦  «        },�n2|
d k    r|
d          "                    ¦   «         n|})|d k    r|d          "                    ¦   «         n|}*|)|k    s|*|k    rSt          j&        d|)|gt          j'        |j        ¬$¦  «        }+t          j&        d|*|gt          j'        |j        ¬$¦  «        },nPt          j&        d|gt          j'        |j        ¬$¦  «        }+t          j&        d|gt          j'        |j        ¬$¦  «        },| *                    d¦  «        }| *                    d¦  «        }| *                    d¦  «        }|dk    r0tW          ||||+|,||||¬%¦	  «	         ,                    d||f¦  «        }-�nÞ|d&k    r|||g} ~~~t[          | |¦  «        }-�n¾|dk    r|||g} ~~~t]          | ||	||¬'¦  «        }-�nš|d	k    r|||g} ~~~t_          | |||¬(¦  «        }-�nw|d k    rD|dk    r>ta          j1        ||||+|,d d |||||¬)¦  «        d          ,                    d||f¦  «        }-�n-|d k    r7te          j1        ||||+|,|||||||¬*¦  «         ,                    d||f¦  «        }-nð|d+k    rêdd,l3m4}. |d k    r|
d k    rtk          |||¦  «        }-nÆ|d k    rB|
d k    r<|. 6                    |g|z  |to          |¦  «        ¦  «        }/tk          ||||/¬-¦  «        }-n~|dk    rv|
d k    r|
d          "                    ¦   «         n|})|d k    r|d          "                    ¦   «         n|}*|. 6                    |)||)z
  g||*dg¦  «        }/tk          ||||/¬-¦  «        }-nJ ‚|-                     |¦  «        }-|dk    rSt          j!        |-t          j8        |-j        d         |g|-j        d#d …         ¢R |-j	        |-j        ¬$¦  «        gd¦  «        }-|-S ).Nzmpay_attention received both attention_mask and causal=True; build a combined mask once and pass causal=False.r›   )r	   Úvdnrj   ri   z"MPS uses SDPA for masked attentionr6   )r7   rc   z (ú)Ú z>[WAN2GP] Attention mask is unsupported by selected attention 'ú'z!. Masked attention will use SDPA.Tr   r   r3   r   Úchipmunk)ÚSparseDiffMlpÚSparseDiffAttn)ÚLayerCounterÚGLOBAL_CONFIG)ri   rk   rc   )NNNc                 ó0   — g | ]\  }}|d d …d |…f         ‘ŒS ©Nr(   ©r   ÚuÚszs      r   r‚   z!pay_attention.<locals>.<listcomp>Ê  ó,   € ÐGÐGÐG¡u q¨"˜˜1˜1˜1˜c˜r˜c˜6œÐGÐGÐGr!   c                 ó0   — g | ]\  }}|d d …d |…f         ‘ŒS rÅ   r(   rÆ   s      r   r‚   z!pay_attention.<locals>.<listcomp>Ë  rÉ   r!   )r4   r/   zBFlash attention 3 is not available, use flash attention 2 instead.rg   re   r1   )r0   Únon_blockingéþÿÿÿrª   r   rk   )r8   rN   rO   )rN   rO   )r   r   r   r   Úcu_seqlens_kÚ	seqused_qÚ	seqused_kr   Úmax_seqlen_kÚsoftmax_scalerO   Údeterministic)r   r   r   r   rÍ   r   rÐ   Ú	dropout_prÑ   rO   Úwindow_sizerÒ   rf   )ÚBlockDiagonalPaddedKeysMask)Ú	attn_bias)9r•   r   rœ   r‹   rs   rJ   r&   Ú_MASKED_ATTENTION_SDPA_WARNEDÚprintr2   r>   Úbfloat16Úbfloat16_supportedrB   Úfloat16rK   ÚsizeÚlenÚexpandr0   ÚtypeÚ
contiguousÚsrc.chipmunk.modulesrÀ   rÁ   Úsrc.chipmunk.utilrÂ   rÃ   rm   r=   ÚsplitÚzipr§   ÚcatÚitemÚFLASH_ATTN_3_AVAILABLEÚwarningsÚwarnÚtensorr¬   Úreshaper·   Úsqueezer    Ú	unflattenr]   rX   ra   Úflash_attn_interfaceÚflash_attn_varlen_funcrl   Úxformers.ops.fmha.attn_biasrÕ   r   Úfrom_seqlensÚlistÚempty)0rL   rÓ   rÑ   rO   rÔ   rÒ   r   Úforce_attentionrN   r8   Úq_lensÚk_lensÚrequested_attnÚsupport_reasonÚdetailrŠ   r   r   r   Ú	out_dtypeÚfinal_paddingÚbrR   rS   ÚbatchrÀ   rÁ   rÂ   rÃ   Úchunk_sizesÚk_sizesÚcurrent_sizeÚcurrent_countÚk_lenÚq_chunksÚk_chunksÚv_chunksrW   Úsub_qÚsub_kÚsub_vÚszqÚszkr   rÍ   ÚxrÕ   r9   s0                                                   r   r§   r§   s  sÁ  € ð& Ð! fÐ!Ýð  Iñ  Jô  Jð  	JØ˜ÒÑØ?NÐRVÒ?VÐ?V�Ô-¨lÔ;Ð;Ð\kˆØ9GÈ>Ð9YÐ9YÕ3Ñ5Ô5Ð5Ð_mˆØ$2°hÒ$>Ð$>˜˜ÀNˆØˆÝð 	sØAˆNˆNØ˜wÒ&Ð&­9¸Ò+<Ð+<ÀÈ4ÂÀÐTZÐ^bÒTbÐTbÝCÀHÈnÐlqÐrÑrÔrˆNØ˜WÒ$Ð$¨Ð)?ÅIÐQUÒDUÐDUÐZ`ÐdhÒZhÐZhÐmsÐw{Òm{Ðm{Ø%ˆOˆOà$ˆOØ Ò'Ð'Õ0MÐ'Ø3AÐIÐ/˜nÐ/Ð/Ð/Ð/Àr�Ýð  RÐWeð  Rð  RÐhnð  Rð  Rð  Rñ  Sô  Sð  SØ04Ð-ØÔ ¥E¤NÒ2Ð2Õ;MÐ2Ø+×.Ò.­u¬}Ñ=Ô=ˆNØ1@À4Ò1GÐ1G�7Ô Ô-Ð-È_€DØ+/°>Ð+AÐ+AÕ%Ñ'Ô'Ð'Àt€Dà�E€A€aˆØ‡N‚NÑÔÐØ”€IØ„w•%”.Ò Ð Õ);Ð Ø�DŠD•”ÑÔˆØ�DŠD•”ÑÔˆØ�DŠD•”ÑÔˆØ€MØ—’�q‘	”	˜1Ÿ6š6 !™9œ9 a§f¢f¨Q¡i¤iˆ2€r€Aà	�ŠˆQŒW‰Œ€AØ	�ŠˆQŒW‰Œ€AÝ�‰FŒF€EÝ
ˆ1�v„v�‚€˜AŸHšH U¨B°°BÑ7Ô7˜Ý
ˆ1�v„v�‚€˜AŸHšH U¨B°°BÑ7Ô7˜Ø„x„}˜ÒÐ¨¯ª©¬¸¿º¹¼ÈÏÊÉÌ Q !˜qØˆzÒÐØFÐFÐFÐFÐFÐFÐFÐFØAÐAÐAÐAÐAÐAÐAÐAØˆxÒÐ˜w˜àˆ1‚u�u�˜4’‘ DÐ,FÐ$FÑ$FØ Ò%Ð%Ð%Ð%à˜Š~ˆ~ˆ~ˆ~ØˆØˆØ˜a”yˆØˆØ˜A˜B˜B”Zð 	%ð 	%ˆEØ˜Ò$Ð$Ø Ñ"��à×"Ò" =Ñ1Ô1Ð1Ø—’˜|Ñ,Ô,Ð,Ø !�Ø$��Ø×Ò˜=Ñ)Ô)Ð)Ø�Š�uÑÔÐÝˆ{ÑÔ˜aÒÐ 6¨!¤9°´¸´
Ò#:Ð#:Ý”k ! [Ñ1Ô1ˆHÝ”k ! [Ñ1Ô1ˆHÝ”k ! [Ñ1Ô1ˆHØ&‰GˆAˆq�!ØGÐGµ°H¸gÑ0FÔ0FÐGÑGÔGˆHØGÐGµ°H¸gÑ0FÔ0FÐGÑGÔGˆHØˆAÝ'*¨8°X¸xÑ'HÔ'Hð 4ð 4Ñ#��u˜eØ! 5¨%Ð0�Ø&6Ñ#��u˜eØ—’�-¨Ñ1Ô1Ñ3Ô3Ð3Ð3Ø+;Ñ(ˆH�h Ý”	˜! 1Ð%Ñ%Ô%ˆAØˆHð $;ð �DŠ.ˆ.˜F dšN˜N°Ð8RÐ0RÐ0RØ�AŠvˆvˆvˆvØ"(¨D¢. .ˆf�QŒi�nŠnÑÔÐ°bˆØ"(¨D¢. .ˆf�QŒi�nŠnÑÔÐ°bˆØ˜S™ˆØˆaˆaˆa��#�ˆgŒJˆØˆaˆaˆa��#�ˆgŒJˆØˆaˆaˆa��#�ˆgŒJˆàÐ˜w¨!š|˜|Õ4J˜|ÝŒØPñ	
ô 	
ð 	
ð ˆV‚|€|�t˜W’}‘}Ø�Š6ˆ6Ø˜Š~ˆ~Ýœ¨ t¨a¡xµu´{ÐCÑCÔC×FÒFÈaÌhÐeiÐFÑjÔj�Ø˜Š~ˆ~Ýœ r d¨Q¡hµe´kÐBÑBÔB×EÒEÈQÌXÐdhÐEÑiÔi�Ø�”	˜"Ð,˜qœw r s sœ|Ð,Ð,Ð,ˆAØ�”	˜"Ð,˜qœw r s sœ|Ð,Ð,Ð,ˆAØ�”	˜"Ð,˜qœw r s sœ|Ð,Ð,Ð,ˆAÝ'¨¨6°2Ñ6Ô6ˆLÝ'¨¨6°2Ñ6Ô6ˆL‰Là&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ�bŠyˆy˜C 2šI˜IÝ$œ|¨Q°°R¨LÅÄÐTUÔT\Ð]Ñ]Ô]�Ý$œ|¨Q°°R¨LÅÄÐTUÔT\Ð]Ñ]Ô]��å$œ|¨Q°¨G½5¼;ÈqÌxÐXÑXÔX�Ý$œ|¨Q°¨G½5¼;ÈqÌxÐXÑXÔX�Ø—	’	˜!‘”ˆAØ—	’	˜!‘”ˆAØ—	’	˜!‘”ˆAð ˆV‚|€|Ý#ØØØØ&Ø'ØØØØ"ð

ñ 

ô 

÷ Š)�A˜˜2�wÑ
Ô
ð 	
‰ð 
ˆwŠˆØ�a˜�7ˆØˆa�Ý˜h¨Ñ+Ô+ˆ‰Ø	ˆwŠˆØ�a˜�7ˆØˆa�Ý˜h¨°iÐP^ÐgmÐnÑnÔnˆ‰Ø	ˆvŠˆØ�q˜!�9ˆØˆq�!Ý˜ 2°nÈVÐTÑTÔTˆ‰Ø	ˆwŠˆ˜7 aš<˜<Ý Ô7ØØØØ&Ø&ØØØØØ'ØØ'ð)ñ )ô )ð *+ô,÷ -6ªI°a¸!¸R¸Ñ,AÔ,Að 	
‰ð 
ˆwŠˆÝÔ-ØØØØ&Ø&ØØØØ'ØØ#Ø'ð)ñ )ô )÷ *3ª°1°q¸"°gÑ)>Ô)>ð 	
ˆð  
ˆzÒ	Ð	ØKÐKÐKÐKÐKÐKØ�TŠ>ˆ>˜f¨šn˜nÝ*¨1¨a°Ñ4Ô4ˆAˆAØ�tŠ^ˆ^ ¨$¢ Ø3×@Ò@À"ÀÈÁÈBÕQUÐV\ÑQ]ÔQ]Ñ_Ô_ˆIÝ*¨1¨a°¸yÐJÑJÔJˆAˆAØ�!ŠVˆVØ&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ&,°¢n n�&˜”)—.’.Ñ"Ô"Ð"¸"ˆCØ3×@Ò@À#ÀrÈCÁxÐAQÐTVÐZ]Ð_`ÐYaÑcÔcˆIÝ*¨1¨a°¸yÐJÑJÔJˆAˆAà�5Ø	�ŠˆyÑÔ€AØ�qÒÐÝŒI�q�%œ+¨¬°¬
°MÐ'QÀAÄGÈBÈCÈCÄLÐ'QÐ'QÐZ[ÔZaÐjkÔjrÐuÑuÔuÐwÐyzÑ{Ô{ˆð €Hr!   )FN)FNF)NFrÅ   )r¸   NFr¹   FNNNFNN)IÚsysÚ
contextlibr   r>   Úimportlib.metadatar   Úmmgpr   Útorch.nn.functionalÚnnÚ
functionalr_   rè   ÚplatformÚhasattrÚbackendsr   r�   rs   rt   ru   ry   rz   rÚ   r×   Úobjectrž   Útritonrv   rƒ   Úxformers.opsr   ÚImportErrorrî   rç   ÚModuleNotFoundErrorrl   ÚFLASH_ATTN_2_AVAILABLEr#   r   r    Úspas_sage_attnr"   Úsg2_versionrØ   Ú
sage2_corer$   rJ   r%   r&   rx   ÚcompilerÚdisablerX   rY   r\   Úsg3_versionrZ   r]   ra   rp   r{   r…   rˆ   r‹   rŽ   r”   r™   r¢   r¦   Ú__all__r·   r§   r(   r!   r   ú<module>r#     sÝ  ðà 
€
€
€
Ø %Ð %Ð %Ð %Ð %Ð %à €€€Ø &Ð &Ð &Ð &Ð &Ð &Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø €€€Ø &Ð &Ð &Ð &Ð &Ð &à
Œ,˜(Ò
"Ð
k w w¨u¬~¸uÑ'EÔ'EÐ
kÈ%Ì.ÔJ\×JiÒJiÑJkÔJk€à ÐLˆvˆv e¤j×&FÒ&FÀtÑ&LÔ&L�€€uØ˜q’jÐ Ø %Ð Øˆ6‰8Œ8€ðØ€M€M€MØÐÐøðØÐÐÐøøøð 
Ø%Ø%Ðo¨5°%¨.Ð<oÐ*oðð ðÐ ð&Ø7Ð7Ð7Ð7Ð7Ð7Ð7øØð &ð &ð &Ø!%ÐÐÐð&øøøð#ØÐÐÐØ!ÐÐøØð #ð #ð #Ø"ÐÐÐð#øøøðØÐÐÐØ!ÐÐøØð ð ð Ø"ÐØ€J€J€Jðøøøð#Ø-Ð-Ð-Ð-Ð-Ð-ð Øð
ð 
ð 
ð 
ð 
øð. ð #ð #ð #Ø"ÐÐÐð#øøøð	Ø;Ð;Ð;Ð;Ð;Ð;Ð;øØð ð ð Ø#'Ð Øð ð	Ø!˜' /Ñ2Ô2ˆKØˆEÐaÑbÔbÐbÐbøØð 	ð 	ð 	ØˆDð	øøøøøðøøøðØmÐmÐmÐmÐmÐmÐmÐmÐmÐmØ)Ð)Ñ+Ô+€O€OøØð 	ð 	ð 	Ø€IØ€OØ-eÐ-eÐ*Øð ð	Ø!˜' /Ñ2Ô2ˆKØ#Ð} U UÐ+|Ñ%}Ô%}Ð%}øøØð 	ð 	ð 	ØˆDð	øøøøøð	øøøð „×ÒÑÔð ØØð$ð $ð $ñ Ôð$ðLØ8Ð8Ð8Ð8Ð8Ð8Øð dØˆÐbÑcÔcÐcøøØð ð ð Ø€IØð ð	Ø!˜'Ð"6Ñ7Ô7ˆKØˆEÐcÑdÔdÐdÐdøØð 	ð 	ð 	ØˆDð	øøøøøðøøøð Ðð	Ø>Ð>Ð>Ð>Ð>Ð>Ð>øØð ð ð Øˆ	Øð 	ðØ%˜gÐ&;Ñ<Ô<�Ø�ÐgÑhÔhÐhÐhøØð ð ð Ø�ðøøøøøðøøøð „×ÒÑÔðð ñ Ôðð\ „×ÒÑÔð Øð	ð ð ñ Ôðð(ð ð ð$ð ð ð.yð yð yðCð Cð Cðð ð ðð ð ð ð =Ð<Ð<Ð ðð ð ð ð ð:ð :ð :ñ „ð:ð Ø15Ð@Vð 	:ð 	:ð 	:ñ „ð	:ð
ð 
ð 
€ðð ð ð „×ÒÑÔð ØØØØØØØØØØðRð Rð Rñ ÔðRð Rð Rs,  ÂB# Â#B)Â:C ÃCÃ
CÃC ÃC ÃC Ã$C+ Ã+	C7Ã6C7Ã;D ÄDÄDÄD Ä	EÄ'D>Ä=EÄ>EÅEÅEÅEÅEÅE% Å%FÅ4FÆFÆFÆFÆFÆFÆFÇG Ç	HÇ#G:Ç9HÇ:HÇ?HÈHÈHÈHÈH È	IÈH6È5IÈ6H>È;IÈ=H>È>IÉI