§
     ’Þj: ã                   óè
  — d Z ddlZddlZej        dk    r: eej        d¦  «        r)ej        j                             ¦   «         r ed¦  «        ‚ddl	m
c mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ 	 ddlmZ n# eeef$ r dZY nw xY w e¦   «          ddlmZmZ ddlmZ  ddl!m"c m#Z# ddl!m$Z% ddl&m$Z' ddl(m$Z) ddl*m$Z+ ddl,m-Z. ddl,m/Z/m0Z0 	 ddl1m2Z2  ee2d¦  «        sej3        j4        Z2n#  ej3        j4        Z2Y nxY w	 ddl1m5Z5  ee5d¦  «        sej3        j6        Z5dZ7n	#  dZ7Y nxY w	 ddl1m8Z8  ee8d¦  «        sej3        j9        Z8dZ:n	#  dZ:Y nxY w	 ddl1m;Z;  ee;d¦  «        sej3        j<        Z;dZ=n	#  dZ=Y nxY wddl>mZ? ddl>m@ZA ddl>mBZB ddl>mCaC ddlDmEZEmFZFmGZGmHZHmIZImJZJ ddlKZKddlLZLd„ ZMddlNmOZO  eOd ¦  «        ZPeP Q                    d!¦  «        ZRddlSZSddlTZTddlUZUd"„ ZVd#„ ZWd$eXd%eXfd&„ZY eZ eW¦   «         ¦  «        Z[ej\         ]                    ¦   «         dk    Z^d'Z_d'Z`d(Zadabi Zcd)ejd        d%eXfd*„Zed)ejd        d%effd+„Zgd)ejd        d%eXfd,„Zhd)ejd        fd-„Zid)ejd        d%ejfd.„Zkd/dejl        dfd0„Zmdud2ejn        dz  d)ejd        efz  dz  d3efd%efdz  fd4„Zodud)ejd        efz  dz  d2ejn        dz  d3efd%ejfd5„Zp	 	 	 	 	 	 dvd3efd6ejd7eHeq         d8ejd9ejd:ejd;eEfd<„Zrd=„ Zsdatd>„ Zud?„ Zvd@dAdddBœZwd@e_dCddBœZxd@dAdDddBœZyd@d'dEddBœZzdFe#j{        j|        v Z}dwdG„Z~dH„ ZdI„ Z€ej�        j‚        dJ„ ¦   «         Zƒej�        j‚        dK„ ¦   «         Z„ej�        j‚        dL„ ¦   «         Z…ej�        j‚        dxdM„¦   «         Z†dN„ Z‡dOe#j{        j|        v Zˆi Z‰dP„ ZŠdQ„ Z‹dR„ ZŒdS„ Z�dT„ ZŽdU„ Z�dV„ Z�dW„ Z‘dX„ Z’ej�        j‚        	 	 	 	 	 	 	 	 dyd3efdZefd6ejd7eHeq         d[ejd8ejd2eHejn                 d:ejd;eEd%ejn        fd\„¦   «         Z“ej�        j‚        	 	 	 dzd]ejn        d^ejn        d_ejn        d`ejn        daejn        dbeXdceXd6ejd7eHeq         d[ejd;eEd%ejn        fdd„¦   «         Z”ej�        j‚        	 	 	 	 	 	 	 	 d{d3efd6ejdgefd7eHeq         dhefd[ejdiejd8ejd;eEd%ejn        fdj„¦   «         Z•ej�        j‚        	 	 	 	 	 	 	 	 	 d|d3efd6ejdgefd7eHeq         dhefd[ejdiejd8ejd9ejd;eEd%ejn        fdk„¦   «         Z–ej�        j‚        	 	 	 	 	 	 	 	 	 	 d}d3efd6ejdgefd7eHeq         dhefd[ejdiejd8ejd9ejd;eEd%ejn        fdn„¦   «         Z—ej�        j‚        	 	 	 	 	 	 	 	 d~d3efd6ejdgefd7eHeq         dhefd[ejd8ejd9ejd;eEd%ejn        fdo„¦   «         Z˜t†          Z™	 	 	 ddqeJejn        ešf         d3efdreqdiejfds„Z›dt„ Zœ eœ¦   «          dS )€a8  
Copyright (c) 2024 by SageAttention team.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
é    NÚdarwinÚmpsz?SageAttention is CUDA-only and is disabled on Apple Silicon MPS)Úcuda_graphs)Ú
token_meané   )Úpatch_sageattention)Úfp16_vÚscale_output)Úqk_quant_triton)Úper_block_int8Úquant_per_block_int8_kernel)r   )Úforward)Úper_thread_int8)Ú"quant_query_per_thread_int8_kernelÚ quant_key_per_thread_int8_kernel)Ú_fusedÚtranspose_pad_permute_cuda)Ú_qattn_sm80Úqk_int8_sv_f16_accum_f32_attnTF)Ú_qattn_sm89Ú2qk_int8_sv_f8_accum_f32_fuse_v_scale_attn_inst_buf)Ú_qattn_sm90)Úper_warp_int8)Úsub_mean)Úper_channel_fp8)ÚAnyÚListÚLiteralÚOptionalÚTupleÚUnionc                  ó¼   — t           j                             ¦   «         } t          | ¦  «        D ]-}t           j                             |¦  «        \  }}|dk     r dS Œ.dS )Né   FT)ÚtorchÚcudaÚdevice_countÚrangeÚget_device_capability)r&   ÚiÚmajorÚminors       ú9C:\Users\johnn\OneDrive\Documents\AI\shared\sage2_core.pyÚis_sage2_supportedr-   Y   s_   € Ý”:×*Ò*Ñ,Ô,€LÝ�<Ñ Ô ð ð ˆÝ”z×7Ò7¸Ñ:Ô:‰ˆˆuØ�1Š9ˆ9Ø�5�5ð àˆ4ó    )ÚversionÚsageattentionz2.2c                  óf  — 	 t          j        ddg¦  «                             ¦   «         } t          j        d| ¦  «        }|rHt          |                     d¦  «        ¦  «        t          |                     d¦  «        ¦  «        }}||fS n'# t          $ r}t          d|¦  «         Y d }~nd }~ww xY wdS )NÚnvccz	--versionzrelease (\d+)\.(\d+)r   é   zFailed to get CUDA version:)NN)	Ú
subprocessÚcheck_outputÚdecodeÚreÚsearchÚintÚgroupÚ	ExceptionÚprint)ÚoutputÚmatchr*   r+   Úes        r,   Úget_cuda_versionr@   h   sÄ   € ð0ÝÔ(¨&°+Ð)>Ñ?Ô?×FÒFÑHÔHˆÝ”	Ð1°6Ñ:Ô:ˆØð 	 Ý˜uŸ{š{¨1™~œ~Ñ.Ô.µ°E·K²KÀ±N´NÑ0CÔ0C�5ˆEØ˜%�<Ðð	 øõ ð 0ð 0ð 0ÝÐ+¨QÑ/Ô/Ð/Ð/Ð/Ð/Ð/Ð/øøøøð0øøøàˆ:s   ‚BB
 Â

B.ÂB)Â)B.c                  óÞ   — g } t          t          j                             ¦   «         ¦  «        D ]>}t          j                             |¦  «        \  }}|                      d|› |› �¦  «         Œ?| S )NÚsm)r'   r$   r%   r&   r(   Úappend)Ú
cuda_archsr)   r*   r+   s       r,   Úget_cuda_arch_versionsrE   s   sp   € Ø€JÝ•5”:×*Ò*Ñ,Ô,Ñ-Ô-ð /ð /ˆÝ”z×7Ò7¸Ñ:Ô:‰ˆˆuØ×ÒÐ-˜uÐ- eÐ-Ð-Ñ.Ô.Ð.Ð.ØÐr.   ÚindexÚreturnc           	      ó€   — t           j                             | ¦  «        }t          |dt          |dd¦  «        ¦  «        S )NÚshared_memory_per_block_optinÚshared_memory_per_blockr   )r$   r%   Úget_device_propertiesÚgetattr)rF   Úpropss     r,   Ú_device_shared_memory_limitrN   {   s8   € ÝŒJ×,Ò,¨UÑ3Ô3€EÝ�5Ð9½7À5ÐJcÐefÑ;gÔ;gÑhÔhÐhr.   é@   i h Údevicec                 óZ   — | j         €t          j                             ¦   «         n| j         S ©N)rF   r$   r%   Úcurrent_device©rP   s    r,   Ú_get_device_indexrU   Š   s$   € Ø*0¬,Ð*>�5Œ:×$Ò$Ñ&Ô&Ð&ÀFÄLÐPr.   c                 ó’   — t          | ¦  «        }|t          t          ¦  «        k     rt          |         S t          ¦   «         |         S rR   )rU   ÚlenÚ_CUDA_ARCHSrE   ©rP   Úidxs     r,   Ú_get_cuda_archr[   Ž   s>   € Ý
˜FÑ
#Ô
#€CØ
�S•ÑÔÒÐÝ˜3ÔÐÝ!Ñ#Ô# CÔ(Ð(r.   c                 óz   — t          | ¦  «        }|t          vrt          |¦  «        t          |<   t          |         S rR   )rU   Ú_SHARED_MEMORY_LIMIT_BY_DEVICErN   rY   s     r,   Ú_get_shared_memory_limitr^   •   s9   € Ý
˜FÑ
#Ô
#€CØ
Õ0Ð0Ð0Ý.IÈ#Ñ.NÔ.NÕ& sÑ+Ý)¨#Ô.Ð.r.   c                 ó¼   — t           rd S t          | ¦  «        }|t          j                             ¦   «         k    r!t          j                             |¦  «         d S d S rR   )Ú_SINGLE_CUDA_DEVICErU   r$   r%   rS   Ú
set_devicerY   s     r,   Ú_maybe_set_devicerb   œ   sY   € Ýð ØˆÝ
˜FÑ
#Ô
#€CØ
�eŒj×'Ò'Ñ)Ô)Ò)Ð)ÝŒ
×Ò˜cÑ"Ô"Ð"Ð"Ð"ð *Ð)r.   c                 ó2   — t          | ¦  «        t          k     S rR   )r^   Ú%_UPSTREAM_MASKED_HEAD128_SHARED_BYTESrT   s    r,   Ú_use_low_shared_masked_tritonre   ¤   s   € Ý# FÑ+Ô+Õ.SÒSÐSr.   ÚHNDc	                 ód  — t           s7t          dt          › dt          › dt	          | j        ¦  «        › d�¦  «         da t          j        | j        || j        ¬¦  «        }	|dk    �r| j        \  }
}}}|j        \  }}}}|  	                    d¦  «        |  	                    d	¦  «        |  	                    d
¦  «        }}}| 	                    d¦  «        | 	                    d	¦  «        | 	                    d
¦  «        }}}| 	                    d¦  «        | 	                    d	¦  «        | 	                    d
¦  «        }}}|	 	                    d¦  «        |	 	                    d	¦  «        |	 	                    d
¦  «        }}}�n/|dk    �r| j        \  }
}}}|j        \  }}}}|  	                    d¦  «        |  	                    d
¦  «        |  	                    d	¦  «        }}}| 	                    d¦  «        | 	                    d
¦  «        | 	                    d	¦  «        }}}| 	                    d¦  «        | 	                    d
¦  «        | 	                    d	¦  «        }}}|	 	                    d¦  «        |	 	                    d
¦  «        |	 	                    d	¦  «        }}}nt          d|› d�¦  «        ‚| 	                    d¦  «        | 	                    d	¦  «        | 	                    d
¦  «        | 	                    d¦  «        f\  }}}} |r)t          j        |
||gt          j        | j        ¬¦  «        n!t          j        dgt          j        d¬¦  «        }!|t          z   d	z
  t          z  ||
f}"t          j        |"         | |||||	||!|||||||||||||||| |||||z  ft          t          |d	|dddœŽ |	|!fS )NzE[SageAttention] Using low-shared-memory masked Triton patch (BLOCK_M=z
, BLOCK_N=z, GPU limit=z bytes).T©ÚdtyperP   rf   r   r   r3   ÚNHDztensor_layout z not supportedé   Úcpué   ©ÚBLOCK_MÚBLOCK_NÚHEAD_DIMÚSTAGEÚ
RETURN_LSEÚ	num_warpsÚ
num_stages)Ú'_LOW_SHARED_MASKED_TRITON_PATCH_PRINTEDr<   Ú_LOW_SHARED_MASKED_BLOCK_MÚ_LOW_SHARED_MASKED_BLOCK_Nr^   rP   r$   ÚemptyÚshapeÚstrideÚ
ValueErrorÚfloat32Úattn_qk_int8_per_blockÚ	_attn_fwd)#ÚqÚkÚvÚq_scaleÚk_scaleÚtensor_layoutÚ	attn_maskÚoutput_dtypeÚ
return_lseÚoÚbÚh_qoÚqo_lenÚhead_dimÚ_Úh_kvÚkv_lenÚstride_bz_qÚ
stride_h_qÚstride_seq_qÚstride_bz_kÚ
stride_h_kÚstride_seq_kÚstride_bz_vÚ
stride_h_vÚstride_seq_vÚstride_bz_oÚ
stride_h_oÚstride_seq_oÚstride_bz_maskÚstride_h_maskÚstride_m_maskÚstride_n_maskÚlseÚgrids#                                      r,   Ú_attn_false_low_shared_maskedr£   ¨   sè  € å2ð 7Ýð  QÕVpð  Qð  Qõ  }Wð  Qð  Qõ  e}ð  ~ô  ~Fñ  eGô  eGð  Qð  Qð  Qñ  	Rô  	Rð  	RØ26Ð/åŒ�A”G <¸¼ÐAÑAÔA€Aà˜ÒÑØ$%¤GÑ!ˆˆ4�˜ØœWÑˆˆ4�˜Ø01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �ZˆØ01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �ZˆØ01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �ZˆØ01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �Zˆ‰Ø	˜%Ò	Ñ	Ø$%¤GÑ!ˆˆ6�4˜ØœWÑˆˆ6�4˜Ø01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �ZˆØ01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �ZˆØ01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �ZˆØ01·²¸±´¸Q¿XºXÀa¹[¼[È!Ï(Ê(ÐSTÉ+Ì+ �ZˆˆåÐG¨-ÐGÐGÐGÑHÔHÐHàBK×BRÒBRÐSTÑBUÔBUÐW`×WgÒWgÐhiÑWjÔWjÐlu×l|Òl|Ð}~ÑlÔlð  BK÷  BRò  BRð  STñ  BUô  BUð  CUÑ?€N�M =°-ØR\ð  V�%Œ+�q˜$ Ð'­u¬}ÀQÄXÐ
NÑ
NÔ
NÐ
NÕbgÔbmÐopÐnqÕy~ô  zGð  PUð  cVñ  cVô  cV€CØÕ0Ñ0°1Ñ4Õ9SÑSÐUYÐ[\Ð]€DÝÔ$ TÔ*Ø	ˆ1ˆa�˜' 1 i°Ø�Z Ø�Z Ø�Z Ø�Z Ø˜ }°mØ�Øˆd�d‰lðõ +Õ4NÐYaØ˜JØØðð ð ð ð ˆcˆ6€Mr.   rj   r†   r…   c                 óœ  — | �| d         j         }t          j                             ¦   «         sdS t          j         |€dn|¦  «        }	 t          j                             t          |¦  «        ¦  «        \  }}|dk     rdt          |¦  «        › d�S t          t          d¦  «        sdS d	t          j
        t          ¦  «        j        vrd
S n# t          t          f$ r Y dS w xY w| €d S | d         }|j        t          j        t          j        fvrd|j        › d�S |j        d         dk    r"|j        d         dk    rd|j        d         › d�S d S )Nr   zCUDA is unavailabler%   r#   zCUDA architecture z! has no masked SageAttention pathr   z*SageAttention Triton kernel is unavailabler†   z1installed SageAttention does not expose attn_maskz6unable to inspect installed SageAttention mask supportzdtype z is unsupportedéÿÿÿÿé€   é   z	head_dim )rP   r$   r%   Úis_availabler(   rU   r[   Úhasattrr~   ÚinspectÚ	signatureÚ
attn_falseÚ
parametersÚ	TypeErrorr|   ri   Úfloat16Úbfloat16rz   )Úqkv_listr†   rP   r…   r*   rŽ   r€   s          r,   Ú&sageattn_attention_mask_support_reasonr²   Õ   sw  € ØÐØ˜!”Ô#ˆÝŒ:×"Ò"Ñ$Ô$ð %Ø$Ð$ÝŒ\ F N˜&˜&¸Ñ?Ô?€Fð	HÝ”:×3Ò3Õ4EÀfÑ4MÔ4MÑNÔN‰ˆˆqØ�1Š9ˆ9Øa­°vÑ(>Ô(>ÐaÐaÐaÐaÝÕ-¨{Ñ;Ô;ð 	@Ø?Ð?Ø�gÔ/µ
Ñ;Ô;ÔFÐFÐFØFÐFð Gøå•zÐ"ð Hð Hð HØGÐGÐGðHøøøàÐØˆtà�Œ€AØ„w•u”}¥e¤nÐ5Ð5Ð5Ø0˜œÐ0Ð0Ð0Ð0Ø„wˆr„{�SÒÐ˜QœW Rœ[¨CÒ/Ð/Ø7˜1œ7 2œ;Ð7Ð7Ð7Ð7Øˆ4s   Á	AC ÂC Â( C ÃC ÃC c                 ó*   — t          ||| |¦  «        d u S rR   )r²   )rP   r±   r†   r…   s       r,   Ú sageattn_supports_attention_maskr´   ð   s   € Ý1°(¸IÀvÈ}Ñ]Ô]ÐaeÐeÐer.   Ú	is_causalÚsm_scalerˆ   Ú	recycle_qÚscale_vÚkwargsc           	      óÚ  — |                      dd¦  «        }t          | d         j        ¦  «        }	|�)t          | ||¬¦  «        }
|
�t	          d|	› d|
› �¦  «        ‚| d         j        d         dk    r-|d	k    s|s|s|�t	          d
¦  «        ‚t          | |||¦  «        S |€-t          j        ¦   «         r0t          | d         j        ¦  «        st          | ||||||¬¦  «        S |	dk    rt          | ||||d¬¦  «        S |	dk    rt          | |||||¬¦  «        S |	dv r t          | f|||||dœt          |	¦  «        ¤ŽS |	dk    rt          | ||||d|¬¦  «        S t	          d|	› �¦  «        ‚)a4  
    Automatically selects the appropriate implementation of the SageAttention kernel based on the GPU compute capability.

    Parameters
    ----------
    q : torch.Tensor
        The query tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    k : torch.Tensor
        The key tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    v : torch.Tensor
        The value tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    tensor_layout : str
        The tensor layout, either "HND" or "NHD".
        Default: "HND".

    is_causal : bool
        Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len.
        Default: False.

    sm_scale : Optional[float]
        The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``.

    return_lse : bool
        Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention.
        Default: False.

    Returns
    -------
    torch.Tensor
        The output tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    torch.Tensor
        The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor).
        Shape: ``[batch_size, num_qo_heads, qo_len]``.
        Only returned if `return_lse` is True.

    Note
    ----
    - ``num_qo_heads`` must be divisible by ``num_kv_heads``.
    - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16``
    - All tensors must be on the same cuda device.
    r†   Nr   ©r…   z9Masked SageAttention is unsupported on CUDA architecture z: r¥   r§   rj   z\SageAttention with head_dim 256 takes NHD tensors, without causal mask, softmax scale or LSE)r…   rµ   r¶   rˆ   r†   r¸   Úsm80Úfp32)r…   rµ   r¶   rˆ   Úpv_accum_dtypeÚsm86)r…   rµ   r¶   rˆ   r¸   ©Úsm89Úsm120)r…   rµ   r¶   rˆ   r·   Úsm90ú	fp32+fp32)r…   rµ   r¶   rˆ   r¾   r·   zUnsupported CUDA architecture: )Úpopr[   rP   r²   r|   rz   Ú_sageattn_head256r   Ú	recordingÚcuda_kernels_capturableÚsageattn_qk_int8_pv_fp16_tritonÚsageattn_qk_int8_pv_fp16_cudaÚsageattn_qk_int8_pv_fp8_cudaÚ_fp8_pv_settingsÚ!sageattn_qk_int8_pv_fp8_cuda_sm90)r±   r…   rµ   r¶   rˆ   r·   r¸   r¹   r†   ÚarchÚsupport_reasons              r,   ÚsageattnrÐ   ó   sa  € ð@ —
’
˜;¨Ñ-Ô-€IÝ˜( 1œ+Ô,Ñ-Ô-€DØÐÝ?ÀÈ)ÐcpÐqÑqÔqˆØÐ%ÝÐqÐY]ÐqÐqÐaoÐqÐqÑrÔrÐrØ�„{Ô˜Ô Ò#Ð#Ø˜EÒ!Ð! YÐ!°*Ð!ÀÐ@TÝÐ{Ñ|Ô|Ð|Ý  ¨9°g¸yÑIÔIÐIØÐ¥Ô 5Ñ 7Ô 7ÐÕ@WÐX`ÐabÔXcÔXjÑ@kÔ@kÐÝ.¨xÀ}Ð`iÐt|ð  JTð  `ið  szð  {ñ  {ô  {ð  	{Øˆv‚~€~Ý,¨XÀ]Ð^gÐrzð  HRð  cið  jñ  jô  jð  	jØ	�ŠˆÝ.¨xÀ}Ð`iÐt|ð  JTð  ^eð  fñ  fô  fð  	fØ	Ð"Ð	"Ð	"Ý+¨Hð  CÀMÐ]fÐqyð  GQð  _hð  Cð  Cõ  l|ð  }Añ  lBô  lBð  Cð  Cð  	CØ	�ŠˆÝ0°ÈÐbkÐv~ð  LVð  grð  @Ið  Jñ  Jô  Jð  	JåÐA¸4ÐAÐAÑBÔBÐBr.   c                 óh  — | \  }}}|                       ¦   «          t          |j        |du|d¦  «        }dt          |                     ¦   «         g|¦  «        t          |g|¦  «        g}dx}}t          ||¦  «        |d<   t          |r|nt          j	        |t          j
        ¬¦  «        |||¦  «        S )a  sageattn of NHD q, k and v with head_dim 256 through the staged functions (no CUDA kernel, and sageattn_qk_int8_pv_fp16_triton's
    launches exceed the shared memory at 256). k is copied: staged_quantize_k subtracts its mean in place and callers may hold it.Nr§   r   )Úmemory_format)ÚclearÚstaged_settingsrP   Ústaged_quantize_kÚcloneÚstaged_quantize_vÚstaged_quantize_qÚstaged_attentionr$   Ú
empty_likeÚcontiguous_format)	r±   r†   r¸   r·   r€   r�   r‚   ÚsettingsÚ	quantizeds	            r,   rÆ   rÆ   K  sÇ   € ð �G€A€qˆ!Ø‡N‚NÑÔÐÝ˜qœx¨¸$Ð)>ÀÈÑMÔM€HØÕ(¨!¯'ª'©)¬)¨°hÑ?Ô?ÕARÐTUÐSVÐX`ÑAaÔAaÐb€IØ€L€AˆÝ$ Q¨Ñ1Ô1€Iˆa�LÝ Ðj˜A˜AµÔ0@ÀÕRWÔRiÐ0jÑ0jÔ0jÐluÐwð  BKñ  Lô  Lð  Lr.   c           
      ó   — t           �€ðt          j        ¦   «         5  t          j        dt          j        | ¬¦  «        }t          j        |j        dt          j        | ¬¦  «        }t          j        dt          j	        | ¬¦  «        t          j        d| ¬¦  «        }}t          j
                             ¦   «         t          j
                             | ¦  «        }}t          j
                             | ¦  «         t          j
                             |¦  «        5  |                     d¬	¦  «         |                     d¦  «         t#          j        |||d
dd¦  «         |                     ¦   «          ddd¦  «         n# 1 swxY w Y   t          j
                             | ¦  «         t)          |dk                         ¦   «         ¦  «        a ~ddd¦  «         n# 1 swxY w Y   t           st-          d¦  «         t           S )a  True when this SageAttention build launches its CUDA kernels on PyTorch's current stream: a CUDA graph can then capture them.
    Builds that launch them on the legacy default stream cannot be captured: blocks recorded for a CUDA graph use the Triton kernels.N)r   r¦   r   r¦   rh   é   )r   r   rm   r   rT   Úrelaxed)Úcapture_error_moder¦   é    r   zØ[Sage] This SageAttention build cannot run its CUDA kernels in CUDA graphs: its slower Triton kernels are used in CUDA Graph mode. SageAttention 2.2.0 post6 or later keeps the CUDA kernels (see docs/INSTALLATION.md).)Ú_capturabler$   Úinference_modeÚzerosr¯   Úfullrz   Úint8ry   r}   r%   Ú	CUDAGraphÚStreamÚsynchronizeÚstreamÚcapture_beginÚfill_r   Úquant_per_warp_int8_cudaÚcapture_endÚboolÚallr<   )rP   r€   Úq_int8rƒ   ÚmarkerÚgraphrë   s          r,   rÈ   rÈ   Z  s2  € õ ÑÝÔ!Ñ#Ô#ð 	ð 	Ý”Ð,µE´MÈ&ÐQÑQÔQˆAÝ”Z ¤¨µ%´*ÀVÐLÑLÔLˆFÝ#œk¨)½5¼=ÐQWÐXÑXÔXÕZ_ÔZeÐfgÐpvÐZwÑZwÔZw�VˆGÝ!œJ×0Ò0Ñ2Ô2µE´J×4EÒ4EÀfÑ4MÔ4M�6ˆEÝŒJ×"Ò" 6Ñ*Ô*Ð*Ý”×"Ò" 6Ñ*Ô*ð $ð $Ø×#Ò#°yÐ#ÑAÔAÐAØ—’˜Q‘”�ÝÔ/°°6¸7ÀCÈÈQÑOÔOÐOØ×!Ò!Ñ#Ô#Ð#ð	$ð $ð $ñ $ô $ð $ð $ð $ð $ð $ð $øøøð $ð $ð $ð $õ
 ŒJ×"Ò" 6Ñ*Ô*Ð*Ý ¨!¢×0Ò0Ñ2Ô2Ñ3Ô3ˆKØð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	õ ð 	kÝð jñ kô kð kåÐs8   œC;GÄAE<Å0GÅ<F 	Æ GÆF 	ÆAGÇGÇGc                 óV   — | dk    rdt           rdndddœS dt           rdndt            dœS )	zGsageattn's settings for the INT8 QK / FP8 PV kernels of sm89 and sm120.rÁ   Ú
per_threadú	fp32+fp16rÄ   F)Úqk_quant_granr¾   Úsmooth_vÚper_warpr½   )Úsg2pp)rÎ   s    r,   rÌ   rÌ   s  sE   € àˆv‚~€~Ø!-ÕPUÐAfÀÀÐ[fÐtyÐzÐzÐzà'Í%Ð;[¸;¸;ÐU[ÕmrÐirÐsÐsÐsr.   Ú	per_blockr¦   )rø   Úblock_mÚlaunchr¸   ©rm   rk   )r#   r   )r#   r3   Úmaskc                 ó>  — |dk    r|rt           nt          }n{|rt          | ¦  «        rt          nt          }n[t          | ¦  «        x}dk    s"t          j        ¦   «         rt          | ¦  «        st          }nt          r|dv rt          |¦  «        ndS |ri |¥ddi¥n|S )a¡  sageattn's quantization settings for the staged_* functions on this device (with a mask: its masked Triton path, smaller
    blocks where shared memory is short), None where sageattn uses other kernels. A block recorded for a CUDA graph gets the Triton path
    when the CUDA kernels cannot be captured (cuda_kernels_capturable), as head_dim 256 always does. scale_v: v scaled by powers of two
    on the Triton path.r§   r¿   rÀ   Nr¸   T)Ú_MASKED_HEAD256_SETTINGSÚ_HEAD256_SETTINGSre   Ú_LOW_SHARED_SETTINGSÚ_PER_BLOCK_SETTINGSr[   r   rÇ   rÈ   ÚSM89_ENABLEDrÌ   )rP   Úmaskedr¸   r�   rÜ   rÎ   s         r,   rÔ   rÔ   ˆ  sÅ   € ð
 �3‚€Ø/5ÐLÕ+Ð+Õ;LˆˆØ	ð ^Ý+HÈÑ+PÔ+PÐiÕ'Ð'ÕViˆˆÝ  Ñ(Ô(Ð
(ˆ$¨VÒ	3Ð	3µ{Ô7LÑ7NÔ7NÐ	3ÕWnÐouÑWvÔWvÐ	3Ý&ˆˆå)5Ð]¸$ÐBSÐ:SÐ:SÕ Ñ%Ô%Ð%ÐY]Ð]Ø,3ÐAÐ(ˆhÐ(˜	 4Ð(Ð(Ð(¸ÐAr.   c                 óê   — t           €dS | d         dk    r!t           j        dddft           j        dddfdœS | d         d	k    r0t           j        | d
         d|dz  z  dz  dft           j        dddfdœS dS )a½  Row layouts of these settings' staged q / k quantization for shared.kernels.qk_quant_triton (the norm and RoPE of q and k fused
    with their quantization): {"q": (layout, block, sm_scale, scale_eps), "k": (...)} as staged_quantize_q / staged_quantize_k
    quantize; None where the fused kernels do not reproduce the core's quantizers (the per-warp CUDA kernels of sm120 round
    differently by one LSB in places and are not validated there).Nrø   rö   râ   ç      ð?gH¯¼šò×z>rO   )r€   r�   rü   rý   ç      à?ç/lîdG÷?g        )Ú_qk_layoutsÚLAYOUT_THREAD_QÚLAYOUT_THREAD_KÚLAYOUT_PER_BLOCK)rÜ   r�   s     r,   Úfused_qk_layoutsr  ˜  s¾   € õ
 ÐØˆtØ�Ô  LÒ0Ð0Ý!Ô1°2°s¸DÐAÍÔIdÐfhÐjmÐosÐHtÐuÐuÐuØ�Ô  KÒ/Ð/Ý!Ô2°H¸YÔ4GÈÈxÐ[^ÉÑI^ÐakÑIkÐmpÐqõ  zEô  zVð  XZð  \_ð  adð  yeð  fð  fð  	fØˆ4r.   c                 ó  — | j         \  }}}}||z   dz
  |z  }t          j        |||ft          j        | j        ¬¦  «        }	t          |||f         | ||	||                      d¦  «        |                      d¦  «        |                      d¦  «        |                     d¦  «        |                     d¦  «        |                     d¦  «        |	                     d¦  «        |	                     d¦  «        |||¬¦  «         |	S )zgper_block_int8's Triton quantization of x (NHD) into x_int8, returns the scales (batch, heads, blocks).r   rh   r   r3   )r¶   ÚCÚBLK)rz   r$   ry   r}   rP   r   r{   )
ÚxÚx_int8Úblockr¶   rŠ   ÚtokensÚheadsr�   ÚblocksÚscales
             r,   Ú_quant_per_blockr  ¦  só   € à!"¤Ñ€A€vˆu�hØ�u‰n˜qÑ  UÑ*€FÝŒK˜˜E 6Ð*µ%´-ÈÌÐQÑQÔQ€EÝ ¨°Ð 2Ô3Ø	ˆ6�5˜& !§(¢(¨1¡+¤+¨q¯xªx¸©{¬{¸A¿HºHÀQ¹K¼KÈÏÊÐWXÑIYÔIYÐ[a×[hÒ[hÐijÑ[kÔ[kÐms×mzÒmzÐ{|Ñm}Ôm}Ø�Š�Q‰Œ˜Ÿš a™œ°8¸xÈUðTñ Tô Tð Tð €Lr.   c                 ó  — |d         dk    r7| d         }|                       ¦   «          g t          ||d         ¦  «        ¢d‘R S t          | d         j        ¦  «         |d         dk    rdnd	}t	          | d
||d         ¬¦  «        S )zšFP8 v (transposed, padded, scaled per channel) with its scale and mean, or on the Triton path fp16 v with its scales (fp16_v) and
    None; v is released.rø   rü   r   r¸   Nr¾   r÷   ç      @ç      |@rj   rù   ©r…   Ú	scale_maxrù   )rÓ   r	   rb   rP   r   )Úv_listrÜ   r‚   r   s       r,   r×   r×   ±  s˜   € ð �Ô  KÒ/Ð/Ø�1ŒIˆØ�Š‰ŒˆØ4•�q˜( 9Ô-Ñ.Ô.Ð4°Ð4Ð4Ð4Ý�f˜Q”iÔ&Ñ'Ô'Ð'Ø Ð!1Ô2°kÒAÐA��Àu€IÝ˜6°À)ÐV^Ð_iÔVjÐkÑkÔkÐkr.   c                 óì  — | d         }|                       ¦   «          t          |j        ¦  «         |j        \  }}}}|                     d¦  «        }|€t          |d¦  «        nA|                     |j        ¦  «                             |dd¦  «         	                    d¦  «        }t          j        |j        t          j        |j        ¬¦  «        }|dz   dz  }	|d	         d
k    rkt          j        |||	ft          j        |j        ¬¦  «        }
t          j        ||                     d¦  «                             ¦   «         ||
dd¦  «         ||
fS |                     |¦  «         |d	         dk    r|t'          ||dd¦  «        fS t          j        |||	dz  ft          j        |j        ¬¦  «        }
t)          |	dz  ||f         |||
||                     d¦  «        |                     d¦  «        |                     d¦  «        |                     d¦  «        |                     d¦  «        |                     d¦  «        |
                     d¦  «        |
                     d¦  «        |d¬¦  «         ||
fS )aN  INT8 k minus its mean over the tokens, with its scales; k is released (the per thread and per block paths subtract the mean in
    place). settings["k_mean"] (batch or 1, heads, head_dim) FP32, when present, is the mean subtracted instead (that of a cached prefix
    whose keys are attended in a separate pass: shared.attention_kit).r   Úk_meanNr   r¥   rh   é?   rO   rø   rú   rü   r	  rm   r3   ©r  r  )rÓ   rb   rP   rz   Úgetr   Útori   ÚexpandÚ	unsqueezer$   ry   rç   r}   r   Ú'quant_per_block_int8_fuse_sub_mean_cudaÚsqueezeÚ
contiguousÚsub_r  r   r{   )Úk_listrÜ   r�   rŠ   r�   r  r�   ÚkmÚk_int8r  r„   s              r,   rÕ   rÕ   ¾  s@  € ð
 	ˆqŒ	€AØ
‡L‚L�N„N€NÝ�a”hÑÔÐØ!"¤Ñ€A€vˆu�hØ	�Š�hÑ	Ô	€BØ˜Z��A�qÑ	Ô	Ð	¨R¯UªU°1´7©^¬^×-BÒ-BÀ1ÀbÈ"Ñ-MÔ-M×-WÒ-WÐXYÑ-ZÔ-Z€BÝŒ[˜œ­¬
¸1¼8ÐDÑDÔD€FØ�r‰k˜bÑ €FØ�Ô  JÒ.Ð.Ý”+˜q %¨Ð0½¼ÈaÌhÐWÑWÔWˆÝÔ6°q¸"¿*º*ÀQ¹-¼-×:RÒ:RÑ:TÔ:TÐV\Ð^eÐgiÐklÑmÔmÐmØ�wˆÐØ‡F‚Fˆ2�J„J€JØ�Ô  KÒ/Ð/ØÕ'¨¨6°2°sÑ;Ô;Ð;Ð;ÝŒk˜1˜e V¨a¡ZÐ0½¼ÈaÌhÐWÑWÔW€GÝ$ f¨q¡j°%¸Ð%;Ô<Ø	ˆ6�7˜F A§H¢H¨Q¡K¤K°·²¸!±´¸a¿hºhÀq¹k¼kÈ6Ï=Ê=ÐYZÑK[ÔK[Ð]c×]jÒ]jÐklÑ]mÔ]mÐou×o|Òo|Ð}~ÑoÔoØ�Š�qÑÔ˜7Ÿ>š>¨!Ñ,Ô,°¸bðBñ Bô Bð Bð �7ˆ?Ðr.   c                 óŒ  — t          | j        ¦  «         | j        \  }}}}t          j        | j        t          j        | j        ¬¦  «        }|d         dk    r#|t          | ||d         d|dz  z  dz  ¦  «        fS |dz   d	z  d
z  }|d         dk    rCt          j        |||ft          j        | j        ¬¦  «        }t          j	        | ||d	dd¦  «         nít          j        |||dz  ft          j        | j        ¬¦  «        }t          |dz  ||f         | ||||                      d¦  «        |                      d¦  «        |                      d¦  «        |                     d¦  «        |                     d¦  «        |                     d¦  «        |                     d¦  «        |                     d¦  «        |d¬¦  «         ||fS )zZINT8 q with its scales; q is not released (staged_attention can write the output into it).rh   rø   rü   rý   r	  r
  r  é   r¦   rm   rú   râ   r   r#   r3   r   r%  )rb   rP   rz   r$   ry   rç   r  r}   r   rî   r   r{   )	r€   rÜ   rŠ   rŒ   r  r�   rò   Úwarpsrƒ   s	            r,   rØ   rØ   Ù  sÔ  € õ �a”hÑÔÐØ!"¤Ñ€A€vˆu�hÝŒ[˜œ­¬
¸1¼8ÐDÑDÔD€FØ�Ô  KÒ/Ð/ØÕ'¨¨6°8¸IÔ3FÈÈhÐZ]ÉoÑH]Ð`jÑHjÑkÔkÐkÐkØ�c‰\˜cÑ! AÑ%€EØ�Ô  JÒ.Ð.Ý”+˜q %¨Ð/µu´}ÈQÌXÐVÑVÔVˆÝÔ'¨¨6°7¸CÀÀQÑGÔGÐGÐGå”+˜q %¨°©Ð3½5¼=ÐQRÔQYÐZÑZÔZˆÝ*¨E°A©I°u¸aÐ+@ÔAØˆv�w ¨¯ª°©¬°Q·X²X¸a±[´[À!Ç(Â(È1Á+Ä+ÈvÏ}Ê}Ð]^ÑO_ÔO_Ðag×anÒanÐopÑaqÔaqÐsy÷  tAò  tAð  BCñ  tDô  tDØ�NŠN˜1ÑÔ˜wŸ~š~¨aÑ0Ô0°HÀ"ð	Fñ 	Fô 	Fð 	Fð �7ˆ?Ðr.   c                 óú  — |\  \  }}\  }}\  }	}
}|                      ¦   «          t          | j        ¦  «         |d         dk    �r<|j        \  }}}}|j        d         }|�|                     ||||¦  «        }|�|f|                     ¦   «         fn
t          rdnd}|d         p	|dk    rd	nd
\  }}|r)t          j        |||ft          j	        | j        ¬¦  «        n!t          j        dgt          j	        d¬¦  «        }t          j        ||d         z   dz
  |d         z  ||f         |||	||| g|d         ¢|‘|                     d¦  «        ‘|                     d¦  «        ‘|                     d¦  «        ‘|                     d¦  «        ‘|                     d¦  «        ‘|                     d¦  «        ‘|	                     d¦  «        ‘|	                     d¦  «        ‘|	                     d¦  «        ‘|                      d¦  «        ‘|                      d¦  «        ‘|                      d¦  «        ‘|d         ¢|‘|‘|‘||j        d         z  ‘R |d         d|d|||dœŽ |
€| nt          | |
d¦  «        } |r| |fn| S |||	| |||
f|d         dk    r|d         r|fndz   } t          |¦  «        g |¢d‘d‘|d         dk    rdnd‘|                      d¦  «        dz  ‘t          |¦  «        ‘R Ž }|r| |fn| S )a  Attention of the staged [(q_int8, q_scale), (k_int8, k_scale), (v, v_scale, v_mean)], written into out and returned. out has
    q's shape with a contiguous last dimension: q itself, or a slice of heads of a larger output. attn_mask (per block settings
    only): SageAttention's mask layout (batch, heads, q, k), broadcastable. return_lse: (out, lse), lse (batch, heads, tokens) FP32 the
    base-2 log-sum-exp of each query's scores without the keys' smoothing mean (attentions merged by it must share that mean).rø   rü   r   N)rR   )r   r   r   r   )© r5  rþ   rO   rÿ   )r#   rm   rh   r   rl   rý   r3   rn   r¾   r½   rù   r5  rö   rk   r¥   ç      à¿)rÓ   rb   rP   rz   r(  r{   Ú_TRITON_MASK_ARGUMENTr$   ry   r}   r~   r   r
   Ú_cuda_kernelÚsizer9   )ÚoutÚquantized_listrÜ   r†   rˆ   rò   rƒ   r0  r„   r‚   Úv_scaleÚv_meanrŠ   rŒ   r  r�   r�   r   rt   ru   r¡   Úinputss                         r,   rÙ   rÙ   í  sa  € ð BPÑ>Ñ€VˆWÑ(˜ Ñ*>¨1¨g°vØ×ÒÑÔÐÝ�c”jÑ!Ô!Ð!Ø�Ô  KÒ/Ñ/Ø%+¤\Ñ"ˆˆ6�5˜(Ø”˜a”ˆØÐ Ø!×(Ò(¨¨E°6¸6ÑBÔBˆIØ5>Ð5J��˜i×.Ò.Ñ0Ô0Ð1Ð1õ  lAð  QOÐPgÐPgð  GOˆØ (¨Ô 2Ð \ÀÈRÂÀ°v°vÐU[Ñˆ	�:ØYcð  ]�eŒk˜1˜e VÐ,µE´MÈ#Ì*ÐUÑUÔUÐUÕinÔitÐvwÐuxõ  AFô  ANð  W\ð  j]ñ  j]ô  j]ˆÝÔ(¨6°H¸YÔ4GÑ+GÈ!Ñ+KÐPXÐYbÔPcÑ*cÐejÐlmÐ)nÔoØ�F˜A˜w¨°ð	OØ7;¸A´wð	OØ@Cð	Oà�MŠM˜!ÑÔð	Oà$Ÿmšm¨AÑ.Ô.ð	Oà06·²¸aÑ0@Ô0@ð	OàBHÇ-Â-ÐPQÑBRÔBRð	OàTZ×TaÒTaÐbcÑTdÔTdð	Oàfl×fsÒfsÐtuÑfvÔfvð	Oð �HŠH�Q‰KŒKð	Oð Ÿš !™œð	Oð '(§h¢h¨q¡k¤kð	Oð 47·:²:¸a±=´=ð	Oð CFÇ*Â*ÈQÁ-Ä-ð	Oð RU×Q[ÒQ[Ð\]ÑQ^ÔQ^ð	Oð bfÐfgÔahð	Oð ð		Oð ð		Oð "ð		Oð $)¨F¬L¸¬OÑ#;ð		Oð 	Oð FNÈiÔEXÐbdÐowð  @AØ!¨YÀ:ð	Oð 	Oð 	Oð 	Oð �_ˆcˆc­,°s¸GÀQÑ*GÔ*GˆØ'Ð0��Sˆzˆz¨SÐ0Ø�f˜a  g¨w¸Ð@ÐQYÐZjÔQkÐouÒQuÐQuð  {Cð  DNô  {OÐQuÀVÀIÀIð  UWñ  X€FØ
 �,�xÑ
 Ô
 ð  N &ð  N¨!ð  N¨Qð  N°X¸oÔ5NÐR^Ò5^Ð5^°°Ðdeð  NÐgj×goÒgoÐprÑgsÔgsÐw{Ñg{ð  Nõ  ~Að  BLñ  ~Mô  ~Mð  Nð  Nð  N€CØ#Ð,ˆC�ˆ:ˆ:¨Ð,r.   c                 ó¢   — | d         dk    rt           j        S | d         dk    rt           j        S | d         rt           j        S t           j        S )Nr¾   r÷   rÄ   rù   )r   Ú2qk_int8_sv_f8_accum_f16_fuse_v_scale_attn_inst_bufr   Ú5qk_int8_sv_f8_accum_f32_fuse_v_scale_fuse_v_mean_attnÚ)qk_int8_sv_f8_accum_f32_fuse_v_scale_attn©rÜ   s    r,   r8  r8    sV   € ØÐ Ô! [Ò0Ð0ÝÔMÐMØÐ Ô! [Ò0Ð0ÝÔMÐMØ�
Ôð QÝÔPÐPÝÔ@Ð@r.   rs   c                 óè   — t                                | ¦  «        }|€Ut          t          | dd¦  «        dd¦  «        }|�t          d„ |j        D ¦   «         ¦  «        n
d| j        pdv x}t           | <   |S )zyWhether a SageAttention CUDA kernel of this build takes return_lse: its operator schema, or a pybind signature naming it.NÚdefaultÚ_schemac              3   ó,   K  — | ]}|j         d k    V — ŒdS )rˆ   N)Úname)Ú.0Úarguments     r,   ú	<genexpr>z_returns_lse.<locals>.<genexpr>  s)   è è € Ð*jÐ*jÈX¨8¬=¸LÒ+HÐ*jÐ*jÐ*jÐ*jÐ*jÐ*jr.   rˆ   Ú )Ú_lse_kernelsr&  rL   ÚanyÚ	argumentsÚ__doc__)ÚkernelÚfoundÚschemas      r,   Ú_returns_lserT    sŸ   € å×Ò˜VÑ$Ô$€EØ€}Ý� ¨°DÑ9Ô9¸9ÀdÑKÔKˆØntð  oA¥sÐ*jÐ*jÐY_ÔYiÐ*jÑ*jÔ*jÑ'jÔ'jÐ'jð  GSð  X^ô  Xfð  Xlð  jlð  Gmð  	mˆ•˜VÑ$Ø€Lr.   c                 ó‚   — t          | |d¦  «        } | d         dk    rt          nt          t          | ¦  «        ¦  «        S )z´Whether this SageAttention build's kernels for the prefix's layout (prefix_settings) return the log-sum-exp (otherwise the kit
    keeps the prefix in q8 and joins it to the keys).Frø   rü   )Úprefix_settingsÚ_TRITON_LSE_ARGUMENTrT  r8  )rÜ   rP   ri   s      r,   Úprefix_supportedrX  &  sC   € õ ˜x¨°Ñ7Ô7€HØ#+¨OÔ#<ÀÒ#KÐ#KÕÐÕQ]Õ^jÐksÑ^tÔ^tÑQuÔQuÐur.   c                 óh   — | d         dk    st          |¦  «        r| S |ri t          ¥ddi¥nt          S )a  The settings with which a cached prefix is kept and every attention with it runs: these, or the Triton per-block path that blocks
    recorded for CUDA graphs use where the CUDA kernels cannot be captured (staged_settings), so that eager and recorded steps keep
    the prefix's layout.rø   rü   r¸   T)rÈ   r  )rÜ   rP   r¸   s      r,   rV  rV  -  sI   € ð �Ô  KÒ/Ð/Õ3JÈ6Ñ3RÔ3RÐ/ØˆØ7>ÐWÐ3Õ!Ð3 9¨dÐ3Ð3Ð3ÕDWÐWr.   c                 ó   — | d         dk    S )Nrø   rü   r5  rC  s    r,   Úprefix_fp8_vr[  6  s   € Ø�OÔ$¨Ò3Ð3r.   c                 óZ   — t          | d¦  «        dd…df                              ¦   «         S )zèThe key mean of a cached prefix k (batch, tokens, heads, head_dim) as settings["k_mean"] takes it: (batch, heads, head_dim) FP32,
    rounded to k's dtype so that the 16-bit and the fused FP32 quantizations subtract the same values.r   Nr   )r   Úfloat)r�   s    r,   Úprefix_meanr^  :  s-   € õ �a˜ÑÔ˜A˜A˜A˜q˜DÔ!×'Ò'Ñ)Ô)Ð)r.   c                 ó2   — | d d …|j         |j        …f         S rR   )ÚstartÚstop)Úmeanr  s     r,   Úprefix_mean_headsrc  @  s   € Ø����5”;˜uœzÐ)Ð)Ô*Ð*r.   c                 ó.   — t          | i |¥d|i¥¦  «        S )z^A prefix's k (released) quantized with mean, as staged_quantize_k makes it: (k_int8, k_scale).r#  )rÕ   )r.  rÜ   rb  s      r,   Úprefix_quantize_kre  D  s"   € å˜VÐ%A¨Ð%A°(¸DÐ%AÐ%AÑBÔBÐBr.   c                 ó8  — | dd…dd…|j         |j        …f         |dd…|j         |j        …f         }} | j        d         |k    r/|                      |ddd¦  «        |                     |dd¦  «        }} |                      ¦   «         |                     ¦   «         fS )zËThe (k_int8, k_scale) of staged_attention for the heads of a cached prefix (k_int8 (batch, tokens, heads, head_dim), k_scale
    (batch, heads, scales)) and batch items (one prefix item serves them all).Nr   r¥   ©r`  ra  rz   r(  r,  )r0  r„   r  Úbatchs       r,   Úprefix_kri  I  s¤   € ð ˜Q˜Q˜Q    5¤;¨u¬zÐ#9Ð9Ô:¸GÀAÀAÀAÀuÄ{ÐSXÔS]ÐG]ÐD]Ô<^ˆG€FØ„|�A„˜%ÒÐØ Ÿ-š-¨¨r°2°rÑ:Ô:¸G¿NºNÈ5ÐRTÐVXÑ<YÔ<Y�ˆØ×ÒÑÔ × 2Ò 2Ñ 4Ô 4Ð4Ð4r.   c                 óÔ  — | dd…dd…|j         |j        …f         |dd…|j         |j        …f         }} |€dn|dd…|j         |j        …f         }| j        d         |k    rJ|                      |ddd¦  «        |                     |dd¦  «        }} |€dn|                     |dd¦  «        }|                      ¦   «         |                     ¦   «         |€dn|                     ¦   «         fS )z~The (v, v_scale, v_mean) of staged_attention for the heads of a cached prefix's FP8 v (batch, head_dim, heads, padded tokens).Nr   r¥   rg  )r‚   r<  r=  r  rh  s        r,   Úprefix_vrk  R  sñ   € à�1�1�1�a�a�a˜œ U¤ZÐ/Ð/Ô0°'¸!¸!¸!¸U¼[ÈÌÐ=SÐ:SÔ2T€w€AØ�^ˆTˆT¨°°°°5´;¸u¼zÐ3IÐ0IÔ)J€FØ„wˆq„z�UÒÐØ—X’X˜e R¨¨RÑ0Ô0°'·.².ÀÈÈBÑ2OÔ2Oˆ7ˆØ˜��¨V¯]ª]¸5À"ÀbÑ-IÔ-IˆØ�<Š<‰>Œ>˜7×-Ò-Ñ/Ô/¸¸°°ÈV×M^ÒM^ÑM`ÔM`Ð`Ð`r.   ÚtritonÚquantization_backendÚsmooth_kc	                 óê
  — | \  }
}}|                       ¦   «          |
j        }|
j        s
J d¦   «         ‚|t          j        t          j        fv s
J d¦   «         ‚|
j        |j        cxk    r|j        k    sn J d¦   «         ‚|
j        |j        cxk    r|j        k    sn J d¦   «         ‚|�6|r
J d¦   «         ‚|j        t          j        k    s|j        |k    s
J d¦   «         ‚t          |j        ¦  «         |
 	                    d¦  «        }|duot          |
j        ¦  «        }|d	k     rt          j        j                             |
d
d	|z
  f¦  «        }
t          j        j                             |d
d	|z
  f¦  «        }t          j        j                             |d
d	|z
  f¦  «        }n£|d	k    r…|dk     rt          j        j                             |
d
d|z
  f¦  «        }
t          j        j                             |d
d|z
  f¦  «        }t          j        j                             |d
d|z
  f¦  «        }n|dk    rt          d|› �¦  «        ‚|
                     d¦  «        dk    r2|                     d¦  «        dk    r|                     d¦  «        dk    s
J d¦   «         ‚|dk    rdnd}|rôt!          ||¦  «        }|rá|dk    r‚t          j        |
                     dd¦  «        |                     dd¦  «                             dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }n\t          j        |
|                     dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }nd}t-          ||d
|f¦  «        \  }}|€d|dz  z  }|dk    rB|r&t/          |
||t0          t2          ||¬¦  «        \  }}}}nLt/          |
||||¬¦  «        \  }}}}n2|dk    rt5          |
||||¬¦  «        \  }}}}nt          d|› �¦  «        ‚~
~~|�Š|dk    r2|j        d
         |j        d         |j        d         |j        d         fn1|j        d
         |j        d         |j        d         |j        d         f}|j        |k    r|                     |¦  «        }|rt;          ||||||||¬¦  «        \  }}nV|rt=          |||||||||¬¦	  «	        \  }}n8|�t?          |||||||||¬¦	  «	        \  }}nt?          ||||||||¬¦  «        \  }}|�tA          ||d|z
  ¦  «         |dd|…f         }|r||r|dz  ||z  z   n|dz  fS |S )a®
  
    SageAttention with per-block INT8 quantization for Q and K, FP16 PV with FP16 accumulation, implemented using Triton.
    scale_v: v scaled by powers of two (shared.sage1_core.fp16_v), for v large enough to overflow the FP16 accumulation.
    The FP16 accumulator is added to a FP32 buffer immediately after each iteration.

    Parameters
    ----------
    q : torch.Tensor
        The query tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    k : torch.Tensor
        The key tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    v : torch.Tensor
        The value tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    tensor_layout : str
        The tensor layout, either "HND" or "NHD".
        Default: "HND".

    quantization_backend : str
        The quantization backend, either "triton" or "cuda".
        "cuda" backend offers better performance due to kernel fusion.

    is_causal : bool
        Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len.
        Default: False.

    sm_scale : Optional[float]
        The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``.

    smooth_k : bool
        Whether to smooth the key tensor by subtracting the mean along the sequence dimension.
        Default: True.

    return_lse : bool
        Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention.
        Default: False.

    Returns
    -------
    torch.Tensor
        The output tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    torch.Tensor
        The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor).
        Shape: ``[batch_size, num_qo_heads, qo_len]``.
        Only returned if `return_lse` is True.

    Note
    ----
    - ``num_qo_heads`` must be divisible by ``num_kv_heads``. 
    - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16``, ``torch.bfloat16`` or ``torch.float32``.
    - All tensors must be on the same cuda device.
    - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances.
    úInput tensors must be on cuda.úAInput tensors must be in dtype of torch.float16 or torch.bfloat16ú'All tensors must be on the same device.ú%All tensors must have the same dtype.Nz?SageAttention does not support attn_mask with causal attention.z(attn_mask must be bool or match q dtype.r¥   rO   r   r¦   úUnsupported head_dim: r   ú#Last dim of qkv must be contiguous.rj   r3   rk   r	  r
  rl  )r/  ÚBLKQÚBLKKr¶   r…   )r/  r¶   r…   r%   z"Unsupported quantization backend: )r…   r‡   rˆ   )r…   r‡   rˆ   r†   .r  )!rÓ   ri   Úis_cudar$   r¯   r°   rP   rð   rb   r9  re   ÚnnÚ
functionalÚpadr|   r{   r   ÚmatmulÚ	transposer+  r'  r}   r	   Úper_block_int8_tritonrw   rx   Úper_block_int8_cudarz   r(  Ú	attn_truer£   r¬   r
   )r±   r…   rm  rµ   r¶   rn  rˆ   r†   r¸   r¹   r€   r�   r‚   ri   Úhead_dim_ogÚmasked_low_sharedÚseq_dimr/  Úlse_correctionr<  rò   rƒ   r0  r„   Útarget_shaper‰   r¡   s                              r,   rÉ   rÉ   [  s�  € ð` �G€A€qˆ!Ø‡N‚NÑÔÐØŒG€EØŒ9Ð6Ð6Ð6Ñ6Ô6ˆ9Ø•U”]¥E¤NÐ3Ð3Ð3Ð3Ð5xÑ3Ô3Ð3ØŒ8�q”xÐ+Ð+Ò+Ð+ 1¤8Ò+Ð+Ð+Ð+Ð+Ð-VÑ+Ô+Ð+ØŒ7�a”gÐ(Ð(Ò(Ð( ¤Ò(Ð(Ð(Ð(Ð(Ð*QÑ(Ô(Ð(ØÐØÐ_Ð_Ð_Ñ_Ô_ˆ}ØŒ¥%¤*Ò,Ð,°	´À5Ò0HÐ0HÐ0HÐJtÑ0HÔ0HÐHõ �a”hÑÔÐà—&’&˜‘*”*€KØ!¨Ð-ÐYÕ2OÐPQÔPXÑ2YÔ2YÐà�RÒÐÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆˆØ	�rÒ	Ð	˜k¨CÒ/Ð/ÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆˆØ	�sÒ	Ð	ÝÐ?°+Ð?Ð?Ñ@Ô@Ð@ð �8Š8�B‰<Œ<˜1ÒÐ §¢¨"¡¤°Ò!2Ð!2°q·x²xÀ±|´|ÀqÒ7HÐ7HÐ7HÐJoÑ7HÔ7HÐHà  EÒ)Ð)ˆaˆa¨q€Gàð Ý˜˜7Ñ#Ô#ˆØð 	cØ Ò%Ð%Ý!&¤¨a¯kªk¸!¸QÑ.?Ô.?ÀÇÂÈaÐQRÑASÔAS×A]ÒA]Ð^_ÐabÑAcÔAcÑ!dÔ!d×!lÒ!lÐmoÑ!pÔ!p×!sÒ!sÕtyô  uBñ  "Cô  "C��å!&¤¨a°·²¸aÀÑ1CÔ1CÑ!DÔ!D×!LÒ!LÈRÑ!PÔ!P×!SÒ!SÕTYÔTaÑ!bÔ!b�øàˆå˜˜7 Q¨ LÑ1Ô1�J€A€wàÐØ˜+¨Ñ,Ñ-ˆà˜xÒ'Ð'Øð 	BÝ/DÀQÈÈbÕWqõ  ySð  ^fð  vCð  0Dñ  0Dô  0DÑ,ˆF�G˜V W Wå/DÀQÈÈbÐ[cð  tAð  0Bñ  0Bô  0BÑ,ˆF�G˜V W WØ	 Ò	'Ð	'Ý+>¸qÀ!ÈÐU]ÐmzÐ+{Ñ+{Ô+{Ñ(ˆ�˜  åÐTÐ>RÐTÐTÑUÔUÐUØ	ˆ!ˆRàÐð  Ò%Ð%ð Œ\˜!Œ_˜fœl¨1œo¨v¬|¸A¬ÀÄÈQÄÐPÐPà”,˜q”/ 6¤<°¤?°F´LÀ´OÀVÄ\ÐRSÄ_ÐUð 	ð
 Œ?˜lÒ*Ð*Ø!×(Ò(¨Ñ6Ô6ˆIàð IÝ˜6 6¨1¨g°wÈmÐjoð  }Gð  Hñ  Hô  H‰ˆˆ3ˆ3Ø	ð IÝ.¨v°v¸qÀ'È7Ðboð  Dð  Q[ð  gpð  qñ  qô  q‰ˆˆ3ˆ3Ø	Ð	Ý˜F F¨A¨w¸È}Ðkpð  ~Hð  T]ð  ^ñ  ^ô  ^‰ˆˆ3ˆ3å˜F F¨A¨w¸È}Ðkpð  ~Hð  Iñ  Iô  I‰ˆˆ3ØÐÝ�Q˜  W¡Ñ-Ô-Ð-à	ˆ#ˆ|�ˆ|Ð
Ô€Aàð ØÀ(Ð`�#˜
Ñ" ^°hÑ%>Ñ>Ð>ÐPSÐV`ÑP`Ð`Ð`àˆr.   r€   r�   r‚   Úcu_seqlens_qÚcu_seqlens_kÚmax_seqlen_qÚmax_seqlen_kc
                 óN  — | j         }| j        s
J d¦   «         ‚|t          j        t          j        fv s
J d¦   «         ‚| j        |j        cxk    r|j        k    sn J d¦   «         ‚| j         |j         cxk    r|j         k    sn J d¦   «         ‚t          |j        ¦  «         |                      d¦  «        }|dk     rt          j        j	         
                    | dd|z
  f¦  «        } t          j        j	         
                    |dd|z
  f¦  «        }t          j        j	         
                    |dd|z
  f¦  «        }n£|dk    r…|dk     rt          j        j	         
                    | dd|z
  f¦  «        } t          j        j	         
                    |dd|z
  f¦  «        }t          j        j	         
                    |dd|z
  f¦  «        }n|dk    rt          d	|› �¦  «        ‚|                      d¦  «        d
k    r2|                     d¦  «        d
k    r|                     d¦  «        d
k    s
J d¦   «         ‚|                     ¦   «         r|                     ¦   «         s
J d¦   «         ‚|t          j        k    s|t          j        k    r|                     t          j        ¦  «        }|	rt!          |d¦  «        }||z
  }|€d|dz  z  }t#          | ||||||¬¦  «        \  }}}}}}|rt%          |||||||||||¬¦  «        }nt'          |||||||||||¬¦  «        }|dd|…f         }|S )a2  

    Parameters
    ----------
    q : torch.Tensor
        The query tensor, shape: ``[cu_seqlens_q[-1], num_qo_heads, head_dim]``.

    k : torch.Tensor
        The key tensor, shape: ``[cu_seqlens_k[-1], num_kv_heads, head_dim]``.

    v : torch.Tensor
        The value tensor, shape: ``[cu_seqlens_k[-1], num_kv_heads, head_dim]``.

    cu_seqlens_q : torch.Tensor
        The cumulative sequence lengths for the query sequences in the batch, used to index into `q`. 
        Shape: ``[batch_size + 1]``, where each entry represents the cumulative length of sequences up to that batch index.

    cu_seqlens_k : torch.Tensor
        The cumulative sequence lengths for the key and value sequences in the batch, used to index into `k` and `v`. 
        Shape: ``[batch_size + 1]``, where each entry represents the cumulative length of sequences up to that batch index.

    max_seqlen_q : int
        The maximum sequence length for the query tensor in the batch.
    
    max_seqlen_k : int
        The maximum sequence length for the key and value tensors in the batch.

    is_causal : bool
        Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len for each sequence.
        Default: False.
    
    sm_scale : Optional[float]
        The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``.

    smooth_k : bool
        Whether to smooth the key tensor by subtracting the mean along the sequence dimension.
        Default: True.

    Returns
    -------
    torch.Tensor
        The output tensor, shape: ``[cu_seqlens_q[-1], num_qo_heads, head_dim]``.

    Note
    ----
    - ``num_qo_heads`` must be divisible by ``num_kv_heads``.
    - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16``, ``torch.bfloat16`` or ``torch.float32``.
    - The tensors `cu_seqlens_q` and `cu_seqlens_k` must have the dtype ``torch.int32`` or ``torch.int64``.
    - All tensors must be on the same cuda device.
    - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances.
    rp  rq  rr  rs  r¥   rO   r   r¦   rt  r   ru  z1cu_seqlens_q and cu_seqlens_k must be contiguous.Nr	  r
  )r¶   )r‡   .)ri   rx  r$   r¯   r°   rP   rb   r9  ry  rz  r{  r|   r{   Úis_contiguousr}   r'  r   Úper_block_int8_varlen_tritonÚattn_true_varlenÚattn_false_varlen)r€   r�   r‚   r†  r‡  rˆ  r‰  rµ   r¶   rn  r¹   ri   r�  r/  rò   rƒ   r0  r„   Úcu_seqlens_q_scaleÚcu_seqlens_k_scaler‰   s                        r,   Úsageattn_varlenr‘    s°  € ðD ŒG€EØŒ9Ð6Ð6Ð6Ñ6Ô6ˆ9Ø•U”]¥E¤NÐ3Ð3Ð3Ð3Ð5xÑ3Ô3Ð3ØŒ8�q”xÐ+Ð+Ò+Ð+ 1¤8Ò+Ð+Ð+Ð+Ð+Ð-VÑ+Ô+Ð+ØŒ7�a”gÐ(Ð(Ò(Ð( ¤Ò(Ð(Ð(Ð(Ð(Ð*QÑ(Ô(Ð(õ �a”hÑÔÐà—&’&˜‘*”*€Kà�RÒÐÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆˆØ	�rÒ	Ð	˜k¨CÒ/Ð/ÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆˆØ	�sÒ	Ð	ÝÐ?°+Ð?Ð?Ñ@Ô@Ð@à�8Š8�B‰<Œ<˜1ÒÐ §¢¨"¡¤°Ò!2Ð!2°q·x²xÀ±|´|ÀqÒ7HÐ7HÐ7HÐJoÑ7HÔ7HÐHØ×%Ò%Ñ'Ô'Ð}¨L×,FÒ,FÑ,HÔ,HÐ}Ð}ÐJ}Ñ}Ô}ÐHà•”ÒÐ %­5¬=Ò"8Ð"8Ø�DŠD•”ÑÔˆàð Ý˜˜1ÑÔˆØ�‰FˆàÐØ˜+¨Ñ,Ñ-ˆåOkÐlmÐopÐr~ð  AMð  O[ð  ]ið  t|ð  P}ñ  P}ô  P}ÑL€FˆG�V˜WÐ&8Ð:Làð iÝ˜V V¨Q°¸lÈLÐZaÐcjÐl~ð  ASð  bgð  hñ  hô  hˆˆå˜f f¨a°¸|È\Ð[bÐdkÐmð  BTð  chð  iñ  iô  iˆà	ˆ#ˆ|�ˆ|Ð
Ô€Aà€Hr.   rö   r½   rø   r¾   rù   c	                 óR  — | \  }
}}|                       ¦   «          |
j        }t          s
J d¦   «         ‚|
j        s
J d¦   «         ‚|t          j        t          j        fv s
J d¦   «         ‚|dv s
J d¦   «         ‚|
j        |j        cxk    r|j        k    sn J d¦   «         ‚|
j        |j        cxk    r|j        k    sn J d¦   «         ‚t          |j        ¦  «         |dk    rd	nd
}|rd
nd	}|dk    rdnd}|rd
nd	}|
 	                    d¦  «        }|dk     rt          j
        j                             |
d	d|z
  f¦  «        }
t          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }n£|dk    r…|dk     rt          j
        j                             |
d	d|z
  f¦  «        }
t          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }n|dk    rt          d|› �¦  «        ‚|
                     d¦  «        d
k    r2|                     d¦  «        d
k    r|                     d¦  «        d
k    s
J d¦   «         ‚|€|dz  }|d	k    rd
nd}|rôt          ||¦  «        }|rá|dk    r‚t	          j        |
                     d
d¦  «        |                     d
d¦  «                             dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }n\t	          j        |
|                     dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }nd}|dk    r=t+          |
|||d|
 	                    d¦  «        dk    r|dk    rdndd¬¦  «        \  }}}}nC|dk    r=t-          |
|||d|
 	                    d¦  «        dk    r|dk    rdnddd¬¦  «        \  }}}}|
 	                    ¦   «         }|
j        }~
~~t	          j        |||¬¦  «        }|dv r|rt1          j        d|› d�¦  «         d}|d k    r>|                     t          j        ¦  «        }t5          j        |||||||||||¦  «        }nÑ|d!k    ru|r5t9          ||¬"¦  «        \  }}~t5          j        ||||||||||||¦  «        }n”|                     t          j        ¦  «        }t5          j        |||||||||||¦  «        }nV|dk    r>|                     t          j        ¦  «        }t5          j        |||||||||||¦  «        }nt          d#|› �¦  «        ‚|d$d|…f         }|r||r|d%z  ||z  z   n|d%z  fS |S )&ah  
    SageAttention with INT8 quantization for Q and K, FP16 PV with FP16/FP32 accumulation, implemented using CUDA.

    Parameters
    ----------
    q : torch.Tensor
        The query tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    k : torch.Tensor
        The key tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    v : torch.Tensor
        The value tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    tensor_layout : str
        The tensor layout, either "HND" or "NHD".
        Default: "HND".

    is_causal : bool
        Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len.
        Default: False.

    qk_quant_gran : str
        The granularity of quantization for Q and K, either "per_warp" or "per_thread".
        Default: "per_thread".

    sm_scale : Optional[float]
        The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``.

    pv_accum_dtype : str
        The dtype of the accumulation of the product of the value tensor and the attention weights, either "fp16", "fp16+fp32" or "fp32".
        - "fp16": PV accumulation is done in fully in FP16. This is the fastest option but may lead to numerical instability. `smooth_v` option will increase the accuracy in cases when the value tensor has a large bias (like in CogVideoX-2b).
        - "fp32": PV accumulation is done in FP32. This is the most accurate option but may be slower than "fp16" due to CUDA core overhead.
        - "fp16+fp32": PV accumulation is done in FP16, but added to a FP32 buffer every few iterations. This offers a balance between speed and accuracy.
        Default: "fp32".

    smooth_k : bool
        Whether to smooth the key tensor by subtracting the mean along the sequence dimension.
        Default: True.
    
    smooth_v : bool
        Whether to smooth the value tensor by subtracting the mean along the sequence dimension.
        smooth_v will be ignored if pv_accum_dtype is "fp32" or "fp16+fp32".
        Default: False.

    return_lse : bool
        Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention.
        Default: False.

    Returns
    -------
    torch.Tensor
        The output tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    torch.Tensor
        The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor).
        Shape: ``[batch_size, num_qo_heads, qo_len]``.
        Only returned if `return_lse` is True.

    Note
    ----
    - ``num_qo_heads`` must be divisible by ``num_kv_heads``. 
    - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16``
    - All tensors must be on the same cuda device.
    - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances.
    zWSM80 kernel is not available. make sure you GPUs with compute capability 8.0 or higher.rp  rq  ©rú   rö   ú8qk_quant_gran must be either 'per_warp' or 'per_thread'.rr  rs  rj   r   r   rö   rk   r3   r¥   rO   r¦   rt  ru  Nr6  rú   ú	fp16+fp32é   râ   ©r…   rv  ÚWARPQrw  ©r…   rv  r˜  rw  ÚWARPKrh   )r½   r•  zpv_accum_dtype is z, smooth_v will be ignored.Fr½   Úfp16r»   zUnsupported pv_accum_dtype: .r  ) rÓ   ri   ÚSM80_ENABLEDrx  r$   r¯   r°   rP   rb   r9  ry  rz  r{  r|   r{   r   r|  r}  r+  r'  r}   Úper_warp_int8_cudaÚper_thread_int8_tritonry   ÚwarningsÚwarnr   r   r   Ú)qk_int8_sv_f16_accum_f16_fuse_v_mean_attnÚqk_int8_sv_f16_accum_f16_attnÚ&qk_int8_sv_f16_accum_f16_attn_inst_buf) r±   r…   rµ   rø   r¶   r¾   rn  rù   rˆ   r¹   r€   r�   r‚   ri   Ú_tensor_layoutÚ	_is_caualÚ_qk_quant_granÚ_return_lser�  rƒ  r/  r„  rò   rƒ   r0  r„   Úq_sizeÚq_devicer‰   r¡   Ú
smoothed_vÚvms                                    r,   rÊ   rÊ   {  sg  € ðt �E€A€aˆØ‡N‚NÑÔÐØŒG€EÝÐrÐrÐrÑrÔrˆ<ØŒ9Ð6Ð6Ð6Ñ6Ô6ˆ9Ø•U”]¥E¤NÐ3Ð3Ð3Ð3Ð5xÑ3Ô3Ð3ØÐ6Ð6Ð6Ð6Ð8rÑ6Ô6Ð6ØŒ8�q”xÐ+Ð+Ò+Ð+ 1¤8Ò+Ð+Ð+Ð+Ð+Ð-VÑ+Ô+Ð+ØŒ7�a”gÐ(Ð(Ò(Ð( ¤Ò(Ð(Ð(Ð(Ð(Ð*QÑ(Ô(Ð(õ �a”hÑÔÐà'¨5Ò0Ð0�Q�Q°a€NØÐ%�� A€IØ'¨<Ò7Ð7�Q�Q¸Q€NØ!Ð(�!�! q€Kà—&’&˜‘*”*€Kà�RÒÐÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆˆØ	�rÒ	Ð	˜k¨CÒ/Ð/ÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆˆØ	�sÒ	Ð	ÝÐ?°+Ð?Ð?Ñ@Ô@Ð@ð �8Š8�B‰<Œ<˜1ÒÐ §¢¨"¡¤°Ò!2Ð!2°q·x²xÀ±|´|ÀqÒ7HÐ7HÐ7HÐJoÑ7HÔ7HÐHàÐØ Ñ$ˆà! QÒ&Ð&ˆaˆa¨A€Gàð Ý˜˜7Ñ#Ô#ˆØð 	cØ Ò%Ð%Ý!&¤¨a¯kªk¸!¸QÑ.?Ô.?ÀÇÂÈaÐQRÑASÔAS×A]ÒA]Ð^_ÐabÑAcÔAcÑ!dÔ!d×!lÒ!lÐmoÑ!pÔ!p×!sÒ!sÕtyô  uBñ  "Cô  "C��å!&¤¨a°·²¸aÀÑ1CÔ1CÑ!DÔ!D×!LÒ!LÈRÑ!PÔ!P×!SÒ!SÕTYÔTaÑ!bÔ!b�øàˆà˜
Ò"Ð"Ý+=¸aÀÀBÐVcÐjmÐ}~÷  ~Dò  ~Dð  EGñ  ~Hô  ~Hð  LOò  ~Oð  ~Oð  Tbð  fqò  Tqð  TqÐvxÐvxð  xzð  BDð  ,Eñ  ,Eô  ,EÑ(ˆ�˜  Ø	˜,Ò	&Ð	&Ý+AÀ!ÀQÈÐZgÐnqð  BC÷  BHò  BHð  IKñ  BLô  BLð  PSò  BSð  BSð  Xfð  juò  Xuð  XuÐz|Ðz|ð  |~ð  FHð  PRð  ,Sñ  ,Sô  ,SÑ(ˆ�˜ à�VŠV‰XŒX€FØŒx€HØ	ˆ!ˆRÝŒ�F %°Ð9Ñ9Ô9€AàÐ.Ð.Ð.°8Ð.ÝŒÐV¨>ÐVÐVÐVÑWÔWÐWØˆà˜ÒÐØ�DŠD•”ÑÔˆÝÔ7¸ÀÈÈ1ÈgÐW^Ð`nÐpyð  |Jð  LTð  Vañ  bô  bˆˆØ	˜6Ò	!Ð	!Øð 	fÝ% a°}ÐEÑEÔE‰NˆJ˜ØÝÔGÈÐPVÐXbÐdeÐgnÐpwÐy{ð  ~Lð  NWð  Ygð  iqð  s~ñ  ô  ˆCˆCà—’•U”]Ñ#Ô#ˆAÝÔ;¸FÀFÈAÈqÐRYÐ[bÐdrÐt}ð  @Nð  PXð  Zeñ  fô  fˆCˆCØ	˜;Ò	&Ð	&Ø�DŠD•”ÑÔˆÝÔ@ÀÈÐQRÐTUÐW^Ð`gÐiwð  zCð  ESð  U]ð  _jñ  kô  kˆˆåÐH¸ÐHÐHÑIÔIÐIà	ˆ#ˆ|�ˆ|Ð
Ô€Aàð ØÀ(Ð`�#˜
Ñ" ^°hÑ%>Ñ>Ð>ÐPSÐV`ÑP`Ð`Ð`àˆr.   c
                 ó¼
  — |d k    rt           rdnd}	 | \  }}}|                      ¦   «          |j        }t          s
J d¦   «         ‚|j        s
J d¦   «         ‚|t
          j        t
          j        fv s
J d¦   «         ‚|dv s
J d¦   «         ‚|j        |j        cxk    r|j        k    sn J d¦   «         ‚|j        |j        cxk    r|j        k    sn J d	¦   «         ‚t          |j        ¦  «         |d
k    rdnd}|rdnd}|dk    rdnd}|rdnd}| 
                    d¦  «        }|dk     rt
          j        j                             |dd|z
  f¦  «        }t
          j        j                             |dd|z
  f¦  «        }t
          j        j                             |dd|z
  f¦  «        }n£|dk    r…|dk     rt
          j        j                             |dd|z
  f¦  «        }t
          j        j                             |dd|z
  f¦  «        }t
          j        j                             |dd|z
  f¦  «        }n|dk    rt          d|› �¦  «        ‚|                     d¦  «        dk    r2|                     d¦  «        dk    r|                     d¦  «        dk    s
J d¦   «         ‚|€|dz  }|dk    rdnd}|dk    r|rt!          j        d¦  «         d}|g}~t           r@|dk    r|rt!          j        d¦  «         d}d}|dk    rd}t%          ||||¬¦  «        \  }}}nt%          |||¬¦  «        \  }}}|rôt'          ||¦  «        }|rá|d
k    r‚t          j        |                     dd¦  «        |                     dd¦  «                             dd¦  «        ¦  «                             d¦  «                             t
          j        ¦  «        }n\t          j        ||                     dd¦  «        ¦  «                             d¦  «                             t
          j        ¦  «        }nd }|dk    rt3          ||||ddd¬¦  «        \  }}}}n"|dk    rt5          ||||dddd¬ ¦  «        \  }}}}| 
                    ¦   «         } |j        }!|	r~~|}"n~~~t          j        | ||!¬!¦  «        }"|d"k    rC|r!t9          j        ||||"|||||||||¦  «        }#nkt9          j        ||||"||||||||¦  «        }#nK|dk    r t9          j        ||||"||||||||¦  «        }#n%|dk    rt9          j         ||||"||||||||¦  «        }#|"d#d |…f         }"|r|"|r|#d$z  ||z  z   n|#d$z  fS |"S )%Nr÷   rÄ   úMSM89 kernel is not available. Make sure you GPUs with compute capability 8.9.rp  rq  r“  r”  rr  rs  rj   r   r   rö   rk   r3   r¥   rO   r¦   rt  ru  r6  ú8pv_accum_dtype is 'fp32+fp32', smooth_v will be ignored.Fz8pv_accum_dtype is 'fp32+fp16', smooth_v will be ignored.r  r  r  ©r…   rù   rú   râ   r—  r™  rh   r½   .r  )!rû   rÓ   ri   r  rx  r$   r¯   r°   rP   rb   r9  ry  rz  r{  r|   r{   rŸ  r   r   r   r|  r}  r+  r'  r}   r�  rž  ry   r   rA  rB  r   r@  )$r±   r…   rµ   rø   r¶   r¾   rn  rù   rˆ   r·   r¹   r€   r�   r‚   ri   r¤  r¥  r¦  r§  r�  rƒ  r!  Úquant_v_scale_maxÚv_fp8r<  r«  r/  r„  rò   rƒ   r0  r„   r¨  r©  r‰   r¡   s$                                       r,   rË   rË   1  s8  € ð ˜ÒÐÝ(-Ð>˜˜°;ˆðHðR �G€A€qˆ!Ø‡N‚NÑÔÐàŒG€EÝÐhÐhÐhÑhÔhˆ<ØŒ9Ð6Ð6Ð6Ñ6Ô6ˆ9Ø•U”]¥E¤NÐ3Ð3Ð3Ð3Ð5xÑ3Ô3Ð3ØÐ6Ð6Ð6Ð6Ð8rÑ6Ô6Ð6ØŒ8�q”xÐ+Ð+Ò+Ð+ 1¤8Ò+Ð+Ð+Ð+Ð+Ð-VÑ+Ô+Ð+ØŒ7�a”gÐ(Ð(Ò(Ð( ¤Ò(Ð(Ð(Ð(Ð(Ð*QÑ(Ô(Ð(õ �a”hÑÔÐà'¨5Ò0Ð0�Q�Q°a€NØÐ%�� A€IØ'¨<Ò7Ð7�Q�Q¸Q€NØ!Ð(�!�! q€Kà—&’&˜‘*”*€Kà�RÒÐÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆˆØ	�rÒ	Ð	˜k¨CÒ/Ð/ÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆˆØ	�sÒ	Ð	ÝÐ?°+Ð?Ð?Ñ@Ô@Ð@ð �8Š8�B‰<Œ<˜1ÒÐ §¢¨"¡¤°Ò!2Ð!2°q·x²xÀ±|´|ÀqÒ7HÐ7HÐ7HÐJoÑ7HÔ7HÐHàÐØ Ñ$ˆØ! QÒ&Ð&ˆaˆa¨A€Gà˜Ò$Ð$¨Ð$ÝŒÐPÑQÔQÐQØˆð ˆS€FØ	Ýð 
eØ˜[Ò(Ð(¨XÐ(ÝŒMÐTÑUÔUÐUØˆHà!ÐØ˜[Ò(Ð(Ø $ÐÝ,¨VÀ=Ð\mð  yAð  Bñ  Bô  BÑˆˆw˜˜å,¨VÀ=Ð[cÐdÑdÔdÑˆˆw˜ð ð Ý˜˜7Ñ#Ô#ˆØð 	cØ Ò%Ð%Ý!&¤¨a¯kªk¸!¸QÑ.?Ô.?ÀÇÂÈaÐQRÑASÔAS×A]ÒA]Ð^_ÐabÑAcÔAcÑ!dÔ!d×!lÒ!lÐmoÑ!pÔ!p×!sÒ!sÕtyô  uBñ  "Cô  "C��å!&¤¨a°·²¸aÀÑ1CÔ1CÑ!DÔ!D×!LÒ!LÈRÑ!PÔ!P×!SÒ!SÕTYÔTaÑ!bÔ!b�øàˆà˜
Ò"Ð"Ý+=¸aÀÀBÐVcÐjmÐuwð  Að  ,Bñ  ,Bô  ,BÑ(ˆ�˜  Ø	˜,Ò	&Ð	&Ý+AÀ!ÀQÈÐZgÐnqÐy{ð  CEð  MOð  ,Pñ  ,Pô  ,PÑ(ˆ�˜ Ø�VŠV‰XŒX€FØŒx€HØð >ØˆbØˆˆàˆa�ÝŒK˜ e°HÐ=Ñ=Ô=ˆØ˜ÒÐØð 	ÝÔSÐTZÐ\bÐdiÐklÐnuÐw~ð  AHð  JLð  N\ð  ^gð  iwð  yAð  CNñ  Oô  OˆCˆCåÔGÈÐPVÐX]Ð_`ÐbiÐkrÐt{ð  ~Lð  NWð  Ygð  iqð  s~ñ  ô  ˆCˆCØ	˜;Ò	&Ð	&ÝÔLÈVÐU[Ð]bÐdeÐgnÐpwð  zAð  CQð  S\ð  ^lð  nvð  xCñ  Dô  DˆˆØ	˜;Ò	&Ð	&ÝÔLÈVÐU[Ð]bÐdeÐgnÐpwð  zAð  CQð  S\ð  ^lð  nvð  xCñ  Dô  Dˆð 	
ˆ#ˆ|�ˆ|Ð
Ô€Aàð ØÀ(Ð`�#˜
Ñ" ^°hÑ%>Ñ>Ð>ÐPSÐV`ÑP`Ð`Ð`àˆr.   rÄ   r¥   c                 óÀ	  — | \  }}}|                       ¦   «          |j        }t          s
J d¦   «         ‚|j        s
J d¦   «         ‚|t          j        t          j        fv s
J d¦   «         ‚|dv s
J d¦   «         ‚|j        |j        cxk    r|j        k    sn J d¦   «         ‚|j        |j        cxk    r|j        k    sn J d¦   «         ‚t          |j        ¦  «         |dk    rd	nd
}|rd
nd	}|dk    rdnd}|rd
nd	}| 	                    d¦  «        }|dk     rt          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }n£|dk    r…|dk     rt          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }n|dk    rt          d|› �¦  «        ‚|                     d¦  «        d
k    r2|                     d¦  «        d
k    r|                     d¦  «        d
k    s
J d¦   «         ‚|€|dz  }|d	k    rd
nd}|dk    r|rt          j        d¦  «         d}|g}~t#          |||¬¦  «        \  }}}|rôt%          ||¦  «        }|rá|dk    r‚t	          j        |                     d
d¦  «        |                     d
d¦  «                             dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }n\t	          j        ||                     dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }nd}|dk    rt1          ||||ddd¬¦  «        \  }}}}n"|dk    rt3          ||||dddd¬¦  «        \  }}}}| 	                    ¦   «         } |j        }!|
r~|}"n~~t	          j        | ||!¬¦  «        }"|dk    rE|r"t7          j        ||||"||||||||||	¦  «        }#nGt7          j        ||||"|||||||||	¦  «        }#n&|dk    r t7          j        ||||"|||||||||	¦  «        }#|"dd|…f         }"|r|"|r|#d z  ||z  z   n|#d z  fS |"S )!a‹  
    SageAttention with INT8 quantization for Q and K, FP8 PV with FP32 accumulation, implemented using CUDA.

    Parameters
    ----------
    q : torch.Tensor
        The query tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    k : torch.Tensor
        The key tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    v : torch.Tensor
        The value tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    tensor_layout : str
        The tensor layout, either "HND" or "NHD".
        Default: "HND".

    is_causal : bool
        Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len.
        Default: False.

    qk_quant_gran : str
        The granularity of quantization for Q and K, either "per_warp" or "per_thread".
        Default: "per_thread".

    sm_scale : Optional[float]
        The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``.

    pv_accum_dtype : str
        The dtype of the accumulation of the product of the value tensor and the attention weights, either "fp32" or "fp32+fp32".
        - "fp32": PV accumulation is done in fully in FP32. However, due to the hardware issue, there are only 22 valid bits in the FP32 accumulator.
        - "fp32+fp32": PV accumulation is done in FP32 (actually FP22), but added to a FP32 buffer every few iterations. This offers a balance between speed and accuracy.
        Default: "fp32+fp32".
        
    smooth_k : bool
        Whether to smooth the key tensor by subtracting the mean along the sequence dimension.
        Default: True.
    
    smooth_v : bool
        Whether to smooth the value tensor by subtracting the mean along the sequence dimension.
        smooth_v will be ignored if pv_accum_dtype is "fp32+fp32".
        Default: False.

    return_lse : bool
        Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention.
        Default: False.

    Returns
    -------
    torch.Tensor
        The output tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

            torch.Tensor
        The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor).
        Shape: ``[batch_size, num_qo_heads, qo_len]``.
        Only returned if `return_lse` is True.

    Note
    ----
    - ``num_qo_heads`` must be divisible by ``num_kv_heads``. 
    - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16``
    - All tensors must be on the same cuda device.
    - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances.
    r­  rp  rq  r“  r”  rr  rs  rj   r   r   rö   rk   r3   r¥   rO   r¦   rt  ru  Nr6  rÄ   r®  Fr¯  rú   râ   r—  r™  rh   r½   .r  )rÓ   ri   r  rx  r$   r¯   r°   rP   rb   r9  ry  rz  r{  r|   r{   rŸ  r   r   r   r|  r}  r+  r'  r}   r�  rž  ry   r   rA  rB  r   )$r±   r…   rµ   rø   r¶   r¾   rn  rù   rˆ   Úwindowr·   r¹   r€   r�   r‚   ri   r¤  r¥  r¦  r§  r�  rƒ  r!  r±  r<  r«  r/  r„  rò   rƒ   r0  r„   r¨  r©  r‰   r¡   s$                                       r,   Ú#sageattn_qk_int8_pv_fp8_window_cudar´  ú  sR  € ðv �E€A€aˆØ‡N‚NÑÔÐØŒG€EÝÐhÐhÐhÑhÔhˆ<ØŒ9Ð6Ð6Ð6Ñ6Ô6ˆ9Ø•U”]¥E¤NÐ3Ð3Ð3Ð3Ð5xÑ3Ô3Ð3ØÐ6Ð6Ð6Ð6Ð8rÑ6Ô6Ð6ØŒ8�q”xÐ+Ð+Ò+Ð+ 1¤8Ò+Ð+Ð+Ð+Ð+Ð-VÑ+Ô+Ð+ØŒ7�a”gÐ(Ð(Ò(Ð( ¤Ò(Ð(Ð(Ð(Ð(Ð*QÑ(Ô(Ð(õ �a”hÑÔÐà'¨5Ò0Ð0�Q�Q°a€NØÐ%�� A€IØ'¨<Ò7Ð7�Q�Q¸Q€NØ!Ð(�!�! q€Kà—&’&˜‘*”*€Kà�RÒÐÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆˆØ	�rÒ	Ð	˜k¨CÒ/Ð/ÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆˆØ	�sÒ	Ð	ÝÐ?°+Ð?Ð?Ñ@Ô@Ð@ð �8Š8�B‰<Œ<˜1ÒÐ §¢¨"¡¤°Ò!2Ð!2°q·x²xÀ±|´|ÀqÒ7HÐ7HÐ7HÐJoÑ7HÔ7HÐHàÐØ Ñ$ˆà! QÒ&Ð&ˆaˆa¨A€Gà˜Ò$Ð$¨Ð$ÝŒÐPÑQÔQÐQØˆàˆS€FØ	Ý(¨¸}ÐW_Ð`Ñ`Ô`Ñ€Eˆ7�Bàð Ý˜˜7Ñ#Ô#ˆØð 	cØ Ò%Ð%Ý!&¤¨a¯kªk¸!¸QÑ.?Ô.?ÀÇÂÈaÐQRÑASÔAS×A]ÒA]Ð^_ÐabÑAcÔAcÑ!dÔ!d×!lÒ!lÐmoÑ!pÔ!p×!sÒ!sÕtyô  uBñ  "Cô  "C��å!&¤¨a°·²¸aÀÑ1CÔ1CÑ!DÔ!D×!LÒ!LÈRÑ!PÔ!P×!SÒ!SÕTYÔTaÑ!bÔ!b�øàˆà˜
Ò"Ð"Ý+=¸aÀÀBÐVcÐjmÐuwð  Að  ,Bñ  ,Bô  ,BÑ(ˆ�˜  Ø	˜,Ò	&Ð	&Ý+AÀ!ÀQÈÐZgÐnqÐy{ð  CEð  MOð  ,Pñ  ,Pô  ,PÑ(ˆ�˜ à�VŠV‰XŒX€FØŒx€HØð >ØØˆˆàˆaÝŒK˜ e°HÐ=Ñ=Ô=ˆà˜ÒÐØð 	HÝÔSÐTZÐ\bÐdiÐklÐnuÐw~ð  AHð  JLð  N\ð  ^gð  iwð  yAð  CNð  PVñ  Wô  WˆCˆCåÔGÈÐPVÐX]Ð_`ÐbiÐkrÐt{ð  ~Lð  NWð  Ygð  iqð  s~ð  AGñ  Hô  HˆCˆCØ	˜;Ò	&Ð	&ÝÔLÈVÐU[Ð]bÐdeÐgnÐpwð  zAð  CQð  S\ð  ^lð  nvð  xCð  FLñ  Mô  Mˆà	ˆ#ˆ|�ˆ|Ð
Ô€Aàð ØÀ(Ð`�#˜
Ñ" ^°hÑ%>Ñ>Ð>ÐPSÐV`ÑP`Ð`Ð`àˆr.   c	                 óL  — | \  }
}}|                       ¦   «          |
j        }t          s
J d¦   «         ‚|
j        s
J d¦   «         ‚|t          j        t          j        fv s
J d¦   «         ‚|dv s
J d¦   «         ‚|
j        |j        cxk    r|j        k    sn J d¦   «         ‚|
j        |j        cxk    r|j        k    sn J d¦   «         ‚t          |j        ¦  «         |dk    rd	nd
}|rd
nd	}|dk    rdnd}|rd
nd	}|
 	                    d¦  «        }|dk     rt          j
        j                             |
d	d|z
  f¦  «        }
t          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }n£|dk    r…|dk     rt          j
        j                             |
d	d|z
  f¦  «        }
t          j
        j                             |d	d|z
  f¦  «        }t          j
        j                             |d	d|z
  f¦  «        }n|dk    rt          d|› �¦  «        ‚|
                     d¦  «        d
k    r2|                     d¦  «        d
k    r|                     d¦  «        d
k    s
J d¦   «         ‚|€|dz  }|d	k    rd
nd}| 	                    |¦  «        }|dz  d	k    rd|dz  z
  nd	}|d	k    rí|dk    rtt	          j        |t	          j        | 	                    d	¦  «        | 	                    d
¦  «        || 	                    d¦  «        |j        |j        ¬¦  «        gd¬¦  «        }nst	          j        |t	          j        | 	                    d	¦  «        || 	                    d¦  «        | 	                    d¦  «        |j        |j        ¬¦  «        gd
¬¦  «        }|g}~t#          ||d¬¦  «        \  }}}|rôt%          ||¦  «        }|rá|dk    r‚t	          j        |
                     d
d¦  «        |                     d
d¦  «                             dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }n\t	          j        |
|                     dd¦  «        ¦  «                             d¦  «                             t          j        ¦  «        }nd}|dk    rt1          |
|||ddd¬¦  «        \  }}}}n"|dk    rt3          |
|||dddd¬¦  «        \  }}}}|
 	                    ¦   «         } |
j        }!|r~|
}"n~
~t	          j        | ||!¬¦  «        }"|dk    rt7          d¦  «        ‚|d k    rt9          j        ||||"||||||||¦  «        }#|"d!d|…f         }"|r|"|r|#d"z  ||z  z   n|#d"z  fS |"S )#a¶  
    SageAttention with INT8 quantization for Q and K, FP8 PV with FP32 accumulation, implemented using CUDA.

    Parameters
    ----------
    q : torch.Tensor
        The query tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

    k : torch.Tensor
        The key tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    v : torch.Tensor
        The value tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``.

    tensor_layout : str
        The tensor layout, either "HND" or "NHD".
        Default: "HND".

    is_causal : bool
        Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len.
        Default: False.

    qk_quant_gran : str
        The granularity of quantization for Q and K, either "per_warp" or "per_thread".
        Default: "per_thread".

    sm_scale : Optional[float]
        The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``.

    pv_accum_dtype : str
        The dtype of the accumulation of the product of the value tensor and the attention weights, either "fp32" or "fp32+fp32".
        - "fp32": PV accumulation is done in fully in FP32. However, due to the hardware issue, there are only 22 valid bits in the FP32 accumulator.
        - "fp32+fp32": PV accumulation is done in FP32 (actually FP22), but added to a FP32 buffer every few iterations. This offers a balance between speed and accuracy.
        Default: "fp32+fp32".
        
    smooth_k : bool
        Whether to smooth the key tensor by subtracting the mean along the sequence dimension.
        Default: True.

    return_lse : bool
        Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention.
        Default: False.

    Returns
    -------
    torch.Tensor
        The output tensor. Shape:
        - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``.
        - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``.

            torch.Tensor
        The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor).
        Shape: ``[batch_size, num_qo_heads, qo_len]``.
        Only returned if `return_lse` is True.

    Note
    ----
    - ``num_qo_heads`` must be divisible by ``num_kv_heads``. 
    - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16``
    - All tensors must be on the same cuda device.
    - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances.
    zMSM90 kernel is not available. Make sure you GPUs with compute capability 9.0.rp  rq  r“  r”  rr  rs  rj   r   r   rö   rk   r3   r¥   rO   r¦   rt  ru  Nr6  rf   rh   )ÚdimFr¯  rú   r–  r—  r™  r½   z/Please use pv_accum_dtype='fp32+fp32' for sm90.rÄ   .r  )rÓ   ri   ÚSM90_ENABLEDrx  r$   r¯   r°   rP   rb   r9  ry  rz  r{  r|   r{   Úcatrå   r   r   r|  r}  r+  r'  r}   r�  rž  ry   ÚNotImplementedErrorr   rB  r   )$r±   r…   rµ   rø   r¶   r¾   rn  rˆ   r·   r¹   r€   r�   r‚   ri   r¤  r¥  r¦  r§  r�  rƒ  r�   Ú	v_pad_lenr!  r±  r<  rŽ   r/  r„  rò   rƒ   r0  r„   r¨  r©  r‰   r¡   s$                                       r,   rÍ   rÍ   °  sm  € ðh �E€A€aˆØ‡N‚NÑÔÐØŒG€EÝÐhÐhÐhÑhÔhˆ<ØŒ9Ð6Ð6Ð6Ñ6Ô6ˆ9Ø•U”]¥E¤NÐ3Ð3Ð3Ð3Ð5xÑ3Ô3Ð3ØÐ6Ð6Ð6Ð6Ð8rÑ6Ô6Ð6ØŒ8�q”xÐ+Ð+Ò+Ð+ 1¤8Ò+Ð+Ð+Ð+Ð+Ð-VÑ+Ô+Ð+ØŒ7�a”gÐ(Ð(Ò(Ð( ¤Ò(Ð(Ð(Ð(Ð(Ð*QÑ(Ô(Ð(å�a”hÑÔÐà'¨5Ò0Ð0�Q�Q°a€NØÐ%�� A€IØ'¨<Ò7Ð7�Q�Q¸Q€NØ!Ð(�!�! q€Kà—&’&˜‘*”*€Kà�RÒÐÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆÝŒHÔ×#Ò# A¨¨2°Ñ+;Ð'<Ñ=Ô=ˆˆØ	�rÒ	Ð	˜k¨CÒ/Ð/ÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆÝŒHÔ×#Ò# A¨¨3°Ñ+<Ð'=Ñ>Ô>ˆˆØ	�sÒ	Ð	ÝÐ?°+Ð?Ð?Ñ@Ô@Ð@ð �8Š8�B‰<Œ<˜1ÒÐ §¢¨"¡¤°Ò!2Ð!2°q·x²xÀ±|´|ÀqÒ7HÐ7HÐ7HÐJoÑ7HÔ7HÐHàÐØ Ñ$ˆà! QÒ&Ð&ˆaˆa¨A€Gð
 �VŠV�G‰_Œ_€FØ(.°©¸Ò(9Ð(9��v ‘|Ñ$Ð$¸q€IØ�1‚}€}Ø˜EÒ!Ð!Ý”	˜1�eœk¨!¯&ª&°©)¬)°Q·V²V¸A±Y´YÀ	È1Ï6Ê6ÐRSÉ9Ì9Ð\]Ô\cÐlmÔltÐuÑuÔuÐvÐ|}Ð~Ñ~Ô~ˆAˆAå”	˜1�eœk¨!¯&ª&°©)¬)°YÀÇÂÀqÁ	Ä	È1Ï6Ê6ÐRSÉ9Ì9Ð\]Ô\cÐlmÔltÐuÑuÔuÐvÐ|}Ð~Ñ~Ô~ˆAàˆS€FØ	Ý'¨¸mÐV[Ð\Ñ\Ô\Ñ€Eˆ7�Aàð Ý˜˜7Ñ#Ô#ˆØð 	cØ Ò%Ð%Ý!&¤¨a¯kªk¸!¸QÑ.?Ô.?ÀÇÂÈaÐQRÑASÔAS×A]ÒA]Ð^_ÐabÑAcÔAcÑ!dÔ!d×!lÒ!lÐmoÑ!pÔ!p×!sÒ!sÕtyô  uBñ  "Cô  "C��å!&¤¨a°·²¸aÀÑ1CÔ1CÑ!DÔ!D×!LÒ!LÈRÑ!PÔ!P×!SÒ!SÕTYÔTaÑ!bÔ!b�øàˆà˜
Ò"Ð"Ý+=¸aÀÀBÐVcÐjlÐtvð  ~Að  ,Bñ  ,Bô  ,BÑ(ˆ�˜  Ø	˜,Ò	&Ð	&Ý+AÀ!ÀQÈÐZgÐnpÐxzð  BEð  MPð  ,Qñ  ,Qô  ,QÑ(ˆ�˜ à�VŠV‰XŒX€FØŒx€HØð >ØØˆˆàˆaÝŒK˜ e°HÐ=Ñ=Ô=ˆà˜ÒÐÝ!Ð"SÑTÔTÐTà	˜;Ò	&Ð	&ÝÔLÈVÐU[Ð]bÐdeÐgnÐpwð  zAð  CQð  S\ð  ^lð  nvð  xCñ  Dô  Dˆà	ˆ#ˆ|�ˆ|Ð
Ô€Aàð ØÀ(Ð`�#˜
Ñ" ^°hÑ%>Ñ>Ð>ÐPSÐV`ÑP`Ð`Ð`àˆr.   r  Ú	v_or_listr   c           	      ó
  — |dk    rdnd}t          | t          ¦  «        r| d         }|                      ¦   «          n| }|j        }|dk    r8|j        \  }}}	}
|	dz   dz  dz  }t          j        |||
|f|j        |¬¦  «        }n=|dk    r7|j        \  }}	}}
|	dz   dz  dz  }t          j        ||
||f|j        |¬¦  «        }t          j	        |||¦  «         ~t          j        |j        t
          j
        |¬¦  «        }t          j        |||
ft
          j        |¬¦  «        }t          j        |||
ft
          j        |¬¦  «        }|rt          j        |||||	||¦  «         |||fS t          j        ||||	||¦  «         ||d fS )Nrj   r   r   rf   r$  rO   rh   )Ú
isinstanceÚlistrÓ   rP   rz   r$   ry   ri   r   r   Úfloat8_e4m3fnr}   Úmean_scale_fuse_quant_cudaÚscale_fuse_quant_cuda)r»  r…   r   rù   r¤  r‚   rP   rŠ   r�   r�   r�   Ú
padded_lenÚv_transposed_permuttedr±  r<  r«  s                   r,   Úxper_channel_fp8rÄ  a  sÐ  € ð (¨5Ò0Ð0�Q�Q°a€NÝ�)�TÑ"Ô"ð Ø�aŒLˆØ�ŠÑÔÐÐàˆØŒX€FØ˜ÒÐØ$%¤GÑ!ˆˆ4�˜Ø˜r‘k bÑ(¨2Ñ-ˆ
Ý!&¤¨a°°xÀÐ-LÐTUÔT[ÐdjÐ!kÑ!kÔ!kÐÐà	˜%Ò	Ð	Ø$%¤GÑ!ˆˆ6�4˜Ø˜r‘k bÑ(¨2Ñ-ˆ
Ý!&¤¨a°¸4ÀÐ-LÐTUÔT[ÐdjÐ!kÑ!kÔ!kÐå
Ô% aÐ)?ÀÑPÔPÐPØ	ÝŒKÐ.Ô4½EÔ<OÐX^Ð_Ñ_Ô_€EåŒk˜1˜d HÐ-µU´]È6ÐRÑRÔR€GÝ	Œ�a˜˜xÐ(µ´ÀfÐ	MÑ	MÔ	M€Bàð $ÝÔ)Ð*@À%ÈÈWÐV\Ð^gÐiwÑxÔxÐxØ�g˜rÐ!Ð!åÔ$Ð%;¸UÀGÈVÐU^Ð`nÑoÔoÐoØ�g˜tÐ#Ð#r.   c                  óÌ   — t           at          j                             d¦  «        } | �t           | _        t          j                             d¦  «        }|�t           |_        d S d S )Nzsageattention.quantzsageattention.core)rÄ  r   ÚsysÚmodulesr&  )Ú
sage_quantÚ	sage_cores     r,   Ú%_install_per_channel_fp8_monkey_patchrÊ  ‡  sY   € å&€OÝ”—’Ð!6Ñ7Ô7€JØÐÝ%5ˆ
Ô"Ý”—’Ð 4Ñ5Ô5€IØÐÝ$4ˆ	Ô!Ð!Ð!ð Ðr.   )NNNrj   )rf   FNFFF)FFr¦   )NF)rf   rl  FNTFNF)FNT)rf   Frö   Nr½   TFF)	rf   Frö   NNTFFF)
rf   Frö   NrÄ   TFFr¥   F)rf   Frö   NrÄ   TFF)rf   r  T)�rP  rÆ  r$   Úplatformr©   Úbackendsr   r¨   ÚImportErrorÚtorch.nn.functionalry  rz  ÚFÚmmgpr   Úshared.kernels.token_meanr   Úsageattention_patchr   Ú
sage1_corer	   r
   Úshared.kernelsr   r  ÚOSErrorÚRuntimeErrorÚ$sageattention.triton.quant_per_blockr   r~  r   Ú+sageattention.triton.quant_per_block_varlenrŒ  Ú+sageattention.triton.attn_qk_int8_per_blockrl  r~   r   r¬   Ú2sageattention.triton.attn_qk_int8_per_block_causalr€  Ú.sageattention.triton.attn_qk_int8_block_varlenrŽ  Ú9sageattention.triton.attn_qk_int8_per_block_causal_varlenr�  Ú%sageattention.triton.quant_per_threadr   rž  r   r   r0   r   ÚopsÚsageattention_fusedr   Úsageattention_qattn_sm80rœ  r   Úsageattention_qattn_sm89r  r   Úsageattention_qattn_sm90r·  Úsageattention.quantr  r   r�  r   r   Útypingr   r   r   r   r    r!   rŸ  Úosr-   Úimportlib.metadatar/   Úsg2_versionÚ
startswithrû   r4   r7   rª   r@   rE   r9   rN   ÚtuplerX   r%   r&   r`   rw   rx   rd   rv   r]   rP   rU   Ústrr[   r^   rb   rð   re   r¯   r£   ÚTensorr²   r´   r]  rÐ   rÆ   rã   rÈ   rÌ   r  r  r  r  r   Ú	arg_namesr7  rÔ   r  r  ÚcompilerÚdisabler×   rÕ   rØ   rÙ   r8  rW  rM  rT  rX  rV  r[  r^  rc  re  ri  rk  rÉ   r‘  rÊ   rË   r´  rÍ   Ú_sage_per_channel_fp8r¾  rÄ  rÊ  r5  r.   r,   ú<module>rð     s/  ððð ð  €
€
€
Ø €€€à„<�8ÒÐ  ¨¬¸Ñ >Ô >ÐÀ5Ä>ÔCU×CbÒCbÑCdÔCdÐØ
ˆ+ÐWÑ
XÔ
XÐXà Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à 0Ð 0Ð 0Ð 0Ð 0Ð 0à 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,ðØ=Ð=Ð=Ð=Ð=Ð=Ð=øØ�W˜lÐ+ð ð ð Ø€K€K€Kðøøøð Ð Ñ Ô Ð à uÐ uÐ uÐ uÐ uÐ uÐ uÐ uØ fÐ fÐ fÐ fÐ fÐ fØ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ MÐ MÐ MÐ MÐ MÐ MØ SÐ SÐ SÐ SÐ SÐ SØ WÐ WÐ WÐ WÐ WÐ WØ aÐ aÐ aÐ aÐ aÐ aà [Ð [Ð [Ð [Ð [Ð [Ø vÐ vÐ vÐ vÐ vÐ vÐ vÐ vð+Ø$Ð$Ð$Ð$Ð$Ð$Øˆ7�6Ð7Ñ8Ô8ð /Ø”Ô.ˆøøð+ØŒYÔ*€F€F€FøøøðØ)Ð)Ð)Ð)Ð)Ð)Øˆ7�;Ð ?Ñ@Ô@ð 9Ø”iÔ8ˆØ€L€LøðØ€L€L€LøøøðØ)Ð)Ð)Ð)Ð)Ð)Øˆ7�;Ð TÑUÔUð 9Ø”iÔ8ˆØ€L€LøðØ€L€L€LøøøðØ)Ð)Ð)Ð)Ð)Ð)Øˆ7�;Ð TÑUÔUð 9Ø”iÔ8ˆØ€L€LøðØ€L€L€Løøøà EÐ EÐ EÐ EÐ EÐ EØ CÐ CÐ CÐ CÐ CÐ CØ (Ð (Ð (Ð (Ð (Ð (Ø /Ð /Ð /Ð /Ð /Ð /à =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ø €€€Ø 	€	€	€	ðð ð ð 'Ð &Ð &Ð &Ð &Ð &Øˆg�oÑ&Ô&€Ø×Ò˜uÑ%Ô%€à Ð Ð Ð Ø 	€	€	€	Ø €€€ð	ð 	ð 	ðð ð ði sð i¨sð ið ið ið ið
 ˆeÐ*Ð*Ñ,Ô,Ñ-Ô-€Ø”j×-Ò-Ñ/Ô/°1Ò4Ð ØÐ ØÐ à(.Ð %Ø*/Ð 'Ø!#Ð ðQ˜eœlð Q¨sð Qð Qð Qð Qð)˜5œ<ð )¨Cð )ð )ð )ð )ð/ U¤\ð /°cð /ð /ð /ð /ð#˜eœlð #ð #ð #ð #ðT¨%¬,ð T¸4ð Tð Tð Tð Tð LQÐ\`ÐotÔo|ð  JOð *ð *ð *ð *ðZð ÀUÄ\ÐTXÑEXð ÐinÔiuÐx{Ñi{ð  Cñ  jCð ð  [^ð ð  knð  quñ  kuð ð ð ð ð6fð f¨U¬\¸CÑ-?À$Ñ-Fð fÐinÔiuÐx|Ñi|ð fð  UXð fð  eið fð fð fð fð
 ØØ $ØØØðUCð UCàðUCð ðUCð �uŒoð	UCð
 ðUCð ðUCð ðUCð ðUCð UCð UCð UCðp	Lð 	Lð 	Lð €ðð ð ð2tð tð tð )4ÀÈtÐ`eÐfÐfÐ Ø)4ÐA[ÐgmÐzð  Að  AÐ à&1¸cÈVÐ`eÐfÐfÐ Ø-8ÀRÐSYÐfkÐlÐlÐ àÐ"8Ô"BÔ"LÐLÐ ðBð Bð Bð Bð ð ð ðð ð ð „Ôð	lð 	lñ Ôð	lð „Ôðð ñ Ôðð4 „Ôðð ñ Ôðð& „Ôð-ð -ð -ñ Ôð-ð:Að Að Að $Ð'=Ô'GÔ'QÐQÐ Ø€ðð ð ðvð vð vðXð Xð Xð4ð 4ð 4ð*ð *ð *ð+ð +ð +ðCð Cð Cð
5ð 5ð 5ðað að að „Ôð Ø (ØØ $ØØØ(,Øðhð hð
 ðhð ðhð ðhð �uŒoðhð ðhð ðhð ˜œÔ%ðhð ðhð ðhð „\ðhð hð hñ ÔðhðT „Ôð Ø $Øðrð rØ„|ðrà„|ðrð „|ðrð ”,ð	rð
 ”,ðrð ðrð ðrð ðrð �uŒoðrð ðrð ðrð „\ðrð rð rñ Ôðrðh „Ôð ØØ%Ø $Ø ØØØðsð sð
 ðsð ðsð ðsð �uŒoðsð ðsð ðsð ðsð ðsð ðsð „\ðsð sð sñ Ôðsðj „Ôð ØØ%Ø $ØØØØØðEð EàðEð ðEð ð	Eð
 �uŒoðEð ðEð ðEð ðEð ðEð ðEð ðEð „\ðEð Eð Eñ ÔðEðP „Ôð ØØ%Ø $Ø%ØØØØØðsð sð
 ðsð ðsð ðsð �uŒoðsð ðsð ðsð ðsð ðsð ðsð ðsð  „\ð!sð sð sñ Ôðsðj „Ôð ØØ%Ø $Ø%ØØØðkð kð
 ðkð ðkð ðkð �uŒoðkð ðkð ðkð ðkð ðkð ðkð „\ðkð kð kñ ÔðkõZ (Ð ð
 ØØð	#$ð #$Ø�U”\ 4Ð'Ô(ð#$àð#$ð ð#$ð ð	#$ð #$ð #$ð #$ðL5ð 5ð 5ð &Ð %Ñ 'Ô 'Ð 'Ð 'Ð 'sB   Á4A; Á;
BÂBÃC2 Ã2DÄ D' Ä'D-Ä1 E ÅEÅ E= Å=F