oFjdZddlZddlZejdkr:eejdr)ejjr edddl m cm Z ddl mZddlmZddlmcmZddlmZddlmZddlmZddlmZddlmZ dd l m!Z!ee!d s ej"j#Z!n#ej"j#Z!YnxYw dd l m$Z$ee$d s ej"j%Z$d Z&n #dZ&YnxYw ddl m'Z'ee'ds ej"j(Z'd Z)n #dZ)YnxYw ddl m*Z*ee*ds ej"j+Z*d Z,n #dZ,YnxYwddl-mZ.ddl-m/Z0ddl-m1Z1ddl-m2a2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9ddl:Z:ddl;Z;dZZ>e>dZ?e?@dZAddlBZBddlCZCddlDZDdZEdZFdeGdeGfdZHeIeFZJejKLdkZMd ZNd ZOd!ZPdaQiZRd"ejSdeGfd#ZTd"ejSdeUfd$ZVd"ejSdeGfd%ZWd"ejSfd&ZXd"ejSdeYfd'ZZd(dej[dfd)Z\dQd+ej]dzd"ejSeUzdzd,eUdeUdzfd-Z^dQd"ejSeUzdzd+ej]dzd,eUdeYfd.Z_ dRd,eUd/eYd0e7e`d1eYd2eYd3e4f d4Zaejbjc dSd,eUd6eUd/eYd0e7e`d7eYd1eYd+e7ej]d3e4dej]fd8Zdejbjc dTd9ej]d:ej]d;ej]deGd?eGd/eYd0e7e`d7eYd3e4dej]fd@Zeejbjc dUd,eUd/eYdCeUd0e7e`dDeUd7eYdEeYd1eYd3e4dej]fdFZfejbjc dVd,eUd/eYdCeUd0e7e`dDeUd7eYdEeYd1eYd2eYd3e4dej]fdGZgejbjc dWd,eUd/eYdCeUd0e7e`dDeUd7eYdEeYd1eYd2eYd3e4dej]fdJZhejbjc dXd,eUd/eYdCeUd0e7e`dDeUd7eYd1eYd2eYd3e4dej]fdKZitdZj dYdMe9ej]ekfd,eUdNe`dEeYfdOZldPZmemdS)Za8 Copyright (c) 2024 by SageAttention team. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Ndarwinmpsz?SageAttention is CUDA-only and is disabled on Apple Silicon MPS)per_block_int8)forward)per_thread_int8)_fusedtranspose_pad_permute_cuda) _qattn_sm80qk_int8_sv_f16_accum_f32_attnTF) _qattn_sm892qk_int8_sv_f8_accum_f32_fuse_v_scale_attn_inst_buf) _qattn_sm90) per_warp_int8)sub_mean)per_channel_fp8)AnyListLiteralOptionalTupleUnionctj}t|D]-}tj|\}}|dkrdS.dS)NFT)torchcuda device_countrangeget_device_capability)rimajorminors *C:\Users\johnn\Wan2GP\shared\sage2_core.pyis_sage2_supportedr#Ks_:**,,L < z77:: u 19955  4)version sageattentionz2.2cf tjddg}tjd|}|rHt |dt |d}}||fSn'#t$r}td|Yd}~nd}~wwxYwdS)Nnvccz --versionzrelease (\d+)\.(\d+)zFailed to get CUDA version:)NN) subprocess check_outputdecoderesearchintgroup Exceptionprint)outputmatchr r!es r"get_cuda_versionr7Zs0(&+)>??FFHH 16::  u{{1~~..EKKNN0C0C5E%<   000 +Q////////0 :sBB B.B))B.cg}ttjD]>}tj|\}}|d||?|S)Nsm)rrrrrappend) cuda_archsrr r!s r"get_cuda_arch_versionsr<espJ 5:**,, - -//z77:: u-u-e--.... r$indexreturnc tj|}t|dt|ddS)Nshared_memory_per_block_optinshared_memory_per_blockr)rrget_device_propertiesgetattr)r=propss r"_device_shared_memory_limitrEms8 J , ,U 3 3E 5975Jcef;g;g h hhr$r)@ihdevicecZ|jtjn|jSN)r=rrcurrent_devicerGs r"_get_device_indexrL|s$*0,*>5: $ $ & & &FLPr$ct|}|ttkr t|St|SrI)rLlen _CUDA_ARCHSr<rGidxs r"_get_cuda_archrRs> F # #C S  3 ! # #C ((r$czt|}|tvrt|t|<t|SrI)rL_SHARED_MEMORY_LIMIT_BY_DEVICErErPs r"_get_shared_memory_limitrUs9 F # #C 000.I#.N.N&s+ )# ..r$ctrdSt|}|tjkr!tj|dSdSrI)_SINGLE_CUDA_DEVICErLrrrJ set_devicerPs r"_maybe_set_devicerYsY F # #C ej'')))) c"""""*)r$c2t|tkSrI)rU%_UPSTREAM_MASKED_HEAD128_SHARED_BYTESrKs r"_use_low_shared_masked_tritonr\s #F + +.S SSr$HNDc dts7tdtdtdt |jddat j|j||j} |dkr|j\} } } } |j\}}}}| d| d | d }}}| d| d | d }}}| d| d | d }}}| d| d | d }}}n/|d kr|j\} } } } |j\}}}}| d| d | d }}}| d| d | d }}}| d| d | d }}}| d| d | d }}}ntd |d | d| d | d | df\}}}} |r)t j| | | gt j |jn!t jdgt j d}!| tzd z tz| | f}"tj |"|||||| ||!|||||||||||||||| | || | |zftt| d |ddd| |!fS)NzE[SageAttention] Using low-shared-memory masked Triton patch (BLOCK_M=z , BLOCK_N=z , GPU limit=z bytes).TdtyperGr]rr)r*NHDztensor_layout z not supportedcpu)BLOCK_MBLOCK_NHEAD_DIMSTAGE RETURN_LSE num_warps num_stages)'_LOW_SHARED_MASKED_TRITON_PATCH_PRINTEDr3_LOW_SHARED_MASKED_BLOCK_M_LOW_SHARED_MASKED_BLOCK_NrUrGremptyshapestride ValueErrorfloat32attn_qk_int8_per_block _attn_fwd)#qkvq_scalek_scale tensor_layout attn_mask output_dtype return_lseobh_qoqo_lenhead_dim_h_kvkv_len stride_bz_q stride_h_q stride_seq_q stride_bz_k stride_h_k stride_seq_k stride_bz_v stride_h_v stride_seq_v stride_bz_o stride_h_o stride_seq_ostride_bz_mask stride_h_mask stride_m_mask stride_n_masklsegrids# r"_attn_false_low_shared_maskedrs 27 QVpQQ}WQQe}~~FeGeGQQQ R R R26/ AG<AAAA$%G!4W401 QXXa[[!((ST++Z 01 QXXa[[!((ST++Z 01 QXXa[[!((ST++Z 01 QXXa[[!((ST++Z %  $%G!64W6401 QXXa[[!((ST++Z 01 QXXa[[!((ST++Z 01 QXXa[[!((ST++Z 01 QXXa[[!((ST++Z G-GGGHHHBKBRBRSTBUBUW`WgWghiWjWjlul|l|}~llBKBRBRSTBUBUCU?NM=-R\ V%+q$'u}QX N N N Nbgbmopnqy~zGPUcVcVcVC 0 01 49S SUY[\ ]D$T* 1a'1iZZZZ }m ddl +4NYaJ     c6Mr$rar|r{cz| |dj}tjsdStj|dn|} tjt |\}}|dkrdt |dSttdsdSd tj tj vrd Sn#ttf$rYd SwxYw|dS|d}|jtjtjfvr d |jd S|jddkrd|jdd SdS)NrzCUDA is unavailablerrzCUDA architecture z! has no masked SageAttention pathruz*SageAttention Triton kernel is unavailabler|z1installed SageAttention does not expose attn_maskz6unable to inspect installed SageAttention mask supportzdtype z is unsupportedz head_dim )rGrr is_availablerrLrRhasattrrtinspect signature attn_false parameters TypeErrorrrr`float16bfloat16rp)qkv_listr|rGr{r rrvs r"&sageattn_attention_mask_support_reasonrsg!# : " " $ $%$$ \FN&& ? ?F H:334Ef4M4MNNq 199av(>(>aaa a-{;; @?? g/ ;;F F FFF G z "HHHGGGHt Awu}en55500000wr{S7172;7777 4s AC C ( C C C c*t||||duSrI)r)rGrr|r{s r" sageattn_supports_attention_maskrs 1(Iv} ] ]ae eer$ is_causalsm_scaler~ recycle_qkwargsc P|dd}t|dj}|)t|||} | t d|d| |t ||||||S|dkrt |||||d S|d krt ||||| S|d krt|||||trdnd|S|dkrt|||||d|S|dkr't|||d||trdnd t | St d|)a4 Automatically selects the appropriate implementation of the SageAttention kernel based on the GPU compute capability. Parameters ---------- q : torch.Tensor The query tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. k : torch.Tensor The key tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. v : torch.Tensor The value tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. tensor_layout : str The tensor layout, either "HND" or "NHD". Default: "HND". is_causal : bool Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len. Default: False. sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``. return_lse : bool Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention. Default: False. Returns ------- torch.Tensor The output tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. torch.Tensor The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). Shape: ``[batch_size, num_qo_heads, qo_len]``. Only returned if `return_lse` is True. Note ---- - ``num_qo_heads`` must be divisible by ``num_kv_heads``. - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16`` - All tensors must be on the same cuda device. r|Nrr{z9Masked SageAttention is unsupported on CUDA architecture z: )r{rrr~r|sm80fp32)r{rrr~pv_accum_dtypesm86)r{rrr~sm89 fp32+fp16 fp32+fp32)r{rrr~rrsm90sm120per_warp)r{r qk_quant_granrr~rsmooth_vrzUnsupported CUDA architecture: ) poprRrGrrrsageattn_qk_int8_pv_fp16_tritonsageattn_qk_int8_pv_fp16_cudasageattn_qk_int8_pv_fp8_cudasg2pp!sageattn_qk_int8_pv_fp8_cuda_sm90) rr{rrr~rrr|archsupport_reasons r"sageattnrsI~ ;--I (1+, - -D?)cpqqq  %qY]qqaoqqrr r.x}`it|JT`ijjj j v~~,X]^grzHRcijjj j .x}`it|JTUUU U +HM]fqyGQqvbGbmbm|GU^___ _ 0bkv~LVgr@IJJJ J +HM]fwALTakLQ}]}H}HW]mrir@IJJJ JA4AABBBr$tritonquantization_backendsmooth_kc  |\} } } || j} | js Jd| tjtjfvs Jd| j| jcxkr | jks nJd| j| jcxkr | jks nJd|6|r Jd|jtjks|j| ks Jdt| j| d} |duot| j}| d krtj j | d d | z f} tj j | d d | z f} tj j | d d | z f} n| d kr| d krtj j | d d | z f} tj j | d d | z f} tj j | d d | z f} n| d krtd | | dd kr2| dd kr| dd ks Jd|dkrd nd}|r| |d}|r|dkrtj| d d|d ddddtj}n\tj| |dddtj}nd}| tjks| tjkr| tj} |d| dzz }|dkrB|r&t-| | |t.t0||\}}}}nLt-| | |||\}}}}n2|dkrt3| | |||\}}}}ntd|~ ~ ~||dkr2|jd |jd|jd |jd fn1|jd |jd |jd|jdf}|j|kr||}|rt9||| |||| |\}}nV|rt;||| |||| || \}}n8|t=||| |||| || \}}nt=||| |||| |\}}|dd| f}|r||r |dz ||zzn|dz fS|S)a5 SageAttention with per-block INT8 quantization for Q and K, FP16 PV with FP16 accumulation, implemented using Triton. The FP16 accumulator is added to a FP32 buffer immediately after each iteration. Parameters ---------- q : torch.Tensor The query tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. k : torch.Tensor The key tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. v : torch.Tensor The value tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. tensor_layout : str The tensor layout, either "HND" or "NHD". Default: "HND". quantization_backend : str The quantization backend, either "triton" or "cuda". "cuda" backend offers better performance due to kernel fusion. is_causal : bool Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len. Default: False. sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``. smooth_k : bool Whether to smooth the key tensor by subtracting the mean along the sequence dimension. Default: True. return_lse : bool Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention. Default: False. Returns ------- torch.Tensor The output tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. torch.Tensor The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). Shape: ``[batch_size, num_qo_heads, qo_len]``. Only returned if `return_lse` is True. Note ---- - ``num_qo_heads`` must be divisible by ``num_kv_heads``. - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16``, ``torch.bfloat16`` or ``torch.float32``. - All tensors must be on the same cuda device. - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances. Input tensors must be on cuda.AInput tensors must be in dtype of torch.float16 or torch.bfloat16'All tensors must be on the same device.%All tensors must have the same dtype.Nz?SageAttention does not support attn_mask with causal attention.z(attn_mask must be bool or match q dtype.rrFrrUnsupported head_dim: r)#Last dim of qkv must be contiguous.rar*Tdimkeepdimrb??r)kmBLKQBLKKrr{)rrr{rz"Unsupported quantization backend: )r{r}r~)r{r}r~r|./ldG?)clearr`is_cudarrrrGboolrYsizer\nn functionalpadrrrqmeanmatmul transposesqueezetorsper_block_int8_tritonrmrnper_block_int8_cudarpexpand attn_truerr)rr{rrrrr~r|rrvrwrxr` head_dim_ogmasked_low_sharedseq_dimrlse_correctionq_int8ryk_int8rz target_shaperrs r"rr9s\GAq! NN GE 9666669 U]EN3 3 3 35x 3 3 3 8qx + + + +18 + + + + +-V + + + 7ag ( ( ( ( ( ( ( ( (*Q ( ( (_____}%*,, 50H0H0HJt0H0HHah&&**K!-Y2OPQPX2Y2YR H  # #A2 +;'< = = H  # #A2 +;'< = = H  # #A2 +;'< = = r  kC// H  # #A3+<'= > > H  # #A3+<'= > > H  # #A3+<'= > > s  ?+??@@@ 88B<<1  "!2!2qxx||q7H7H7HJo7H7H H E))aaqG VVV . .  c%%!&akk!Q.?.?aQRASASA]A]^_abAcAc!d!d!l!lmo!p!p!s!styuB"C"C!&aa1C1C!D!D!L!LR!P!P!S!STYTa!b!b  %5="8"8 DD  +,-x''  B/DQbWqyS^fvC0D0D0D ,FGVWW/DQb[ctA0B0B0B ,FGVWW  ' '+>q!U]mz+{+{+{(T>RTTUUU !R%%\!_fl1ov|A Q P P,q/6<?FLOV\RS_U  ?l * *!((66II661gwmjo}GHHH33 I.vvq'7boDQ[gpqqq33  FFAw}kp~HT]^^^33FFAw}kp~HIII3 #| | A(`# "^h%>>>PSV`P```r$rvrwrx cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_kc \|j} |js Jd| tjtjfvs Jd|j|jcxkr |jks nJd|j|jcxkr |jks nJdt |j|d} | dkrtjj |dd| z f}tjj |dd| z f}tjj |dd| z f}n| dkr| dkrtjj |dd| z f}tjj |dd| z f}tjj |dd| z f}n| dkrtd | | dd kr2| dd kr| dd ks Jd | r| s Jd | tjks| tjkr|tj}| r|dd } || z }|d| dzz }t#|||||||\}}}}}}|rt%|||||||||||  }nt'|||||||||||  }|dd| f}|S)a2 Parameters ---------- q : torch.Tensor The query tensor, shape: ``[cu_seqlens_q[-1], num_qo_heads, head_dim]``. k : torch.Tensor The key tensor, shape: ``[cu_seqlens_k[-1], num_kv_heads, head_dim]``. v : torch.Tensor The value tensor, shape: ``[cu_seqlens_k[-1], num_kv_heads, head_dim]``. cu_seqlens_q : torch.Tensor The cumulative sequence lengths for the query sequences in the batch, used to index into `q`. Shape: ``[batch_size + 1]``, where each entry represents the cumulative length of sequences up to that batch index. cu_seqlens_k : torch.Tensor The cumulative sequence lengths for the key and value sequences in the batch, used to index into `k` and `v`. Shape: ``[batch_size + 1]``, where each entry represents the cumulative length of sequences up to that batch index. max_seqlen_q : int The maximum sequence length for the query tensor in the batch. max_seqlen_k : int The maximum sequence length for the key and value tensors in the batch. is_causal : bool Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len for each sequence. Default: False. sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``. smooth_k : bool Whether to smooth the key tensor by subtracting the mean along the sequence dimension. Default: True. Returns ------- torch.Tensor The output tensor, shape: ``[cu_seqlens_q[-1], num_qo_heads, head_dim]``. Note ---- - ``num_qo_heads`` must be divisible by ``num_kv_heads``. - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16``, ``torch.bfloat16`` or ``torch.float32``. - The tensors `cu_seqlens_q` and `cu_seqlens_k` must have the dtype ``torch.int32`` or ``torch.int64``. - All tensors must be on the same cuda device. - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances. rrrrrrFrrrr)rz1cu_seqlens_q and cu_seqlens_k must be contiguous.TrNrr)r)r}.)r`rrrrrGrYrrrrrrrq is_contiguousrsrrper_block_int8_varlen_tritonattn_true_varlenattn_false_varlen)rvrwrxrrrrrrrrr`rrrryrrzcu_seqlens_q_scalecu_seqlens_k_scalers r"sageattn_varlenrsD GE 9666669 U]EN3 3 3 35x 3 3 3 8qx + + + +18 + + + + +-V + + + 7ag ( ( ( ( ( ( ( ( (*Q ( ( (ah&&**KR H  # #A2 +;'< = = H  # #A2 +;'< = = H  # #A2 +;'< = = r  kC// H  # #A3+<'= > > H  # #A3+<'= > > H  # #A3+<'= > > s  ?+??@@@ 88B<<1  "!2!2qxx||q7H7H7HJo7H7H H  % % ' '}L,F,F,H,H}}J}}} H %5="8"8 DD   VV4V ( ( F+,-Oklmopr~AMO[]it|P}P}P}LFGVW&8:Li VVQ lLZacjl~ASbg h h h ffa|\[bdkmBTch i i i #| | A Hr$ per_threadrrrrc  ` |\} } } || j} ts Jd| js Jd| tjtjfvs Jd|dvs Jd| j| jcxkr | jks nJd| j| jcxkr | jks nJdt| j|dkrd nd }|rd nd }|d krd nd }|rd nd }| d}|dkrtj j | d d|z f} tj j | d d|z f} tj j | d d|z f} n|dkr|dkrtj j | d d|z f} tj j | d d|z f} tj j | d d|z f} n|dkrtd|| dd kr2| dd kr| dd ks Jd||dz}|d krd nd }|r| |d}|r|dkrt j| d d |d d d d dtj}n\t j| |d d dtj}nd}|dkr=t+| | ||d| ddkr|dkrdndd\}}}}nC|d kr=t-| | ||d| ddkr|dkrdnddd\}}}}| }| j}~ ~ ~t j|| |}|dvr|rt1jd|d d!}|d"kr>| tj} t5j||| |||||||| }n|d#kru|r5t9| |$\}}~ t5j|||||||||||| }n| tj} t5j||| |||||||| }nV|dkr>| tj} t5j||| |||||||| }ntd%||d&d|f}|r||r |d'z ||zzn|d'z fS|S)(ah SageAttention with INT8 quantization for Q and K, FP16 PV with FP16/FP32 accumulation, implemented using CUDA. Parameters ---------- q : torch.Tensor The query tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. k : torch.Tensor The key tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. v : torch.Tensor The value tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. tensor_layout : str The tensor layout, either "HND" or "NHD". Default: "HND". is_causal : bool Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len. Default: False. qk_quant_gran : str The granularity of quantization for Q and K, either "per_warp" or "per_thread". Default: "per_thread". sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``. pv_accum_dtype : str The dtype of the accumulation of the product of the value tensor and the attention weights, either "fp16", "fp16+fp32" or "fp32". - "fp16": PV accumulation is done in fully in FP16. This is the fastest option but may lead to numerical instability. `smooth_v` option will increase the accuracy in cases when the value tensor has a large bias (like in CogVideoX-2b). - "fp32": PV accumulation is done in FP32. This is the most accurate option but may be slower than "fp16" due to CUDA core overhead. - "fp16+fp32": PV accumulation is done in FP16, but added to a FP32 buffer every few iterations. This offers a balance between speed and accuracy. Default: "fp32". smooth_k : bool Whether to smooth the key tensor by subtracting the mean along the sequence dimension. Default: True. smooth_v : bool Whether to smooth the value tensor by subtracting the mean along the sequence dimension. smooth_v will be ignored if pv_accum_dtype is "fp32" or "fp16+fp32". Default: False. return_lse : bool Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention. Default: False. Returns ------- torch.Tensor The output tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. torch.Tensor The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). Shape: ``[batch_size, num_qo_heads, qo_len]``. Only returned if `return_lse` is True. Note ---- - ``num_qo_heads`` must be divisible by ``num_kv_heads``. - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16`` - All tensors must be on the same cuda device. - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances. zWSM80 kernel is not available. make sure you GPUs with compute capability 8.0 or higher.rrrr8qk_quant_gran must be either 'per_warp' or 'per_thread'.rrrarr)rrbr*rrFrrrNTrr fp16+fp32 r{rWARPQrr{rrrWARPKr_)rrzpv_accum_dtype is z, smooth_v will be ignored.Frfp16rzUnsupported pv_accum_dtype: .r) rr` SM80_ENABLEDrrrrrGrYrrrrrrrqrrrrrrsper_warp_int8_cudaper_thread_int8_tritonrowarningswarnr r r)qk_int8_sv_f16_accum_f16_fuse_v_mean_attnqk_int8_sv_f16_accum_f16_attn&qk_int8_sv_f16_accum_f16_attn_inst_buf) rr{rrrrrrr~rrvrwrxr`_tensor_layout _is_caual_qk_quant_gran _return_lserrrrrryrrzq_sizeq_devicerr smoothed_vvms r"rrVsmt EAa NN GE rrrrr< 9666669 U]EN3 3 3 35x 3 3 3 6 6 6 68r 6 6 6 8qx + + + +18 + + + + +-V + + + 7ag ( ( ( ( ( ( ( ( (*Q ( ( (ah'500QQaN%AI'<77QQQN!(!!qK&&**KR H  # #A2 +;'< = = H  # #A2 +;'< = = H  # #A2 +;'< = = r  kC// H  # #A3+<'= > > H  # #A3+<'= > > H  # #A3+<'= > > s  ?+??@@@ 88B<<1  "!2!2qxx||q7H7H7HJo7H7H H$!Q&&aaAG VVV . .  c%%!&akk!Q.?.?aQRASASA]A]^_abAcAc!d!d!l!lmo!p!p!s!styuB"C"C!&aa1C1C!D!D!L!LR!P!P!S!STYTa!b!b  ""+=aBVcjm}~~D~DEG~H~HLO~O~OTbfqTqTqvxvxxzBD,E,E,E( , & &+A!QZgnqBCBHBHIKBLBLPSBSBSXfjuXuXuz|z||~FHPR,S,S,S( VVXXFxH !R F%999A...8. V>VVVWWW DD  71gW^`npy|JLTVabb 6 ! !  f%a}EEENJGPVXbdegnpwy{~LNWYgiqs~CCU]##A;FFAqRY[bdrt}@NPXZeffCC ; & & DD  @QRTUW^`giwzCESU]_jkkHHHIII #| | A(`# "^h%>>>PSV`P```r$c  |dkr trdnd} |\} } } || j}ts Jd| js Jd|t jt jfvs Jd|dvs Jd| j| jcxkr | jks nJd| j| jcxkr | jks nJd t| j|d krd nd }|rd nd }|d krdnd}|rd nd }| d}|dkrt j j | d d|z f} t j j | d d|z f} t j j | d d|z f} n|dkr|dkrt j j | d d|z f} t j j | d d|z f} t j j | d d|z f} n|dkrtd|| dd kr2| dd kr| dd ks Jd||dz}|d krd nd}|dkr|rt!jdd}| g}~ tr@|dkr|rt!jdd}d}|dkrd}t%||||\}}}nt%|||\}}}|r| |d}|r|d krt j| d d|d ddddt j}n\t j| |dddt j}nd}|dkrt3| | ||dd d!\}}}}n"|d krt5| | ||dd dd"\}}}}| } | j}!| r~ ~| }"n~ ~ ~t j| ||!#}"|d$krC|r!t9j||||"||||||||| }#nkt9j||||"|||||||| }#nK|dkr t9j||||"|||||||| }#n%|dkrt9j ||||"|||||||| }#|"d%d|f}"|r|"|r |#d&z ||zzn|#d&z fS|"S)'NrrMSM89 kernel is not available. Make sure you GPUs with compute capability 8.9.rrrrrrrarr)rrbr*rrFrrrr8pv_accum_dtype is 'fp32+fp32', smooth_v will be ignored.Fz8pv_accum_dtype is 'fp32+fp16', smooth_v will be ignored.|@g@)r{ scale_maxrr{rTrrrrrr_r.r)!rrr` SM89_ENABLEDrrrrrGrYrrrrrrrqr r rrrrrrrsr r ror 5qk_int8_sv_f8_accum_f32_fuse_v_scale_fuse_v_mean_attn)qk_int8_sv_f8_accum_f32_fuse_v_scale_attnr 2qk_int8_sv_f8_accum_f16_fuse_v_scale_attn_inst_buf)$rr{rrrrrrr~rrrvrwrxr`rrrrrrv_listquant_v_scale_maxv_fp8v_scalerrrrryrrzrrrrs$ r"rr s>(->;HRGAq! NN GE hhhhh< 9666669 U]EN3 3 3 35x 3 3 3 6 6 6 68r 6 6 6 8qx + + + +18 + + + + +-V + + + 7ag ( ( ( ( ( ( ( ( (*Q ( ( (ah'500QQaN%AI'<77QQQN!(!!qK&&**KR H  # #A2 +;'< = = H  # #A2 +;'< = = H  # #A2 +;'< = = r  kC// H  # #A3+<'= > > H  # #A3+<'= > > H  # #A3+<'= > > s  ?+??@@@ 88B<<1  "!2!2qxx||q7H7H7HJo7H7H H$!Q&&aaAG$$$ PQQQSF  e [ ( (X ( MT U U UH! [ ( ( $ ,V=\myABBBw,V=[cdddw VVV . .  c%%!&akk!Q.?.?aQRASASA]A]^_abAcAc!d!d!l!lmo!p!p!s!styuB"C"C!&aa1C1C!D!D!L!LR!P!P!S!STYTa!b!b  ""+=aBVcjmuwA,B,B,B( , & &+A!QZgnqy{CEMO,P,P,P( VVXXFxH> b  a KeH = = =  STZ\bdiklnuw~AHJLN\^giwyACNOOCCGPVX]_`bikrt{~LNWYgiqs~CC ; & &LVU[]bdegnpwzACQS\^lnvxCDD ; & &LVU[]bdegnpwzACQS\^lnvxCDD #| | A(`# "^h%>>>PSV`P```r$rrc  |\} } }|| j}ts Jd| js Jd|tjtjfvs Jd|dvs Jd| j| jcxkr |jks nJd| j| jcxkr |jks nJdt|j|dkrd nd }|rd nd }|d krd nd }|rd nd }| d}|dkrtj j | d d|z f} tj j | d d|z f} tj j |d d|z f}n|dkr|dkrtj j | d d|z f} tj j | d d|z f} tj j |d d|z f}n|dkrtd|| dd kr2| dd kr|dd ks Jd||dz}|d krd nd }|dkr|rtjdd}|g}~t#|||\}}}|r| |d}|r|dkrt j| d d |d d d d dtj}n\t j| |d d dtj}nd}|dkrt1| | ||ddd\}}}}n"|d krt3| | ||dddd\}}}}| } | j}!| r~ | }"n~ ~ t j| ||!}"|d krE|r"t7j||||"|||||||||| }#nGt7j||||"||||||||| }#n&|dkr t7j||||"||||||||| }#|"d!d|f}"|r|"|r |#d"z ||zzn|#d"z fS|"S)#a SageAttention with INT8 quantization for Q and K, FP8 PV with FP32 accumulation, implemented using CUDA. Parameters ---------- q : torch.Tensor The query tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. k : torch.Tensor The key tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. v : torch.Tensor The value tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. tensor_layout : str The tensor layout, either "HND" or "NHD". Default: "HND". is_causal : bool Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len. Default: False. qk_quant_gran : str The granularity of quantization for Q and K, either "per_warp" or "per_thread". Default: "per_thread". sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``. pv_accum_dtype : str The dtype of the accumulation of the product of the value tensor and the attention weights, either "fp32" or "fp32+fp32". - "fp32": PV accumulation is done in fully in FP32. However, due to the hardware issue, there are only 22 valid bits in the FP32 accumulator. - "fp32+fp32": PV accumulation is done in FP32 (actually FP22), but added to a FP32 buffer every few iterations. This offers a balance between speed and accuracy. Default: "fp32+fp32". smooth_k : bool Whether to smooth the key tensor by subtracting the mean along the sequence dimension. Default: True. smooth_v : bool Whether to smooth the value tensor by subtracting the mean along the sequence dimension. smooth_v will be ignored if pv_accum_dtype is "fp32+fp32". Default: False. return_lse : bool Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention. Default: False. Returns ------- torch.Tensor The output tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. torch.Tensor The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). Shape: ``[batch_size, num_qo_heads, qo_len]``. Only returned if `return_lse` is True. Note ---- - ``num_qo_heads`` must be divisible by ``num_kv_heads``. - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16`` - All tensors must be on the same cuda device. - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances. rrrrrrrrarr)rrbr*rrFrrrNrrrFrTrrrrrr_r.r)rr`rrrrrrGrYrrrrrrrqr r rrrrrrrsr r ror rr r )$rr{rrrrrrr~windowrrrvrwrxr`rrrrrrr"r$r%rrrrryrrzrrrrs$ r"#sageattn_qk_int8_pv_fp8_window_cudar(sXv EAa NN GE hhhhh< 9666669 U]EN3 3 3 35x 3 3 3 6 6 6 68r 6 6 6 8qx + + + +18 + + + + +-V + + + 7ag ( ( ( ( ( ( ( ( (*Q ( ( (ah'500QQaN%AI'<77QQQN!(!!qK&&**KR H  # #A2 +;'< = = H  # #A2 +;'< = = H  # #A2 +;'< = = r  kC// H  # #A3+<'= > > H  # #A3+<'= > > H  # #A3+<'= > > s  ?+??@@@ 88B<<1  "!2!2qxx||q7H7H7HJo7H7H H$!Q&&aaAG$$$ PQQQSF (}W_```E7B VVV . .  c%%!&akk!Q.?.?aQRASASA]A]^_abAcAc!d!d!l!lmo!p!p!s!styuB"C"C!&aa1C1C!D!D!L!LR!P!P!S!STYTa!b!b  ""+=aBVcjmuwA,B,B,B( , & &+A!QZgnqy{CEMO,P,P,P( VVXXFxH>  a KeH = = =  HSTZ\bdiklnuw~AHJLN\^giwyACNPVWWCCGPVX]_`bikrt{~LNWYgiqs~AGHHCC ; & &LVU[]bdegnpwzACQS\^lnvxCFLMM #| | A(`# "^h%>>>PSV`P```r$c  Z |\} } } || j} ts Jd| js Jd| tjtjfvs Jd|dvs Jd| j| jcxkr | jks nJd| j| jcxkr | jks nJdt| j|dkrd nd }|rd nd }|d krd nd }|rd nd }| d}|dkrtj j | d d|z f} tj j | d d|z f} tj j | d d|z f} n|dkr|dkrtj j | d d|z f} tj j | d d|z f} tj j | d d|z f} n|dkrtd|| dd kr2| dd kr| dd ks Jd||dz}|d krd nd }| |}|dzd krd|dzz nd }|d kr|dkrtt j| t j| d | d || d | j| jgd } nst j| t j| d || d | d | j| jgd } | g}~ t#||d\}}}|r| |d}|r|dkrt j| d d |d d d d dtj}n\t j| |d d dtj}nd}|dkrt1| | ||ddd\}}}}n"|d krt3| | ||dddd\}}}}| } | j}!|r~ | }"n~ ~ t j| | |!}"|d krt7d!|d"krt9j||||"|||||||| }#|"d#d|f}"|r|"|r |#d$z ||zzn|#d$z fS|"S)%a SageAttention with INT8 quantization for Q and K, FP8 PV with FP32 accumulation, implemented using CUDA. Parameters ---------- q : torch.Tensor The query tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. k : torch.Tensor The key tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. v : torch.Tensor The value tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_kv_heads, kv_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, kv_len, num_kv_heads, head_dim]``. tensor_layout : str The tensor layout, either "HND" or "NHD". Default: "HND". is_causal : bool Whether to apply causal mask to the attention matrix. Only applicable when qo_len == kv_len. Default: False. qk_quant_gran : str The granularity of quantization for Q and K, either "per_warp" or "per_thread". Default: "per_thread". sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``. pv_accum_dtype : str The dtype of the accumulation of the product of the value tensor and the attention weights, either "fp32" or "fp32+fp32". - "fp32": PV accumulation is done in fully in FP32. However, due to the hardware issue, there are only 22 valid bits in the FP32 accumulator. - "fp32+fp32": PV accumulation is done in FP32 (actually FP22), but added to a FP32 buffer every few iterations. This offers a balance between speed and accuracy. Default: "fp32+fp32". smooth_k : bool Whether to smooth the key tensor by subtracting the mean along the sequence dimension. Default: True. return_lse : bool Whether to return the log sum of the exponentiated attention weights. Used for cases like Ring Attention. Default: False. Returns ------- torch.Tensor The output tensor. Shape: - If `tensor_layout` is "HND": ``[batch_size, num_qo_heads, qo_len, head_dim]``. - If `tensor_layout` is "NHD": ``[batch_size, qo_len, num_qo_heads, head_dim]``. torch.Tensor The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). Shape: ``[batch_size, num_qo_heads, qo_len]``. Only returned if `return_lse` is True. Note ---- - ``num_qo_heads`` must be divisible by ``num_kv_heads``. - The tensors `q`, `k`, and `v` must have the dtype ``torch.float16`` or ``torch.bfloat16`` - All tensors must be on the same cuda device. - `smooth_k` will introduce slight overhead but will improve the accuracy under most circumstances. zMSM90 kernel is not available. Make sure you GPUs with compute capability 9.0.rrrrrrrarr)rrbr*rrFrrrNrr]r_)rFrTrrrrrrz/Please use pv_accum_dtype='fp32+fp32' for sm90.r.r)rr` SM90_ENABLEDrrrrrGrYrrrrrrrqcatzerosrrrrrrrsr r roNotImplementedErrorrr r )$rr{rrrrrr~rrrvrwrxr`rrrrrrr v_pad_lenr"r$r%rrrrryrrzrrrrs$ r"rrssh EAa NN GE hhhhh< 9666669 U]EN3 3 3 35x 3 3 3 6 6 6 68r 6 6 6 8qx + + + +18 + + + + +-V + + + 7ag ( ( ( ( ( ( ( ( (*Q ( ( (ah'500QQaN%AI'<77QQQN!(!!qK&&**KR H  # #A2 +;'< = = H  # #A2 +;'< = = H  # #A2 +;'< = = r  kC// H  # #A3+<'= > > H  # #A3+<'= > > H  # #A3+<'= > > s  ?+??@@@ 88B<<1  "!2!2qxx||q7H7H7HJo7H7H H$!Q&&aaAG VVG__F(. (9(9v|$$qI1}} E ! ! 1ek!&&))QVVAYY 166RS99\]\clmltuuuv|}~~~AA 1ek!&&))Yq 166RS99\]\clmltuuuv|}~~~ASF 'mV[\\\E7A VVV . .  c%%!&akk!Q.?.?aQRASASA]A]^_abAcAc!d!d!l!lmo!p!p!s!styuB"C"C!&aa1C1C!D!D!L!LR!P!P!S!STYTa!b!b  ""+=aBVcjltv~A,B,B,B( , & &+A!QZgnpxzBEMP,Q,Q,Q( VVXXFxH>  a KeH = = =!"STTT ; & &LVU[]bdegnpwzACQS\^lnvxCDD #| | A(`# "^h%>>>PSV`P```r$r v_or_listrc  |dkrdnd}t|tr|d}|n|}|j}|dkr8|j\}}} } | dzdzdz} t j||| | f|j|} n=|dkr7|j\}} }} | dzdzdz} t j|| || f|j|} tj || |~t j| jt j |} t j||| ft j |}t j||| ft j |}|rtj | | ||| ||| ||fStj | | || ||| |dfS)Nrarr)r]?rFr_) isinstancelistrrGrprror`rr float8_e4m3fnrsmean_scale_fuse_quant_cudascale_fuse_quant_cuda)r/r{rrrrxrGrrrr padded_lenv_transposed_permuttedr$r%rs r"xper_channel_fp8r9<s (500QQaN)T"" aL  XF$%G!4rkb(2- !&ax-LTUT[dj!k!k!k %  $%G!64rkb(2- !&a4-LTUT[dj!k!k!k %a)?PPP K.4Eras-   <8 > >5>CUCbCbCdCd +W X XXXXXXXXffffffLLLLLLLLLMMMMMMSSSSSSWWWWWWaaaaaa[[[[[[+$$$$$$ 767 8 8/.+ Y *FFF)))))) 7; ? @ @9i8 LLLLL)))))) 7; T U U9i8 LLLLL)))))) 7; T U U9i8 LLLLLEEEEEECCCCCC((((((//////================ '&&&&&go&& u%%    isisiiii e**,,-- j--//14(.%*/'!#QelQsQQQQ)5<)C))))/U\/c////#el####T%,T4TTTTLQ\`oto|JO****ZU\TXEXiniux{i{CjC[^knquku6ffU\C-?$-Ffiniux|i|fUXfeiffff  $ RCRCRCRCuo RC  RC  RCRCRCRCRCh  ( $(,ee  e  eeuoeee %ee \eeeeN $r r  |r  |r  |r , r , r  r r r uor r r  \r r r r h % $ ss  s  ssuossssss \ssssj% $EEEE E uo E  EEEEEE \EEEEP % $% ss  s  ssuosssssss  \!ssssj % $%kk  k  kkuokkkkkk \kkkkZ(  #$#$U\4'(#$#$#$ #$#$#$#$L555&%'''''s0 B,,B< C!!C'+ D D D77D=