ó
    oM‡jªg ã                  ó¦   • % S SK Jr  S SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
JrJrJr  S SKrS SKJr  S SKJs  Jr  \R                   R-                  5       (       a	  S SKJs  Jr  SSKJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(  SSK)J*r*J+r+  SS	K,J-r-J.r.  S
SK/J0r0  \(       a  S
SK/J1r1  Sr2Sr3Sr4Sr5Sr6\" \75      r8\" 5       =(       a	    \" S\25      r9\" 5       r:\!" 5       =(       a	    \"" S\35      r;\$" S\45      r<\#" 5       r=\%" 5       =(       a	    \&" S\55      r>\'" 5       =(       a	    \(" S\65      r?\$" SS5      r@\9(       a   S SKAJBrBJCrC  S SKDJErEJFrF  OSrBSrCSrESrF\:(       a   S SKLJBrM  S SKLJCrN  OSrMSrN\;(       a   S SKOJPrPJQrQJRrRJSrSJTrTJUrU  OSrPSrSSrTSrQSrRSrU\<(       a   S SKVJs  JWs  JXrX  OSrX\=(       a   S S KYJZrZ  OSrZ\>(       a   S S"K[J\r]  OSr]\?(       a   S SK^J_r`  OSr`\RÂ                  S%:¼  a-  \RÄ                  RÆ                  rd\RÄ                  RÊ                  rfOS°SSS&.S' jjrgS°SS
S(.S) jjrh\grd\hrf " S* S+\i\	5      rj " S, S-5      rk\ " S. S/5      5       rl\jRÚ                  \l" S0S1S2S3S
S49\jRÜ                  \l" S0S5S2S3S
S49\jRÞ                  \l" S6S1S7S8S
S49\jRà                  \l" S6S5S9S:S
S49\jRâ                  \l" S;S<S
S=9\jRä                  \l" S>S1S S=9\jRæ                  \l" S?S1S
S=90rtS@\uSA'   \Rì                  \jRî                  4S±SB jj5       rx      S²SSSD.                       S³SE jjjryS´SF jrzSµSG jr{SµSH jr|S¶SI jr}SµSJ jr~SµSK jrSµSL jr€ S°         S·SM jjr�S¸SN jr‚\-" SOSP9 S°       S¹SQ jj5       rƒ S°       SºSR jjr„  S»           S¼SS jjr…S½ST jr†S¾SU jr‡SV rˆS¿SW jr‰S¸SX jrŠ\d" SYSZS[S\9            SÀ                               SÁS] jj5       r‹\f" SY5                  SÀ                               SÁS^ jj5       rŒ        SÂ                       SÃS` jjr�    SÄSa jrŽ        SÂ                       SÃSb jjr�SSSSSSc.             SÅSd jjr�        SÂ                       SÃSe jjr‘SSSSSSc.             SÅSf jjr’        SÂSgSh.                         SÆSi jjjr“SSSSSSc.             SÅSj jjr”        SÂSgSh.                         SÆSk jjjr•SSSSSSc.             SÅSl jjr–        SÂSgSh.                         SÆSm jjjr—SSSSSSc.             SÅSn jjr˜        SÂSgSCS
SSS So.                                   SÇSp jjjr™SSSSSSc.             SÅSq jjrš        SÂSgSCS
SSS So.                                   SÇSr jjjr›SSSSSSc.             SÅSs jjrœ        SÂ                       SÃSt jjr�        SÂ                       SÃSu jjrž    SÄSv jrŸS°SÈSw jjr         SÂ                       SÃSx jjr¡    SÄSy jr¢Sz r£SÉS{ jr¤SÊSËS| jjr¥ " S} S~\GRL                  GRN                  5      r¨SÌS jr©SÍS€ jrªSÌS� jr«SÍS‚ jr¬SÎSƒ jr­\.      SÏS„ j5       r®\.SÏS… j5       r¯ " S† S‡\GRL                  GRN                  5      r° " Sˆ S‰\GRL                  GRN                  5      r± " SŠ S‹\GRL                  GRN                  5      r² " SŒ S�\GRL                  GRN                  5      r³   SÐ                       SÑSŽ jjr´      SÒSS�.                   SÓS� jjjrµ\kGRm                  \jGRn                  \{\\�/S_S‘9       SÔ                     SÕS’ jj5       r¸\kGRm                  \jRÞ                  \{\\�/S_S‘9       SÔ                     SÕS“ jj5       r¹\kGRm                  \jRà                  \{\\�/S_S‘9       SÖ                     S×S” jj5       rº\kGRm                  \jGRv                  \{\\�/S•9       SÖ                     S×S– jj5       r¼\kGRm                  \jGRz                  \{\\�/S•9     SØ                 SÙS— jj5       r¾\kGRm                  \jRÚ                  \{\\�/S_S‘9        SÚ                       SÛS˜ jj5       r¿\kGRm                  \jRÜ                  \{\\�/S_S‘9     SØ                 SÙS™ jj5       rÀ\kGRm                  \jRä                  \{\\�/SS‘9     SØ                 SÙSš jj5       rÁ\kGRm                  \jGR„                  \{\\�/S•9     SØ                 SÙS› jj5       rÃ\kGRm                  \jRæ                  \|\€\�/S•9      S²                   SÜSœ jj5       rÄ\kGRm                  \jGRŠ                  \z\{\�/S•9      SÝ                   SÞS� jj5       rÆ Sß       SàSž jjrÇ\kGRm                  \jRî                  \{\�/S_S‘9       Sá                     SâSŸ jj5       rÈ\kGRm                  \jGR’                  \{\\�/S_S‘9       Sá                     SâS  jj5       rÊ\kGRm                  \jGR–                  \{\�/S•9       Sá                     SâS¡ jj5       rÌ\kGRm                  \jGRš                  \{\\�/S_S‘9       Sá                     SâS¢ jj5       rÎ\kGRm                  \jGRž                  \{\�/S•9       Sá                     SâS£ jj5       rÐ\kGRm                  \jGR¢                  \{\\�/S_S‘9     Sã                 SäS¤ jj5       rÒ\kGRm                  \jGR¦                  \{\�/S•9    Så               SæS¥ jj5       rÔ\kGRm                  \jGRª                  \|\\�/S_S‘9     Sç                 SèS¦ jj5       rÖ\kGRm                  \jRâ                  \|\\�/S_S‘9     Sç                 SèS§ jj5       r×\kGRm                  \jGR°                  \|\\�/S•9     Sç                 SèS¨ jj5       rÙ\kGRm                  \jGR´                  \}" S©S 5      \�/S•9     Sç                 SèSª jj5       rÛ\kGRm                  \jGR¸                  \}" S©S 5      \�/S•9     Sç                 SèS« jj5       rÝ\kGRm                  \jGR¼                  \}" S¬S 5      \�/S•9     Sç                 SèS­ jj5       rß\kGRm                  \jGRÀ                  \}" S¬S 5      \�/S•9     Sç                 SèS® jj5       rá\kGRm                  \jGRÄ                  \z\{\�/S•9       Sá                     SâS¯ jj5       rãg! \G\H\I4 a*  rJ\8R—                  S\J S35        Sr9SrBSrCSrESrF SrJCJGNÄSrJCJff = f! \G\H\I4 a&  rJ\8R—                  S\J S35        Sr:SrMSrN SrJCJGNÝSrJCJff = f! \G\H\I4 a.  rJ\8R—                  S\J S35        Sr;SrPSrQSrRSrSSrTSrU SrJCJGNòSrJCJff = f! \G\H\I4 a$  rJ\8R—                  S\J S35        Sr<SrX SrJCJGNSrJCJff = f! \G\H\I4 a$  rJ\8R—                  S!\J S35        Sr=SrZ SrJCJGN*SrJCJff = f! \G\H\I4 a$  rJ\8R—                  S#\J S35        Sr>Sr] SrJCJGNISrJCJff = f! \G\H\I4 a$  rJ\8R—                  S$\J S35        Sr?Sr` SrJCJGNhSrJCJff = f)éé    )ÚannotationsN)Ú	dataclass)ÚEnum)ÚTYPE_CHECKINGÚAnyÚCallableé   )Ú
get_loggerÚis_flash_attn_3_availableÚis_flash_attn_availableÚis_flash_attn_versionÚis_kernels_availableÚis_kernels_versionÚis_sageattention_availableÚis_sageattention_versionÚis_torch_npu_availableÚis_torch_versionÚis_torch_xla_availableÚis_torch_xla_versionÚis_xformers_availableÚis_xformers_version)ÚDIFFUSERS_ATTN_BACKENDÚDIFFUSERS_ATTN_CHECKS)Úlru_cache_unless_exportÚmaybe_allow_in_graphé   )Úgather_size_by_comm)ÚParallelConfigz2.6.3z2.1.1z2.5.0z2.2z0.0.29ú>=z2.9.0)Úflash_attn_funcÚflash_attn_varlen_func)Ú_wrapped_flash_attn_backwardÚ_wrapped_flash_attn_forwardz.flash_attn is installed but failed to import: z+. Falling back to native PyTorch attention.F)r    )r!   zflash_attn_3 failed to import: z#. Falling back to native attention.)ÚsageattnÚsageattn_qk_int8_pv_fp8_cudaÚ!sageattn_qk_int8_pv_fp8_cuda_sm90Úsageattn_qk_int8_pv_fp16_cudaÚsageattn_qk_int8_pv_fp16_tritonÚsageattn_varlenz sageattention failed to import: z!flex_attention failed to import: )Únpu_fusion_attentionztorch_npu failed to import: )Úflash_attentionztorch_xla failed to import: zxformers failed to import: z2.4.0)Údevice_typesÚschemac              ó   • S nUc  U$ U$ )Nc                ó   • U $ ©N© ©Úfuncs    Ú±C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffusers/models/attention_dispatch.pyÚwrapÚcustom_op_no_op.<locals>.wrapÀ   ó   € ØˆKó    r1   )ÚnameÚfnÚmutates_argsr,   r-   r5   s         r4   Úcustom_op_no_opr<   ¿   ó   € ò	ð ‘zˆtÐ) rÐ)r8   )ÚlibÚ_stacklevelc              ó   • S nUc  U$ U$ )Nc                ó   • U $ r0   r1   r2   s    r4   r5   Ú!register_fake_no_op.<locals>.wrapÆ   r7   r8   r1   )Úopr:   r>   r?   r5   s        r4   Úregister_fake_no_oprD   Å   r=   r8   c                  ó|   • \ rS rSrSrSrSrSrSrSr	Sr
S	rS
rSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrg)ÚAttentionBackendNameéÖ   ÚflashÚ	flash_hubÚflash_varlenÚflash_varlen_hubÚflash_4_hubÚ_flash_3Ú_flash_varlen_3Ú_flash_3_hubÚ_flash_3_varlen_hubÚaiter_fa2_hubÚflexÚnativeÚ_native_cudnnÚ_native_efficientÚ_native_flashÚ_native_mathÚ_native_npuÚ_native_xlaÚsageÚsage_hubÚsage_varlenÚ_sage_qk_int8_pv_fp8_cudaÚ_sage_qk_int8_pv_fp8_cuda_sm90Ú_sage_qk_int8_pv_fp16_cudaÚ_sage_qk_int8_pv_fp16_tritonÚxformersr1   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚFLASHÚ	FLASH_HUBÚFLASH_VARLENÚFLASH_VARLEN_HUBÚFLASH_4_HUBÚ_FLASH_3Ú_FLASH_VARLEN_3Ú_FLASH_3_HUBÚ_FLASH_3_VARLEN_HUBÚAITER_FA2_HUBÚFLEXÚNATIVEÚ_NATIVE_CUDNNÚ_NATIVE_EFFICIENTÚ_NATIVE_FLASHÚ_NATIVE_MATHÚ_NATIVE_NPUÚ_NATIVE_XLAÚSAGEÚSAGE_HUBÚSAGE_VARLENÚ_SAGE_QK_INT8_PV_FP8_CUDAÚ_SAGE_QK_INT8_PV_FP8_CUDA_SM90Ú_SAGE_QK_INT8_PV_FP16_CUDAÚ_SAGE_QK_INT8_PV_FP16_TRITONÚXFORMERSÚ__static_attributes__r1   r8   r4   rF   rF   Ö   s•   † ð €EØ€IØ!€LØ)ÐØ€KØ€HØ'€OØ!€LØ/Ðð $€Mð €DØ€FØ#€MØ+ÐØ#€MØ!€LØ€KØ€Kð €DØ€HØ€KØ ;ÐØ%EÐ"Ø!=ÐØ#AÐ ð ƒHr8   rF   c                  ó¸   • \ rS rSr0 r0 r0 r\" 5       r\	" \
5      r\r\  S	     S
S jj5       r\S 5       r\SS j5       r\S 5       r\    SS j5       rSrg)Ú_AttentionBackendRegistryi  Nc                óV   ^ ^^^• [         R                  ST ST 35        UU UU4S jnU$ )NzRegistering attention backend: z with constraints: c                ó6  >• U TR                   T'   T=(       d    / TR                  T'   [        [        R                  " U 5      R
                  R                  5       5      TR                  T'   T(       a%  TR                  R                  TR                  5        U $ r0   )Ú	_backendsÚ_constraintsÚsetÚinspectÚ	signatureÚ
parametersÚkeysÚ_supported_arg_namesÚ_supports_context_parallelÚaddÚvalue)r3   ÚbackendÚclsÚconstraintsÚsupports_context_parallels    €€€€r4   Ú	decoratorÚ5_AttentionBackendRegistry.register.<locals>.decorator  su   ø€ Ø%)ˆC�M‰M˜'Ñ"Ø(3×(9°rˆC×Ñ˜WÑ%Ü03´G×4EÒ4EÀdÓ4K×4VÑ4V×4[Ñ4[Ó4]Ó0^ˆC×$Ñ$ WÑ-Þ(Ø×.Ñ.×2Ñ2°7·=±=ÔAàˆKr8   )ÚloggerÚdebug)r‘   r�   r’   r“   r”   s   ```` r4   ÚregisterÚ"_AttentionBackendRegistry.register	  s4   û€ ô 	�‰Ð6°w°iÐ?RÐS^ÐR_Ð`Ôa÷	ð 	ð Ðr8   c                óL   • U R                   U R                  U R                      4$ r0   )Ú_active_backendr…   ©r‘   s    r4   Úget_active_backendÚ,_AttentionBackendRegistry.get_active_backend  s"   € à×"Ñ" C§M¡M°#×2EÑ2EÑ$FÐFÐFr8   c                ó   • Xl         g r0   )r›   )r‘   r�   s     r4   Úset_active_backendÚ,_AttentionBackendRegistry.set_active_backend!  s   € à%Õr8   c                óH   • [        U R                  R                  5       5      $ r0   )Úlistr…   r‹   rœ   s    r4   Úlist_backendsÚ'_AttentionBackendRegistry.list_backends%  s   € ä�C—M‘M×&Ñ&Ó(Ó)Ð)r8   c                ó8   • UR                   U R                  ;   nU$ r0   )r�   r�   )r‘   r�   r“   s      r4   Ú_is_context_parallel_availableÚ8_AttentionBackendRegistry._is_context_parallel_available)  s   € ð
 %,§M¡M°S×5SÑ5SÑ$SÐ!Ø(Ð(r8   r1   )NF)r�   rF   r’   zlist[Callable] | Noner“   Úbool)r�   Ústr)r�   rF   Úreturnr©   )rb   rc   rd   re   r…   r†   rŒ   r‡   r�   rF   r   r›   r   Ú_checks_enabledÚclassmethodr˜   r�   r    r¤   r§   r€   r1   r8   r4   r‚   r‚     sÄ   † Ø€IØ€LØÐÙ!$£ÐÙ*Ð+AÓB€OØ+€Oàð .2Ø*/ð	à%ðð +ðð $(ô	ó ðð& ñGó ðGð ó&ó ð&ð ñ*ó ð*ð ð)à%ð)ð 
ó)ó ó)r8   r‚   c                  ó�   • \ rS rSr% SrS\S'   S\S'   SrS\S'   SrS	\S
'   SrS\S'   Sr	S\S'   Sr
S\S'   SrS\S'   SrS\S'   Srg)Ú_HubKernelConfigi2  zEConfiguration for downloading and using a hub-based attention kernel.rª   Úrepo_idÚfunction_attrNz
str | NoneÚrevisionz
int | NoneÚversionzCallable | NoneÚ	kernel_fnÚwrapped_forward_attrÚwrapped_backward_attrÚwrapped_forward_fnÚwrapped_backward_fnr1   )rb   rc   rd   re   Ú__doc__Ú__annotations__r²   r³   r´   rµ   r¶   r·   r¸   r€   r1   r8   r4   r¯   r¯   2  sZ   ‡ áOàƒLØÓØ€HˆjÓØ€GˆZÓØ!%€IˆÓ%Ø'+Ð˜*Ó+Ø(,Ð˜:Ó,Ø*.Ð˜Ó.Ø+/Ð˜Ö/r8   r¯   zkernels-community/flash-attn3r    z(flash_attn_interface._flash_attn_forwardz)flash_attn_interface._flash_attn_backward)r°   r±   rµ   r¶   r³   r!   zkernels-community/flash-attn2z0flash_attn_interface._wrapped_flash_attn_forwardz1flash_attn_interface._wrapped_flash_attn_backwardz7flash_attn_interface._wrapped_flash_attn_varlen_forwardz8flash_attn_interface._wrapped_flash_attn_varlen_backwardz kernels-community/sage-attentionr$   )r°   r±   r³   zkernels-community/flash-attn4z%kernels-community/aiter-flash-attn-ckz.dict['AttentionBackendName', _HubKernelConfig]Ú_HUB_KERNELS_REGISTRYc              #  óF  #   • U [         R                  ;  a  [        SU  S35      e[        U 5      n [	        U 5        [        U 5        [         R                  n[         R                  U 5         Sv •  [         R                  U5        g! [         R                  U5        f = f7f)z6
Context manager to set the active attention backend.
zBackend z is not registered.N)r‚   r…   Ú
ValueErrorrF   Ú%_check_attention_backend_requirementsÚ"_maybe_download_kernel_for_backendr›   r    )r�   Úold_backends     r4   Úattention_backendrÁ   q  s„   é € ð
 Ô/×9Ñ9Ó9Ü˜8 G 9Ð,?Ð@ÓAÐAä" 7Ó+€GÜ)¨'Ô2Ü& wÔ/ä+×;Ñ;€KÜ×0Ñ0°Ô9ðBÛä!×4Ñ4°[ÕAøÔ!×4Ñ4°[ÕAüs   ‚A*B!Á-B Á1B!ÂBÂB!ç        )r�   Úparallel_configc	               ó”  • U=(       d    0 nU	c  [         R                  5       u  p¼O*[        U	5      n[         R                  R	                  U5      nU UUUUUUS.UESU
0En[
        (       a  X}S'   [         R                  (       ay  [        U5      [        [         R                  U   5      -
  nU(       a  [        R                  SU SU S35        [         R                  R	                  U5       H  nU" S0 UD6  M     UR                  5        VVs0 s H"  u  nnU[         R                  U   ;   d  M  UU_M$     nnnU" S0 UD6$ s  snnf )N)ÚqueryÚkeyr�   Ú	attn_maskÚ	dropout_pÚ	is_causalÚscaleÚ_parallel_configÚ
enable_gqaz5Removing unsupported arguments for attention backend z: Ú.r1   )r‚   r�   rF   r…   ÚgetÚ_CAN_USE_FLEX_ATTNr¬   r‡   rŒ   r–   Úwarningr†   Úitems)rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   Úattention_kwargsr�   rÃ   Úbackend_nameÚ
backend_fnÚkwargsÚremoved_kwargsÚcheckÚkÚvs                     r4   Údispatch_attention_fnrÚ   †  sN  € ð (×-¨2Ðà�ô $=×#OÑ#OÓ#QÑ ˆ�jä+¨GÓ4ˆÜ.×8Ñ8×<Ñ<¸\ÓJˆ
ð ØØØØØØñ
ð ð
ð 	˜Oñ
€F÷ ÒØ)ˆ|Ñä ×0×0Ü˜V›¤sÔ+D×+YÑ+YÐZfÑ+gÓ'hÑhˆÞÜ�N‰NÐRÐS_ÐR`Ð`bÐcqÐbrÐrsÐtÔuÜ.×;Ñ;×?Ñ?ÀÖMˆEÙ‰O�FŒOñ Nð  &Ÿ|™|œ~Ôsš~‘t�q˜!°Ô6O×6dÑ6dÐeqÑ6rÑ1r‹dˆa�Šd™~€FÑsáÑ˜ÑÐùó ts   ÄEÄ2Ec                ó0   • U b  U(       a  [        S5      eg g )Nz8`is_causal` cannot be True when `attn_mask` is not None.)r½   )rÇ   rÉ   rÕ   s      r4   Ú_check_attn_mask_or_causalrÜ   ¾  s   € ØÑ¦ÜÐSÓTÐTð "+Ðr8   c                ó   • U R                   UR                   :w  d  U R                   UR                   :w  a  [        S5      eU R                  UR                  :w  d  U R                  UR                  :w  a  [        S5      eg )Nz1Query, key, and value must be on the same device.z/Query, key, and value must have the same dtype.)Údevicer½   Údtype©rÅ   rÆ   r�   rÕ   s       r4   Ú_check_devicerá   Ã  s]   € Ø‡|�|�s—z‘zÓ! U§\¡\°U·\±\Ó%AÜÐLÓMÐMØ‡{�{�c—i‘iÓ 5§;¡;°%·+±+Ó#=ÜÐJÓKÐKð $>r8   c                óf   • [        XU5        U R                  R                  S:w  a  [        S5      eg )NÚcudaz/Query, key, and value must be on a CUDA device.)rá   rÞ   Útyper½   rà   s       r4   Ú_check_device_cudarå   Ê  s/   € Ü�%˜eÔ$Ø‡|�|×Ñ˜FÓ"ÜÐJÓKÐKð #r8   c                ó   ^ ^• SU U4S jjnU$ )Nc                ó    >• [        XU5        [        R                  R                  U R                  5      TT4:  a  [        ST ST S35      eg )NzJQuery, key, and value must be on a CUDA device with compute capability >= rÍ   )rå   Útorchrã   Úget_device_capabilityrÞ   r½   )rÅ   rÆ   r�   rÕ   ÚmajorÚminors       €€r4   Úcheck_device_cudaÚ:_check_device_cuda_atleast_smXY.<locals>.check_device_cudaÑ  sW   ø€ Ü˜5 uÔ-Ü�:‰:×+Ñ+¨E¯L©LÓ9¸UÀE¸NÓJÜØ\Ð]bÐ\cÐcdÐejÐdkÐklÐmóð ð Kr8   ©rÅ   útorch.TensorrÆ   rï   r�   rï   r«   ÚNoner1   )rê   rë   rì   s   `` r4   Ú_check_device_cuda_atleast_smXYrñ   Ð  s   ù€ ÷ð ð Ðr8   c                ó˜   • U R                   UR                   :w  a  [        S5      eU R                   UR                   :w  a  [        S5      eg )Nz'Query and key must have the same dtype.z)Query and value must have the same dtype.)rß   r½   rà   s       r4   Ú_check_qkv_dtype_matchró   Û  s?   € Ø‡{�{�c—i‘iÓÜÐBÓCÐCØ‡{�{�e—k‘kÓ!ÜÐDÓEÐEð "r8   c                óŽ   • [        XU5        U R                  [        R                  [        R                  4;  a  [        S5      eg )Nz9Query, key, and value must be either bfloat16 or float16.)ró   rß   rè   Úbfloat16Úfloat16r½   rà   s       r4   Ú_check_qkv_dtype_bf16_or_fp16r÷   â  s6   € Ü˜5 uÔ-Ø‡{�{œ5Ÿ>™>¬5¯=©=Ð9Ó9ÜÐTÓUÐUð :r8   c                ón   • [        XU5        U R                  [        R                  :w  a  [	        S5      eg )Nz'Query, key, and value must be bfloat16.)ró   rß   rè   rõ   r½   rà   s       r4   Ú_check_qkv_dtype_bf16rù   è  s-   € Ü˜5 uÔ-Ø‡{�{”e—n‘nÓ$ÜÐBÓCÐCð %r8   c                ó  • U R                   S   UR                   S   :w  a  [        S5      eUR                   S   UR                   S   :w  a  [        S5      eUb,  UR                   S   UR                   S   :w  a  [        S5      eg g )Néÿÿÿÿz0Query and key must have the same head dimension.éýÿÿÿz1Key and value must have the same sequence length.z4Attention mask must match the key's sequence length.)Úshaper½   )rÅ   rÆ   r�   rÇ   rÕ   s        r4   Ú_check_shaperþ   î  s   € ð ‡{�{�2�˜#Ÿ)™) B™-Ó'ÜÐKÓLÐLØ
‡y�y��}˜Ÿ™ B™Ó'ÜÐLÓMÐMØÑ §¡°Ñ!4¸¿	¹	À"¹Ó!EÜÐOÓPÐPð "FÐr8   c                ó4  • U [         R                  [         R                  4;   a,  [        (       d   [	        SU R
                   S[         S35      eg U [         R                  [         R                  4;   a%  [        (       d  [	        SU R
                   S35      eg U [         R                  [         R                  [         R                  [         R                  [         R                  [         R                  [         R                   4;   a’  [#        5       (       d  [	        SU R
                   S35      e[%        SS	5      (       d  [	        SU R
                   S
35      eU [         R                  :X  a+  [%        SS5      (       d  [	        SU R
                   S35      eg g U [         R&                  [         R(                  [         R*                  [         R,                  [         R.                  [         R0                  4;   a,  [2        (       d   [	        SU R
                   S[4         S35      eg U [         R6                  :X  a%  [8        (       d  [	        SU R
                   S35      eg U [         R:                  :X  a%  [<        (       d  [	        SU R
                   S35      eg U [         R>                  :X  a,  [@        (       d   [	        SU R
                   S[B         S35      eg U [         RD                  :X  a,  [F        (       d   [	        SU R
                   S[H         S35      eg g )NzFlash Attention backend 'zb' is not usable because of missing package or the version is too old. Please install `flash-attn>=z`.zFlash Attention 3 backend 'zp' is not usable because of missing package or the version is too old. Please build FA3 beta release from source.z	Backend 'zl' is not usable because the `kernels` package isn't available. Please install it with `pip install kernels`.r   z0.12zj' needs to be used with a `kernels` version of at least 0.12. Please update with `pip install -U kernels`.z0.12.3zl' needs to be used with a `kernels` version of at least 0.12.3. Please update with `pip install -U kernels`.zSage Attention backend 'ze' is not usable because of missing package or the version is too old. Please install `sageattention>=zFlex Attention backend 'zd' is not usable because of missing package or the version is too old. Please install `torch>=2.5.0`.zNPU Attention backend 'za' is not usable because of missing package or the version is too old. Please install `torch_npu`.zXLA Attention backend 'za' is not usable because of missing package or the version is too old. Please install `torch_xla>=zXformers Attention backend 'z`' is not usable because of missing package or the version is too old. Please install `xformers>=)%rF   rf   rh   Ú_CAN_USE_FLASH_ATTNÚRuntimeErrorr�   Ú_REQUIRED_FLASH_VERSIONrk   rl   Ú_CAN_USE_FLASH_ATTN_3rg   ri   rm   rn   ry   rj   ro   r   r   rx   rz   r{   r|   r}   r~   Ú_CAN_USE_SAGE_ATTNÚ_REQUIRED_SAGE_VERSIONrp   rÏ   rv   Ú_CAN_USE_NPU_ATTNrw   Ú_CAN_USE_XLA_ATTNÚ_REQUIRED_XLA_VERSIONr   Ú_CAN_USE_XFORMERS_ATTNÚ_REQUIRED_XFORMERS_VERSION)r�   s    r4   r¾   r¾     se  € ØÔ'×-Ñ-Ô/C×/PÑ/PÐQÓQß"Ò"ÜØ+¨G¯M©M¨?ð  ;]ô  ^uð  ]vð  vxð  yóð ð #ð
 
Ô)×2Ñ2Ô4H×4XÑ4XÐYÓ	Yß$Ò$ÜØ-¨g¯m©m¨_ð  =mð  nóð ð %ð
 
Ü×&Ñ&Ü×-Ñ-Ü×)Ñ)Ü×0Ñ0Ü×%Ñ%Ü×(Ñ(Ü×*Ñ*ðó 
ô $×%Ñ%ÜØ˜GŸM™M˜?ð  +Wð  Xóð ô " $¨×/Ñ/ÜØ˜GŸM™M˜?ð  +Uð  Vóð ð Ô*×6Ñ6Ó6Ô?QÐRVÐX`×?aÑ?aÜØ˜GŸM™M˜?ð  +Wð  Xóð ð @bÐ6ð
 
Ü×!Ñ!Ü×(Ñ(Ü×6Ñ6Ü×;Ñ;Ü×7Ñ7Ü×9Ñ9ðó 
÷ "Ò!ÜØ*¨7¯=©=¨/ð  :_ô  `vð  _wð  wyð  zóð ð "ð
 
Ô(×-Ñ-Ó	-ß!Ò!ÜØ*¨7¯=©=¨/ð  :^ð  _óð ð "ð
 
Ô(×4Ñ4Ó	4ß Ò ÜØ)¨'¯-©-¨ð  9Zð  [óð ð !ð
 
Ô(×4Ñ4Ó	4ß Ò ÜØ)¨'¯-©-¨ð  9Zô  [pð  Zqð  qsð  tóð ð !ð
 
Ô(×1Ñ1Ó	1ß%Ò%ÜØ.¨w¯}©}¨oð  >^ô  _yð  ^zð  z|ð  }óð ð &ð 
2r8   é€   )Úmaxsizec                ó  • [         R                  " U 4U[         R                  US9n[         R                  " U 4U[         R                  US9n[         R                  " U S-   [         R                  US9n[         R                  " U S-   [         R                  US9n[         R                  " USS9USS & [         R                  " USS9USS & UR                  5       R                  5       nUR                  5       R                  5       n	XE4Xg4X‰44$ )N©rß   rÞ   r   r   ©Údim)rè   ÚfullÚint32ÚzerosÚcumsumÚmaxÚitem)
Ú
batch_sizeÚ	seq_len_qÚ
seq_len_kvrÞ   Ú	seqlens_qÚ	seqlens_kÚcu_seqlens_qÚcu_seqlens_kÚmax_seqlen_qÚmax_seqlen_ks
             r4   Ú3_prepare_for_flash_attn_or_sage_varlen_without_maskr   P  sÙ   € ô —
’
˜J˜=¨)¼5¿;¹;ÈvÑV€IÜ—
’
˜J˜=¨*¼E¿K¹KÐPVÑW€IÜ—;’;˜z¨A™~´U·[±[ÈÑP€LÜ—;’;˜z¨A™~´U·[±[ÈÑP€LÜ—|’| I°1Ñ5€L��ÐÜ—|’| I°1Ñ5€L��ÐØ—=‘=“?×'Ñ'Ó)€LØ—=‘=“?×'Ñ'Ó)€LØÐ! LÐ#?À,ÐA]Ð]Ð]r8   c                ó  • [         R                  " U 4U[         R                  US9nUR                  S[         R                  S9n[         R                  " U S-   [         R                  US9n[         R                  " U S-   [         R                  US9n[         R
                  " USS9USS & [         R
                  " USS9USS & UR                  5       R                  5       nUR                  5       R                  5       n	XE4Xg4X‰44$ )Nr  r   )r  rß   r   r  )rè   r  r  Úsumr  r  r  r  )
r  r  rÇ   rÞ   r  r  r  r  r  r  s
             r4   Ú0_prepare_for_flash_attn_or_sage_varlen_with_maskr#  b  sÔ   € ô —
’
˜J˜=¨)¼5¿;¹;ÈvÑV€IØ—‘ !¬5¯;©;�Ð7€IÜ—;’;˜z¨A™~´U·[±[ÈÑP€LÜ—;’;˜z¨A™~´U·[±[ÈÑP€LÜ—|’| I°1Ñ5€L��ÐÜ—|’| I°1Ñ5€L��ÐØ—=‘=“?×'Ñ'Ó)€LØ—=‘=“?×'Ñ'Ó)€LØÐ! LÐ#?À,ÐA]Ð]Ð]r8   c                ó8   • Uc  [        XX$5      $ [        XX45      $ r0   )r   r#  )r  r  r  rÇ   rÞ   s        r4   Ú&_prepare_for_flash_attn_or_sage_varlenr%  s  s&   € ð ÑÜBÀ:ÐZdÓmÐmÜ;¸JÐS\ÓeÐer8   c                óÆ   • [         R                  " X#-  /U R                  SS Q7U R                  U R                  S.6nXU'   UR
                  " X#/U R                  SS Q76 $ )zQscatter a packed `(nnz, ...)` tensor back to padded `(batch_size, seq_len, ...)`.r   Nr  )rè   r  rý   rß   rÞ   Úview)ÚpackedÚindicesr  Úseq_lenÚoutputs        r4   Ú_unpad_to_paddedr,    s[   € ä�[Š[˜Ñ-Ðk°·±¸Q¸RÐ0@ÑkÈÏÉÐ]c×]jÑ]jÒk€FØˆ7�OØ�;Š;�zÐ>¨V¯\©\¸!¸"Ð-=Ò>Ð>r8   c           	     óº  • U R                   [        R                  :w  a  [        SU R                    S35      eU R                  S:X  a"  U R                  S5      R                  X5      n GOAU R                  S:X  aH  U R                  S5      SU4;  a  [        SU R                  S    SU S35      eU R                  X5      n OéU R                  S	:X  aW  U R                  S5      SU4;  a  [        SU R                  S    SU S
35      eU R                  SS9n U R                  X5      n O‚U R                  S:X  aZ  U R                  S5      SU4;  a  [        SU R                  S    SU S35      eU R                  USSU5      n U R                  SS9n O[        SU R                   35      eU R                  X4:w  a  [        SU R                   SU SU S35      eU $ )zÅ
Normalize an attention mask to shape [batch_size, seq_len_k] (bool) suitable for inferring seqlens_[q|k] in
FlashAttention/Sage varlen.

Supports 1D to 4D shapes and common broadcasting patterns.
z)Attention mask must be of type bool, got rÍ   r   r   r	   zattn_mask.shape[0] (z) must be 1 or z for 2D attention mask.é   z for 3D attention mask.r  é   z for 4D attention mask.rû   )r   r	   z"Unsupported attention mask shape: z.Normalized attention mask shape mismatch: got z, expected (z, Ú))
rß   rè   r©   r½   ÚndimÚ	unsqueezeÚexpandÚsizerý   Úany)rÇ   r  Ú	seq_len_ks      r4   Ú_normalize_attn_maskr7  †  sö  € ð ‡�œ%Ÿ*™*Ó$ÜÐDÀYÇ_Á_ÐDUÐUVÐWÓXÐXà‡~�~˜Óà×'Ñ'¨Ó*×1Ñ1°*ÓHŠ	à	�‰˜1Ó	à�>‰>˜!Ó Q¨
 OÓ3ÜØ& y§¡°qÑ'9Ð&:¸/È*ÈÐUlÐmóð ð ×$Ñ$ ZÓ;‰	à	�‰˜1Ó	ð �>‰>˜!Ó Q¨
 OÓ3ÜØ& y§¡°qÑ'9Ð&:¸/È*ÈÐUlÐmóð ð —M‘M a�MÐ(ˆ	Ø×$Ñ$ ZÓ;‰	à	�‰˜1Ó	à�>‰>˜!Ó Q¨
 OÓ3ÜØ& y§¡°qÑ'9Ð&:¸/È*ÈÐUlÐmóð ð ×$Ñ$ Z°°R¸ÓCˆ	Ø—M‘M f�MÐ-‰	ô Ð=¸i¿o¹oÐ=NÐOÓPÐPà‡�˜:Ð1Ó1ÜØ<¸Y¿_¹_Ð<MÈ\ÐZdÐYeÐegÐhqÐgrÐrsÐtó
ð 	
ð Ðr8   c                ó
   • X#:¬  $ r0   r1   )Ú	batch_idxÚhead_idxÚq_idxÚkv_idxs       r4   Ú_flex_attention_causal_mask_modr=  º  s
   € Ø‰?Ðr8   c                ó¨   • U nUR                  S5       H:  n[        X#5      (       d  [        SU R                   SU S35      e[	        X#5      nM<     U$ )NrÍ   zKernel module 'z"' does not define attribute path 'z'.)ÚsplitÚhasattrÚAttributeErrorrb   Úgetattr)ÚmoduleÚ	attr_pathÚtargetÚattrs       r4   Ú_resolve_kernel_attrrG  ¿  sY   € Ø€FØ—‘ Ö$ˆÜ�v×$Ñ$Ü  ?°6·?±?Ð2CÐCeÐfoÐepÐprÐ!sÓtÐtÜ˜Ó&Šñ %ð €Mr8   c                ó’  • U [         ;  a  g [         U    nUR                  S L nUR                  S L=(       a    UR                  S L nUR                  S L=(       a    UR
                  S L nU(       d  U(       d  U(       d  g  SSKJn  U" UR                  UR                  UR                  S9nU(       a  [        XaR                  5      Ul        U(       a  [        XaR                  5      Ul        U(       a  [        XaR                  5      Ul        g g ! [         a+  n[        R                  SUR                   SU 35        e S nAff = f)Nr   )Ú
get_kernel)r²   r³   z)An error occurred while fetching kernel 'z' from the Hub: )r»   r´   rµ   r·   r¶   r¸   ÚkernelsrI  r°   r²   r³   rG  r±   Ú	Exceptionr–   Úerror)r�   ÚconfigÚneeds_kernelÚneeds_wrapped_forwardÚneeds_wrapped_backwardrI  Úkernel_moduleÚes           r4   r¿   r¿   È  s  € ØÔ+Ó+ØÜ" 7Ñ+€Fà×#Ñ# tÐ+€LØ"×7Ñ7¸tÐC×iÈ×HaÑHaÐeiÐHiÐØ#×9Ñ9ÀÐE×lÈ&×JdÑJdÐhlÐJlÐæÖ1Ö5KØðÝ&á" 6§>¡>¸F¿O¹OÐU[×UcÑUcÑdˆÞÜ3°M×CWÑCWÓXˆFÔæ Ü(<¸]×LgÑLgÓ(hˆFÔ%æ!Ü)=¸m×MiÑMiÓ)jˆFÕ&ð "øô ó Ü�‰Ð@ÀÇÁÐ@PÐP`ÐabÐ`cÐdÔeØûðús   Â BD Ä
EÄ&EÅEz,_diffusers_flash_attn_3::_flash_attn_forwardr1   rã   )r;   r,   c                ó¶   • Sn[        S0 SU _SU_SU_SU_SU_SU_SU_S	U_S
U_SU_SU	_SU
_SU_SU_SU_SU_SS_6nUtnnnUR                  SSS5      nUU4$ )N©rû   rû   ÚqrØ   rÙ   Úsoftmax_scaleÚcausalÚqvÚ	q_descaleÚ	k_descaleÚ	v_descaleÚwindow_sizeÚattention_chunkÚsoftcapÚ
num_splitsÚpack_gqaÚdeterministicÚ	sm_marginÚreturn_attn_probsTr   r	   r   r1   )Úflash_attn_3_funcÚpermute)rU  rØ   rÙ   rV  rW  rX  rY  rZ  r[  r]  r^  r_  r`  ra  rb  r\  ÚresultÚoutÚlseÚ_s                       r4   Ú_wrapped_flash_attn_3rj  ê  sá   € ð& €KÜò Ù
ðá
ðñ ðñ $ð	ñ
 ðñ ðñ ðñ ðñ ðñ  ðñ (ðñ ðñ ðñ ðñ $ðñ  ð!ñ" ð#€Fð& €L€CˆˆqØ
�+‰+�a˜˜AÓ
€CØ�ˆ8€Or8   c                ó€   • SnU R                   u  nnnnUUU4n[        R                  " U 5      U R                  U5      4$ )NrT  )rý   rè   Ú
empty_likeÚ	new_empty)rU  rØ   rÙ   rV  rW  rX  rY  rZ  r[  r]  r^  r_  r`  ra  rb  r\  r  r*  Ú	num_headsÚhead_dimÚ	lse_shapes                        r4   ri  ri    sH   € ð$ €Kð 01¯w©wÑ,€J�˜ HØ˜W iÐ0€IÜ×Ò˜AÓ §¡¨IÓ 6Ð6Ð6r8   Tc                ó8  • U	(       a  [        S5      eU
(       a0  U R                  XU5        X@l        XPl        X`l        Xpl        X€l        S XU4 5       u  pn[        R                  R                  R                  UUUUUUUUS9nUR                  SSSS5      nU$ )Nz1Native attention does not support return_lse=Truec              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7f©r   r	   r   r.  N©re  ©Ú.0Úxs     r4   Ú	<genexpr>Ú/_native_attention_forward_op.<locals>.<genexpr>N  ó"   é € ÐLÒ8K°1Ÿ™ 1 a¨¨A×.Ð.Ò8Kùó   ‚ "©rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   r   r	   r   r.  )r½   Úsave_for_backwardrÇ   rÈ   rÉ   rÊ   rÌ   rè   ÚnnÚ
functionalÚscaled_dot_product_attentionre  ©ÚctxrÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   Ú
return_lseÚ	_save_ctxrË   rg  s                r4   Ú_native_attention_forward_opr…  3  s§   € ö ÜÐLÓMÐMö Ø×Ñ˜e¨%Ô0Ø!ŒØ!ŒØ!ŒØŒ	Ø#ŒáL¸ÀUÑ8KÓLÑ€E�Ü
�(‰(×
Ñ
×
:Ñ
:ØØØØØØØØð ;ð 	€Cð �+‰+�a˜˜A˜qÓ
!€Cà€Jr8   c                ó°  • U R                   u  pEnUR                  S5        UR                  S5        UR                  S5        [        R                  " 5          S XEU4 5       u  pxn	[        R                  R
                  R                  UUU	U R                  U R                  U R                  U R                  U R                  S9n
U
R                  SSSS5      n
[        R                  R                  X§X‰/USS	9u  p¼nS S S 5        WR                  SSSS5      nWR                  SSSS5      nWR                  SSSS5      nXïU4$ ! , (       d  f       NN= f)
NTc              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7frs  rt  ru  s     r4   rx  Ú0_native_attention_backward_op.<locals>.<genexpr>k  s"   é € Ð"VÒBU¸Q§9¡9¨Q°°1°a×#8Ð#8ÒBUùr{  r|  r   r	   r   r.  F)ÚoutputsÚinputsÚgrad_outputsÚretain_graph)Úsaved_tensorsÚrequires_grad_rè   Úenable_gradr~  r  r€  rÇ   rÈ   rÉ   rÊ   rÌ   re  ÚautogradÚgrad)r‚  Úgrad_outÚargsrÕ   rÅ   rÆ   r�   Úquery_tÚkey_tÚvalue_trg  Úgrad_query_tÚ
grad_key_tÚgrad_value_tÚ
grad_queryÚgrad_keyÚ
grad_values                    r4   Ú_native_attention_backward_opr�  ^  sH  € ð ×)Ñ)Ñ€E�à	×Ñ˜ÔØ×Ñ�tÔØ	×Ñ˜Ôä	×	Ò	Õ	Ù"VÀ5ÈuÑBUÓ"VÑˆ˜Ü�h‰h×!Ñ!×>Ñ>ØØØØ—m‘mØ—m‘mØ—m‘mØ—)‘)Ø—~‘~ð ?ð 	
ˆð �k‰k˜!˜Q  1Ó%ˆä16·±×1DÑ1DØ¨%Ð 9ÈÐ_dð 2Eð 2
Ñ.ˆ ,÷ 
ð$ ×%Ñ% a¨¨A¨qÓ1€JØ×!Ñ! ! Q¨¨1Ó-€HØ×%Ñ% a¨¨A¨qÓ1€Jà Ð+Ð+÷- 
Õ	ús   ÁB'EÅ
Ec                óZ  • U(       a  [        S5      e[        S XU4 5       5      nUS L=(       a    UR                  R                  S:„  nU	=(       d    U=(       d    UnUR	                  SS5      R                  5       nUR	                  SS5      R                  5       nUR	                  SS5      R                  5       n[        R                  R                  R                  UUUUUUUSUS9	u	  nnnnnnnnnUR	                  SS5      R                  5       nU(       a<  [        (       a  UR                  S5      nUR	                  SS5      R                  5       nU
(       a>  U R                  XUUUUUUU5	        XPl        X`l        Xpl        X@l        UU l        UU l        U	(       a  UU4$ U$ )Nz6`enable_gqa` is not yet supported for cuDNN attention.c              3  ó8   #   • U  H  oR                   v •  M     g 7fr0   ©Úrequires_gradru  s     r4   rx  Ú._cudnn_attention_forward_op.<locals>.<genexpr>�  ó   é € ÐDÒ0C¨1—–Ò0Cùó   ‚r   r	   F)	rÅ   rÆ   r�   Ú	attn_biasÚcompute_log_sumexprÈ   rÉ   Úreturn_debug_maskrÊ   rû   )r½   r5  Úcontext_parallel_configÚ_world_sizeÚ	transposeÚ
contiguousrè   ÚopsÚatenÚ#_scaled_dot_product_cudnn_attentionÚ_CUDNN_LSE_HAS_TRAILING_DIMÚsqueezer}  rÈ   rÉ   rÊ   rÇ   Úmax_qÚmax_k)r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   Úgrad_enabledÚ
cp_enabledr¦  r”  r•  r–  rg  rh  Ú	cum_seq_qÚ	cum_seq_kr±  r²  Úphilox_seedÚphilox_offsetÚdebug_attn_masks                              r4   Ú_cudnn_attention_forward_oprº  †  s›  € ö ÜÐQÓRÐRô ÑD°¸UÑ0CÓDÓD€LØ!¨Ð-×jÐ2B×2ZÑ2Z×2fÑ2fÐijÑ2j€JØ#×A |×A°zÐð
 �o‰o˜a Ó#×.Ñ.Ó0€GØ�M‰M˜!˜QÓ×*Ñ*Ó,€EØ�o‰o˜a Ó#×.Ñ.Ó0€Gô 	�	‰	�‰×:Ñ:ØØØØØ1ØØØ#Øð 	;ð 
	
ñ ^€Cˆˆi˜ E¨5°+¸}Èoð �-‰-˜˜1Ó
×
(Ñ
(Ó
*€CÞ÷ 'Ò&Ø—+‘+˜b“/ˆCØ�m‰m˜A˜qÓ!×,Ñ,Ó.ˆö Ø×Ñ˜e¨%°°c¸9ÀiÐQ\Ð^kÔlØ!ŒØ!ŒØŒ	Ø!ŒØˆŒ	ØˆŒ	æ#ˆC�ˆ:Ð,¨Ð,r8   ©rÅ   rÆ   r�   rg  rh  c               ó<  • U R                   u	  pšp¼pÞnnnUc  U	OUnUc  U
OUnUc  UOUnUc  UOUnUc  UOUnUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       n[        (       a  UR	                  S5      n[
        R                  R                  R                  UUUUUUUUU R                  UUU R                  U R                  U R                  U R                  U R                  S9u  nnnS UUU4 5       u  nnnUUU4$ )Nr   r	   rû   )Ú	logsumexpr·  r¸  r¥  rµ  r¶  r±  r²  rÈ   rÉ   rÊ   c              3  ó`   #   • U  H$  oR                  S S5      R                  5       v •  M&     g7f©r   r	   N©rª  r«  ru  s     r4   rx  Ú/_cudnn_attention_backward_op.<locals>.<genexpr>  ó'   é € Ð'sÒPrÈ1¯©°A°qÓ(9×(DÑ(D×(FÐ(FÒPrùó   ‚,.)r�  rª  r«  r¯  r2  rè   r¬  r­  Ú,_scaled_dot_product_cudnn_attention_backwardrÇ   r±  r²  rÈ   rÉ   rÊ   ©r‚  r’  rÅ   rÆ   r�   rg  rh  r“  rÕ   Úsaved_queryÚ	saved_keyÚsaved_valueÚ	saved_outÚ	saved_lserµ  r¶  r·  r¸  rš  r›  rœ  s                        r4   Ú_cudnn_attention_backward_oprË  Î  s   € ð 	×Ññ p€K˜K°IÈ)ÐU`Ðboð !™=‰K¨e€EØ‘{‰)¨€CØ ™=‰K¨e€EØ‘{‰)¨€CØ‘{‰)¨€Cà×!Ñ! ! QÓ'×2Ñ2Ó4€HØ�O‰O˜A˜qÓ!×,Ñ,Ó.€EØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ�O‰O˜A˜qÓ!×,Ñ,Ó.€EØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€Cà
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€Cß"Ò"Ø�m‰m˜BÓˆô (-§y¡y§~¡~×'bÑ'bØØØØØØØØ#Ø—-‘-ØØØ�i‰iØ�i‰iØ—-‘-Ø—-‘-Ø�i‰ið! (cð (Ñ$€J�˜*ñ$ (tÐQ[Ð]eÐgqÑPrÓ'sÑ$€J�˜*à�x Ð+Ð+r8   c                ój  • U(       a  [        S5      eUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       n[        R                  R
                  R                  UUUUUSUS9u	  nnnnnnnnnUR                  SS5      R                  5       nUR                  SS5      R                  5       nU
(       a7  U R                  XX?UUUUU5	        XPl        X`l	        Xpl
        UU l        UU l        U	(       a  UU4$ U$ )Nz=`enable_gqa` is not yet supported for native flash attention.r   r	   F)rÅ   rÆ   r�   rÈ   rÉ   r§  rÊ   )r½   rª  r«  rè   r¬  r­  Ú#_scaled_dot_product_flash_attentionr}  rÈ   rÉ   rÊ   r±  r²  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   r”  r•  r–  rg  rh  rµ  r¶  r±  r²  r·  r¸  r¹  s                           r4   Ú"_native_flash_attention_forward_oprÎ  	  s-  € ö ÜÐXÓYÐYð �o‰o˜a Ó#×.Ñ.Ó0€GØ�M‰M˜!˜QÓ×*Ñ*Ó,€EØ�o‰o˜a Ó#×.Ñ.Ó0€Gô 	�	‰	�‰×:Ñ:ØØØØØØ#Øð 	;ð 	
ñ ^€Cˆˆi˜ E¨5°+¸}Èoð �-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€Cö Ø×Ñ˜e¨%°c¸9ÀiÐQ\Ð^kÔlØ!ŒØ!ŒØŒ	ØˆŒ	ØˆŒ	æ#ˆC�ˆ:Ð,¨Ð,r8   c               óî  • U R                   u	  pšp¼pÞnnnUc  U	OUnUc  U
OUnUc  UOUnUc  UOUnUc  UOUnUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       nUR                  SS5      R                  5       n[        R                  R
                  R                  UUUUUUUUUUU R                  U R                  U R                  U R                  U R                  S9u  nnnS UUU4 5       u  nnnUUU4$ )Nr   r	   )
r½  r·  r¸  rµ  r¶  r±  r²  rÈ   rÉ   rÊ   c              3  ó`   #   • U  H$  oR                  S S5      R                  5       v •  M&     g7fr¿  rÀ  ru  s     r4   rx  Ú6_native_flash_attention_backward_op.<locals>.<genexpr>k  rÂ  rÃ  )r�  rª  r«  rè   r¬  r­  Ú,_scaled_dot_product_flash_attention_backwardr±  r²  rÈ   rÉ   rÊ   rÅ  s                        r4   Ú#_native_flash_attention_backward_oprÓ  =  sƒ  € ð 	×Ññ p€K˜K°IÈ)ÐU`Ðboð !™=‰K¨e€EØ‘{‰)¨€CØ ™=‰K¨e€EØ‘{‰)¨€CØ‘{‰)¨€Cà×!Ñ! ! QÓ'×2Ñ2Ó4€HØ�O‰O˜A˜qÓ!×,Ñ,Ó.€EØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ�O‰O˜A˜qÓ!×,Ñ,Ó.€EØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€CØ
�-‰-˜˜1Ó
×
(Ñ
(Ó
*€Cä',§y¡y§~¡~×'bÑ'bØØØØØØØØ#ØØØ�i‰iØ�i‰iØ—-‘-Ø—-‘-Ø�i‰ið (cð (Ñ$€J�˜*ñ" (tÐQ[Ð]eÐgqÑPrÓ'sÑ$€J�˜*à�x Ð+Ð+r8   rT  ©r\  c               óh  • Ub  [        S5      eU(       a  [        S5      eSnS nSn[        S XU4 5       5      nUc  UR                  S   S-  nU(       d  Ub$  UR                  R                  S:”  a
  US	:”  a  UOS
n[
        R                  " U5         [        UUUUUUUS	   US   UUU	5      u  nnnnUR                  S	SS5      nS S S 5        U
(       a?  U R                  XUWWW5        XPl
        Xpl        X`l        XÀl        XÐl        Xàl        Xðl        U	(       a  WW4$ W$ ! , (       d  f       Na= f)Nz2`attn_mask` is not yet supported for flash-attn 2.z3`enable_gqa` is not yet supported for flash-attn 2.rÂ   Fc              3  ó8   #   • U  H  oR                   v •  M     g 7fr0   r   ru  s     r4   rx  Ú._flash_attention_forward_op.<locals>.<genexpr>‰  r£  r¤  rû   ç      à¿r   r   ç ÂëþKH´9r	   )r½   r5  rý   r¨  r©  rè   Úset_grad_enabledr#   re  r}  rÈ   rÊ   rÉ   r\  r^  Úalibi_slopesra  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   r\  r^  rÛ  ra  r³  rg  rh  ÚS_dmaskÚ	rng_states                        r4   Ú_flash_attention_forward_oprÞ  q  sJ  € ð  ÑÜÐMÓNÐNÞÜÐNÓOÐOà€GØ€LØ€MÜÑD°¸UÑ0CÓDÓD€Là�}Ø—‘˜B‘ DÑ)ˆö Ð(Ñ4Ð9I×9aÑ9a×9mÑ9mÐpqÓ9qØ!*¨Q£‘I°Eˆ	ä	×	Ò	 Õ	-Ü'BØØØØØØØ˜‰NØ˜‰NØØØó(
Ñ$ˆˆS�'˜9ð �k‰k˜!˜Q Ó"ˆ÷ 
.ö  Ø×Ñ˜e¨%°°c¸9ÔEØ!ŒØŒ	Ø!ŒØ%ŒØŒØ'ÔØ)Ôæ#ˆC�ˆ:Ð,¨Ð,÷5 
.Õ	-ús   Â4D#Ä#
D1c               ó   • U R                   u  pšp¼pÞUc  U	OUnUc  U
OUnUc  UOUnUc  UOUnUc  UOUnUR                  SSS5      R                  5       n[        R                  " U5      [        R                  " U5      [        R                  " U5      nnn[        UUUUUUUUUU R                  U R                  U R                  U R                  S   U R                  S   U R                  U R                  U R                  U5      nUSS UR                  S   24   nUSS UR                  S   24   nUSS UR                  S   24   nUUU4$ )Nr   r	   r   .rû   )r�  re  r«  rè   rl  r"   rÈ   rÊ   rÉ   r\  r^  rÛ  ra  rý   )r‚  r’  rÅ   rÆ   r�   rg  rh  r“  rÕ   rÆ  rÇ  rÈ  rÉ  rÊ  rÝ  rš  r›  rœ  Úlse_ds                      r4   Ú_flash_attention_backward_oprá  ¯  sr  € ð LO×K\ÑK\ÑH€K˜K°IØ ™=‰K¨e€EØ‘{‰)¨€CØ ™=‰K¨e€EØ‘{‰)¨€CØ‘{‰)¨€Cà
�+‰+�a˜˜AÓ
×
)Ñ
)Ó
+€CÜ',×'7Ò'7¸Ó'>Ä×@PÒ@PÐQTÓ@UÔW\×WgÒWgÐhmÓWn˜*�€Jä(ØØØØØØØØØØ�‰Ø�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ×ÑØó%€Eð, ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r8   c               ó  • Ub  [        S5      eU(       a  [        S5      e[        [        R                     nUR                  nUR
                  nUb  Uc  [        S5      eUc  UR                  S   S-  nSnS nSn[        S XU4 5       5      nU(       d  Ub$  UR                  R                  S	:”  a
  US
:”  a  UOSn[        R                  " U5         U" UUUUUUUS
   US	   UUU	5      u  nnnnUR                  S
SS	5      R                  5       nS S S 5        U
(       aB  U R                  XUWWW5        XPl        Xpl        X`l        XÀl        UU l        UU l        UU l        U	(       a  WW4$ W$ ! , (       d  f       Nd= f)Nz<`attn_mask` is not yet supported for flash-attn hub kernels.z=`enable_gqa` is not yet supported for flash-attn hub kernels.zˆFlash attention hub kernels must expose `_wrapped_flash_attn_forward` and `_wrapped_flash_attn_backward` for context parallel execution.rû   rØ  rÂ   Fc              3  ó8   #   • U  H  oR                   v •  M     g 7fr0   r   ru  s     r4   rx  Ú2_flash_attention_hub_forward_op.<locals>.<genexpr>  r£  r¤  r   r   rÙ  r	   )r½   r»   rF   rg   r·   r¸   r  rý   r5  r¨  r©  rè   rÚ  re  r«  r}  rÈ   rÊ   rÉ   r\  r^  rÛ  ra  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   r\  rM  r·   r¸   r^  rÛ  ra  r³  rg  rh  rÜ  rÝ  s                           r4   Ú_flash_attention_hub_forward_oprå  ä  s   € ð  ÑÜÐWÓXÐXÞÜÐXÓYÐYä"Ô#7×#AÑ#AÑB€FØ×2Ñ2ÐØ ×4Ñ4ÐØÑ!Ð%8Ñ%@Üð.ó
ð 	
ð
 �}Ø—‘˜B‘ DÑ)ˆà€GØ€LØ€MÜÑD°¸UÑ0CÓDÓD€LæÐ(Ñ4Ð9I×9aÑ9a×9mÑ9mÐpqÓ9qØ!*¨Q£‘I°Eˆ	ä	×	Ò	 Õ	-Ù'9ØØØØØØØ˜‰NØ˜‰NØØØó(
Ñ$ˆˆS�'˜9ð �k‰k˜!˜Q Ó"×-Ñ-Ó/ˆ÷ 
.ö  Ø×Ñ˜e¨%°°c¸9ÔEØ!ŒØŒ	Ø!ŒØ%ŒØˆŒØ'ˆÔØ)ˆÔæ#ˆC�ˆ:Ð,¨Ð,÷5 
.Õ	-ús   Ã?E1Å1
E?c               óþ  • [         [        R                     n	U	R                  n
U
c  [	        S5      eU R
                  u  p¼pÞnnUc  UOUnUc  UOUnUc  UOUnUc  UOUnUc  UOUnUR                  SSS5      R                  5       n[        R                  " U5      [        R                  " U5      [        R                  " U5      nnnU
" UUUUUUUUUU R                  U R                  U R                  U R                  S   U R                  S   U R                  U R                  U R                   U5      nUSS UR"                  S   24   nUSS UR"                  S   24   nUSS UR"                  S   24   nUUU4$ )NzfFlash attention hub kernels must expose `_wrapped_flash_attn_backward` for context parallel execution.r   r	   r   .rû   )r»   rF   rg   r¸   r  r�  re  r«  rè   rl  rÈ   rÊ   rÉ   r\  r^  rÛ  ra  rý   )r‚  r’  rÅ   rÆ   r�   rg  rh  r“  rÕ   rM  r¸   rÆ  rÇ  rÈ  rÉ  rÊ  rÝ  rš  r›  rœ  ri  s                        r4   Ú _flash_attention_hub_backward_oprç  *  s©  € ô #Ô#7×#AÑ#AÑB€FØ ×4Ñ4ÐØÑ"ÜØtó
ð 	
ð
 LO×K\ÑK\ÑH€K˜K°I¸yØ ™=‰K¨e€EØ‘{‰)¨€CØ ™=‰K¨e€EØ‘{‰)¨€CØ‘{‰)¨€Cà
�+‰+�a˜˜AÓ
×
)Ñ
)Ó
+€CÜ',×'7Ò'7¸Ó'>Ä×@PÒ@PÐQTÓ@UÔW\×WgÒWgÐhmÓWn˜*�€JáØØØØØØØØØØ�‰Ø�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ×ÑØó%	€Að* ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r8   c               óF  • U(       a  [        S5      e[        [        R                     nUR                  nUR
                  nUb  Uc  [        S5      eUc  UR                  S   S-  nSnS nSn[        S XU4 5       5      nU(       d  Ub$  UR                  R                  S:”  a
  US	:”  a  UOS
nUR                  u  nnnnUR                  u  nn  nUb·  [        UUU5      n[        UUXAR                  5      u  u  nnu  nnu  nnUR                  5       R                  SS9R                  5       nUR                  S	S5      nUR                   " S/UR                  SS  Q76 U   nUR                   " S/UR                  SS  Q76 U   n Un!O][#        UUUUR                  5      u  u  nnu  nnu  n!nUR                  S	S5      nUR                  S	S5      nUR                  S	S5      n S n[$        R&                  " U5         U" UUU UUU!UUUUUS	   US   UUU	5      u  n"n#nn$S S S 5        W"R(                  " UU/U"R                  SS  Q76 n%U
(       a‚  U R+                  UUU U"W#W$UU5        UU l        Ub  WOS U l        U!U l        UU l        UU l        UU l        UU l        UU l        XPl        Xpl        X`l         XÀl!        UU l"        UU l#        UU l$        W#R)                  UUU5      RK                  SSS	5      RM                  5       n&U	(       a  U%U&4$ U%$ ! , (       d  f       N÷= f)NzD`enable_gqa` is not yet supported for flash-attn varlen hub kernels.z�Flash attention varlen hub kernels must expose `_wrapped_flash_attn_varlen_forward` and `_wrapped_flash_attn_varlen_backward` for context parallel execution.rû   rØ  rÂ   Fc              3  ó8   #   • U  H  oR                   v •  M     g 7fr0   r   ru  s     r4   rx  Ú9_flash_varlen_attention_hub_forward_op.<locals>.<genexpr>…  r£  r¤  r   r   rÙ  ©Úas_tupler	   )'r½   r»   rF   ri   r·   r¸   r  rý   r5  r¨  r©  r7  r#  rÞ   ÚflattenÚnonzeroÚreshaper   rè   rÚ  r'  r}  r  Ú	indices_kr  r  r  r  r  rn  rÈ   rÊ   rÉ   r\  r^  rÛ  ra  re  r«  )'r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   r\  rM  r·   r¸   r^  rÛ  ra  r³  r  r  rn  ri  r  r  r  r  r  rð  Úquery_packedÚ
key_packedÚvalue_packedr  Ú
out_packedrh  rÝ  rg  Úlse_sps'                                          r4   Ú&_flash_varlen_attention_hub_forward_oprö  c  sj  € ö  ÜÐ_Ó`Ð`ä"Ô#7×#HÑ#HÑI€FØ×2Ñ2ÐØ ×4Ñ4ÐØÑ!Ð%8Ñ%@ÜðTó
ð 	
ð
 �}Ø—‘˜B‘ DÑ)ˆà€GØ€LØ€MÜÑD°¸UÑ0CÓDÓD€LæÐ(Ñ4Ð9I×9aÑ9a×9mÑ9mÐpqÓ9qØ!*¨Q£‘I°Eˆ	à*/¯+©+Ñ'€J�	˜9 aØŸ)™)Ñ€A€z�1�aàÑÜ(¨°JÀ
ÓKˆ	ä<¸ZÈÐT]×_kÑ_kÓlñ 	H‰ˆˆIÑ4˜ |Ñ6G°q¸,ð ×%Ñ%Ó'×/Ñ/¸Ð/Ð?×GÑGÓIˆ	Ø—}‘} Q¨Ó*ˆØ—[’[ Ð4 c§i¡i°° mÒ4°YÑ?ˆ
Ø—}’} RÐ:¨%¯+©+°a°b¨/Ò:¸9ÑEˆØ ‰ô @À
ÈIÐWaÐch×coÑcoÓpñ 	S‰ˆˆIÑ4˜ |Ñ6R°|À\ð —}‘} Q¨Ó*ˆØ—[‘[  AÓ&ˆ
Ø—}‘} Q¨Ó*ˆØˆ	ä	×	Ò	 Õ	-Ù(:ØØØØØØØØØØØ˜‰NØ˜‰NØØØó)
Ñ%ˆ
�C˜˜I÷ 
.ð& �/Š/˜* iÐ
G°*×2BÑ2BÀ1À2Ð2FÒ
G€CæØ×ÑØ˜* l°JÀÀYÐP\Ð^jô	
ð "ˆŒØ%.Ñ%:™	ÀˆŒØ'ˆÔØ'ˆÔØ#ˆŒØ!ˆŒØ#ˆŒØ!ˆŒØ!ŒØŒ	Ø!ŒØ%ŒØˆŒØ'ˆÔØ)ˆÔð �X‰X�i ¨YÓ7×?Ñ?ÀÀ1ÀaÓH×SÑSÓU€Fæ&ˆC�ˆ=Ð/¨CÐ/÷Y 
.Õ	-ús   Ç?"LÌ
L c               ó°  • [         [        R                     n	U	R                  n
U
c  [	        S5      eU R
                  u  p¼pÞnnnnUc  UOUnUc  UOUnUc  UOUnUc  UOUR                  SS5      nUc  UO:UR                  SSS5      R                  U R                  S5      R                  5       nUR                  SS5      n[        R                  " U5      [        R                  " U5      [        R                  " U5      nnnU
" UUUUUUUUUUUU R                  U R                  U R                  U R                   U R"                  U R$                  S   U R$                  S   U R&                  U R(                  U R*                  U5      nUR,                  " U R.                  U R0                  /UR2                  SS  Q76 nU R4                  bY  [7        UU R8                  U R.                  U R:                  5      n[7        UU R8                  U R.                  U R:                  5      nOjUR,                  " U R.                  U R:                  /UR2                  SS  Q76 nUR,                  " U R.                  U R:                  /UR2                  SS  Q76 nUSS UR2                  S   24   nUSS UR2                  S   24   nUSS UR2                  S   24   nUUU4$ )NztFlash attention varlen hub kernels must expose `_wrapped_flash_attn_varlen_backward` for context parallel execution.r   r   r	   rû   .)r»   rF   ri   r¸   r  r�  rí  re  rï  rn  r«  rè   rl  r  r  rÈ   rÊ   rÉ   r\  r^  rÛ  ra  r'  r  r  rý   r  r,  rð  r  )r‚  r’  rÅ   rÆ   r�   rg  rh  r“  rÕ   rM  r¸   rÆ  rÇ  rÈ  rÉ  rÊ  rÝ  r  r  rñ  rò  ró  rô  Úgrad_out_packedrš  r›  rœ  ri  s                               r4   Ú'_flash_varlen_attention_hub_backward_oprù  Ï  sÀ  € ô #Ô#7×#HÑ#HÑI€FØ ×4Ñ4ÐØÑ"Üð.ó
ð 	
ð 	×Ññ e€K˜K°I¸yÈ,ÐXdð #(¡-‘;°U€LØ!™k‘¨s€JØ"'¡-‘;°U€LØ!™k‘¨s¯{©{¸1¸aÓ/@€Jà‘{‰)¨¯©°A°q¸!Ó(<×(DÑ(DÀSÇ]Á]ÐTVÓ(W×(bÑ(bÓ(d€Cà×&Ñ& q¨!Ó,€Oä×Ò˜Ó&Ü×Ò˜Ó$Ü×Ò˜Ó&ð %�€Jñ 	ØØØØØØØØØØØØ×ÑØ×ÑØ�‰Ø�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ×ÑØó-	€Að2 —’ §¡°·±ÐVÀ×AQÑAQÐRSÐRTÐAUÒV€Jà
‡}�}Ñ Ü# H¨c¯m©m¸S¿^¹^ÈSÏ^É^Ó\ˆÜ% j°#·-±-ÀÇÁÐQT×Q_ÑQ_Ó`‰
à—=’= §¡°·±ÐUÀ(Ç.Á.ÐQRÐQSÐBTÒUˆØ—_’_ S§^¡^°S·^±^Ð[Àj×FVÑFVÐWXÐWYÐFZÒ[ˆ
à˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r8   ©r\  r^  r_  r`  ra  rb  c               ó.  • Ub  [        S5      eUS:w  a  [        S5      eU(       a  [        S5      e[        [        R                     nUR                  nUc  [        S5      eUc  UR                  S   S-  nU" UUUS S S S S S S S S S S S S S S S S S S S U4UUS   US	   SUUUUS
.6tnnnUR                  SSS	5      R                  5       nU
(       a:  U R                  XUUU5        Xpl
        X`l        XÀl        XÐl        UU l        UU l        U	(       a  UU4$ U$ )Nz>`attn_mask` is not yet supported for flash-attn 3 hub kernels.rÂ   z>`dropout_p` is not yet supported for flash-attn 3 hub kernels.z?`enable_gqa` is not yet supported for flash-attn 3 hub kernels.ztFlash attention 3 hub kernels must expose `flash_attn_interface._flash_attn_forward` for context parallel execution.rû   rØ  r   r   )rW  Úwindow_size_leftÚwindow_size_rightr]  r^  r_  r`  rb  r	   )r½   r»   rF   rm   r·   r  rý   re  r«  r}  rÊ   rÉ   r\  r^  ra  rb  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   r\  r^  r_  r`  ra  rb  rM  r·   rg  Úsoftmax_lseri  rh  s                           r4   Ú!_flash_attention_3_hub_forward_oprÿ  !  ss  € ð* ÑÜÐYÓZÐZØ�CÓÜÐYÓZÐZÞÜÐZÓ[Ð[ä"Ô#7×#DÑ#DÑE€FØ×2Ñ2ÐØÑ!Üð.ó
ð 	
ð
 �}Ø—‘˜B‘ DÑ)ˆá-ØØØØØØØØØØØØØØØØØØØØØØØØð1!ð2 Ø$ Q™Ø% a™.ØØØØØòA!Ð€Cˆ�qðJ ×
Ñ
˜a  AÓ
&×
1Ñ
1Ó
3€CæØ×Ñ˜e¨%°°cÔ:ØŒ	Ø!ŒØ%ŒØŒØ)ˆÔØ!ˆŒæ#ˆC�ˆ:Ð,¨Ð,r8   c               óð  • [         [        R                     n	U	R                  n
U
c  [	        S5      eU R
                  u  p¼pÞnUc  UOUnUc  UOUnUc  UOUnUc  UOUnUc  UOUnUR                  SSS5      R                  5       n[        R                  " U5      n[        R                  " U5      n[        R                  " U5      nU
" UUUUUUS S S S S S UUUU R                  U R                  U R                  S   U R                  S   U R                  U R                  U R                  5        USS UR                   S   24   nUSS UR                   S   24   nUSS UR                   S   24   nUUU4$ )NzuFlash attention 3 hub kernels must expose `flash_attn_interface._flash_attn_backward` for context parallel execution.r   r	   r   .rû   )r»   rF   rm   r¸   r  r�  re  r«  rè   rl  rÊ   rÉ   r\  r^  ra  rb  rý   )r‚  r’  rÅ   rÆ   r�   rg  rh  r“  rÕ   rM  r¸   rÆ  rÇ  rÈ  rÉ  rÊ  rþ  rš  r›  rœ  s                       r4   Ú"_flash_attention_3_hub_backward_opr  {  s¬  € ô #Ô#7×#DÑ#DÑE€FØ ×4Ñ4ÐØÑ"Üð.ó
ð 	
ð AD×@QÑ@QÑ=€K˜K°IØ ™=‰K¨e€EØ‘{‰)¨€CØ ™=‰K¨e€EØ‘{‰)¨€CØ‘{‰)¨€Cà—+‘+˜a  AÓ&×1Ñ1Ó3€KÜ×!Ò! %Ó(€JÜ×Ò Ó$€HÜ×!Ò! %Ó(€JáØØØØØØØØØØØØØØØØ�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ�‰ô-ð2 ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r8   c               óŠ  • US:w  a  [        S5      eU(       a  [        S5      e[        [        R                     nUR                  nUR
                  nUb  Uc  [        S5      eUc  UR                  S   S-  nUR                  u  nnnnUR                  u  nn  nUb·  [        UUU5      n[        UUXAR                  5      u  u  nnu  nnu  nnUR                  5       R                  SS9R                  5       nUR                  S	S
5      nUR                  " S/UR                  SS  Q76 U   n UR                  " S/UR                  SS  Q76 U   n!Un"O][        UUUUR                  5      u  u  nnu  nnu  n"nUR                  S	S
5      nUR                  S	S
5      n UR                  S	S
5      n!S nU" UU U!S S S S UUS S S U"US S S S S S S S S U4UUS	   US
   S	USS UUUS.
6tn#n$nU#R                  " UU/U#R                  S
S  Q76 n%U
(       az  U R!                  UU U!U#U$UU5        UU l        Ub  WOS U l        U"U l        UU l        UU l        UU l        UU l        UU l        Xpl        X`l        XÀl        XÐl        UU l        UU l        U$R                  UUU5      R?                  S
SS	5      RA                  5       n&U	(       a  U%U&4$ U%$ )NrÂ   zE`dropout_p` is not yet supported for flash-attn 3 varlen hub kernels.zF`enable_gqa` is not yet supported for flash-attn 3 varlen hub kernels.z«Flash attention 3 varlen hub kernels must expose `flash_attn_interface._flash_attn_forward` and `flash_attn_interface._flash_attn_backward` for context parallel execution.rû   rØ  Frë  r   r   r	   T)
rW  rü  rý  r]  r^  Úrotary_interleavedÚscheduler_metadatar_  r`  rb  )!r½   r»   rF   rn   r·   r¸   r  rý   r7  r#  rÞ   rí  rî  rï  r   r'  r}  r  rð  r  r  r  r  r  rn  rÊ   rÉ   r\  r^  ra  rb  re  r«  )'r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   r\  r^  r_  r`  ra  rb  rM  r·   r¸   r  r  rn  ri  r  r  r  r  r  rð  rñ  rò  ró  r  rô  rþ  rg  rõ  s'                                          r4   Ú(_flash_attention_3_varlen_hub_forward_opr  »  sI  € ð* �CÓÜÐ`ÓaÐaÞÜÐaÓbÐbä"Ô#7×#KÑ#KÑL€FØ×2Ñ2ÐØ ×4Ñ4ÐØÑ!Ð%8Ñ%@ÜðZó
ð 	
ð
 �}Ø—‘˜B‘ DÑ)ˆà*/¯+©+Ñ'€J�	˜9 aØŸ)™)Ñ€A€z�1�aàÑÜ(¨°JÀ
ÓKˆ	ä<¸ZÈÐT]×_kÑ_kÓlñ 	H‰ˆˆIÑ4˜ |Ñ6G°q¸,ð ×%Ñ%Ó'×/Ñ/¸Ð/Ð?×GÑGÓIˆ	Ø—}‘} Q¨Ó*ˆØ—[’[ Ð4 c§i¡i°° mÒ4°YÑ?ˆ
Ø—}’} RÐ:¨%¯+©+°a°b¨/Ò:¸9ÑEˆØ ‰ô @À
ÈIÐWaÐch×coÑcoÓpñ 	S‰ˆˆIÑ4˜ |Ñ6R°|À\ð —}‘} Q¨Ó*ˆØ—[‘[  AÓ&ˆ
Ø—}‘} Q¨Ó*ˆØˆ	á"4ØØØØØØØØØØØØØØØØØØØØØØØØð1##ð2 Ø$ Q™Ø% a™.ØØØØØØØòE##Ð€J�˜aðJ �/Š/˜* iÐ
G°*×2BÑ2BÀ1À2Ð2FÒ
G€CæØ×ÑØ˜* l°JÀÈ\Ð[gô	
ð "ˆŒØ%.Ñ%:™	ÀˆŒØ'ˆÔØ'ˆÔØ#ˆŒØ!ˆŒØ#ˆŒØ!ˆŒØŒ	Ø!ŒØ%ŒØŒØ)ˆÔØ!ˆŒð ×Ñ˜i¨°YÓ?×GÑGÈÈ1ÈaÓP×[Ñ[Ó]€Fæ&ˆC�ˆ=Ð/¨CÐ/r8   c               óš  • [         [        R                     n	U	R                  n
U
c  [	        S5      eU R
                  u  p¼pÞnnnUc  UOUnUc  UOUnUc  UOUnUc  UOUR                  SS5      nUc  UO:UR                  SSS5      R                  U R                  S5      R                  5       nUR                  SS5      n[        R                  " U5      [        R                  " U5      [        R                  " U5      nnnU
" UUUUUUUUS S U R                  U R                  UUUU R                  U R                   U R"                  S   U R"                  S   U R$                  U R&                  U R(                  5        UR*                  " U R,                  U R.                  /UR0                  SS  Q76 nU R2                  bY  [5        UU R6                  U R,                  U R8                  5      n[5        UU R6                  U R,                  U R8                  5      nOjUR*                  " U R,                  U R8                  /UR0                  SS  Q76 nUR*                  " U R,                  U R8                  /UR0                  SS  Q76 nUSS UR0                  S   24   nUSS UR0                  S   24   nUSS UR0                  S   24   nUUU4$ )Nz|Flash attention 3 varlen hub kernels must expose `flash_attn_interface._flash_attn_backward` for context parallel execution.r   r   r	   rû   .)r»   rF   rn   r¸   r  r�  rí  re  rï  rn  r«  rè   rl  r  r  rÊ   rÉ   r\  r^  ra  rb  r'  r  r  rý   r  r,  rð  r  )r‚  r’  rÅ   rÆ   r�   rg  rh  r“  rÕ   rM  r¸   rÆ  rÇ  rÈ  rÉ  rÊ  r  r  rñ  rò  ró  rô  rþ  rø  rš  r›  rœ  s                              r4   Ú)_flash_attention_3_varlen_hub_backward_opr  7  s²  € ô #Ô#7×#KÑ#KÑL€FØ ×4Ñ4ÐØÑ"Üð.ó
ð 	
ð ]`×\mÑ\mÑY€K˜K°I¸|È\Ø"'¡-‘;°U€LØ!™k‘¨s€JØ"'¡-‘;°U€LØ!™k‘¨s¯{©{¸1¸aÓ/@€Jà"™{‘)°·±¸A¸qÀ!Ó0D×0LÑ0LÈSÏ]É]Ð\^Ó0_×0jÑ0jÓ0l€Kà×&Ñ& q¨!Ó,€Oä×Ò˜Ó&Ü×Ò˜Ó$Ü×Ò˜Ó&ð %�€Jñ ØØØØØØØØØØØ×ÑØ×ÑØØØØ�	‰	Ø�‰Ø�‰˜ÑØ�‰˜ÑØ�‰Ø×ÑØ�‰ô-ð2 —’ §¡°·±ÐVÀ×AQÑAQÐRSÐRTÐAUÒV€Jà
‡}�}Ñ Ü# H¨c¯m©m¸S¿^¹^ÈSÏ^É^Ó\ˆÜ% j°#·-±-ÀÇÁÐQT×Q_ÑQ_Ó`‰
à—=’= §¡°·±ÐUÀ(Ç.Á.ÐQRÐQSÐBTÒUˆØ—_’_ S§^¡^°S·^±^Ð[Àj×FVÑFVÐWXÐWYÐFZÒ[ˆ
à˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€JØ˜Ð1˜xŸ~™~¨bÑ1Ð1Ð1Ñ2€HØ˜CÐ!5 8§>¡>°"Ñ#5Ð!5Ð5Ñ6€Jà�x Ð+Ð+r8   c           
     óÚ   • Ub  [        S5      eUS:”  a  [        S5      eU(       a  [        S5      e[        UUUSUUU	S9nS nU	(       a  UtpÍnUR                  SSS	5      nU	(       a  XÍ4$ U$ ©
Nz4`attn_mask` is not yet supported for Sage attention.rÂ   z4`dropout_p` is not yet supported for Sage attention.z5`enable_gqa` is not yet supported for Sage attention.ÚNHD©rU  rØ   rÙ   Útensor_layoutrÉ   Úsm_scalerƒ  r   r	   r   )r½   r$   re  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   rg  rh  ri  s                  r4   Ú_sage_attention_forward_opr  ‡  s‹   € ð ÑÜÐOÓPÐPØ�3ƒÜÐOÓPÐPÞÜÐPÓQÐQä
Ø
Ø
Ø
ØØØØñ€Cð €CÞØˆˆ�1Ø�k‰k˜!˜Q Ó"ˆæ#ˆCˆ:Ð,¨Ð,r8   c           
     ó2  • Ub  [        S5      eUS:”  a  [        S5      eU(       a  [        S5      e[        [        R                     R                  nU" UUUSUUU	S9nS nU	(       a%  UtpÞnUR                  SSS	5      R                  5       nU	(       a  XÞ4$ U$ r	  )r½   r»   rF   ry   r´   re  r«  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  r„  rË   r3   rg  rh  ri  s                   r4   Ú_sage_attention_hub_forward_opr  ­  s«   € ð ÑÜÐOÓPÐPØ�3ƒÜÐOÓPÐPÞÜÐPÓQÐQä Ô!5×!>Ñ!>Ñ?×IÑI€DÙ
Ø
Ø
Ø
ØØØØñ€Cð €CÞØˆˆ�1Ø�k‰k˜!˜Q Ó"×-Ñ-Ó/ˆæ#ˆCˆ:Ð,¨Ð,r8   c                ó   • [        S5      e)Nz4Backward pass is not implemented for Sage attention.©ÚNotImplementedError©r‚  r’  r“  s      r4   Ú_sage_attention_backward_opr  Õ  s   € ô
 ÐTÓ
UÐUr8   c                ó®  • Ub   [         R                  " US:g  5      (       a  S nUGb-  UR                  S:X  a«  UR                  S   U R                  S   :X  a‹  UR                  S   UR                  S   :X  ak  UR                  S   U R                  S   UR                  S   pTnUR	                  S5      R                  X4U5      R	                  S5      R                  5       nORUR                  S:X  aB  UR                  SS S:X  a/  UR                  SSU R                  S   S5      R                  5       nUR                  [         R                  5      ) nU$ )Nr   r	   r   r/  r.  )r   r   rû   )	rè   Úallr1  rý   r2  r3  r«  Útor©   )rÅ   rÆ   rÇ   r  r  r  s         r4   Ú_maybe_modify_attn_mask_npur  Ý  s%  € àÑ¤§¢¨9¸©>×!:Ñ!:Øˆ	ð ÒØ�>‰>˜QÓ 9§?¡?°1Ñ#5¸¿¹ÀQ¹Ó#GÈIÏOÉOÐ\]ÑL^Ðbe×bkÑbkÐlmÑbnÓLnØ09·±ÀÑ0BÀEÇKÁKÐPQÁNÐTW×T]ÑT]Ð^_ÑT` :ˆJØ!×+Ñ+¨AÓ.×5Ñ5°jÈZÓX×bÑbÐcdÓe×pÑpÓr‰IØ�^‰^˜qÓ  Y§_¡_°Q°qÐ%9¸VÓ%CØ!×(Ñ(¨¨R°·±¸Q±ÀÓD×OÑOÓQˆIà—\‘\¤%§*¡*Ó-Ð-ˆ	àÐr8   c                óè   • U	(       a  [        S5      e[        XU5      n[        UUUUR                  S5      USS Uc&  S[        R
                  " UR                  S   5      -  OUSSSU-
  SSS	9S   nU$ )
NúANPU attention backend does not support setting `return_lse=True`.r	   ÚBSNDç      ð?rû   é   Fr   ©	Ú
atten_maskÚinput_layoutÚpserÊ   Úpre_tockensÚnext_tockensÚ	keep_probÚsyncÚinner_precise)r½   r  r*   r4  ÚmathÚsqrtrý   r�  s                r4   Ú_npu_attention_forward_opr*  ð  s‰   € ö ÜÐ\Ó]Ð]ä+¨E¸	ÓB€Iä
ØØØØ�
‰
�1‹ØØØØ27±-ˆc”D—I’I˜eŸk™k¨"™oÓ.Ò.ÀUØØØ˜	‘/ØØñð ñ	€Cð  €Jr8   c                ó   • [        S5      e)Nz:Backward pass is not implemented for Npu Fusion Attention.r  )r‚  r’  r“  rÕ   s       r4   Ú_npu_attention_backward_opr,    s   € ô ÐZÓ
[Ð[r8   c                ód   • [        U [        R                  5      (       a  U R                  5       n U $ r0   )Ú
isinstanceÚfuncolÚAsyncCollectiveTensorÚwait)Útensors    r4   Ú_wait_tensorr3  '  s%   € Ü�&œ&×6Ñ6×7Ñ7Ø—‘“ˆØ€Mr8   c                ó¨   • U R                   nU R                  5       n [        R                  " U S S U5      n U R	                  U5      n [        U 5      n U $ r0   )rý   rí  r/  Úall_to_all_singlerï  r3  )rw  Úgrouprý   s      r4   Ú_all_to_all_singler7  -  sJ   € Ø�G‰G€Eð
 	
�	‰	‹€AÜ× Ò   D¨$°Ó6€AØ	�	‰	�%Ó€AÜ�Q‹€AØ€Hr8   c                ó  • [         R                  R                  U5      nUS:X  a¦  US:X  a   U R                  u  pVpxXd-  n	Xt-  n
U R	                  XVXJU5      R                  SS5      R                  5       nUS:”  a
  [        X³S9nOUnUR	                  X•X¨5      R                  SSSS5      R                  5       nUR	                  XYX¨5      nU$ US:X  a¬  US:X  a¦  U R                  u  pYp¨X¤-  nX”-  nU R	                  XTXjU5      R                  SSSSS5      R	                  XJXeU5      nUS:”  a  [        X³5      nOUnUR	                  XvXX5      R                  SS5      R                  5       nUR	                  XVXx5      nU$ [        S5      e)u  
Perform dimension sharding / reassembly across processes using _all_to_all_single.

This utility reshapes and redistributes tensor `x` across the given process group, across sequence dimension or
head dimension flexibly by accepting scatter_idx and gather_idx.

Args:
    x (torch.Tensor):
        Input tensor. Expected shapes:
        - When scatter_idx=2, gather_idx=1: (batch_size, seq_len_local, num_heads, head_dim)
        - When scatter_idx=1, gather_idx=2: (batch_size, seq_len, num_heads_local, head_dim)
    scatter_idx (int) :
        Dimension along which the tensor is partitioned before all-to-all.
    gather_idx (int):
        Dimension along which the output is reassembled after all-to-all.
    group :
        Distributed process group for the Ulysses group.

Returns:
    torch.Tensor: Tensor with globally exchanged dimensions.
        - For (scatter_idx=2 â†’ gather_idx=1): (batch_size, seq_len, num_heads_local, head_dim)
        - For (scatter_idx=1 â†’ gather_idx=2): (batch_size, seq_len_local, num_heads, head_dim)
r	   r   r   ©r6  r.  r/  z<Invalid scatter/gather indices for _all_to_all_dim_exchange.)
rè   ÚdistributedÚget_world_sizerý   rï  rª  r«  r7  re  r½   )rw  Úscatter_idxÚ
gather_idxr6  Úgroup_world_sizer  Úseq_len_localrn  ro  r*  Únum_heads_localÚx_temprg  r+  s                 r4   Ú_all_to_all_dim_exchangerB  :  sš  € ô0 ×(Ñ(×7Ñ7¸Ó>Ðà�aÓ˜J¨!›Oð :;¿¹Ñ6ˆ
 9ØÑ2ˆØ#Ñ7ˆð �I‰I�jÐ1AÐT\Ó]ß‰Y�q˜!‹_ß‰Z‹\ð 	ð ˜aÓÜ$ VÑ9‰CàˆCà�k‰k˜'¨ÓI×QÑQÐRSÐUVÐXYÐ[\Ó]×hÑhÓjˆØ�k‰k˜*¨ÓIˆØˆ
Ø	˜Ó	˜j¨A›oð :;¿¹Ñ6ˆ
˜_Ø#Ñ6ˆ	ØÑ3ˆð �I‰I�j°MÐT\Ó]ß‰W�Q˜˜1˜a Ó#ß‰WÐ%¸ÐS[Ó\ð 	ð ˜aÓÜ'¨Ó6‰FàˆFØ—‘ 	¸*ÓO×YÑYÐZ[Ð]^Ó_×jÑjÓlˆØ—‘ 
¸9ÓOˆØˆäÐWÓXÐXr8   c                  ó<   • \ rS rSrSr\SS j5       r\S 5       rSrg)ÚSeqAllToAllDimiƒ  zƒ
all_to_all operation for unified sequence parallelism. uses _all_to_all_dim_exchange, see _all_to_all_dim_exchange
for more info.
c                ó>   • Xl         X0l        X@l        [        X#XA5      $ r0   )r6  Ú
scatter_idÚ	gather_idrB  )r‚  r6  ÚinputrF  rG  s        r4   ÚforwardÚSeqAllToAllDim.forward‰  s   € àŒ	Ø#ŒØ!ŒÜ'¨¸9ÓLÐLr8   c                óz   • [         R                  U R                  UU R                  U R                  5      nS US S 4$ r0   )rD  Úapplyr6  rG  rF  )r‚  r‹  Ú
grad_inputs      r4   ÚbackwardÚSeqAllToAllDim.backward�  s;   € ä#×)Ñ)Ø�I‰IØØ�M‰MØ�N‰Nó	
ˆ
ð �j $¨Ð-Ð-r8   r1   N)r	   r   )	rb   rc   rd   re   r¹   ÚstaticmethodrI  rN  r€   r1   r8   r4   rD  rD  ƒ  s1   † ñð
 óMó ðMð ñ.ó ó.r8   rD  c                óÖ   • [         R                  " US9nSnX-  S:w  aI  X1U-  -
  nX-   U-  nXE:  d   SU SU 35       e[        R                  " U SSSU45      R	                  5       n X4$ )zÜMaybe pad the head dimension to be divisible by world_size.
x: torch.Tensor, shape (B, S_LOCAL, H, D) H: int, original global head num return: tuple[torch.Tensor, int], padded
tensor (B, S_LOCAL, H + H_PAD, D) and H_PAD
r9  r   úPadding head num ú( should be less than new local head num )Údistr;  ÚFÚpadr«  )rw  ÚHr6  Ú
world_sizeÚH_PADÚNEW_H_LOCALs         r4   Ú_maybe_pad_qkv_headr[  œ  sˆ   € ô
 ×$Ò$¨5Ñ1€JØ€EØ�~˜ÓØ *™nÑ-ˆØ‘y ZÑ/ˆð Ó"ÐtÐ&7¸°wÐ>fÐgrÐfsÐ$tÓtÐ"Ü�EŠE�!�a˜˜A˜uÐ%Ó&×1Ñ1Ó3ˆØˆ8€Or8   c                ó²   • [         R                  " US9n[         R                  " US9nUS:”  a  X4S-
  :X  a  U SS2SS2SU* 2SS24   n U R                  5       $ )z¶Maybe unpad the head dimension.
x: torch.Tensor, shape (B, S_GLOBAL, H_LOCAL + H_PAD, D) H_PAD: int, head padding num return: torch.Tensor,
unpadded tensor (B, S_GLOBAL, H_LOCAL, D)
r9  r   r   N)rT  Úget_rankr;  r«  )rw  rY  r6  ÚrankrX  s        r4   Ú_maybe_unpad_qkv_headr_  ­  sW   € ô
 �=Š=˜uÑ%€DÜ×$Ò$¨5Ñ1€Jàˆqƒy�T¨!™^Ó+ØŠa’�G�e�V�GšQÐÑˆØ�<‰<‹>Ðr8   c                ó  • Uc  U S4$ [         R                  " US9n[         R                  " US9nSnX-  S:w  aQ  XAU-  -
  nX-   U-  nXV:  d   SU SU 35       eX4S-
  :X  a)  [        R                  " U SSSU45      R                  5       n X4$ )zêMaybe pad the head dimension to be divisible by world_size.
x: torch.Tensor, shape (B, S_GLOBAL, H_LOCAL, D) H: int, original global head num return: tuple[torch.Tensor, int],
padded tensor (B, S_GLOBAL, H_LOCAL + H_PAD, D) and H_PAD
r   r9  rR  rS  r   )rT  r]  r;  rU  rV  r«  )rw  rW  r6  r^  rX  rY  rZ  s          r4   Ú_maybe_pad_o_headra  º  s®   € ð
 	�yØ�!ˆtˆä�=Š=˜uÑ%€DÜ×$Ò$¨5Ñ1€JØ€Eà�~˜Óð  *™nÑ-ˆØ‘y ZÑ/ˆØÓ"ÐtÐ&7¸°wÐ>fÐgrÐfsÐ$tÓtÐ"Ø ‘>Ó!Ü—’�a˜!˜Q  5Ð)Ó*×5Ñ5Ó7ˆAØˆ8€Or8   c                óR   • US:”  a  U SS2SS2SU* 2SS24   n U R                  5       $ )z¶Maybe unpad the head dimension.
x: torch.Tensor, shape (B, S_LOCAL, H_GLOBAL + H_PAD, D) H_PAD: int, head padding num return: torch.Tensor,
unpadded tensor (B, S_LOCAL, H_GLOBAL, D)
r   N)r«  )rw  rY  r6  s      r4   Ú_maybe_unpad_o_headrc  Ñ  s0   € ð
 ˆqƒyØŠa’�G�e�V�GšQÐÑˆØ�<‰<‹>Ðr8   c                ó’   • [        U R                  5      S:X  d   S5       e0 nU R                  S   US'   U R                  S   US'   U$ )Nr/  zEQuery tensor must be 4-dimensional of shape (B, S_LOCAL, H_GLOBAL, D)r	   ÚNUM_QO_HEADr   Ú	Q_S_LOCAL)Úlenrý   )rÅ   rÕ   Úextra_kwargss      r4   Úulysses_anything_metadatari  Û  sN   € äˆu�{‰{Ó˜qÓ ÐiÐ"iÓiÐ Ø€LØ"'§+¡+¨a¡.€L�ÑØ %§¡¨A¡€L�ÑàÐr8   c                óv  ^ ^^• [         R                  " TS9nT R                  u  pEpg[        T UT5      u  m mUT-   U-  nT R	                  XEX8U5      R                  SSSSS5      R                  5       m U/U-  n	[        UT5      n
T R                  SS5      m [        R                  " T X©T5      m SUUU 4S jjnU$ )	zc
x: torch.Tensor, shape (B, S_LOCAL, H, D) return: Callable that returns (B, S_GLOBAL, H_LOCAL, D)
r9  r	   r   r   r.  r/  c                 ó|   >• [        T5      mTR                  SSSS5      R                  5       m[        TT T5      mT$ )Nr   r   r	   r.  )r3  re  r«  r_  )rY  r6  rw  s   €€€r4   r1  Ú-all_to_all_single_any_qkv_async.<locals>.waitû  s>   ø€ ä˜‹Oˆð �I‰I�a˜˜A˜qÓ!×,Ñ,Ó.ˆÜ! ! U¨EÓ2ˆØˆr8   ©r«   rï   )rT  r;  rý   r[  rï  re  r«  r   rí  r/  r5  )rw  r6  rÕ   rX  ÚBÚS_LOCALrW  ÚDÚH_LOCALÚinput_split_sizesÚoutput_split_sizesr1  rY  s   ``          @r4   Úall_to_all_single_any_qkv_asyncrt  å  sÂ   ú€ ô ×$Ò$¨5Ñ1€JØ—w‘wÑ€A�Ü" 1 a¨Ó/�H€A€uØ�5‰y˜ZÑ'€Gà	�	‰	�!˜j°1Ó5×=Ñ=¸aÀÀAÀqÈ!ÓL×WÑWÓY€Aà ˜	 JÑ.Ðô -¨W°eÓ<ÐØ	�	‰	�!�Q‹€AÜ× Ò  Ð$6È5ÓQ€A÷ñ ð €Kr8   c           	     ó€  ^ ^^	^
^^^^• UR                  SS5      n[        R                  " TS9m[        T UT5      u  m mT R                  nUu  m	nmm
UR                  S5      m[        TT5      nT R                  SSSS5      R                  5       m T/T-  n[        R                  " T XvT5      m S
U	U
UUUUUU 4S	 jjnU$ )zj
x: torch.Tensor, shape (B, S_GLOBAL, H_LOCAL, D) return: Callable that returns (B, S_LOCAL, H_GLOBAL, D)
re  Nr9  rf  r   r   r	   r.  c                 óÖ   >• [        T5      mTR                  TTT TT5      mTR                  SSSSS5      R                  5       mTR                  T TTT-  T5      m[	        TTT5      mT$ )Nr	   r   r   r.  r/  )r3  rï  re  r«  rc  )rn  rp  rq  rY  ro  r6  rX  rw  s   €€€€€€€€r4   r1  Ú+all_to_all_single_any_o_async.<locals>.wait#	  sm   ø€ ä˜‹OˆØ�I‰I�j '¨1¨g°qÓ9ˆØ�I‰I�a˜˜A˜q !Ó$×/Ñ/Ó1ˆØ�I‰I�a˜ *¨wÑ"6¸Ó:ˆÜ  5¨%Ó0ˆØˆr8   rm  )
rÎ   rT  r;  ra  rý   r   re  r«  r/  r5  )rw  r6  rÕ   rW  rý   ÚS_GLOBALrr  rs  r1  rn  rp  rq  rY  ro  rX  s   ``       @@@@@@r4   Úall_to_all_single_any_o_asyncry  	  s»   ÿ€ ð 	�
‰
�= $Ó'€AÜ×$Ò$¨5Ñ1€Jä   A uÓ-�H€A€uØ�G‰G€EØ %Ñ€Qˆ�'˜1ð �j‰j˜Ó%€GÜ+¨G°UÓ;ÐØ	�	‰	�!�Q˜˜1Ó×(Ñ(Ó*€AØ!˜ ZÑ/ÐÜ× Ò  Ð$6È5ÓQ€A÷ö ð €Kr8   c                  ót   • \ rS rSr\ S                     SS jj5       r\    SS j5       rSrg)	ÚTemplatedRingAttentioni/	  Nc                óp  • UR                   R                  nUR                   R                  nUR                   R                  nUS-   U-  nS =nnX l        X°l        UR                  U l        UR                  U l        XÀl	        [        R                  " UR                  5       UR                  5       /5      R                  5       n[        R                  " USUR!                  5       S9nUR#                  U5      n[%        U5       GHK  nUS:”  aE  UU   nUR'                  5       nUS U R)                  U5      nUUS  R)                  U5      nUS-   U-  nU
" U UUUUUUUUSUS:H  US9u  nnUR                   R*                  (       a>  UR-                  [        R.                  5      nUR-                  [        R.                  5      nUR0                  S:X  a  UR3                  S5      nUbd  U[        R4                  R6                  R9                  UU-
  5      UU-
  -  -
  nU[        R4                  R6                  R;                  UU-
  5      -
  nUnUnGMN     WR-                  UR<                  5      nWR?                  S5      nURA                  5       U l!        URA                  5       U l"        U	(       a  UU4$ U$ )Nr   r   ©Ú
gather_dimr6  T©r„  rË   r.  rû   )#r¨  Ú
_ring_meshÚ_ring_local_rankÚring_degreeÚ
forward_opÚbackward_oprý   Úq_shapeÚkv_shaperË   rè   Úcatrí  r«  r/  Úall_gather_tensorÚ	get_groupÚchunkÚrangeÚnumelÚ
reshape_asÚconvert_to_fp32r  Úfloat32r1  r2  r~  r  ÚsigmoidÚ
logsigmoidrß   r°  ÚdetachÚring_outÚring_lse)r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rƒ  r„  rË   Ú	ring_meshr^  rX  Ú	next_rankÚprev_outÚprev_lseÚ	kv_bufferÚiÚkvÚ	key_numelrg  rh  s                            r4   rI  ÚTemplatedRingAttention.forward0	  se  € ð  %×<Ñ<×GÑGˆ	Ø×7Ñ7×HÑHˆØ%×=Ñ=×IÑIˆ
Ø˜A‘X Ñ+ˆ	Ø"Ð"ˆ�8à#ŒØ%ŒØ—k‘kˆŒØ—y‘yˆŒØ/Ôä—I’I˜sŸ{™{›}¨e¯m©m«oÐ>Ó?×JÑJÓLˆ	Ü×,Ò,¨YÀ1ÈI×L_ÑL_ÓLaÑbˆ	Ø—O‘O JÓ/ˆ	ä�z×"ˆAØ�1‹uØ˜yÑ)�ØŸI™I›K�	Ø˜˜)�n×/Ñ/°Ó4�Ø˜9˜:˜×1Ñ1°%Ó8�Ø&¨™]¨jÑ8�	á!ØØØØØØØØØØØ˜q™&Ø!1ñ‰HˆC�ð  ×7Ñ7×G×GØ—f‘fœUŸ]™]Ó+�Ø—f‘fœUŸ]™]Ó+�ð �x‰x˜1‹}Ø—m‘m BÓ'�ØÑ#Ø¤§¡×!4Ñ!4×!<Ñ!<¸SÀ8¹^Ó!LÐPXÐ[^ÑP^Ñ!_Ñ_�Ø¤§¡×!4Ñ!4×!?Ñ!?ÀÈ3ÁÓ!OÑO�ØˆHØ‹HñM #ðP �f‰f�U—[‘[Ó!ˆØ�k‰k˜"‹oˆð —z‘z“|ˆŒØ—z‘z“|ˆŒæ'��SˆzÐ0¨SÐ0r8   c                óÊ  ^• U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  nUS-   U-  n[        [        SU5      5      S/-   nU R                   R                  R                  (       a  [        R                  OTR                  n[        R                  " U R                  UTR                  S9n	[        R                  " U R                  UTR                  S9n
[        R                  " U R                  UTR                  S9nS nU R                  tpÞnnU R                   U R"                  nn[        R$                  " UR'                  5       UR'                  5       /5      R)                  5       n[*        R,                  " USUR/                  5       S9nUR1                  U5      n[        U5       GH/  nUS:”  aE  UU   nUR3                  5       nUS U R5                  U5      nUUS  R5                  U5      nUS-   U-  nU R7                  U TXÞUUUS9tnnnnUS:”  aC  [9        U5      nU
R3                  5       nUS U R5                  U
5      n
UUS  R5                  U5      nU	U-  n	U
U-  n
UU-  nUUS-
  :  d  MÊ  [        R$                  " U
R'                  5       UR'                  5       /5      R)                  5       n[*        R:                  " UXsR/                  5       S9nGM2     [        R$                  " U
R'                  5       UR'                  5       /5      R)                  5       n[9        [*        R:                  " UXsR/                  5       S95      nU
R3                  5       nUS U R5                  U
5      n
UUS  R5                  U5      nU4S jXšU4 5       u  pšnXšUS S S S S S S S S 4$ )Nr   r   r  r}  r»  r9  c              3  óX   >#   • U  H  oR                  TR                  5      v •  M!     g 7fr0   )r  rß   )rv  rw  r’  s     €r4   rx  Ú2TemplatedRingAttention.backward.<locals>.<genexpr>Ã	  s!   øé € Ð+mÒJlÀQ¯D©D°·±×,@Ð,@ÒJlùs   ƒ'*)rË   r¨  r€  r�  r‚  r£   r‹  rŽ  rè   r�  rß   r  r…  rÞ   r†  r�  r“  r”  r‡  rí  r«  r/  rˆ  r‰  rŠ  rŒ  r�  r„  r3  Úpermute_tensor)r‚  r’  r“  r•  r^  rX  r–  Ú
next_ranksÚaccum_dtyperš  r›  rœ  Únext_grad_kvrÅ   rÆ   r�   ri  rg  rh  r™  rš  r›  rœ  Úgrad_query_opÚgrad_key_opÚgrad_value_opÚgrad_kv_bufferÚgrad_key_numels    `                          r4   rN  ÚTemplatedRingAttention.backward„	  so  ø€ ð ×(Ñ(×@Ñ@×KÑKˆ	Ø×#Ñ#×;Ñ;×LÑLˆØ×)Ñ)×AÑA×MÑMˆ
Ø˜A‘X Ñ+ˆ	Üœ%  :Ó.Ó/°1°#Ñ5ˆ
à'*×';Ñ';×'SÑ'S×'c×'c”e—m’mÐiq×iwÑiwˆÜ—[’[ §¡°KÈÏÉÑXˆ
Ü—;’;˜sŸ|™|°;ÀxÇÁÑWˆÜ—[’[ §¡°[ÈÏÉÑYˆ
Øˆð
 !$× 1Ñ 1Ðˆ�E˜AØ—<‘< §¡ˆSˆÜ—I’I˜sŸ{™{›}¨e¯m©m«oÐ>Ó?×JÑJÓLˆ	Ü×,Ò,¨YÀ1ÈI×L_ÑL_ÓLaÑbˆ	Ø—O‘O JÓ/ˆ	ä�z×"ˆAØ�1‹uØ˜yÑ)�ØŸI™I›K�	Ø˜˜)�n×/Ñ/°Ó4�Ø˜9˜:˜×1Ñ1°%Ó8�Ø&¨™]¨jÑ8�	à<?¿O¹OØ�X U¸5ÀcÈsð =Lð =Ð9ˆM˜;¨¸ð �1‹uÜ!-¨lÓ!;�Ø!)§¡Ó!1�Ø)¨/¨>Ð:×EÑEÀhÓO�Ø+¨N¨OÐ<×GÑGÈ
ÓS�
à˜-Ñ'ˆJØ˜Ñ#ˆHØ˜-Ñ'ˆJà�: ‘>Õ!Ü!&§¢¨H×,<Ñ,<Ó,>À
×@RÑ@RÓ@TÐ+UÓ!V×!aÑ!aÓ!c�Ü%×4Ò4°^ÀZ×WjÑWjÓWlÑm“ñ1 #ô< Ÿš H×$4Ñ$4Ó$6¸
×8JÑ8JÓ8LÐ#MÓN×YÑYÓ[ˆÜ%¤f×&;Ò&;¸NÈJ×^qÑ^qÓ^sÑ&tÓuˆØ!Ÿ™Ó)ˆØ! / >Ð2×=Ñ=¸hÓGˆØ# N OÐ4×?Ñ?À
ÓKˆ
ä+mÈ:ÐakÑJlÓ+mÑ(ˆ
˜jà Z°°t¸TÀ4ÈÈtÐUYÐ[_ÐaeÐeÐer8   r1   r0   ©r‚  ú#torch.autograd.function.FunctionCtxrÅ   rï   rÆ   rï   r�   rï   rÇ   útorch.Tensor | NonerÈ   ÚfloatrÉ   r©   rÊ   úfloat | NonerÌ   r©   rƒ  r©   rË   ú'ParallelConfig' | None©r‚  r¬  r’  rï   ©rb   rc   rd   re   rP  rI  rN  r€   r1   r8   r4   r{  r{  /	  s½   † Øð 59ðQ1Ø0ðQ1àðQ1ð ðQ1ð ð	Q1ð
 'ðQ1ð ðQ1ð ðQ1ð ðQ1ð ðQ1ð ðQ1ð 2ôQ1ó ðQ1ðf ð@fØ0ð@fàó@fó ó@fr8   r{  c                  ót   • \ rS rSr\ S                     SS jj5       r\    SS j5       rSrg)	ÚTemplatedUlyssesAttentioniÈ	  Nc                óˆ  ^• UR                   R                  nUR                   R                  nUR                  5       mX l        X°l        XÀl        UR                  u  nnnnUR                  u  nn  nUU-  nUR                  UUUUU5      R                  SSSSS5      R                  5       nUR                  UUUUU5      R                  SSSSS5      R                  5       nUR                  UUUUU5      R                  SSSSS5      R                  5       nU4S jXU4 5       u  pnS XU4 5       u  pnUbm  UR                  S   U:X  aZ  [        U5       Vs/ s H  n[        R                  " U5      PM     nn[        R                  " UUTS	9  [        R                   " USS
9nU
" U UUUUUUUUU	SUS9nU	(       a  UtnnnUR                  XþUUU5      R                  SSSSS5      R                  5       n[#        UT5      nUR%                  SS5      R                  SSSS5      R                  5       nU	(       aq  WR                  XþUU5      R                  SSSS5      R                  5       n[#        UT5      nUR%                  SS5      R                  SSS5      R                  5       nOS nU	(       a  UU4$ U$ s  snf )Nr	   r   r   r.  r/  c              3  ó<   >#   • U  H  n[        UT5      v •  M     g 7fr0   ©r7  ©rv  rw  r6  s     €r4   rx  Ú4TemplatedUlyssesAttention.forward.<locals>.<genexpr>ç	  s   øé € ÐWÒCV¸aÔ/°°5×9Ð9ÒCVùó   ƒc              3  ó„   #   • U  H6  oR                  S S5      R                  SS SS5      R                  5       v •  M8     g7f©r   r   r	   r.  N©rí  re  r«  ru  s     r4   rx  r¹  è	  s8   é € ÐkÒWjÐRSŸY™Y q¨!›_×4Ñ4°Q¸¸1¸aÓ@×KÑK×MÐMÒWjùó   ‚>A rû   r9  r  Tr  )r¨  Ú_ulysses_meshÚulysses_degreer‰  rƒ  r„  rË   rý   rï  re  r«  r‹  rè   rl  rT  Ú
all_gatherr‡  r7  rí  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rƒ  r„  rË   Úulysses_meshrX  rn  Ú	S_Q_LOCALrW  rp  ri  Ú
S_KV_LOCALrq  Ú	mask_listrg  rh  r6  s                            @r4   rI  Ú!TemplatedUlyssesAttention.forwardÉ	  s¿  ø€ ð  (×?Ñ?×MÑMˆØ%×=Ñ=×LÑLˆ
Ø×&Ñ&Ó(ˆà#ŒØ%ŒØ/Ôà"Ÿ[™[Ñˆˆ9�a˜Ø!Ÿi™iÑˆˆ:�q˜!Ø�z‘/ˆØ—‘˜a ¨J¸ÀÓC×KÑKÈAÈqÐRSÐUVÐXYÓZ×eÑeÓgˆØ�k‰k˜!˜Z¨°W¸aÓ@×HÑHÈÈAÈqÐRSÐUVÓW×bÑbÓdˆØ—‘˜a ¨Z¸À!ÓD×LÑLÈQÐPQÐSTÐVWÐYZÓ[×fÑfÓhˆÜWÀEÐPUÑCVÓWÑˆ�EÙkÐX]ÐdiÑWjÓkÑˆ�EàÑ  Y§_¡_°RÑ%8¸JÓ%Fä>CÀJÔ>OÓPÒ>O¸œ×)Ò)¨)Ö4Ñ>OˆIÐPÜ�OŠO˜I y¸Ò>ÜŸ	š	 )°Ñ4ˆIáØØØØØØØØØØØØ-ñ
ˆö ØˆLˆC��qà�k‰k˜!¨°G¸QÓ?×GÑGÈÈ1ÈaÐQRÐTUÓV×aÑaÓcˆÜ   eÓ,ˆØ�k‰k˜!˜QÓ×'Ñ'¨¨1¨a°Ó3×>Ñ>Ó@ˆæØ—+‘+˜a¨Y¸Ó@×HÑHÈÈAÈqÐRSÓT×_Ñ_ÓaˆCÜ$ S¨%Ó0ˆCØ—+‘+˜a Ó#×+Ñ+¨A¨q°!Ó4×?Ñ?ÓA‰CàˆCæ'��SˆzÐ0¨SÐ0ùòA Qs   Å J?c                ój  ^^^^^^• U R                   R                  R                  nU R                   R                  R                  mUR	                  5       mUR
                  u  mmnmUT-  mUR                  TTTTT5      R                  SSSSS5      R                  5       n[        UT5      nUR                  SS5      R                  SSSS5      R                  5       nU R                  X5      tpVpxUUUUU4S jXVU4 5       u  pšnU4S jXšU4 5       u  pšnS XšU4 5       u  pšnXšUS S S S S S S S S 4$ )	Nr	   r   r   r.  r/  c              3  ó�   >#   • U  H;  nUR                  TTTTT5      R                  S SSSS5      R                  5       v •  M=     g7f)r   r.  r   r	   r/  N)rï  re  r«  )rv  rw  rn  rp  rq  ro  rX  s     €€€€€r4   rx  Ú5TemplatedUlyssesAttention.backward.<locals>.<genexpr>!
  sH   øé € ð ,
â@�ð �I‰I�a˜ W¨g°qÓ9×AÑAÀ!ÀQÈÈ1ÈaÓP×[Ñ[×]Ð]Ú@ùs   ƒAAc              3  ó<   >#   • U  H  n[        UT5      v •  M     g 7fr0   r·  r¸  s     €r4   rx  rÉ  %
  s   øé € Ð+uÒRtÈQÔ,>¸qÀ%×,HÐ,HÒRtùrº  c              3  ó„   #   • U  H6  oR                  S S5      R                  SSS S5      R                  5       v •  M8     g7fr¼  r½  ru  s     r4   rx  rÉ  &
  s9   é € ð ,
ÚFhÀ�I‰I�a˜‹O×#Ñ# A q¨!¨QÓ/×:Ñ:×<Ð<ÒFhùr¾  )rË   r¨  r¿  rÀ  r‰  rý   rï  re  r«  r7  rí  r„  )r‚  r’  r“  rÂ  rW  r¥  r¦  r§  ri  rš  r›  rœ  rn  rp  rq  ro  r6  rX  s               @@@@@@r4   rN  Ú"TemplatedUlyssesAttention.backward
  sS  ý€ ð ×+Ñ+×CÑC×QÑQˆØ×)Ñ)×AÑA×PÑPˆ
Ø×&Ñ&Ó(ˆà#Ÿ>™>Ñˆˆ7�A�qØ�z‘/ˆà×#Ñ# A w°
¸GÀQÓG×OÑOÐPQÐSTÐVWÐYZÐ\]Ó^×iÑiÓkˆÜ% h°Ó6ˆØ×#Ñ# A qÓ)×1Ñ1°!°Q¸¸1Ó=×HÑHÓJˆà8;¿¹ÈÓ8VÐ5ˆ M÷,
ð ,
à#°-Ñ@ó,
Ñ(ˆ
˜jô ,vÐS]ÐisÑRtÓ+uÑ(ˆ
˜jñ,
ØGQÐ]gÑFhó,
Ñ(ˆ
˜jð  Z°°t¸TÀ4ÈÈtÐUYÐ[_ÐaeÐeÐer8   r1   r0   r«  r±  r²  r1   r8   r4   r´  r´  È	  s¹   † Øð 59ðB1Ø0ðB1àðB1ð ðB1ð ð	B1ð
 'ðB1ð ðB1ð ðB1ð ðB1ð ðB1ð ðB1ð 2ôB1ó ðB1ðH ðfØ0ðfàófó ófr8   r´  c                  ót   • \ rS rSr\ S                     SS jj5       r\    SS j5       rSrg)	ÚTemplatedRingAnythingAttentioni-
  Nc                ó  ^ • Ub  [        S5      eUR                  R                  nUR                  5       nUR                  R                  nUR                  R
                  nUS-   U-  nS =nnX l        X°l        UR                  U l	        UR                  U l
        XÀl        UR                  S   n[        UU5      n[        U5      m S
U 4S jjnU" U5      nU" U5      n[        R                  " UR!                  5       UR!                  5       /5      R#                  5       n[$        R&                  " USUS9nUR)                  U5      nUR+                  5       n[-        U5       GHi  nUS:”  aM  UU   nUU   nUS U R/                  U5      S S 2S U24   nUUS  R/                  U5      S S 2S U24   nUS-   U-  nOUS S 2S U24   nUS S 2S U24   nU
" U UUUUUUUUSUS:H  US9u  nnUR                  R0                  (       a>  UR3                  [        R4                  5      nUR3                  [        R4                  5      nUR6                  S:X  a  UR9                  S	5      nUbd  U[        R:                  R<                  R?                  UU-
  5      UU-
  -  -
  nU[        R:                  R<                  RA                  UU-
  5      -
  nUnUnGMl     WR3                  URB                  5      nWRE                  S	5      nU	(       a  UU4$ U$ )Nz•TemplatedRingAnythingAttention does not support non-None attn_mask: non-uniform sequence lengths across ranks make cross-rank mask slicing ambiguous.r   c                óÄ   >• TU R                   S   -
  nUS:X  a  U $ U R                   S   U/U R                   SS  Q7n[        R                  " X R                  U5      /SS9$ )Nr   r   r	   r  )rý   rè   r‡  Ú	new_zeros)ÚtÚpad_lenÚ	pad_shapeÚs_maxs      €r4   Úpad_to_s_maxÚ<TemplatedRingAnythingAttention.forward.<locals>.pad_to_s_maxV
  s_   ø€ Ø˜aŸg™g a™jÑ(ˆGØ˜!‹|Ø�ØŸ™ ™ WÐ;¨q¯w©w°q°r¨{Ñ;ˆIÜ—9’9˜a§¡¨YÓ!7Ð8¸aÑ@Ð@r8   r   r}  Tr  r.  rû   )rÒ  rï   r«   rï   )#r½   r¨  r€  r‰  r�  r‚  rƒ  r„  rý   r…  r†  rË   r   r  rè   r‡  rí  r«  r/  rˆ  rŠ  rŒ  r‹  r�  rŽ  r  r�  r1  r2  r~  r  r�  r‘  rß   r°  )!r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rƒ  r„  rË   r•  r6  r^  rX  r–  r—  r˜  Ú
kv_seq_lenÚall_kv_seq_lensrÖ  Ú
key_paddedÚvalue_paddedr™  Úkv_padded_numelrš  Útrue_seq_lenr›  rg  rh  rÕ  s!                                   @r4   rI  Ú&TemplatedRingAnythingAttention.forward.
  s  ø€ ð" Ñ Üðdóð ð %×<Ñ<×GÑGˆ	Ø×#Ñ#Ó%ˆØ×7Ñ7×HÑHˆØ%×=Ñ=×IÑIˆ
Ø˜A‘X Ñ+ˆ	Ø"Ð"ˆ�8à#ŒØ%ŒØ—k‘kˆŒØ—y‘yˆŒØ/Ôà—Y‘Y˜q‘\ˆ
Ü-¨j¸%Ó@ˆÜ�OÓ$ˆ÷	Añ " #Ó&ˆ
Ù# EÓ*ˆä—I’I˜z×1Ñ1Ó3°\×5IÑ5IÓ5KÐLÓM×XÑXÓZˆ	Ü×,Ò,¨YÀ1ÈEÑRˆ	Ø—O‘O JÓ/ˆ	ð %×*Ñ*Ó,ˆä�z×"ˆAØ�1‹uØ.¨yÑ9�Ø˜yÑ)�àÐ)˜/Ð*×5Ñ5°jÓAÂ!À]ÀlÀ]ÐBRÑS�Ø˜?Ð+Ð,×7Ñ7¸ÓEÂaÈÈ,ÈÐFVÑW�Ø&¨™]¨jÑ8‘	ð !¢ K Z K Ñ0�Ø$¢Q¨¨¨ ^Ñ4�á!ØØØØØØØØØØØ˜q™&Ø!1ñ‰HˆC�ð  ×7Ñ7×G×GØ—f‘fœUŸ]™]Ó+�Ø—f‘fœUŸ]™]Ó+�ð
 �x‰x˜1‹}Ø—m‘m BÓ'�ØÑ#Ø¤§¡×!4Ñ!4×!<Ñ!<¸SÀ8¹^Ó!LÐPXÐ[^ÑP^Ñ!_Ñ_�Ø¤§¡×!4Ñ!4×!?Ñ!?ÀÈ3ÁÓ!OÑO�ØˆHØ‹HñS #ðV �f‰f�U—[‘[Ó!ˆØ�k‰k˜"‹oˆæ'��SˆzÐ0¨SÐ0r8   c                ó   • [        S5      e)NzNBackward pass for Ring Anything Attention in diffusers is not implemented yet.r  r  s      r4   rN  Ú'TemplatedRingAnythingAttention.backward˜
  s   € ô "Ð"rÓsÐsr8   r1   r0   r«  r±  r²  r1   r8   r4   rÎ  rÎ  -
  s¹   † Øð 59ðg1Ø0ðg1àðg1ð ðg1ð ð	g1ð
 'ðg1ð ðg1ð ðg1ð ðg1ð ðg1ð ðg1ð 2ôg1ó ðg1ðR ðtØ0ðtàótó ótr8   rÎ  c                  ót   • \ rS rSr\ S                     SS jj5       r\    SS j5       rSrg)	Ú!TemplatedUlyssesAnythingAttentioni¡
  Nc                ó2  • UR                   R                  nUR                  5       nX l        X°l        XÀl        UR                  u  nn  n[        U5      n[        X40 UD6n[        X/40 UD6n[        X?40 UD6nU" 5       nU" 5       nU" 5       nUbð  UR                  S   U:X  aÝ  [        UR                  S   U5      n[        U5      nUR                  S   U:  a)  [        R                  " USUUR                  S   -
  45      n[        [        R                  " US95       Vs/ s H  n[         R"                  " U5      PM     nn[        R$                  " UXOS9  [         R&                  " USS9nUSS [)        U5      24   nU
" U UUUUUUUUU	SUS9nU	(       a  Utnnn[+        UU40 UD6nU	(       aL  WR-                  S5      n[+        UU40 UD6nU" 5       nU" 5       nUR/                  S5      R1                  5       nO	U" 5       nS nU	(       a  UU4$ U$ s  snf )Nrû   r   r9  r  .Fr  )r¨  r¿  r‰  rƒ  r„  rË   rý   ri  rt  r   r  rU  rV  r‹  rT  r;  rè   rl  rÁ  r‡  r"  ry  r2  r°  r«  )r‚  rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rƒ  r„  rË   rÕ   rÂ  r6  ri  rÄ  ÚmetadataÚ
query_waitÚkey_waitÚ
value_waitÚmask_local_sizesÚ	max_localrÅ  rg  rh  Úout_waitÚlse_waits                                r4   rI  Ú)TemplatedUlyssesAnythingAttention.forward¢
  s$  € ð" (×?Ñ?×MÑMˆØ×&Ñ&Ó(ˆà#ŒØ%ŒØ/Ôà!Ÿi™iÑˆˆ:�q˜!ä,¨UÓ3ˆÜ4°UÑNÀXÑNˆ
Ü2°3ÑJÀÑJˆÜ4°UÑNÀXÑNˆ
á“ˆÙ‹jˆÙ“ˆàÑ  Y§_¡_°RÑ%8¸JÓ%Fô  3°9·?±?À2Ñ3FÈÓNÐÜÐ,Ó-ˆIØ�‰˜rÑ" YÓ.ÜŸEšE )¨a°¸Y¿_¹_ÈRÑ=PÑ1PÐ-QÓR�	Ü>CÄD×DWÒDWÐ^cÑDdÔ>eÓfÒ>e¸œ×)Ò)¨)Ö4Ñ>eˆIÐfÜ�OŠO˜I yÒ>ÜŸ	š	 )°Ñ4ˆIØ! #Ð'>¬Ð-=Ó)>Ð'>Ð">Ñ?ˆIáØØØØØØØØØØØØ-ñ
ˆö ØˆLˆC��qô 1°°eÑH¸xÑHˆæà—-‘- Ó#ˆCÜ4°S¸%ÑLÀ8ÑLˆHÙ“*ˆCÙ“*ˆCØ—+‘+˜b“/×,Ñ,Ó.‰Cá“*ˆCØˆCæ'��SˆzÐ0¨SÐ0ùòI gs   Ä# Hc                ó   • [        S5      e)NzQBackward pass for Ulysses Anything Attention in diffusers is not implemented yet.r  r  s      r4   rN  Ú*TemplatedUlyssesAnythingAttention.backwardó
  s   € ô "Ð"uÓvÐvr8   r1   r0   )r‚  r¬  rÅ   rï   rÆ   rï   r�   rï   rÇ   rï   rÈ   r®  rÉ   r©   rÊ   r®  rÌ   r©   rƒ  r©   rË   r°  r±  r²  r1   r8   r4   râ  râ  ¡
  s¹   † Øð 59ðN1Ø0ðN1àðN1ð ðN1ð ð	N1ð
  ðN1ð ðN1ð ðN1ð ðN1ð ðN1ð ðN1ð 2ôN1ó ðN1ð` ðwØ0ðwàówó ówr8   râ  c                ó  • UR                   R                  nUR                  5       n[        R	                  XðXÍ5      n [        R	                  XñXÍ5      n[        R	                  XòXÍ5      n[
        R	                  U UUUUUUUUU	U
U5      nU(       a  UtnnnOUn[        R	                  UUUU5      nU(       aM  WR                  S:X  a  UR                  S5      n[        R	                  UUXÜ5      nUR                  S5      nUU4$ U$ )zt
Unified Sequence Parallelism attention combining Ulysses and ring attention. See: https://arxiv.org/abs/2405.07719
r.  rû   )	r¨  r¿  r‰  rD  rL  r{  r1  r2  r°  )rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rƒ  r„  rË   r<  r=  rÂ  Úulysses_grouprg  Úcontext_layerrh  ri  r+  s                        r4   Ú_templated_unified_attentionrò  ü
  s  € ð& $×;Ñ;×IÑI€LØ ×*Ñ*Ó,€Mä× Ñ  °{ÓO€EÜ
×
Ñ
˜}°;Ó
K€CÜ× Ñ  °{ÓO€EÜ
 ×
&Ñ
&ØØØØØØØØØØØØó€Cö Ø!$Ðˆ�s™Qàˆä×!Ñ!ØØØØó	€Fö ð
 �8‰8�q‹=Ø—-‘- Ó#ˆCÜ×"Ñ" =°#°zÓOˆØ�k‰k˜"‹oˆØ˜ˆ}ÐØ€Mr8   )rË   c	               óÈ  • U(       a  [        S5      eU(       a  [        S5      eUR                  R                  S:”  a0  UR                  R                  S:”  a  [	        U UUUUUUUUU	U
U5      $ UR                  R                  S:”  a[  UR                  R
                  (       a   [        R                  U UUUUUUUUU	U
U5      $ [        R                  U UUUUUUUUU	U
U5      $ UR                  R                  S:”  a[  UR                  R                  (       a   [        R                  U UUUUUUUUU	U
U5      $ [        R                  U UUUUUUUUU	U
U5      $ [        S5      e)Nz>Causal attention is not yet supported for templated attention.z1GQA is not yet supported for templated attention.r   z@Reaching this branch of code is unexpected. Please report a bug.)r½   r¨  r‚  rÀ  rò  Úring_anythingrÎ  rL  r{  Úulysses_anythingrâ  r´  )rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rƒ  r„  rË   s               r4   Ú%_templated_context_parallel_attentionrö  ;  sµ  € ö ÜÐYÓZÐZÞÜÐLÓMÐMð 	×0Ñ0×<Ñ<¸qÓ@Ø×4Ñ4×CÑCÀaÓGä+ØØØØØØØØØØØØó
ð 	
ð 
×	1Ñ	1×	=Ñ	=ÀÓ	AØ×3Ñ3×A×AÜ1×7Ñ7ØØØØØØØØØØØØ óð ô *×/Ñ/ØØØØØØØØØØØØ óð ð 
×	1Ñ	1×	@Ñ	@À1Ó	DØ×3Ñ3×D×Dä4×:Ñ:ØØØØØØØØØØØØ óð ô -×2Ñ2ØØØØØØØØØØØØ óð ô Ð[Ó\Ð\r8   )r’   r“   c
                óð   • S n
Ub  [        S5      eU	c  [        U UUUUUUUS9nU(       a  UtpºnO<[        R                  " [        US9n[        U UUS UUUSUU[        U	S9nU(       a  Uu  pºU(       a  Xº4$ U$ ©Nz.`attn_mask` is not supported for flash-attn 2.)rU  rØ   rÙ   rÈ   rV  rW  r\  rc  rÔ  F©rƒ  r„  rË   )r½   r    Ú	functoolsÚpartialrÞ  rö  rá  )rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   r\  rƒ  rË   rh  rg  ri  rƒ  s                 r4   Ú_flash_attentionrü  ¨  s¯   € ð" €CØÑÜÐIÓJÐJàÑÜØØØØØØØ#Ø(ñ	
ˆö ØˆLˆC�qøä×&Ò&Ô'BÐP[Ñ\ˆ
Ü3ØØØØØØØØØØ!Ü4Ø-ñ
ˆö Ø‰HˆCæ#ˆCˆ:Ð,¨Ð,r8   c
                ó,  • S n
Ub  [        S5      e[        [        R                     R                  nU	c  U" U UUUUUUUS9nU(       a  UtpÊnO<[
        R                  " [        US9n[        U UUS UUUSUU[        U	S9nU(       a  Uu  pÊU(       a  XÊ4$ U$ rø  )
r½   r»   rF   rg   r´   rú  rû  rå  rö  rç  )rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   r\  rƒ  rË   rh  r3   rg  ri  rƒ  s                  r4   Ú_flash_attention_hubrþ  à  sÆ   € ð" €CØÑÜÐIÓJÐJä Ô!5×!?Ñ!?Ñ@×JÑJ€DØÑÙØØØØØØØ#Ø(ñ	
ˆö ØˆLˆC�qøä×&Ò&Ô'FÐT_Ñ`ˆ
Ü3ØØØØØØØØØØ!Ü8Ø-ñ
ˆö Ø‰HˆCæ#ˆCˆ:Ð,¨Ð,r8   c
                óÖ  • Ub(  U	b%  U	R                   R                  S:”  a  [        S5      eS n
U R                  u  p¼  nUR                  u  pÞ  nU	GcO  Ub¡  [	        X;U5      n[        X¼X0R                  5      u  u    nu  nnu  nnUR                  5       R                  SS9R                  5       nUR                  " S/UR                  SS  Q76 U   nUR                  " S/UR                  SS  Q76 U   nOG[        X¼XàR                  5      u  u    nu  nnu  nnUR                  SS5      nUR                  SS5      nU R                  SS5      n[        [        R                     R                  nU" UUUUUUUUUUUUS9nU(       a  Utnp­UR                  SUS45      nO=[         R"                  " [$        US	9n['        U UUUUUUSUU[(        U	S
9nU(       a  Uu  nn
U(       a  UU
4$ U$ )Nr   zY`attn_mask` is not yet supported for the FLASH_VARLEN_HUB backend with `ring_degree > 1`.Frë  rû   r	   r   ©rU  rØ   rÙ   r  r  r  r  rÈ   rV  rW  r\  rc  rÔ  rù  )r¨  r‚  r  rý   r7  r#  rÞ   rí  rî  rï  r   r»   rF   ri   r´   Ú	unflattenrú  rû  rö  rö  rù  )rÅ   rÆ   r�   rÇ   rÈ   rÊ   rÉ   r\  rƒ  rË   rh  r  r  ri  r  r  r  r  r  rð  rò  ró  rñ  r3   rg  rƒ  s                             r4   Ú_flash_varlen_attention_hubr    s2  € ð( 	ÑØÑ(Ø×4Ñ4×@Ñ@À1ÓDä!Øgó
ð 	
ð €CØ"'§+¡+Ñ€J˜1˜aØŸ)™)Ñ€A�1�aàÒØÑ Ü,¨YÀJÓOˆIä@ÀÐXa×coÑcoÓpñ O‰FˆQ�Ñ0�\ <Ñ2N°<Àð "×)Ñ)Ó+×3Ñ3¸UÐ3ÐC×KÑKÓMˆIØŸš RÐ8¨#¯)©)°A°B¨-Ò8¸ÑCˆJØ Ÿ=š=¨Ð>¨e¯k©k¸!¸"¨oÒ>¸yÑI‰Lô DÀJÐ[e×gsÑgsÓtñ O‰FˆQ�Ñ0�\ <Ñ2N°<Àð Ÿ™ Q¨Ó*ˆJØ Ÿ=™=¨¨AÓ.ˆLà—}‘} Q¨Ó*ˆä$Ô%9×%JÑ%JÑK×UÑUˆÙØØØØ%Ø%Ø%Ø%ØØØØ#Ø(ñ
ˆö ØˆLˆC�Ø�m‰m˜A 
¨BÐ/Ó0‰ä×&Ò&Ô'MÐ[fÑgˆ
Ü3ØØØØØØØØØØ!Ü?Ø-ñ
ˆö Ø‰HˆC�æ#ˆC�ˆ:Ð,¨Ð,r8   )r’   c
                ó  • U R                   u  p«  nUR                   u  pÍ  nUb  [        X:U5      n[        X«XÓU R                  S9u  u  pÎu  nnu  nn/ / nn[	        U
5       H8  nUU   nUR                  UUS U24   5        UR                  UUS U24   5        M:     U R                  SS5      n[        R                  " USS9n[        R                  " USS9n[        UUUUUUUUUUUUS9nUR                  SU
S45      nU$ )N©rÇ   rÞ   r   r   r  r   rû   )rý   r7  r%  rÞ   r‹  Úappendrí  rè   r‡  r!   r  )rÅ   rÆ   r�   rÇ   rÈ   rÊ   rÉ   r\  rƒ  rË   r  r  ri  r  r  r  r  r  r  Ú	key_validÚvalue_validÚbÚ	valid_lenrñ  rò  ró  rg  s                              r4   Ú_flash_varlen_attentionr
  s  s8  € ð  #(§+¡+Ñ€J˜1˜aØŸ)™)Ñ€A�1�aàÑÜ(¨À
ÓKˆ	ô 	/Ø :È5Ï<É<ñ	
ñ O�N€QÑ0�\ <Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Ö0ñ ð
 —=‘=  AÓ&€LÜ—’˜9¨!Ñ,€JÜ—9’9˜[¨aÑ0€Lä
 Ø
Ø
Ø
Ø!Ø!Ø!Ø!ØØØØØ$ñ€Cð �-‰-˜˜J¨Ð+Ó
,€Cà€Jr8   c                óT   • Ub  [        S5      e[        U UUUUS9u  p‰U(       a  X‰4$ U$ )Nú.`attn_mask` is not supported for flash-attn 3.©rU  rØ   rÙ   rV  rW  )r½   rj  )
rÅ   rÆ   r�   rÇ   rÊ   rÉ   rƒ  rË   rg  rh  s
             r4   Ú_flash_attention_3r  ¬  sB   € ð ÑÜÐIÓJÐJä$Ø
Ø
Ø
ØØñ�H€Cö $ˆCˆ:Ð,¨Ð,r8   c                ó¸  • Ub  [        S5      e[        [        R                     R                  nU
cI  U" S0 SU _SU_SU_SU_SU_SS _SS _S	S _S
S _SU_SU_SS_SS _SU_SS_SU	_6nU	(       a
  US   US   4$ U$ [
        R                  " [        UUSS USS9n[
        R                  " [        UUSS USS9n[        U UUS SUUSU	UUU
S9nU	(       a  Uu  pÏXÏ4$ U$ )Nr  rU  rØ   rÙ   rV  rW  rX  rY  rZ  r[  r\  r^  r_  r   r`  ra  rb  r   rc  rú  rÂ   Frù  r1   )
r½   r»   rF   rm   r´   rú  rû  rÿ  r  rö  )rÅ   rÆ   r�   rÇ   rÊ   rÉ   r\  r^  ra  rc  rË   r3   rg  rƒ  r„  rh  s                   r4   Ú_flash_attention_3_hubr  Ç  s‰  € ð$ ÑÜÐIÓJÐJä Ô!5×!BÑ!BÑC×MÑM€DØÑÙò 
Ùð
áð
ñ ð
ñ  ð	
ñ
 ð
ñ ð
ñ ð
ñ ð
ñ ð
ñ $ð
ñ ð
ñ ð
ñ ð
ñ (ð
ñ ð
ñ  0ð!
ˆö$ $5��A‘˜˜A™ÐÐ=¸#Ð=ä×"Ò"Ü)ØØØØØ#Øñ€Jô ×#Ò#Ü*ØØØØØ#Øñ€Kô 0ØØØØØØØØØØØØ)ñ€Cö Ø‰ˆØˆxˆà€Jr8   c                óö  • Ub(  Ub%  UR                   R                  S:”  a  [        S5      eU R                  u  p‰  n
UR                  u  p«  n
UGc\  UbŸ  [	        X8U5      n[        X‰X0R                  5      u  u    n
u  pÍu  pïUR                  5       R                  SS9R                  5       nUR                  " S/UR                  SS  Q76 U   nUR                  " S/UR                  SS  Q76 U   nOE[        X‰X°R                  5      u  u    n
u  pÍu  pïUR                  SS5      nUR                  SS5      nU R                  SS5      n[        [        R                     R                  nU" UUUUUUUUUS9	n[        U[         5      (       a  Utnnn
OUnS nUR#                  SUS45      nOB[$        R&                  " [(        S	S
SS SSS9n[+        U UUUS
UUSUU[,        US9nU(       a  Uu  nnU(       a  UW4$ U$ )Nr   z\`attn_mask` is not yet supported for the _FLASH_3_VARLEN_HUB backend with `ring_degree > 1`.Frë  rû   r	   r   )	rU  rØ   rÙ   r  r  r  r  rV  rW  rT  rÂ   rú  rù  )r¨  r‚  r  rý   r7  r#  rÞ   rí  rî  rï  r   r»   rF   rn   r´   r.  Útupler  rú  rû  r  rö  r  )rÅ   rÆ   r�   rÇ   rÊ   rÉ   rƒ  rË   r  r  ri  r  r  r  r  r  rð  rò  ró  rñ  r3   rf  rg  rh  rƒ  s                            r4   Ú_flash_attention_3_varlen_hubr    sE  € ð$ 	ÑØÑ(Ø×4Ñ4×@Ñ@À1ÓDä!Øjó
ð 	
ð #(§+¡+Ñ€J˜1˜aØŸ)™)Ñ€A�1�aàÒØÑ Ü,¨YÀJÓOˆIä@ÀÐXa×coÑcoÓpñ O‰FˆQ�Ñ0�\Ñ2N°<ð "×)Ñ)Ó+×3Ñ3¸UÐ3ÐC×KÑKÓMˆIØŸš RÐ8¨#¯)©)°A°B¨-Ò8¸ÑCˆJØ Ÿ=š=¨Ð>¨e¯k©k¸!¸"¨oÒ>¸yÑI‰Lô DÀJÐ[e×gsÑgsÓtñ O‰FˆQ�Ñ0�\Ñ2N°<ð Ÿ™ Q¨Ó*ˆJØ Ÿ=™=¨¨AÓ.ˆLà—}‘} Q¨Ó*ˆä$Ô%9×%MÑ%MÑN×XÑXˆÙØØØØ%Ø%Ø%Ø%ØØñ

ˆô �fœe×$Ñ$Ø!ˆLˆC�‘qàˆCØˆCØ�m‰m˜A 
¨BÐ/Ó0‰ä×&Ò&Ü4Ø ØØØØØñ
ˆ
ô 4ØØØØØØØØØØ!ÜAØ-ñ
ˆö Ø‰HˆC�æ#ˆC�ˆ:Ð,¨Ð,r8   c                óÎ   • Ub  [        S5      e[        [        R                     R                  nU" U UUUUS9n	[        U	[        5      (       a  U(       a
  U	S   U	S   4$ U	S   $ U	$ )Nz.`attn_mask` is not supported for flash-attn 4.r  r   r   )r½   r»   rF   rj   r´   r.  r  )
rÅ   rÆ   r�   rÇ   rÊ   rÉ   rƒ  rË   r3   rg  s
             r4   Ú_flash_attention_4_hubr  x  sx   € ð ÑÜÐIÓJÐJä Ô!5×!AÑ!AÑB×LÑL€DÙ
Ø
Ø
Ø
ØØñ€Cô �#”u×ÑÞ#-��A‘˜˜A™ÐÐ9°3°q±6Ð9Ø€Jr8   c                óR  • U R                   u  p‰  n
UR                   u  p«  n
Ub  [        X8U5      n[        X‰X³U R                  S9u  u  p¬u  pÞu  nn/ / nn[	        U5       H8  nUU   nUR                  UUS U24   5        UR                  UUS U24   5        M:     U R                  SS5      n[        R                  " USS9n[        R                  " USS9n[        UUUUUUUUUUS9
n[        U[        5      (       a  Utnnn
OUnS nUR                  SUS45      nU(       a  UU4$ U$ )Nr  r   r   r  )
rU  rØ   rÙ   r  r  r  r  rV  rW  rc  rû   )rý   r7  r%  rÞ   r‹  r  rí  rè   r‡  Úflash_attn_3_varlen_funcr.  r  r  )rÅ   rÆ   r�   rÇ   rÊ   rÉ   rƒ  rË   r  r  ri  r  r  r  r  r  r  r  r  r  r	  rñ  rò  ró  rf  rg  rh  s                              r4   Ú_flash_varlen_attention_3r  —  s^  € ð #(§+¡+Ñ€J˜1˜aØŸ)™)Ñ€A�1�aàÑÜ(¨À
ÓKˆ	ô 	/Ø :È5Ï<É<ñ	
ñ O�N€QÑ0�\Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Ö0ñ ð
 —=‘=  AÓ&€LÜ—’˜9¨!Ñ,€JÜ—9’9˜[¨aÑ0€Lä%Ø
Ø
Ø
Ø!Ø!Ø!Ø!ØØØ$ñ€Fô �&œ%× Ñ ØˆˆˆS‘1àˆØˆØ
�-‰-˜˜J¨Ð+Ó
,€Cæ#ˆC�ˆ:Ð,¨Ð,r8   c	           
     ó˜   • Ub  [        S5      e[        [        R                     R                  n	U	" U UUUUUUS9n
U(       a  U
tp«nX«4$ U
$ )Nz0`attn_mask` is not supported for aiter attention)rU  rØ   rÙ   rÈ   rV  rW  rƒ  )r½   r»   rF   ro   r´   )rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rƒ  rË   r3   rg  rh  ri  s                r4   Ú_aiter_flash_attentionr  Ñ  sf   € ð" ÑÜÐKÓLÐLä Ô!5×!CÑ!CÑD×NÑN€DÙ
Ø
Ø
Ø
ØØØØñ€Cö Øˆˆ�1ØˆxˆØ€Jr8   c	                ó  ^• S n	S n
U R                   u  p¼pÞUR                   u  pï  nTb  [        T[        R                  5      (       a  Tn
OùU(       a(  [        R                  " [
        X½XÏU R                  5      n
OÊ[        R                  " T5      (       a¤  TR                  S:X  a2  TR                  TR                  S5      STR                  S5      S5      mTR                  X½XÏ5      mTR                  [        R                  :X  a+  U4S jn[        R                  " UUS XÏU R                  5      n
OU4S jn	O[        S5      eS XU4 5       u  pn[        R                  " U UUU	U
UUUS9nUR!                  SSSS	5      nU$ )
Nr	   r   r   c                ó   >• TXX#4   $ r0   r1   )r9  r:  r;  r<  rÇ   s       €r4   Úmask_modÚ(_native_flex_attention.<locals>.mask_mod  s   ø€ Ø  °eÐ!CÑDÐDr8   c                ó   >• U TXX44   -   $ r0   r1   )Úscorer9  r:  r;  r<  rÇ   s        €r4   Ú	score_modÚ)_native_flex_attention.<locals>.score_mod   s   ø€ Ø˜y¨¸eÐ)KÑLÑLÐLr8   zCAttention mask must be either None, a BlockMask, or a 2D/4D tensor.c              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7frs  rt  ru  s     r4   rx  Ú)_native_flex_attention.<locals>.<genexpr>%  rz  r{  )rÅ   rÆ   r�   r!  Ú
block_maskrÊ   rÌ   rƒ  r.  )rý   r.  Úflex_attentionÚ	BlockMaskÚcreate_block_maskr=  rÞ   rè   Ú	is_tensorr1  r'  r4  r3  rß   r©   r½   re  )rÅ   rÆ   r�   rÇ   rÉ   rÊ   rÌ   rƒ  rË   r!  r%  r  r  rn  ri  r  r  rg  s      `              r4   Ú_native_flex_attentionr*  õ  se  ø€ ð  €IØ€JØ*/¯+©+Ñ'€J˜9ØŸ)™)Ñ€A�1�aàÑœJ y´.×2JÑ2J×KÑKØ‰
Þ	Ü#×5Ò5Ü+¨ZÀIÐ[`×[gÑ[gó
‰
ô 
�Š˜×	#Ñ	#Ø�>‰>˜QÓØ!Ÿ™ y§~¡~°aÓ'8¸!¸Y¿^¹^ÈAÓ=NÐPQÓRˆIà×$Ñ$ Z¸IÓRˆ	à�?‰?œeŸj™jÓ(õEô (×9Ò9Ø˜* d¨IÀ5Ç<Á<ó‰Jö
Mô Ð^Ó_Ð_áL¸ÀUÑ8KÓLÑ€E�Ü
×
'Ò
'ØØØØØØØØñ	€Cð �+‰+�a˜˜A˜qÓ
!€CØ€Jr8   c                ó  • U R                   [        R                  :X  a1  [        R                  " U S[	        S5      5      n U R                  US9n OU R                  US9n U(       a"  U R                  u  p4U R                  USSU5      n U $ )a÷  
Convert a 2D attention mask to an additive mask, optionally reshaping to 4D for SDPA.

This helper is used by both native SDPA and xformers backends to handle both boolean and additive masks.

Args:
    attn_mask: 2D tensor [batch_size, seq_len_k]
               - Boolean: True means attend, False means mask out
               - Additive: 0.0 means attend, -inf means mask out
    target_dtype: The dtype to convert the mask to (usually query.dtype)
    reshape_4d: If True, reshape from [batch_size, seq_len_k] to [batch_size, 1, 1, seq_len_k] for broadcasting

Returns:
    Additive mask tensor where 0.0 means attend and -inf means mask out. Shape is [batch_size, seq_len_k] if
    reshape_4d=False, or [batch_size, 1, 1, seq_len_k] if reshape_4d=True.
rÂ   ú-inf)rß   r   )rß   rè   r©   Úwherer®  r  rý   r'  )rÇ   Útarget_dtypeÚ
reshape_4dr  r6  s        r4   Ú_prepare_additive_attn_maskr0  4  sw   € ð( ‡�œ%Ÿ*™*Ó$ä—K’K 	¨3´°f³Ó>ˆ	à—L‘L |�LÐ4‰	ð —L‘L |�LÐ4ˆ	ö Ø )§¡Ñˆ
Ø—N‘N :¨q°!°YÓ?ˆ	àÐr8   c
                óò  • U(       a  [        S5      eUbp  UR                  S:X  a`  UR                  S   U R                  S   :X  a@  UR                  S   UR                  S   :X  a   UR                  S5      R                  S5      nU	cR  S XU4 5       u  pn[        R
                  R                  R                  U UUUUUUUS9n
U
R                  SSSS5      n
U
$ [        U UUUUUUUU[        [        U	S9n
U
$ )	NzDNative attention backend does not support setting `return_lse=True`.r	   r   r   c              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7frs  rt  ru  s     r4   rx  Ú$_native_attention.<locals>.<genexpr>z  ó"   é € ÐPÒ<O°qŸY™Y q¨!¨Q°×2Ð2Ò<Oùr{  r|  r.  rù  )r½   r1  rý   r2  rè   r~  r  r€  re  rö  r…  r�  ©rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rË   rg  s              r4   Ú_native_attentionr6  Y  s  € ö" ÜÐ_Ó`Ð`ð
 	ÑØ�N‰N˜aÓØ�O‰O˜AÑ %§+¡+¨a¡.Ó0Ø�O‰O˜AÑ #§)¡)¨A¡,Ó.ð ×'Ñ'¨Ó*×4Ñ4°QÓ7ˆ	àÑÙP¸UÈÑ<OÓPÑˆ�EÜ�h‰h×!Ñ!×>Ñ>ØØØØØØØØ!ð ?ð 	
ˆð �k‰k˜!˜Q  1Ó%ˆð" €Jô 4ØØØØØØØØØÜ3Ü5Ø-ñ
ˆð €Jr8   c
                ó   • S n
U	c¶  U(       d¯  S XU4 5       u  pn[         R                  R                  R                  [         R                  R                  R                  R
                  5         [         R                  R                  R                  U UUUUUUUS9nS S S 5        WR                  SSSS5      nO'[        U UUUUUUUU[        [        U	S9nU(       a  Uu  pºU(       a  Xº4$ U$ ! , (       d  f       NV= f)Nc              3  ód   #   • U  H&  oR                  S SSS5      R                  5       v •  M(     g7frs  )re  r«  ru  s     r4   rx  Ú*_native_cudnn_attention.<locals>.<genexpr>¬  s+   é € Ð]ÒI\ÀAŸY™Y q¨!¨Q°Ó2×=Ñ=×?Ð?ÒI\ùs   ‚.0r|  r   r	   r   r.  rù  )rè   r~  Ú	attentionÚsdpa_kernelÚ
SDPBackendÚCUDNN_ATTENTIONr  r€  re  rö  rº  rË  ©rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rË   rh  rg  s               r4   Ú_native_cudnn_attentionr?  ™  sû   € ð" €CØÑ®
Ù]È%ÐV[ÑI\Ó]Ñˆ�EÜ�X‰X×Ñ×+Ñ+¬E¯H©H×,>Ñ,>×,IÑ,I×,YÑ,YÕZÜ—(‘(×%Ñ%×BÑBØØØØ#Ø#Ø#ØØ%ð Cð 	ˆC÷ [ð �k‰k˜!˜Q  1Ó%‰ä3ØØØØØØØØØÜ2Ü4Ø-ñ
ˆö Ø‰HˆCæ#ˆCˆ:Ð,¨Ð,÷= [ÕZús   Á0/C/Ã/
C=c
                ó¨  • U(       a  [        S5      eS XU4 5       u  pn[        R                  R                  R	                  [        R                  R                  R
                  R                  5         [        R                  R                  R                  U UUUUUUUS9n
S S S 5        W
R                  SSSS5      n
U
$ ! , (       d  f       N$= f)NzNNative efficient attention backend does not support setting `return_lse=True`.c              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7frs  rt  ru  s     r4   rx  Ú._native_efficient_attention.<locals>.<genexpr>à  rz  r{  r|  r   r	   r   r.  )
r½   rè   r~  r:  r;  r<  ÚEFFICIENT_ATTENTIONr  r€  re  r5  s              r4   Ú_native_efficient_attentionrD  Î  s·   € ö  ÜÐiÓjÐjÙL¸ÀUÑ8KÓLÑ€E�Ü	�‰×	Ñ	×	'Ñ	'¬¯©×(:Ñ(:×(EÑ(E×(YÑ(YÕ	ZÜ�h‰h×!Ñ!×>Ñ>ØØØØØØØØ!ð ?ð 	
ˆ÷ 
[ð �+‰+�a˜˜A˜qÓ
!€CØ€J÷ 
[Õ	Zúó   Á6/CÃ
Cc
                ó  • Ub  [        S5      eS n
U	c¶  U(       d¯  S XU4 5       u  pn[        R                  R                  R	                  [        R                  R                  R
                  R                  5         [        R                  R                  R                  U UUS UUUUS9nS S S 5        WR                  SSSS5      nO'[        U UUS UUUUU[        [        U	S9nU(       a  Uu  pºU(       a  Xº4$ U$ ! , (       d  f       NV= f)	Nz7`attn_mask` is not supported for native flash attentionc              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7frs  rt  ru  s     r4   rx  Ú*_native_flash_attention.<locals>.<genexpr>  r4  r{  r|  r   r	   r   r.  rù  )r½   rè   r~  r:  r;  r<  ÚFLASH_ATTENTIONr  r€  re  rö  rÎ  rÓ  r>  s               r4   Ú_native_flash_attentionrJ  ð  s  € ð" ÑÜÐRÓSÐSà
€CØÑ®
ÙP¸UÈÑ<OÓPÑˆ�EÜ�X‰X×Ñ×+Ñ+¬E¯H©H×,>Ñ,>×,IÑ,I×,YÑ,YÕZÜ—(‘(×%Ñ%×BÑBØØØØØ#Ø#ØØ%ð Cð 	ˆC÷ [ð �k‰k˜!˜Q  1Ó%‰ä3ØØØØØØØØØÜ9Ü;Ø-ñ
ˆö Ø‰HˆCæ#ˆCˆ:Ð,¨Ð,÷= [ÕZús   Á>/C=Ã=
Dc
                ó¨  • U(       a  [        S5      eS XU4 5       u  pn[        R                  R                  R	                  [        R                  R                  R
                  R                  5         [        R                  R                  R                  U UUUUUUUS9n
S S S 5        W
R                  SSSS5      n
U
$ ! , (       d  f       N$= f)NzINative math attention backend does not support setting `return_lse=True`.c              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7frs  rt  ru  s     r4   rx  Ú)_native_math_attention.<locals>.<genexpr>:  rz  r{  r|  r   r	   r   r.  )
r½   rè   r~  r:  r;  r<  ÚMATHr  r€  re  r5  s              r4   Ú_native_math_attentionrO  (  s·   € ö  ÜÐdÓeÐeÙL¸ÀUÑ8KÓLÑ€E�Ü	�‰×	Ñ	×	'Ñ	'¬¯©×(:Ñ(:×(EÑ(E×(JÑ(JÕ	KÜ�h‰h×!Ñ!×>Ñ>ØØØØØØØØ!ð ?ð 	
ˆ÷ 
Lð �+‰+�a˜˜A˜qÓ
!€CØ€J÷ 
LÕ	KúrE  c                ó*  • U(       a  [        S5      eUca  [        XU5      n[        U UUU R                  S5      USS Uc&  S[        R
                  " U R                  S   5      -  OUSSSU-
  SSS	9S   nU$ [        U UUUUS US U[        [        US
9nU$ )Nr  r	   r  r  rû   r  Fr   r  rù  )
r½   r  r*   r4  r(  r)  rý   rö  r*  r,  )	rÅ   rÆ   r�   rÇ   rÈ   rÊ   rƒ  rË   rg  s	            r4   Ú_native_npu_attentionrQ  J  sÆ   € ö ÜÐ\Ó]Ð]ØÑÜ/°¸IÓFˆ	ä"ØØØØ�J‰J�q‹MØ ØØØ6;±m�#œŸ	š	 %§+¡+¨b¡/Ó2Ò2ÈØØØ˜I‘oØØñ
ð ñˆð< €Jô 4ØØØØØØØØØÜ0Ü2Ø-ñ
ˆð €Jr8   c                óî   • Ub  [        S5      eU(       a  [        S5      eS XU4 5       u  pnU [        R                  " U R                  S   5      -  n [	        U UUUS9nUR                  SSSS	5      nU$ )
Nz.`attn_mask` is not supported for XLA attentionzAXLA attention backend does not support setting `return_lse=True`.c              3  óH   #   • U  H  oR                  S SSS5      v •  M     g7frs  rt  ru  s     r4   rx  Ú(_native_xla_attention.<locals>.<genexpr>‘  rz  r{  rû   )rU  rØ   rÙ   rW  r   r	   r   r.  )r½   r(  r)  rý   Úxla_flash_attentionre  )rÅ   rÆ   r�   rÇ   rÉ   rƒ  rË   rg  s           r4   Ú_native_xla_attentionrV  €  s…   € ð ÑÜÐIÓJÐJÞÜÐ\Ó]Ð]ÙL¸ÀUÑ8KÓLÑ€E�Ø”D—I’I˜eŸk™k¨"™oÓ.Ñ.€EÜ
Ø
Ø
Ø
Øñ	€Cð �+‰+�a˜˜A˜qÓ
!€CØ€Jr8   c                óÄ   • Ub  [        S5      eS nUc  [        U UUSUUUS9n	U(       a  U	tp˜n
O'[        U UUS SUUSU[        [        US9n	U(       a  U	u  p˜U(       a  X˜4$ U	$ ©Nú/`attn_mask` is not supported for sage attentionr
  r  rÂ   Frù  )r½   r$   rö  r  r  )rÅ   rÆ   r�   rÇ   rÉ   rÊ   rƒ  rË   rh  rg  ri  s              r4   Ú_sage_attentionrZ  �  s˜   € ð ÑÜÐJÓKÐKØ
€CØÑÜØØØØØØØ!ñ
ˆö ØˆLˆC�qøä3ØØØØØØØØØÜ1Ü3Ø-ñ
ˆö Ø‰HˆCæ#ˆCˆ:Ð,¨Ð,r8   c                ó   • Ub  [        S5      eS n[        [        R                     R                  n	Uc  U	" U UUSUUUS9n
U(       a  U
tp¨nO'[        U UUS SUUSU[        [        US9n
U(       a  U
u  p¨U(       a  X¨4$ U
$ rX  )r½   r»   rF   ry   r´   rö  r  r  )rÅ   rÆ   r�   rÇ   rÉ   rÊ   rƒ  rË   rh  r3   rg  ri  s               r4   Ú_sage_attention_hubr\  Ð  s¯   € ð ÑÜÐJÓKÐKØ
€CÜ Ô!5×!>Ñ!>Ñ?×IÑI€DØÑÙØØØØØØØ!ñ
ˆö ØˆLˆC�qøä3ØØØØØØØØØÜ5Ü3Ø-ñ
ˆö Ø‰HˆCæ#ˆCˆ:Ð,¨Ð,r8   c                ó   • U(       a  [        S5      eU R                  u  p‰  n
UR                  u  p«  n
Ub  [        X8U5      n[        X‰X³U R                  S9u  u  p¬u  pÞu  nn/ / nn[        U5       H8  nUU   nUR                  UUS U24   5        UR                  UUS U24   5        M:     U R                  SS5      n[        R                  " USS9n[        R                  " USS9n[        UUUUUUUUUS9	nUR                  SUS45      nU$ )Nz?Sage varlen backend does not support setting `return_lse=True`.r  r   r   r  )	rU  rØ   rÙ   r  r  r  r  rÉ   r  rû   )r½   rý   r7  r%  rÞ   r‹  r  rí  rè   r‡  r)   r  )rÅ   rÆ   r�   rÇ   rÉ   rÊ   rƒ  rË   r  r  ri  r  r  r  r  r  r  r  r  r  r	  rñ  rò  ró  rg  s                            r4   Ú_sage_varlen_attentionr^    s<  € ö ÜÐZÓ[Ð[à"'§+¡+Ñ€J˜1˜aØŸ)™)Ñ€A�1�aàÑÜ(¨À
ÓKˆ	ô 	/Ø :È5Ï<É<ñ	
ñ O�N€QÑ0�\Ñ2N°<Àð   ˆ{€IÜ�:ÖˆØ˜a‘Lˆ	Ø×Ñ˜˜Q 
  
˜]Ñ+Ô,Ø×Ñ˜5  J Y J Ñ/Ö0ñ ð
 —=‘=  AÓ&€LÜ—’˜9¨!Ñ,€JÜ—9’9˜[¨aÑ0€Lä
Ø
Ø
Ø
Ø!Ø!Ø!Ø!ØØñ
€Cð �-‰-˜˜J¨Ð+Ó
,€Cà€Jr8   é	   c           
     ó<   • Ub  [        S5      e[        U UUSUUUS9$ ©NrY  r
  r  )r½   r%   ©rÅ   rÆ   r�   rÇ   rÉ   rÊ   rƒ  rË   s           r4   Ú#_sage_qk_int8_pv_fp8_cuda_attentionrc  ;  s8   € ð ÑÜÐJÓKÐKÜ'Ø
Ø
Ø
ØØØØñð r8   c           
     ó<   • Ub  [        S5      e[        U UUSUUUS9$ ra  )r½   r&   rb  s           r4   Ú(_sage_qk_int8_pv_fp8_cuda_sm90_attentionre  V  s8   € ð ÑÜÐJÓKÐKÜ,Ø
Ø
Ø
ØØØØñð r8   é   c           
     ó<   • Ub  [        S5      e[        U UUSUUUS9$ ra  )r½   r'   rb  s           r4   Ú$_sage_qk_int8_pv_fp16_cuda_attentionrh  q  s8   € ð ÑÜÐJÓKÐKÜ(Ø
Ø
Ø
ØØØØñð r8   c           
     ó<   • Ub  [        S5      e[        U UUSUUUS9$ ra  )r½   r(   rb  s           r4   Ú&_sage_qk_int8_pv_fp16_triton_attentionrj  Œ  s8   € ð ÑÜÐJÓKÐKÜ*Ø
Ø
Ø
ØØØØñð r8   c
                óò  • U(       a  [        S5      eU R                  u  p«pÍUR                  u  pÞpýU(       a  [        R                  " 5       nOøUbõ  UR                  S:X  a™  UR                  S5      nUS-   S-  S-  n[        R                  " X¬UU4U R                  U R                  S9n[        X0R                  S9nUUS S 2S S 2S S 2S U24'   [        S5      US S 2S S 2S S 2US 24'   US S 2S S 2S S 2S U24   nOLUR                  S	:w  a  [        S
5      eUR                  S	:X  a!  UR                  X¬X¾5      R                  U 5      nU(       ay  XÏ-  S:w  a  [        S5      eXÏ-  nU R                  SUS45      n UR                  SUS45      R                  SSSUS5      nUR                  SUS45      R                  SSSUS5      n[        R                  " XX#XF5      nU(       a  UR!                  SS5      nU$ )NzFxformers attention backend does not support setting `return_lse=True`.r	   r   é   rf  r  )r.  r,  r/  zDOnly 2D and 4D attention masks are supported for xformers attention.r   zKNumber of heads in query must be divisible by number of heads in key/value.rû   r.  )r½   rý   ÚxopsÚLowerTriangularMaskr1  r4  rè   r  rß   rÞ   r0  r®  r3  Útype_asr  Úmemory_efficient_attentionrí  )rÅ   rÆ   r�   rÇ   rÈ   rÉ   rÊ   rÌ   rƒ  rË   r  r  Únum_heads_qri  r  Únum_heads_kvÚoriginal_seq_lenÚaligned_seq_lenÚaligned_maskÚmask_additiveÚnum_heads_per_grouprg  s                         r4   Ú_xformers_attentionrx  §  sý  € ö  ÜÐaÓbÐbà,1¯K©KÑ)€J˜;Ø%(§Y¡YÑ"€A�<æÜ×,Ò,Ó.‰	Ø	Ñ	Ø�>‰>˜QÓð
  )Ÿ~™~¨aÓ0ÐØ 0°1Ñ 4¸Ñ:¸aÑ?ˆOô !Ÿ;š;Ø¨)°_ÐEØ—k‘kØ—|‘|ñˆLô 8Ø¯©ñˆMð 8EˆLššAšqÐ"3Ð#3Ð"3Ð3Ñ4ä7<¸V³}ˆLššAšqÐ"2Ñ"3Ð3Ñ4ð %¢Qªª1¨k¨z¨kÐ%9Ñ:‰IØ�^‰^˜qÓ ÜÐcÓdÐdØ�^‰^˜qÓ Ø!×(Ñ(¨À)ÓX×`Ñ`ÐafÓgˆIæØÑ%¨Ó*ÜÐjÓkÐkØ)Ñ9ÐØ—‘  L°"Ð#5Ó6ˆØ�m‰m˜A ¨bÐ1Ó2×9Ñ9¸"¸bÀ"ÐFYÐ[]Ó^ˆØ—‘  L°"Ð#5Ó6×=Ñ=¸bÀ"ÀbÐJ]Ð_aÓbˆä
×
)Ò
)¨%°eÈ	Ó
Y€CæØ�k‰k˜!˜QÓˆà€Jr8   r0   )r�   zstr | AttentionBackendName)NrÂ   FNFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  rÌ   r©   rÒ   zdict[str, Any] | Noner�   zAttentionBackendName | NonerÃ   r°  r«   rï   )rÇ   r­  rÉ   r©   r«   rð   rî   )rê   Úintrë   ry  r«   r   )
rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  r«   rð   )r�   rF   r«   rð   )r  ry  r  ry  r  ry  rÞ   útorch.device | None)r  ry  r  ry  rÇ   rï   rÞ   rz  )NN)r  ry  r  ry  r  ry  rÇ   r­  rÞ   rz  r«   rð   )
r(  rï   r)  rï   r  ry  r*  ry  r«   rï   )rÇ   rï   r  ry  r6  ry  r«   rï   )rD  rª   )NFNNNNr   rÂ   r   NFr   ) rU  rï   rØ   rï   rÙ   rï   rV  r¯  rW  r©   rX  r­  rY  r­  rZ  r­  r[  r­  r]  ry  r^  r®  r_  ry  r`  úbool | Nonera  r©   rb  ry  r«   z!tuple[torch.Tensor, torch.Tensor])NrÂ   FNFFTN)r‚  r¬  rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  rÌ   r©   rƒ  r©   r„  r©   rË   r°  r±  )r‚  r¬  r’  rï   rÅ   r­  rÆ   r­  r�   r­  rg  r­  rh  r­  )r‚  r¬  rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  rÌ   r©   rƒ  r©   r„  r©   rË   r°  r\  útuple[int, int])$r‚  r¬  rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  rÌ   r©   rƒ  r©   r„  r©   rË   r°  r\  r|  r^  r®  r_  ry  r`  r{  ra  r©   rb  ry  )rÅ   rï   rÆ   rï   rÇ   r­  )rw  rï   r«   rï   )r	   r   N)rw  rï   r<  ry  r=  ry  r«   rï   )rw  rï   rW  ry  r6  údist.ProcessGroupr«   ztuple[torch.Tensor, int])rw  rï   rY  ry  r6  r}  r«   rï   )rÅ   rï   r«   Údict)rw  rï   r6  r}  r«   zCallable[..., torch.Tensor])Nr	   r   )rÅ   rï   rÆ   rï   r�   rï   rÇ   rï   rÈ   r®  rÉ   r©   rÊ   r®  rÌ   r©   rƒ  r©   rË   r°  r<  ry  r=  ry  )NrÂ   FNFF)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  rÌ   r©   rƒ  r©   rË   r°  )NrÂ   FNrT  FN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  r\  r|  rƒ  r©   rË   r°  r«   rï   )NrÂ   NFrT  FN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÊ   r¯  rÉ   r©   r\  r|  rƒ  r©   rË   r°  r«   rï   )NNFFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÊ   r¯  rÉ   r©   rƒ  r©   rË   r°  r«   rï   )NNFrT  rÂ   FFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÊ   r¯  rÉ   r©   r\  r|  r^  r®  ra  r©   rc  r©   rË   r°  r«   rï   )rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  rƒ  r©   rË   r°  r«   rï   )NFNFFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   z0torch.Tensor | 'flex_attention.BlockMask' | NonerÉ   r©   rÊ   r¯  rÌ   r©   rƒ  r©   rË   r°  r«   rï   )T)rÇ   rï   r.  ztorch.dtyper/  r©   r«   rï   )NrÂ   FNFFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÉ   r©   rÊ   r¯  rÌ   r©   rƒ  r©   rË   r°  r«   rï   )NrÂ   NFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÈ   r®  rÊ   r¯  rƒ  r©   rË   r°  r«   rï   )NFFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÉ   r©   rƒ  r©   rË   r°  r«   rï   )NFNFN)rÅ   rï   rÆ   rï   r�   rï   rÇ   r­  rÉ   r©   rÊ   r¯  rƒ  r©   rË   r°  r«   rï   )äÚ
__future__r   Ú
contextlibrú  rˆ   r(  Údataclassesr   Úenumr   Útypingr   r   r   rè   Útorch.distributedr:  rT  Útorch.nn.functionalr~  r  rU  Úis_availableÚ)torch.distributed._functional_collectivesÚ_functional_collectivesr/  Úutilsr
   r   r   r   r   r   r   r   r   r   r   r   r   r   Úutils.constantsr   r   Úutils.torch_utilsr   r   Ú_modeling_parallelr   r   r  r  Ú_REQUIRED_FLEX_VERSIONr  r
  rb   r–   r   r  r  rÏ   r  r  r	  r¯  Ú
flash_attnr    r!   Úflash_attn.flash_attn_interfacer"   r#   ÚImportErrorÚOSErrorr  rR  rÐ   Úflash_attn_interfacerd  r  Úsageattentionr$   r%   r&   r'   r(   r)   Ú!torch.nn.attention.flex_attentionr:  r&  Ú	torch_npur*   Ú$torch_xla.experimental.custom_kernelr+   rU  Úxformers.opsr¬  rm  Ú__version__ÚlibraryÚ	custom_opÚ
_custom_opÚregister_fakeÚ_register_faker<   rD   rª   rF   r‚   r¯   rm   rn   rg   ri   ry   rj   ro   r»   rº   Úcontextmanagerrq   rÁ   rÚ   rÜ   rá   rå   rñ   ró   r÷   rù   rþ   r¾   r   r#  r%  r,  r7  r=  rG  r¿   rj  ri  r…  r�  rº  rË  rÎ  rÓ  rÞ  rá  rå  rç  rö  rù  rÿ  r  r  r  r  r  r  r  r*  r,  r3  r7  rB  r�  ÚFunctionrD  r[  r_  ra  rc  ri  rt  ry  r{  r´  rÎ  râ  rò  rö  r˜   rf   rü  rþ  r  rh   r
  rk   r  r  r  r  rl   r  r  rp   r*  r0  r6  rr   r?  rs   rD  rt   rJ  ru   rO  rv   rQ  rw   rV  rx   rZ  r\  rz   r^  r{   rc  r|   re  r}   rh  r~   rj  r   rx  r1   r8   r4   Ú<module>r      s‡'  ðö #ã Û Û Û Ý !Ý ß /Ñ /ã Ý  ß Ð ð 	×Ñ×!Ñ!×#Ñ#ß>Ð>÷÷ ÷ ÷ ÷  Lß MÝ 3ö Ý2à!Ð Ø Ð Ø Ð ØÐ Ø%Ð á	�HÓ	€á-Ó/×hÑ4IÈ$ÐPgÓ4hÐ Ù1Ó3Ð Ù/Ó1×lÑ6NÈtÐUkÓ6lÐ Ù% dÐ,BÓCÐ Ù*Ó,Ð Ù*Ó,×bÑ1EÀdÐLaÓ1bÐ Ù.Ó0×jÑ5HÈÐOiÓ5jÐ ñ /¨t°WÓ=Ð ö ð+ßFßmÐmð €OØ!ÐØ#'Ð Ø"&Ðö ð(ÝMÞ[ð ÐØ#Ðæð÷	
÷ 	
ð 	
ð$ €HØ$(Ð!Ø&*Ð#Ø#'Ð Ø(,Ð%Ø€Oö ð÷ 	CÔBð €Nö ð$Þ2ð  Ðö ð#Þ_ð Ðö ðÞ#ð €Dð 	×Ñ˜ÓØ—‘×(Ñ(€JØ—]‘]×0Ñ0�Nð*ÈÐUYö *ð*°4ÀQö *ð !€JØ(€Nô(˜3 ô (÷V.)ñ .)ðb ÷0ð 0ó ð0ð  ×%Ñ%Ñ'7Ø/Ø'ØGØIØñ(ð ×,Ñ,Ñ.>Ø/Ø.ØGØIØñ/ð ×"Ñ"Ñ$4Ø/Ø'ØOØQØñ%ð ×)Ñ)Ñ+;Ø/Ø.ØVØXØñ,ð ×!Ñ!Ñ#3Ø2Ø Øñ$ð
 ×$Ñ$Ñ&6Ø/Ø'Øñ'ð
 ×&Ñ&Ñ(8Ø7Ø'Øñ)ðO,IÐ ÐEó ,ð^ ×ÑØ<P×<WÑ<Wõ Bó ðBð0 &*ØØØØØ.2ð1 ð ,0Ø/3ñ1 Øð1 à	ð1 ð ð1 ð #ð	1 ð
 ð1 ð ð1 ð ð1 ð ð1 ð ,ð1 ð )ð1 ð -ð1 ð ö1 ôpUô
LôLôôFôVôDð &*ð	QØðQà	ðQð ðQð #ð	Qð 
õQô0GñT  Ñ%ð
 #'ð	^Øð^àð^ð ð^ð  ô	^ó &ð^ð* #'ð	^Øð^àð^ð ð^ð  õ	^ð* &*Ø"&ð	fØð	fàð	fð ð	fð #ð		fð
  ð	fð 
õ	fô?ô1òhô
ôñD Ð:ÈÐZ`Ñað
 #'ØØ"Ø%)Ø%)Ø%)ØØØØ ØØð(Øð(àð(ð ð(ð  ð	(ð
 ð(ð 	ð(ð #ð(ð #ð(ð #ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð  'ô!(ó bð(ñV Ð>Ó?ð
 #'ØØ"Ø%)Ø%)Ø%)ØØØØ ØØð7Øð7àð7ð ð7ð  ð	7ð
 ð7ð 	ð7ð #ð7ð #ð7ð #ð7ð ð7ð ð7ð ð7ð ð7ð ð7ð ð7ð  'ô!7ó @ð7ðB &*ØØØØØØØ04ð(Ø	,ð(àð(ð 
ð(ð ð	(ð
 #ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð ð(ð .õ(ðV",Ø	,ð",àô",ðZ &*ØØØØØØØ04ðC-Ø	,ðC-àðC-ð 
ðC-ð ð	C-ð
 #ðC-ð ðC-ð ðC-ð ðC-ð ðC-ð ðC-ð ðC-ð .õC-ðX "&Ø#Ø!%Ø#Ø#ñ5,Ø	,ð5,àð5,ð ð	5,ð
 
ð5,ð ð5,ð 
ð5,ð 
õ5,ð@ &*ØØØØØØØ04ð.-Ø	,ð.-àð.-ð 
ð.-ð ð	.-ð
 #ð.-ð ð.-ð ð.-ð ð.-ð ð.-ð ð.-ð ð.-ð .õ.-ðp "&Ø#Ø!%Ø#Ø#ñ0,Ø	,ð0,àð0,ð ð	0,ð
 
ð0,ð ð0,ð 
ð0,ð 
õ0,ðr &*ØØØØØØØ04ð;-ð $,ñ;-Ø	,ð;-àð;-ð 
ð;-ð ð	;-ð
 #ð;-ð ð;-ð ð;-ð ð;-ð ð;-ð ð;-ð ð;-ð .ð;-ð !ö;-ðD "&Ø#Ø!%Ø#Ø#ñ2,Ø	,ð2,àð2,ð ð	2,ð
 
ð2,ð ð2,ð 
ð2,ð 
õ2,ðt &*ØØØØØØØ04ðC-ð $,ñC-Ø	,ðC-àðC-ð 
ðC-ð ð	C-ð
 #ðC-ð ðC-ð ðC-ð ðC-ð ðC-ð ðC-ð ðC-ð .ðC-ð !öC-ðT "&Ø#Ø!%Ø#Ø#ñ6,Ø	,ð6,àð6,ð ð	6,ð
 
ð6,ð ð6,ð 
ð6,ð 
õ6,ð| &*ØØØØØØØ04ði0ð $,ñi0Ø	,ði0àði0ð 
ði0ð ð	i0ð
 #ði0ð ði0ð ði0ð ði0ð ði0ð ði0ð ði0ð .ði0ð !öi0ð` "&Ø#Ø!%Ø#Ø#ñO,Ø	,ðO,àðO,ð ð	O,ð
 
ðO,ð ðO,ð 
ðO,ð 
õO,ðn &*ØØØØØØØ04ðW-ð $,ØØØ ØØñ'W-Ø	,ðW-àðW-ð 
ðW-ð ð	W-ð
 #ðW-ð ðW-ð ðW-ð ðW-ð ðW-ð ðW-ð ðW-ð .ðW-ð !ðW-ð ðW-ð  ð!W-ð" ð#W-ð$ ð%W-ð& ö'W-ð| "&Ø#Ø!%Ø#Ø#ñ=,Ø	,ð=,àð=,ð ð	=,ð
 
ð=,ð ð=,ð 
ð=,ð 
õ=,ðJ &*ØØØØØØØ04ðy0ð $,ØØØ ØØñ'y0Ø	,ðy0àðy0ð 
ðy0ð ð	y0ð
 #ðy0ð ðy0ð ðy0ð ðy0ð ðy0ð ðy0ð ðy0ð .ðy0ð !ðy0ð ðy0ð  ð!y0ð" ð#y0ð$ ð%y0ð& ö'y0ð@ "&Ø#Ø!%Ø#Ø#ñM,Ø	,ðM,àðM,ð ð	M,ð
 
ðM,ð ðM,ð 
ðM,ð 
õM,ðj &*ØØØØØØØ04ð#-Ø	,ð#-àð#-ð 
ð#-ð ð	#-ð
 #ð#-ð ð#-ð ð#-ð ð#-ð ð#-ð ð#-ð ð#-ð .õ#-ðV &*ØØØØØØØ04ð%-Ø	,ð%-àð%-ð 
ð%-ð ð	%-ð
 #ð%-ð ð%-ð ð%-ð ð%-ð ð%-ð ð%-ð ð%-ð .õ%-ðPVØ	,ðVàôVöð0 &*ØØØØØØØ04ð#Ø	,ð#àð#ð 
ð#ð ð	#ð
 #ð#ð ð#ð ð#ð ð#ð ð#ð ð#ð ð#ð .õ#ðN\Ø	,ð\àô\ò ô
öFYôR.�U—^’^×,Ò,ô .ô2ô"
ôô.ôð ðØðØ-ðà óó ððB ó$ó ð$ôNVf˜UŸ^š^×4Ò4ô Vfôrbf §¢× 7Ò 7ô bfôJqt U§^¢^×%<Ò%<ô qtôhXw¨¯ª×(?Ò(?ô XwðN 15ØØð<Øð<à	ð<ð ð<ð ð	<ð
 ð<ð ð<ð ð<ð ð<ð ð<ð .ð<ð ð<ð õ<ðF &*ØØØØØðg]ð 15ñg]Øðg]à	ðg]ð ðg]ð #ð	g]ð
 ðg]ð ðg]ð ðg]ð ðg]ð ðg]ð .ög]ðZ ×#Ò#Ø×ÒØÐ =¸|ÐLØ"ð $ð ð &*ØØØØ#+ØØ04ð0-Øð0-à	ð0-ð ð0-ð #ð	0-ð
 ð0-ð ð0-ð ð0-ð !ð0-ð ð0-ð .ð0-ð ô0-óð
0-ðf ×#Ò#Ø×"Ñ"ØÐ =¸|ÐLØ"ð $ð ð &*ØØØØ#+ØØ04ð1-Øð1-à	ð1-ð ð1-ð #ð	1-ð
 ð1-ð ð1-ð ð1-ð !ð1-ð ð1-ð .ð1-ð ô1-óð
1-ðh ×#Ò#Ø×)Ñ)ØÐ =¸|ÐLØ"ð $ð ð &*ØØØØ#+ØØ04ðR-ØðR-à	ðR-ð ðR-ð #ð	R-ð
 ðR-ð ðR-ð ðR-ð !ðR-ð ðR-ð .ðR-ð ôR-óð
R-ðj ×#Ò#Ø×%Ò%ØÐ =¸|ÐLð $ð ð &*ØØØØ#+ØØ04ð2Øð2à	ð2ð ð2ð #ð	2ð
 ð2ð ð2ð ð2ð !ð2ð ð2ð .ð2ð ô2ó	ð2ðj ×#Ò#Ø×!Ò!ØÐ =¸|ÐLð $ð ð &*ØØØØ04ð-Øð-à	ð-ð ð-ð #ð	-ð
 ð-ð ð-ð ð-ð .ð-ð ô-ó	ð-ð. ×#Ò#Ø×%Ñ%ØÐ =¸|ÐLØ"ð $ð ð &*ØØØ#+ØØØ#Ø04ðJØðJà	ðJð ðJð #ð	Jð
 ðJð ðJð !ðJð ðJð ðJð ðJð .ðJð ôJóð
JðZ ×#Ò#Ø×,Ñ,ØÐ =¸|ÐLØ"ð $ð ð &*ØØØØ04ðW-ØðW-à	ðW-ð ðW-ð #ð	W-ð
 ðW-ð ðW-ð ðW-ð .ðW-ð ôW-óð
W-ðt ×#Ò#Ø×$Ñ$ØÐ =¸|ÐLØ#ð $ð ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð ôóð
ð4 ×#Ò#Ø×(Ò(ØÐ =¸|ÐLð $ð ð &*ØØØØ04ð3-Øð3-à	ð3-ð ð3-ð #ð	3-ð
 ð3-ð ð3-ð ð3-ð .ð3-ð ô3-ó	ð3-ðl ×#Ò#Ø×&Ñ&ð $Ð%:¸LÐIð	 $ð ð &*ØØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð ðð .ðð ôóðð< ×#Ò#Ø×ÒØ+¨]¸LÐIð $ð ð CGØØØØØ04ð8Øð8à	ð8ð ð8ð @ð	8ð
 ð8ð ð8ð ð8ð ð8ð .ð8ð ô8ó	ð8ðx LPð"Øð"Ø+6ð"ØDHð"àõ"ðJ ×#Ò#Ø×ÑØ Ð-Ø"ð $ð ð &*ØØØØØØ04ð8Øð8à	ð8ð ð8ð #ð	8ð
 ð8ð ð8ð ð8ð ð8ð ð8ð .ð8ð ô8óð
8ðv ×#Ò#Ø×&Ò&ØÐ =¸|ÐLØ"ð $ð ð &*ØØØØØØ04ð--Øð--à	ð--ð ð--ð #ð	--ð
 ð--ð ð--ð ð--ð ð--ð ð--ð .ð--ð ô--óð
--ð` ×#Ò#Ø×*Ò*Ø Ð-ð $ð ð &*ØØØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð ðð ðð .ðð ôó	ðð< ×#Ò#Ø×&Ò&ØÐ =¸|ÐLØ"ð $ð ð &*ØØØØØØ04ð0-Øð0-à	ð0-ð ð0-ð #ð	0-ð
 ð0-ð ð0-ð ð0-ð ð0-ð ð0-ð .ð0-ð ô0-óð
0-ðf ×#Ò#Ø×%Ò%Ø Ð-ð $ð ð &*ØØØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð ðð ðð .ðð ôó	ðð< ×#Ò#Ø×$Ò$ØÐ =¸|ÐLØ"ð $ð ð &*ØØØØ04ð-Øð-à	ð-ð ð-ð #ð	-ð
 ð-ð ð-ð ð-ð .ð-ð ô-óð
-ðb ×#Ò#Ø×$Ò$Ø Ð-ð $ð ð &*ØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð .ðð ôó	ðð2 ×#Ò#Ø×ÒØ#Ð%BÀLÐQØ"ð $ð ð &*ØØØØ04ð+-Øð+-à	ð+-ð ð+-ð #ð	+-ð
 ð+-ð ð+-ð ð+-ð .ð+-ð ô+-óð
+-ð\ ×#Ò#Ø×!Ñ!Ø#Ð%BÀLÐQØ"ð $ð ð &*ØØØØ04ð,-Øð,-à	ð,-ð ð,-ð #ð	,-ð
 ð,-ð ð,-ð ð,-ð .ð,-ð ô,-óð
,-ð^ ×#Ò#Ø×$Ò$Ø#Ð%BÀLÐQð $ð ð &*ØØØØ04ð0Øð0à	ð0ð ð0ð #ð	0ð
 ð0ð ð0ð ð0ð .ð0ð ô0ó	ð0ðf ×#Ò#Ø×2Ò2Ù0°°AÓ6¸ÐEð $ð ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð ôó	ðð. ×#Ò#Ø×7Ò7Ù0°°AÓ6¸ÐEð $ð ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð ôó	ðð. ×#Ò#Ø×3Ò3Ù0°°AÓ6¸ÐEð $ð ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð ôó	ðð. ×#Ò#Ø×5Ò5Ù0°°AÓ6¸ÐEð $ð ð &*ØØØØ04ðØðà	ðð ðð #ð	ð
 ðð ðð ðð .ðð ôó	ðð. ×#Ò#Ø×!Ò!Ø+¨]¸LÐIð $ð ð &*ØØØØØØ04ð@Øð@à	ð@ð ð@ð #ð	@ð
 ð@ð ð@ð ð@ð ð@ð ð@ð .ð@ð ô@ó	ñ@øðsB ˜ ,Ð/ó +ð 	�‰ÐGÈÀsÐJuÐvÔwØ#ÐØˆØ!%ÐØ'+Ð$Ø&*Ö#ûð+ûð( ˜ ,Ð/ó (Ø�‰Ð8¸¸Ð;^Ð_Ô`Ø %ÐØ ÐØ#'Ö ûð	(ûð& ˜ ,Ð/ó Ø�‰Ð9¸!¸Ð<_Ð`ÔaØ"ÐØˆØ'+Ð$Ø,0Ð)Ø(,Ð%Ø*.Ð'ØŽûðûð0 ˜ ,Ð/ó Ø�‰Ð:¸1¸#Ð=`ÐaÔbØ"ÐØŽûðûð ˜ ,Ð/ó $Ø�‰Ð5°a°SÐ8[Ð\Ô]Ø!ÐØ#Öûð$ûð ˜ ,Ð/ó #Ø�‰Ð5°a°SÐ8[Ð\Ô]Ø!ÐØ"Öûð#ûð ˜ ,Ð/ó Ø�‰Ð4°Q°CÐ7ZÐ[Ô\Ø!&ÐØŽûðús³   Å{1 Å#|' Å<} Æ!~ Æ8 Ç	3 ÇA@# û1	|$û:|ü|$ü'	}ü0}ý}ý	~ý"#~þ~þ	 þ~;þ; ÿ	0ÿ+ÿ+0ÿ3	A@ ÿ<A@Á@A@ Á@#	AAÁ@,AAÁAAA