ó
    Këgj+‰  ã                   ój  • S r SSKrSSKJs  Jr  SSKrSSKJr	  SSK
Jr  SSKJrJr  SSKJrJrJrJrJr  \(       a  / SQO/ SQrS\R,                  S	\4S
 jrS\R,                  S	\4S jrS\R,                  S	\R,                  4S jrS`S\R,                  S\S	\R,                  4S jjr\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4S j5       5       r!\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4S j5       5       r"\" S5      S`S\R,                  S\S	\R,                  4S jj5       r#\" S5      S`S\R,                  S\R,                  S\S	\R,                  4S jj5       r$ " S S\RJ                  RL                  5      r'\'RP                  r)\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4S j5       5       r*\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4S  j5       5       r+\" S5      SbS\R,                  S!\,S\S	\R,                  4S" jj5       r-\" S5        SbS\R,                  S\R,                  S!\,S\S	\R,                  4
S# jj5       r. " S$ S%\RJ                  RL                  5      r/ScS\R,                  S!\,S	\R,                  4S& jjr0\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4S' j5       5       r1\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4S( j5       5       r2\" S5      S`S\R,                  S\S	\R,                  4S) jj5       r3\" S5      S`S\R,                  S\R,                  S\S	\R,                  4S* jj5       r4 " S+ S,\RJ                  RL                  5      r5\5RP                  r6\Rn                  S- 5       r8\Rn                  S. 5       r9 " S/ S0\RJ                  RL                  5      r:\:RP                  r;\Rn                  S1 5       r<\Rn                  S2 5       r= " S3 S4\RJ                  RL                  5      r>\>RP                  r?\Rn                  S5 5       r@\Rn                  S6 5       rA\Rn                  S7 5       rB " S8 S9\RJ                  RL                  5      rC\CRP                  rD\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4S: j5       5       rE\RŒ                  " S;S< 05      \R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  S;\	R@                  4S= j5       5       5       rG\" S5      S`S\R,                  S>\R,                  S\S	\R,                  4S? jj5       rH\" S5        SdS\R,                  S>\R,                  S@\R,                  SA\S\4
SB jj5       rI " SC SD\RJ                  RL                  5      rJ " SE SF\RJ                  RL                  5      rK\JRP                  rL\" S5      SG 5       rM\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4SH j5       5       rN\RŒ                  " S;SI 05      \R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  S;\	R@                  4SJ j5       5       5       rO\R                   R¢                  S`S\R,                  S>\R,                  S\S	\R,                  4SK jj5       rR\R                   R¢                    SdS\R,                  S>\R,                  S@\R,                  SA\S\4
SL jj5       rS " SM SN\RJ                  RL                  5      rT " SO SP\RJ                  RL                  5      rU\TRP                  rV\URP                  rW\R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  4SQ j5       5       rX\RŒ                  " S;SR 05      \R:                  " SaS V Vs/ s H  n \  H  n\R=                  SU 0US9PM     M!     snn S/S.\D6\R>                  " S/S9S\	R@                  S\	R@                  S;\	R@                  4SS j5       5       5       rY\" S5      SeS\R,                  S>\R,                  ST\,S\S	\R,                  4
SU jj5       rZ\" S5         SfS\R,                  S>\R,                  S@\R,                  ST\,SA\S\4SV jj5       r[ " SW SX\RJ                  RL                  5      r\ " SY SZ\RJ                  RL                  5      r]SgS\R,                  S>\R,                  ST\,S	\R,                  4S[ jjr^\" S5       SgS\R,                  S>\R,                  S\\R,                  S]\R,                  ST\,S	\R,                  4S^ jj5       r_\RÀ                  \)\0\6\6\D\?\;S_.rags  snn f s  snn f s  snn f s  snn f s  snn f s  snn f s  snn f s  snn f s  snn f s  snn f s  snn f s  snn f )hun  Fused activation kernels.

The kernels address their inputs through the row stride instead of assuming a fully contiguous buffer.
An inner-contiguous input â€” such as one half of ``x.chunk(2, dim=-1)`` â€” is therefore read in place, sparing the extra
``.contiguous()`` copy (and its memory traffic) that a plain flat element-wise kernel would force on every call.
é    N)Údispatch)ÚexpÚlog)ÚIS_AMDÚautocast_custom_bwdÚautocast_custom_fwdÚautotune_cache_kwargsÚinput_guard)é   é   é   é   é   )r   r   r   r   r   é    ÚxÚreturnc                 óª   • U R                   S:  a  g[        R                  R                  5       (       a  U R                  S   $ U R                  S5      $ )a-  Get the row stride for viewing a tensor as 2D (num_rows, D) where D = shape[-1].

Returns stride(-2) if the tensor is at least 2D, or 0 for 1D tensors.
The caller must ensure the tensor is "inner-contiguous" (stride(-1) == 1 and
higher dims are contiguous relative to dim -2) before using this value.
r   r   éÿÿÿÿéþÿÿÿ)ÚndimÚtorchÚcompilerÚis_compilingÚshapeÚstride©r   s    Ú¥C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\fla/modules/activations.pyÚ_get_strider      sA   € ð 	‡v�v�ƒzØÜ‡~�~×"Ñ"×$Ñ$Ø�w‰w�r‰{ÐØ�8‰8�B‹<Ðó    c                 ór  • U R                   nUS:  a  gU R                  S5      S:w  a  gUS:X  a  gUS:X  a3  U R                  S5      U R                  S5      U R                  S   -  :H  $ US	:X  ah  U R                  S5      U R                  S5      U R                  S   -  :w  a  gU R                  S5      U R                  S5      U R                  S   -  :H  $ U R                  S5      U R                  S   -  n[        US-
  SS5       H+  nU R                  U5      U:w  a    gX R                  U   -  nM-     g)
zÏCheck if a tensor can be safely viewed as 2D (num_rows, D) with row stride = stride(-2).

This holds when stride(-1) == 1 and all dimensions above -2 are contiguous
with respect to the dimension below them.
r   Tr   r   Fé   r   r   r   )r   r   r   Úrange)r   r   ÚexpectedÚds       r   Ú_is_inner_contiguousr%   )   s  € ð �6‰6€DØˆaƒxØØ‡x�x�ƒ|�qÓØØˆqƒyàØˆqƒyà�x‰x˜‹{˜aŸh™h r›l¨Q¯W©W°R©[Ñ8Ñ8Ð8Øˆqƒyà�8‰8�A‹;˜!Ÿ(™( 2›,¨¯©°©Ñ4Ó4ØØ�x‰x˜‹{˜aŸh™h q›k¨A¯G©G°A©JÑ6Ñ6Ð6à�x‰x˜‹|˜aŸg™g b™kÑ)€HÜ�4˜!‘8˜R Ö$ˆØ�8‰8�A‹;˜(Ó"ÙØ—G‘G˜A‘JÑŠñ %ð r   c                 ó¬   • [         R                  R                  5       (       a  U R                  5       $ [	        U 5      (       a  U $ U R                  5       $ )z5Make the tensor inner-contiguous if it isn't already.)r   r   r   Ú
contiguousr%   r   s    r   Ú_ensure_inner_contiguousr(   H   s=   € ä‡~�~×"Ñ"×$Ñ$Ø�|‰|‹~ÐÜ˜A×ÑØˆØ�<‰<‹>Ðr   r'   c                 ór   • U(       a  U R                  U R                  5      $ [        R                  " U 5      $ )zöAllocate the output: a fresh contiguous buffer, or ``empty_like`` otherwise.

``empty_like`` keeps the input's memory format only when it is dense; a non-dense
strided view (e.g. a ``chunk`` slice) falls back to contiguous, not the input stride.
)Ú	new_emptyr   r   Ú
empty_like)r   r'   s     r   Ú_alloc_outputr,   Q   s+   € ö Ø�{‰{˜1Ÿ7™7Ó#Ð#Ü×Ò˜AÓÐr   )i   i   i   i   i    ÚB)Ú	num_warpsÚD)ÚconfigsÚkeyÚT)Údo_not_specializec                 óÞ  • [         R                  " S5      R                  [         R                  5      nXv-  [         R                  " SU5      -   nX„:  n	X…-  n
X…-  n[         R
                  " X
U-  -   U-   U	SS9R                  [         R                  5      n[         R                  " U5      n[         R                  " XU-  -   U-   UR                  UR                  R                  5      U	S9  g ©Nr   ç        ©ÚmaskÚother©r8   ©ÚtlÚ
program_idÚtoÚint64ÚarangeÚloadÚfloat32ÚsigmoidÚstoreÚdtypeÚ
element_ty©r   ÚyÚstride_x_rowÚstride_y_rowr2   r/   r-   Úi_nÚoffsr8   ÚrowÚcolÚb_xÚb_ys                 r   Úsigmoid_fwd_kernelrQ   \   s·   € ô$ �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�*Š*�S‹/€CÜ‡H‚HˆQ�|Ñ#Ñ# cÑ)¨3¯6©6°!·'±'×2DÑ2DÓ+EÈDÓQr   c	                 ój  • [         R                  " S5      R                  [         R                  5      n	X˜-  [         R                  " SU5      -   n
X¦:  nX§-  nX§-  n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R                  " U5      nUU-  SU-
  -  n[         R                  " X,U-  -   U-   UR                  UR                  R                  5      US9  g ©Nr   r6   r7   ç      ð?r:   r;   ©r   ÚdyÚdxrI   Ústride_dy_rowÚstride_dx_rowr2   r/   r-   rK   rL   r8   rM   rN   rO   Úb_dyÚb_sÚb_dxs                     r   Úsigmoid_bwd_kernelr]   x   sø   € ô& �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ�7Š7�2˜mÑ+Ñ+¨cÑ1¸ÀBÑG×JÑJÌ2Ï:É:ÓV€DÜ
�*Š*�S‹/€CØ�#‰:˜˜s™Ñ#€DÜ‡H‚HˆR˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÓUr   ÚmodulesÚoutput_contiguousc           	      óÄ   ^• [        U 5      n U R                  5       U R                  S   smn[        X5      n[        U4S j   " U U[        U 5      [        U5      TUS9  U$ )Nr   c                 ó:   >• [         R                  " TU S   5      4$ ©Nr-   ©ÚtritonÚcdiv©Úmetar2   s    €r   Ú<lambda>Úsigmoid_fwd.<locals>.<lambda>œ   ó   ø€ ¤V§[¢[°°D¸±IÓ%>Ñ$@r   ©r   rH   rI   rJ   r2   r/   )r(   Únumelr   r,   rQ   r   ©r   r_   r/   rH   r2   s       @r   Úsigmoid_fwdrn   —   s_   ø€ ä  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ�aÓ+€AÜÔ@ÒAØ
Ø
Ü  “^Ü  “^Ø
Ø
òð €Hr   rV   c                 óð   ^• [        U 5      n [        U5      nU R                  5       U R                  S   smn[        X5      n[        U4S j   " U UU[        U 5      [        U5      [        U5      TUS9  U$ )Nr   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Úsigmoid_bwd.<locals>.<lambda>­   rj   r   ©r   rV   rW   rI   rX   rY   r2   r/   )r(   rl   r   r,   r]   r   ©r   rV   r_   r/   rW   r2   s        @r   Úsigmoid_bwdrt   §   ss   ø€ ä  Ó#€AÜ	! "Ó	%€BØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ	�qÓ	,€BÜÔ@ÒAØ
ØØÜ  “^Ü! "“oÜ! "“oØ
Ø
ò	ð €Ir   c                   óX   • \ rS rSr\\" SS9S 5       5       r\\" SS9S 5       5       rSrg)ÚSigmoidFunctionéº   T©Úno_guard_contiguousc                 ó:   • U R                  U5        [        U5      $ ©N)Úsave_for_backwardrn   ©Úctxr   s     r   ÚforwardÚSigmoidFunction.forward¼   s   € ð 	×Ñ˜aÔ Ü˜1‹~Ðr   c                 ó4   • U R                   u  n[        X!5      $ r{   )Úsaved_tensorsrt   ©r~   Údoutr   s      r   ÚbackwardÚSigmoidFunction.backwardÂ   s   € ð ×Ñ‰ˆÜ˜1Ó#Ð#r   © N©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr
   r   r…   Ú__static_attributes__r‡   r   r   rv   rv   º   sB   † àÙ TÑ*ñó +ó ðð Ù TÑ*ñ$ó +ó ó$r   rv   c                 óH  • [         R                  " S5      R                  [         R                  5      nX‡-  [         R                  " SU5      -   n	X•:  n
X–-  nX–-  n[         R
                  " XU-  -   U-   U
SS9R                  [         R                  5      n[         R                  " SU5      nS[        [         R                  " U5      * 5      -   nU[        U5      -
  U-  n[         R                  " XU-  -   U-   UR                  UR                  R                  5      U
S9  g rS   )r<   r=   r>   r?   r@   rA   rB   Úminimumr   Úabsr   rD   rE   rF   )r   rH   rI   rJ   Útemperaturer2   r/   r-   rK   rL   r8   rM   rN   rO   Úb_mÚb_zrP   s                    r   Úlogsigmoid_fwd_kernelr•   Ì   så   € ô( �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�*Š*�R˜Ó
€CØ
Œs”B—F’F˜3“K�<Ó Ñ
 €CØ”�S“‰>˜[Ñ
(€CÜ‡H‚HˆQ�|Ñ#Ñ# cÑ)¨3¯6©6°!·'±'×2DÑ2DÓ+EÈDÓQr   c
                 óf  • [         R                  " S5      R                  [         R                  5      n
X©-  [         R                  " SU	5      -   nX·:  nX¸-  nX¸-  n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      nUS[         R                  " U5      -
  U-  -  n[         R                  " X-U-  -   U-   UR                  UR                  R                  5      US9  g rS   r;   )r   rV   rW   rI   rX   rY   r’   r2   r/   r-   rK   rL   r8   rM   rN   rO   rZ   r\   s                     r   Úlogsigmoid_bwd_kernelr—   ì   sõ   € ô, �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ�7Š7�2˜mÑ+Ñ+¨cÑ1¸ÀBÑG×JÑJÌ2Ï:É:ÓV€DØ�BœŸš C›Ñ(¨KÑ7Ñ8€DÜ‡H‚HˆR˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÓUr   r’   c           
      óÆ   ^• [        U 5      n U R                  5       U R                  S   smn[        X5      n[        U4S j   " U U[        U 5      [        U5      UTUS9  U$ )Nr   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Ú logsigmoid_fwd.<locals>.<lambda>  ó   ø€ ¬¯ª°A°t¸C±yÓ(AÑ'Cr   )r   rH   rI   rJ   r’   r2   r/   )r(   rl   r   r,   r•   r   )r   r’   r_   r/   rH   r2   s        @r   Úlogsigmoid_fwdrœ     sb   ø€ ä  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ�aÓ+€AÜÔCÒDØ
Ø
Ü  “^Ü  “^ØØ
Ø
òð €Hr   c                 óò   ^• [        U 5      n [        U5      nU R                  5       U R                  S   smn[        X5      n[        U4S j   " U UU[        U 5      [        U5      [        U5      UTUS9	  U$ )Nr   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Ú logsigmoid_bwd.<locals>.<lambda>)  r›   r   )	r   rV   rW   rI   rX   rY   r’   r2   r/   )r(   rl   r   r,   r—   r   )r   rV   r’   r_   r/   rW   r2   s         @r   Úlogsigmoid_bwdr      sx   ø€ ô 	! Ó#€AÜ	! "Ó	%€BØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ	�qÓ	,€BÜÔCÒDØ
ØØÜ  “^Ü! "“oÜ! "“oØØ
Ø
ò
ð €Ir   c                   óX   • \ rS rSr\\" SS9S 5       5       r\\" SS9S 5       5       rSrg)ÚLogSigmoidFunctioni7  Trx   c                 óF   • U R                  U5        X l        [        X5      $ r{   )r|   r’   rœ   )r~   r   r’   s      r   r   ÚLogSigmoidFunction.forward9  s"   € ð 	×Ñ˜aÔ Ø%ŒÜ˜aÓ-Ð-r   c                 óN   • U R                   u  n[        X!U R                  5      S 4$ r{   )r‚   r    r’   )r~   rV   r   s      r   r…   ÚLogSigmoidFunction.backward@  s'   € ð ×Ñ‰ˆÜ˜a S§_¡_Ó5°tÐ;Ð;r   r‡   Nrˆ   r‡   r   r   r¢   r¢   7  sB   † àÙ TÑ*ñ.ó +ó ð.ð
 Ù TÑ*ñ<ó +ó ó<r   r¢   c                 ó,   • [         R                  X5      $ r{   )r¢   Úapply)r   r’   s     r   Ú
logsigmoidr©   G  s   € Ü×#Ñ# AÓ3Ð3r   c                 óä  • [         R                  " S5      R                  [         R                  5      nXv-  [         R                  " SU5      -   nX„:  n	X…-  n
X…-  n[         R
                  " X
U-  -   U-   U	SS9R                  [         R                  5      nU[         R                  " U5      -  n[         R                  " XU-  -   U-   UR                  UR                  R                  5      U	S9  g r5   r;   rG   s                 r   Úswish_fwd_kernelr«   K  s¼   € ô$ �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CØ
”—
’
˜3“Ñ
€CÜ‡H‚HˆQ�|Ñ#Ñ# cÑ)¨3¯6©6°!·'±'×2DÑ2DÓ+EÈDÓQr   c	                 óv  • [         R                  " S5      R                  [         R                  5      n	X˜-  [         R                  " SU5      -   n
X¦:  nX§-  nX§-  n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R                  " U5      nUU-  SUSU-
  -  -   -  n[         R                  " X,U-  -   U-   UR                  UR                  R                  5      US9  g rS   r;   rU   s                     r   Úswish_bwd_kernelr­   g  s  € ô& �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ�7Š7�2˜mÑ+Ñ+¨cÑ1¸ÀBÑG×JÑJÌ2Ï:É:ÓV€DÜ
�*Š*�S‹/€CØ�#‰:˜˜s c¨C¡iÑ0Ñ0Ñ1€DÜ‡H‚HˆR˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÓUr   c           	      óÄ   ^• [        U 5      n U R                  5       U R                  S   smn[        X5      n[        U4S j   " U U[        U 5      [        U5      TUS9  U$ )Nr   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Úswish_fwd.<locals>.<lambda>‹  ó   ø€ ¤6§;¢;¨q°$°s±)Ó#<Ñ">r   rk   )r(   rl   r   r,   r«   r   rm   s       @r   Ú	swish_fwdr²   †  s_   ø€ ä  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ�aÓ+€AÜÔ>Ò?Ø
Ø
Ü  “^Ü  “^Ø
Ø
òð €Hr   c                 óð   ^• [        U 5      n [        U5      nU R                  5       U R                  S   smn[        X5      n[        U4S j   " U UU[        U 5      [        U5      [        U5      TUS9  U$ )Nr   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Úswish_bwd.<locals>.<lambda>œ  r±   r   rr   )r(   rl   r   r,   r­   r   rs   s        @r   Ú	swish_bwdr¶   –  ss   ø€ ä  Ó#€AÜ	! "Ó	%€BØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ	�qÓ	,€BÜÔ>Ò?Ø
ØØÜ  “^Ü! "“oÜ! "“oØ
Ø
ò	ð €Ir   c                   óX   • \ rS rSr\\" SS9S 5       5       r\\" SS9S 5       5       rSrg)ÚSwishFunctioni©  Trx   c                 ó:   • U R                  U5        [        U5      $ r{   )r|   r²   r}   s     r   r   ÚSwishFunction.forward«  s   € ð 	×Ñ˜aÔ Ü˜‹|Ðr   c                 ó4   • U R                   u  n[        X!5      $ r{   )r‚   r¶   rƒ   s      r   r…   ÚSwishFunction.backward±  s   € ð ×Ñ‰ˆÜ˜Ó!Ð!r   r‡   Nrˆ   r‡   r   r   r¸   r¸   ©  sB   † àÙ TÑ*ñó +ó ðð Ù TÑ*ñ"ó +ó ó"r   r¸   c                 ó”   • X-   nUS-  S[         R                  " SU-  SSU-  U-  -   -  5      -   -  R                  U R                  S9$ ©Nç      à?rT   ç Þe3Eˆé?r   ç÷Hmâä¦?©rE   ©r   Útanhr>   rE   )rH   Úbiasr   s      r   Ú	bias_gelurÆ   Â  sR   € à‰€AØ�‰G�sœUŸZšZ¨
°Q©¸!¸hÈ¹lÈQÑ>NÑ:NÑ(OÓPÑPÑQ×UÑUÐ\]×\cÑ\cÐUÐdÐdr   c                 ó  • X!-   n[         R                  " SU-  SSU-  U-  -   -  5      nSU-  SXD-  -
  SSU-  U-  -   -  -  SSU-   -  -   nXP-  nUR                  UR                  S9UR	                  SUR                  S94$ )	z7Assume that y has shape (B, D=D) and bias has shape (D)rÀ   r   rÁ   r¿   ç6”ü¾vf»?rÂ   r   )ÚdimrE   )r   rÄ   r>   rE   Úsum)ÚgrH   rÅ   r   Útanh_outÚffÚgrad_ys          r   Úbias_gelu_bwdrÏ   Ë  s¤   € ð 	‰€AÜ�zŠz˜* q™.¨A°¸1±¸qÑ0@Ñ,@ÑAÓB€Hà	ˆq‰�Q˜Ñ,Ñ,°¸lÈQÑ>NÐQRÑ>RÑ1RÑSÑ	TÐWZØ	ˆH‰ñXñ 
€Bð ‰V€FØ�9‰9˜1Ÿ7™7ˆ9Ð# V§Z¡Z°Q¸t¿z¹z ZÐ%JÐJÐJr   c                   ó4   • \ rS rSr\S 5       r\S 5       rSrg)ÚGeLUFunctioniØ  c                 ó:   • U R                  X5        [        X5      $ r{   )r|   rÆ   )r~   ÚinputrÅ   s      r   r   ÚGeLUFunction.forwardÚ  s   € ð 	×Ñ˜eÔ*Ü˜Ó%Ð%r   c                 ó6   • U R                   u  p#[        XU5      $ r{   )r‚   rÏ   )r~   Úgrad_outputrÓ   rÅ   s       r   r…   ÚGeLUFunction.backwardà  s   € à×'Ñ'‰ˆÜ˜[°Ó6Ð6r   r‡   N©r‰   rŠ   r‹   rŒ   r�   r   r…   rŽ   r‡   r   r   rÑ   rÑ   Ø  s(   † àñ&ó ð&ð ñ7ó ó7r   rÑ   c                 óŒ   • U S-  S[         R                  " SU -  SSU -  U -  -   -  5      -   -  R                  U R                  S9$ r¾   rÃ   r   s    r   Úgelu_fwdrÚ   ì  sK   € à�‰G�sœUŸZšZ¨
°Q©¸!¸hÈ¹lÈQÑ>NÑ:NÑ(OÓPÑPÑQ×UÑUÐ\]×\cÑ\cÐUÐdÐdr   c                 óÆ   • [         R                  " SU-  SSU-  U-  -   -  5      nSU-  SX"-  -
  SSU-  U-  -   -  -  SSU-   -  -   nX0-  R                  UR                  S9$ )NrÀ   r   rÁ   r¿   rÈ   rÂ   rÃ   )rË   r   rÌ   rÍ   s       r   Úgelu_bwdrÜ   ô  s€   € ä�zŠz˜* q™.¨A°¸1±¸qÑ0@Ñ,@ÑAÓB€Hà	ˆq‰�Q˜Ñ,Ñ,°¸lÈQÑ>NÐQRÑ>RÑ1RÑSÑ	TÐWZØ	ˆH‰ñXñ 
€Bð ‰F�;‰;˜QŸW™Wˆ;Ð%Ð%r   c                   ó4   • \ rS rSr\S 5       r\S 5       rSrg)ÚFastGeLUFunctioniþ  c                 ó:   • U R                  U5        [        U5      $ r{   )r|   rÚ   ©r~   rÓ   s     r   r   ÚFastGeLUFunction.forwardÿ  s   € ð 	×Ñ˜eÔ$Ü˜‹Ðr   c                 ó8   • U R                   u  n[        X5      nU$ r{   )r‚   rÜ   )r~   rÖ   rÓ   Útmps       r   r…   ÚFastGeLUFunction.backward  s   € à×$Ñ$‰ˆÜ�{Ó*ˆØˆ
r   r‡   NrØ   r‡   r   r   rÞ   rÞ   þ  s(   † Øñó ðð ñó ór   rÞ   c                 óf   • [         R                  " US:¬  U S5      R                  UR                  S9$ )Nr   r6   rÂ   )r   Úwherer>   rE   ©rË   r   s     r   Úrelu_bwdrè     s+   € ä�;Š;�q˜A‘v˜q #Ó&×)Ñ)°·±Ð)Ð8Ð8r   c                 ó€   • [         R                  " U R                  5       5      nX-  R                  U R                  S9$ )NrÂ   ©ÚFÚreluÚfloatr>   rE   )r   Úrs     r   Ú
sqrelu_fwdrï     s-   € ä	�Šˆq�w‰w‹yÓ€AØ‰E�:‰:˜AŸG™Gˆ:Ð$Ð$r   c                 ó„   • SU -  [         R                  " UR                  5       5      -  R                  UR                  S9$ )Nç       @rÂ   rê   rç   s     r   Ú
sqrelu_bwdrò     s2   € à�!‰G”a—f’f˜QŸW™W›YÓ'Ñ'×+Ñ+°!·'±'Ð+Ð:Ð:r   c                   ó4   • \ rS rSr\S 5       r\S 5       rSrg)ÚSquaredReLUFunctioni  c                 ó:   • U R                  U5        [        U5      $ r{   )r|   rï   rà   s     r   r   ÚSquaredReLUFunction.forward!  s   € à×Ñ˜eÔ$Ü˜%Ó Ð r   c                 ó4   • U R                   u  n[        X5      $ r{   )r‚   rò   )r~   rÖ   rÓ   s      r   r…   ÚSquaredReLUFunction.backward&  s   € à×"Ñ"‰ˆÜ˜+Ó-Ð-r   r‡   NrØ   r‡   r   r   rô   rô     s(   † àñ!ó ð!ð ñ.ó ó.r   rô   c	                 ó`  • [         R                  " S5      R                  [         R                  5      n	X˜-  [         R                  " SU5      -   n
X¦:  nX§-  nX§-  n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      nU[         R                  " U5      -  U-  n[         R                  " X,U-  -   U-   UR                  UR                  R                  5      US9  g r5   r;   ©r   rH   ÚzrI   rJ   Ústride_z_rowr2   r/   r-   rK   rL   r8   rM   rN   rO   rP   r”   s                    r   Úswiglu_fwd_kernelrý   /  sï   € ô& �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CØ
”—
’
˜3“Ñ
 #Ñ
%€CÜ‡H‚HˆQ�|Ñ#Ñ# cÑ)¨3¯6©6°!·'±'×2DÑ2DÓ+EÈDÓQr   Ú
HAS_WEIGHTc                 ó   • U S   S L$ ©Nrû   r‡   ©Úargss    r   rh   rh   N  ó   € ˜t C™y°Ñ4r   c                 ó8  • [         R                  " S5      R                  [         R                  5      nUU-  [         R                  " SU5      -   nUU:  nUU-  nUU-  n[         R
                  " U UU-  -   U-   USS9R                  [         R                  5      n[         R
                  " UUU-  -   U-   USS9R                  [         R                  5      n[         R
                  " UUU-  -   U-   USS9R                  [         R                  5      n[         R                  " U5      nUU-  nUU-  SUSU-
  -  -   -  U-  nUU-  n[         R                  " UUU	-  -   U-   UR                  UR                  R                  5      US9  [         R                  " UUU
-  -   U-   UR                  UR                  R                  5      US9  U(       aH  UU-  n[         R                  " UUU-  -   U-   UR                  UR                  R                  5      US9  g g rS   r;   )r   rH   rË   rW   rV   rû   rI   rJ   Ústride_g_rowrY   rX   rü   r2   r/   r-   rþ   rK   rL   r8   rM   rN   rO   rP   Úb_gr[   Úb_xsr\   rZ   r”   s                                r   Úswiglu_fwdbwd_kernelr  M  sÎ  € ô4 �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ�‰7”R—Y’Y˜q !“_Ñ$€DØ�!‰8€DØ
�!‰)€CØ
�‰(€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€Cä
�*Š*�S‹/€CØ�‰9€DØ�‰9˜˜c S¨3¡YÑ/Ñ/Ñ0°3Ñ6€DØ�‰:€Dä‡H‚HˆR�#˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÒUÜ‡H‚HˆR�#˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÒUÞØ�S‰jˆÜ
�Š��S˜<Ñ'Ñ'¨#Ñ-¨s¯v©v°a·g±g×6HÑ6HÓ/IÐPTÓUð r   rH   c                 óf  ^• U R                   UR                   :X  d!   SU R                    SUR                    35       e[        U 5      n [        U5      nU R                  5       U R                   S   smn[        X5      n[        U4S j   " U UU[        U 5      [        U5      [        U5      TUS9  U$ )Nzswiglu_fwd: shape mismatch x=ú y=r   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Úswiglu_fwd.<locals>.<lambda>ƒ  ó   ø€ ¤F§K¢K°°4¸±9Ó$=Ñ#?r   ©r   rH   rû   rI   rJ   rü   r2   r/   )r   r(   rl   r,   rý   r   ©r   rH   r_   r/   rû   r2   s        @r   Ú
swiglu_fwdr  |  s¤   ø€ à�7‰7�a—g‘gÓÐTÐ!>¸q¿w¹w¸iÀsÈ1Ï7É7È)ÐTÓTÐÜ  Ó#€AÜ  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ�aÓ+€AÜÔ?Ò@Ø
Ø
Ø
Ü  “^Ü  “^Ü  “^Ø
Ø
ò	ð €Hr   rË   Ú
use_weightc                 ó`  ^	• U R                   UR                   s=:X  a  UR                   :X  d0  O   SU R                    SUR                    SUR                    35       e[        U 5      n [        U5      n[        U5      nU R                  5       U R                   S   sm	n[        X5      n[        X5      nU(       a  [        X5      nOS n[        U	4S j   " U UUUUU[        U 5      [        U5      [        U5      [        U5      [        U5      Ub  [        U5      OST	US9  U(       a  XgU4$ Xg4$ )Nz swiglu_fwdbwd: shape mismatch x=r
  ú g=r   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Úswiglu_fwdbwd.<locals>.<lambda>£  ó   ø€ ¤v§{¢{°1°d¸3±iÓ'@Ñ&Br   r   ©r   rH   rË   rW   rV   rû   rI   rJ   r  rY   rX   rü   r2   r/   )r   r(   rl   r,   r  r   ©
r   rH   rË   r  r_   r/   rW   rV   rû   r2   s
            @r   Úswiglu_fwdbwdr  �  s  ø€ ð �7‰7�a—g‘gÕ( §¡Õ(ÐnÐ,LÈQÏWÉWÈIÐUXÐYZ×Y`ÑY`ÐXaÐadÐef×elÑelÐdmÐ*nÓnÐ(Ü  Ó#€AÜ  Ó#€AÜ  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ	�qÓ	,€BÜ	�qÓ	,€BÞÜ˜!Ó/‰àˆÜÔBÒCØ
Ø
Ø
ØØØ
Ü  “^Ü  “^Ü  “^Ü! "“oÜ! "“oØ'(¡}”[ ”^¸!Ø
Ø
òö  Ø�qˆyÐØˆ6€Mr   c                   ó\   • \ rS rSrSr\\" SS9S 5       5       r\\" SS9S 5       5       rSr	g)	ÚSwiGLUFunctioni¸  z{
Swish-Gated Linear Unit (SwiGLU) function.

.. math::
    \text{SwiGLU}(x, y) = swish(x) * y = \frac{x}{1 + \exp(-x)} * y
Trx   c                 ó:   • U R                  X5        [        X5      $ r{   )r|   r  ©r~   r   rH   s      r   r   ÚSwiGLUFunction.forwardÀ  s   € ð 	×Ñ˜aÔ#Ü˜!ÓÐr   c                 ó6   • U R                   u  p#[        X#U5      $ r{   )r‚   r  ©r~   r„   r   rH   s       r   r…   ÚSwiGLUFunction.backwardÆ  s   € ð × Ñ ‰ˆÜ˜Q 4Ó(Ð(r   r‡   N©
r‰   rŠ   r‹   rŒ   Ú__doc__r�   r
   r   r…   rŽ   r‡   r   r   r  r  ¸  sI   † ñð Ù TÑ*ñ ó +ó ð ð Ù TÑ*ñ)ó +ó ó)r   r  c                   óp   • \ rS rSrSr\\" SS9\S 5       5       5       r\\" SS9\	S 5       5       5       r
Srg)	ÚSwiGLULinearFunctioniÍ  zê
Swish-Gated Linear Unit (SwiGLU) function followed by a linear transformation.

.. math::
    \text{SwiGLULinear}(x, y, W, b) = (swish(x) * y) W + b

This simple wrap discards the intermediate results of SwiGLU(x, y) to save memory.
Trx   c                 ó~   • [        XSS9n[        R                  " XSU5      nU R                  XU5        US L U l        U$ ©NT)r_   )r  rë   Úlinearr|   Úlinear_bias_is_none©r~   r   rH   ÚweightrÅ   rû   Úouts          r   r   ÚSwiGLULinearFunction.forward×  sA   € ô �q¨tÑ4ˆÜ�hŠh�q $Ó'ˆØ×Ñ˜a FÔ+Ø"&¨$ ,ˆÔØˆ
r   c           	      óš  • U R                   u  p4nUR                  SUR                  S   5      n[        R                  " XR                  5       5      R                  U5      n[        X4USSS9u  pxn	[        R                  " SXR                  SU	R                  S   5      5      n
U R                  (       a  S OUR                  S5      nXxX«4$ ©Nr   T)r  r_   z	bo,bi->oir   )r‚   Úreshaper   rë   r(  ÚtÚview_asr  r   Úeinsumr)  rÊ   ©r~   r„   r  r   rH   r+  ÚdzrW   rV   rû   Údlinear_weightÚdlinear_biass               r   r…   ÚSwiGLULinearFunction.backwardá  s£   € ð ×(Ñ(‰ˆˆfØ�|‰|˜B §
¡
¨2¡Ó/ˆÜ�XŠX�dŸH™H›JÓ'×/Ñ/°Ó2ˆÜ! !¨°tÈtÑT‰	ˆ�ÜŸš k°4¿¹À2ÀqÇwÁwÈrÁ{Ó9SÓTˆØ"×6×6‘t¸D¿H¹HÀQ»KˆØ�~Ð3Ð3r   r‡   N©r‰   rŠ   r‹   rŒ   r#  r�   r
   r   r   r   r…   rŽ   r‡   r   r   r%  r%  Í  óY   † ñð Ù TÑ*Øñó ó +ó ðð Ù TÑ*Øñ4ó ó +ó ó4r   r%  c                 ó.   • [         R                  XX#5      $ r{   )r%  r¨   )r   rH   r+  rÅ   s       r   Úswiglu_linearr<  ñ  s   € ä×%Ñ% a¨FÓ9Ð9r   c	                 óZ  • [         R                  " S5      R                  [         R                  5      n	X˜-  [         R                  " SU5      -   n
X¦:  nX§-  nX§-  n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R                  " U5      U-  n[         R                  " X,U-  -   U-   UR                  UR                  R                  5      US9  g r5   r;   rú   s                    r   Úsigmoidglu_fwd_kernelr>  ö  sê   € ô& �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�*Š*�S‹/˜CÑ
€CÜ‡H‚HˆQ�|Ñ#Ñ# cÑ)¨3¯6©6°!·'±'×2DÑ2DÓ+EÈDÓQr   c                 ó   • U S   S L$ r   r‡   r  s    r   rh   rh     r  r   c                 ó"  • [         R                  " S5      R                  [         R                  5      nUU-  [         R                  " SU5      -   nUU:  nUU-  nUU-  n[         R
                  " U UU-  -   U-   USS9R                  [         R                  5      n[         R
                  " UUU-  -   U-   USS9R                  [         R                  5      n[         R
                  " UUU-  -   U-   USS9R                  [         R                  5      n[         R                  " U5      nUU-  SU-
  -  U-  nUU-  n[         R                  " UUU	-  -   U-   UR                  UR                  R                  5      US9  [         R                  " UUU
-  -   U-   UR                  UR                  R                  5      US9  U(       aH  UU-  n[         R                  " UUU-  -   U-   UR                  UR                  R                  5      US9  g g rS   r;   )r   rH   rË   rW   rV   rû   rI   rJ   r  rY   rX   rü   r2   r/   r-   rþ   rK   rL   r8   rM   rN   rO   rP   r  r[   r\   rZ   r”   s                               r   Úsigmoidglu_fwdbwd_kernelrA    s»  € ô4 �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ�‰7”R—Y’Y˜q !“_Ñ$€DØ�!‰8€DØ
�!‰)€CØ
�‰(€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€Cä
�*Š*�S‹/€CØ�‰9˜˜c™	Ñ" SÑ(€DØ�‰9€Dä‡H‚HˆR�#˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÒUÜ‡H‚HˆR�#˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÒUÞØ�C‰iˆÜ
�Š��S˜<Ñ'Ñ'¨#Ñ-¨s¯v©v°a·g±g×6HÑ6HÓ/IÐPTÓUð r   c                 óf  ^• U R                   UR                   :X  d!   SU R                    SUR                    35       e[        U 5      n [        U5      nU R                  5       U R                   S   smn[        X5      n[        U4S j   " U UU[        U 5      [        U5      [        U5      TUS9  U$ )Nz!sigmoidglu_fwd: shape mismatch x=r
  r   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Ú sigmoidglu_fwd.<locals>.<lambda>I  r›   r   r  )r   r(   rl   r,   r>  r   r  s        @r   Úsigmoidglu_fwdrE  B  s¤   ø€ à�7‰7�a—g‘gÓÐXÐ!BÀ1Ç7Á7À)È3ÈqÏwÉwÈiÐXÓXÐÜ  Ó#€AÜ  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ�aÓ+€AÜÔCÒDØ
Ø
Ø
Ü  “^Ü  “^Ü  “^Ø
Ø
ò	ð €Hr   c                 ó`  ^	• U R                   UR                   s=:X  a  UR                   :X  d0  O   SU R                    SUR                    SUR                    35       e[        U 5      n [        U5      n[        U5      nU R                  5       U R                   S   sm	n[        X5      n[        X5      nU(       a  [        X5      nOS n[        U	4S j   " U UUUUU[        U 5      [        U5      [        U5      [        U5      [        U5      Ub  [        U5      OST	US9  U(       a  XgU4$ Xg4$ )Nz$sigmoidglu_fwdbwd: shape mismatch x=r
  r  r   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Ú#sigmoidglu_fwdbwd.<locals>.<lambda>i  s   ø€ ¬6¯;ª;°q¸$¸s¹)Ó+DÑ*Fr   r   r  )r   r(   rl   r,   rA  r   r  s
            @r   Úsigmoidglu_fwdbwdrI  V  s   ø€ ð �7‰7�a—g‘gÕ( §¡Õ(ÐrÐ,PÐQR×QXÑQXÐPYÐY\Ð]^×]dÑ]dÐ\eÐehÐij×ipÑipÐhqÐ*rÓrÐ(Ü  Ó#€AÜ  Ó#€AÜ  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ	�qÓ	,€BÜ	�qÓ	,€BÞÜ˜!Ó/‰àˆÜÔFÒGØ
Ø
Ø
ØØØ
Ü  “^Ü  “^Ü  “^Ü! "“oÜ! "“oØ'(¡}”[ ”^¸!Ø
Ø
òö  Ø�qˆyÐØˆ6€Mr   c                   ó\   • \ rS rSrSr\\" SS9S 5       5       r\\" SS9S 5       5       rSr	g)	ÚSigmoidGLUFunctioni~  z‡
Sigmoid-Gated Linear Unit (SigmoidGLU) function.

.. math::
    \text{SigmoidGLU}(x, y) = sigmoid(x) * y = \frac{1}{1 + \exp(-x)} * y
Trx   c                 ó:   • U R                  X5        [        X5      $ r{   )r|   rE  r  s      r   r   ÚSigmoidGLUFunction.forward†  s   € ð 	×Ñ˜aÔ#Ü˜aÓ#Ð#r   c                 ó6   • U R                   u  p#[        X#U5      $ r{   )r‚   rI  r   s       r   r…   ÚSigmoidGLUFunction.backwardŒ  s   € ð × Ñ ‰ˆÜ   tÓ,Ð,r   r‡   Nr"  r‡   r   r   rK  rK  ~  sI   † ñð Ù TÑ*ñ$ó +ó ð$ð Ù TÑ*ñ-ó +ó ó-r   rK  c                   óp   • \ rS rSrSr\\" SS9\S 5       5       5       r\\" SS9\	S 5       5       5       r
Srg)	ÚSigmoidGLULinearFunctioni“  zú
Sigmoid-Gated Linear Unit (SigmoidGLU) function followed by a linear transformation.

.. math::
    \text{SigmoidGLULinear}(x, y, W, b) = (sigmoid(x) * y) W + b

This simple wrap discards the intermediate results of SigmoidGLU(x, y) to save memory.
Trx   c                 ó~   • [        XSS9n[        R                  " XSU5      nU R                  XU5        US L U l        U$ r'  )rE  rë   r(  r|   r)  r*  s          r   r   Ú SigmoidGLULinearFunction.forward�  sA   € ô ˜1°4Ñ8ˆÜ�hŠh�q $Ó'ˆØ×Ñ˜a FÔ+Ø"&¨$ ,ˆÔØˆ
r   c           	      óš  • U R                   u  p4nUR                  SUR                  S   5      n[        R                  " XR                  5       5      R                  U5      n[        X4USSS9u  pxn	[        R                  " SXR                  SU	R                  S   5      5      n
U R                  (       a  S OUR                  S5      nXxX«4$ r/  )r‚   r0  r   rë   r(  r1  r2  rI  r   r3  r)  rÊ   r4  s               r   r…   Ú!SigmoidGLULinearFunction.backward§  s¤   € ð ×(Ñ(‰ˆˆfØ�|‰|˜B §
¡
¨2¡Ó/ˆÜ�XŠX�dŸH™H›JÓ'×/Ñ/°Ó2ˆÜ% a¨B¸4ÐSWÑX‰	ˆ�ÜŸš k°4¿¹À2ÀqÇwÁwÈrÁ{Ó9SÓTˆØ"×6×6‘t¸D¿H¹HÀQ»KˆØ�~Ð3Ð3r   r‡   Nr9  r‡   r   r   rQ  rQ  “  r:  r   rQ  c
                 ó>  • [         R                  " S5      R                  [         R                  5      n
X©-  [         R                  " SU	5      -   nX·:  nX¸-  nX¸-  n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R
                  " XU-  -   U-   USS9R                  [         R                  5      n[         R                  " U5      nUS:„  n[         R                  " UUS5      n[         R                  " U5      nUUS-   -  n[        U[        U5      -  5      n[         R                  " UUU-  UU-  5      nUU-  n[         R                  " X-U-  -   U-   UR                  UR                  R                  5      US9  g rS   )r<   r=   r>   r?   r@   rA   rB   rC   ræ   Úsqrtr   r   rD   rE   rF   )r   rH   rû   rI   rJ   rü   Úmr2   r/   r-   rK   rL   r8   rM   rN   rO   rP   r[   Úb_posÚb_xpÚb_sqrtÚb_pÚb_powr  r”   s                            r   Úpowglu_fwd_kernelr^  º  sP  € ô( �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ‰7”R—Y’Y˜q !“_Ñ$€DØ‰8€DØ
‰)€CØ
‰(€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�*Š*�S‹/€CØ�!‰G€Eä�8Š8�E˜3 Ó$€DÜ�WŠW�T‹]€FØ
ˆv˜‰|Ñ
€CÜ�”c˜$“i‘Ó €EÜ
�(Š(�5˜% #™+ s¨S¡yÓ
1€CØ
�‰)€CÜ‡H‚HˆQ�|Ñ#Ñ# cÑ)¨3¯6©6°!·'±'×2DÑ2DÓ+EÈDÓQr   c                 ó   • U S   S L$ r   r‡   r  s    r   rh   rh   â  r  r   c                 ó   • [         R                  " S5      R                  [         R                  5      nUU-  [         R                  " SU5      -   nUU:  nUU-  nUU-  n[         R
                  " U UU-  -   U-   USS9R                  [         R                  5      n[         R
                  " UUU-  -   U-   USS9R                  [         R                  5      n[         R
                  " UUU-  -   U-   USS9R                  [         R                  5      n[         R                  " U5      nUS:„  n[         R                  " UUS5      n[         R                  " U5      n[        U5      nUUS-   -  n[        UU-  5      nUU-  n U* SU-  US-   -  US-   -  -  n!U U!U-  UU-  -   S-   U-
  -  n"UU-  n#USUSU-
  -  -   -  n$[         R                  " UU U#5      n%[         R                  " UU"U$5      n&UU-  U&-  n'UU%-  n([         R                  " UUU	-  -   U-   U'R                  UR                  R                  5      US9  [         R                  " UUU
-  -   U-   U(R                  UR                  R                  5      US9  U(       aH  U%U-  n)[         R                  " UUU-  -   U-   U)R                  UR                  R                  5      US9  g g )Nr   r6   r7   rT   rñ   r:   )r<   r=   r>   r?   r@   rA   rB   rC   ræ   rW  r   r   rD   rE   rF   )*r   rH   rË   rW   rV   rû   rI   rJ   r  rY   rX   rü   rX  r2   r/   r-   rþ   rK   rL   r8   rM   rN   rO   rP   r  r[   rY  rZ  r[  Úb_lnr\  r]  Ú
b_gate_posÚb_pprimeÚb_dgate_posÚ
b_gate_negÚb_dgate_negÚb_gateÚb_dgater\   rZ   r”   s*                                             r   Úpowglu_fwdbwd_kernelri  á  s�  € ô6 �-Š-˜Ó
×
Ñ
œbŸh™hÓ
'€CØ�‰7”R—Y’Y˜q !“_Ñ$€DØ�!‰8€DØ
�!‰)€CØ
�‰(€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€CÜ
�'Š'�!�c˜LÑ(Ñ(¨3Ñ.°TÀÑ
D×
GÑ
GÌÏ
É
Ó
S€Cä
�*Š*�S‹/€CØ�!‰G€EÜ�8Š8�E˜3 Ó$€DÜ�WŠW�T‹]€FÜˆt‹9€DØ
ˆv˜‰|Ñ
€CÜ��d‘
‹O€Eà˜‘€Jàˆr�S˜6‘\ V¨c¡\Ñ2°f¸s±lÑCÑD€HØ ¨4¡°#¸±*Ñ <¸sÑ BÀSÑ HÑI€KØ�s‘€JØ˜˜s c¨C¡iÑ0Ñ0Ñ1€Kä�XŠX�e˜Z¨Ó4€FÜ�hŠh�u˜k¨;Ó7€Gà�‰9�wÑ€DØ�‰<€Dä‡H‚HˆR�#˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÒUÜ‡H‚HˆR�#˜Ñ%Ñ%¨Ñ+¨T¯W©W°R·X±X×5HÑ5HÓ-IÐPTÒUÞØ�s‰lˆÜ
�Š��S˜<Ñ'Ñ'¨#Ñ-¨s¯v©v°a·g±g×6HÑ6HÓ/IÐPTÓUð r   Úpowerc                 óh  ^• U R                   UR                   :X  d!   SU R                    SUR                    35       e[        U 5      n [        U5      nU R                  5       U R                   S   smn[        X5      n[        U4S j   " U UU[        U 5      [        U5      [        U5      UTUS9	  U$ )Nzpowglu_fwd: shape mismatch x=r
  r   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Úpowglu_fwd.<locals>.<lambda>(  r  r   )	r   rH   rû   rI   rJ   rü   rX  r2   r/   )r   r(   rl   r,   r^  r   )r   rH   rj  r_   r/   rû   r2   s         @r   Ú
powglu_fwdrn  !  s§   ø€ à�7‰7�a—g‘gÓÐTÐ!>¸q¿w¹w¸iÀsÈ1Ï7É7È)ÐTÓTÐÜ  Ó#€AÜ  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ�aÓ+€AÜÔ?Ò@Ø
Ø
Ø
Ü  “^Ü  “^Ü  “^Ø
Ø
Ø
ò
ð €Hr   c                 ób  ^
• U R                   UR                   s=:X  a  UR                   :X  d0  O   SU R                    SUR                    SUR                    35       e[        U 5      n [        U5      n[        U5      nU R                  5       U R                   S   sm
n[        X5      n[        X5      nU(       a  [        X5      n	OS n	[        U
4S j   " U UUUUU	[        U 5      [        U5      [        U5      [        U5      [        U5      U	b  [        U	5      OSUT
US9  U(       a  XxU	4$ Xx4$ )Nz powglu_fwdbwd: shape mismatch x=r
  r  r   c                 ó:   >• [         R                  " TU S   5      4$ rb   rc   rf   s    €r   rh   Úpowglu_fwdbwd.<locals>.<lambda>J  r  r   r   )r   rH   rË   rW   rV   rû   rI   rJ   r  rY   rX   rü   rX  r2   r/   )r   r(   rl   r,   ri  r   )r   rH   rË   rj  r  r_   r/   rW   rV   rû   r2   s             @r   Úpowglu_fwdbwdrr  6  s  ø€ ð �7‰7�a—g‘gÕ( §¡Õ(ÐnÐ,LÈQÏWÉWÈIÐUXÐYZ×Y`ÑY`ÐXaÐadÐef×elÑelÐdmÐ*nÓnÐ(Ü  Ó#€AÜ  Ó#€AÜ  Ó#€AØ�7‰7‹9�a—g‘g˜b‘k€D€A€qÜ	�qÓ	,€BÜ	�qÓ	,€BÞÜ˜!Ó/‰àˆÜÔBÒCØ
Ø
Ø
ØØØ
Ü  “^Ü  “^Ü  “^Ü! "“oÜ! "“oØ'(¡}”[ ”^¸!Ø
Ø
Ø
òö" Ø�qˆyÐØˆ6€Mr   c                   ó\   • \ rS rSrSr\\" SS9S 5       5       r\\" SS9S 5       5       rSr	g)	ÚPowGLUFunctioni`  a   
Power-Gated Linear Unit (PowGLU) function.

.. math::
    \text{PowGLU}(x, y) = g(x) * y,\quad
    g(x) = \begin{cases} x^{power/(\sqrt{x}+1)}\,\sigma(x) & x > 0 \\ x\,\sigma(x) & x \le 0 \end{cases}

For ``x <= 0`` the gate reduces to swish, matching SwiGLU; for large ``x > 0`` it saturates instead of
growing, replacing SwiGLU's quadratic amplification with bounded growth (Power Linear Unit, arXiv:2605.25704).
Trx   c                 óH   • U R                  X5        X0l        [        XU5      $ r{   )r|   rj  rn  )r~   r   rH   rj  s       r   r   ÚPowGLUFunction.forwardl  s$   € ð 	×Ñ˜aÔ#ØŒ	Ü˜! Ó&Ð&r   c                 óV   • U R                   u  p#[        X#XR                  5      u  pEXES 4$ r{   )r‚   rr  rj  )r~   r„   r   rH   rW   rV   s         r   r…   ÚPowGLUFunction.backwards  s-   € ð × Ñ ‰ˆÜ˜q T¯9©9Ó5‰ˆØ�tˆ|Ðr   r‡   Nr"  r‡   r   r   rt  rt  `  sI   † ñ	ð Ù TÑ*ñ'ó +ó ð'ð
 Ù TÑ*ñó +ó ór   rt  c                   óp   • \ rS rSrSr\\" SS9\S 5       5       5       r\\" SS9\	S 5       5       5       r
Srg)	ÚPowGLULinearFunctioni{  zæ
Power-Gated Linear Unit (PowGLU) function followed by a linear transformation.

.. math::
    \text{PowGLULinear}(x, y, W, b) = (g(x) * y) W + b

This simple wrap discards the intermediate results of PowGLU(x, y) to save memory.
Trx   c                 óŒ   • [        XUSS9n[        R                  " XcU5      nU R                  XU5        US L U l        XPl        U$ r'  )rn  rë   r(  r|   r)  rj  )r~   r   rH   r+  rÅ   rj  rû   r,  s           r   r   ÚPowGLULinearFunction.forward…  sH   € ô �q˜U°dÑ;ˆÜ�hŠh�q $Ó'ˆØ×Ñ˜a FÔ+Ø"&¨$ ,ˆÔØŒ	Øˆ
r   c           	      ó°  • U R                   u  p4nUR                  SUR                  S   5      n[        R                  " XR                  5       5      R                  U5      n[        X4X`R                  SSS9u  pxn	[        R                  " SXR                  SU	R                  S   5      5      n
U R                  (       a  S OUR                  S5      nXxX«S 4$ r/  )r‚   r0  r   rë   r(  r1  r2  rr  rj  r   r3  r)  rÊ   r4  s               r   r…   ÚPowGLULinearFunction.backward�  sª   € ð ×(Ñ(‰ˆˆfØ�|‰|˜B §
¡
¨2¡Ó/ˆÜ�XŠX�dŸH™H›JÓ'×/Ñ/°Ó2ˆÜ! !¨¯I©IÀ$ÐZ^Ñ_‰	ˆ�ÜŸš k°4¿¹À2ÀqÇwÁwÈrÁ{Ó9SÓTˆØ"×6×6‘t¸D¿H¹HÀQ»KˆØ�~°TÐ9Ð9r   r‡   Nr9  r‡   r   r   rz  rz  {  sY   † ñð Ù TÑ*Øñó ó +ó ðð Ù TÑ*Øñ:ó ó +ó ó:r   rz  c                 ó.   • [         R                  XU5      $ r{   )rt  r¨   )r   rH   rj  s      r   Úpowglur€  �  s   € Ü×Ñ  eÓ,Ð,r   r+  rÅ   c                 ó0   • [         R                  XX#U5      $ r{   )rz  r¨   )r   rH   r+  rÅ   rj  s        r   Úpowglu_linearr‚  ¡  s   € ô  ×%Ñ% a¨F¸%Ó@Ð@r   )rì   rC   r©   ÚsiluÚswishÚsqreluÚgelurÆ   )Fr‡   )rT   F)rT   )FF)ç      @F)r‡  FF)r‡  )br#  r   Útorch.nn.functionalÚnnÚ
functionalrë   rd   Útriton.languageÚlanguager<   Úfla.modules.backendsr   Úfla.ops.utils.opr   r   Ú	fla.utilsr   r   r   r	   r
   ÚNUM_WARPS_AUTOTUNEÚTensorÚintr   Úboolr%   r(   r,   ÚautotuneÚConfigÚjitÚ	constexprrQ   r]   rn   rt   ÚautogradÚFunctionrv   r¨   rC   r•   r—   rí   rœ   r    r¢   r©   r«   r­   r²   r¶   r¸   r„  ÚcompilerÆ   rÏ   rÑ   Úbias_gelu_implrÚ   rÜ   rÞ   Úfast_gelu_implrè   rï   rò   rô   r…  rý   Ú
heuristicsr  r  r  r  r%  Úswiglur<  r>  rA  r   ÚdisablerE  rI  rK  rQ  Ú
sigmoidgluÚsigmoidglu_linearr^  ri  rn  rr  rt  rz  r€  r‚  rì   ÚACT2FN)Úbsr.   s   00r   Ú<module>r¤     sÈ  ðñó ß Ð Û Ý å )ß %ß jÕ jæ)/Ó%Ò5IÐ ð�5—<‘<ð  Cô ð˜EŸL™Lð ¨Tô ð> §¡ð °·±ô ñ�U—\‘\ð ¨tð ÀÇÁõ ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðRð
 
‡|�|ðRð 
‡|�|óRó %óðRð$ ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðVð 
‡|�|ðVð 
‡|�|óVó %óðVñ* 
ˆ)Óñ�5—<‘<ð °Dð ÀUÇ\Á\ô ó ðñ 
ˆ)Óñ�5—<‘<ð  U§\¡\ð Àdð ÐW\×WcÑWcô ó ðô$$�e—n‘n×-Ñ-ô $ð ×
Ñ
€ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðRð 
‡|�|ðRð 
‡|�|óRó %óðRð, ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðVð 
‡|�|ðVð 
‡|�|óVó %óðVñ. 
ˆ)Óñ�e—l‘lð °ð ÐPTð Ðaf×amÑamô ó ðñ  
ˆ)Óð Ø#ñ	Ø‡|�|ðà�‰ðð ðð ð	ð
 ‡\�\ôó ðô0<˜Ÿ™×0Ñ0ô <ñ 4�%—,‘,ð 4¨Uð 4¸E¿L¹Lõ 4ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðRð
 
‡|�|ðRð 
‡|�|óRó %óðRð$ ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðVð 
‡|�|ðVð 
‡|�|óVó %óðVñ* 
ˆ)Óñ�—‘ð °$ð À5Ç<Á<ô ó ðñ 
ˆ)Óñ�—‘ð  5§<¡<ð ÀDð ÐUZ×UaÑUaô ó ðô$"�E—N‘N×+Ñ+ô "ð 	×Ñ€ð ‡�ñeó ðeð ‡�ñ	Kó ð	Kô7�5—>‘>×*Ñ*ô 7ð ×#Ñ#€ð ‡�ñeó ðeð ‡�ñ&ó ð&ô�u—~‘~×.Ñ.ô ð "×'Ñ'€ð ‡�ñ9ó ð9ð ‡�ñ%ó ð%ð
 ‡�ñ;ó ð;ô
.˜%Ÿ.™.×1Ñ1ô 
.ð 
×	"Ñ	"€ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðRð 
‡|�|ðRð 
‡|�|óRó %óðRð( ×ÒØÑ4ðó ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðVð 
‡|�|ðVð 
‡|�|ðVð —‘óVó %óóðVñD 
ˆ)Óñ�%—,‘,ð  5§<¡<ð ÀDð ÐUZ×UaÑUaô ó ðñ& 
ˆ)Óð
 Ø#ñ$Ø‡|�|ð$à‡|�|ð$ð ‡|�|ð$ð ð	$ð
 ô$ó ð$ôN)�U—^‘^×,Ñ,ô )ô*4˜5Ÿ>™>×2Ñ2ô 4ðB 
×	Ñ	€ñ 
ˆ)Óñ:ó ð:ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðRð 
‡|�|ðRð 
‡|�|óRó %óðRð( ×ÒØÑ4ðó ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðVð 
‡|�|ðVð 
‡|�|ðVð —‘óVó %óóðVðB ‡�×Ññ�e—l‘lð  u§|¡|ð Èð ÐY^×YeÑYeô ó ðð& ‡�×Ñð
 Ø#ñ$Ø‡|�|ð$à‡|�|ð$ð ‡|�|ð$ð ð	$ð
 ô$ó ð$ôN-˜Ÿ™×0Ñ0ô -ô*4˜uŸ~™~×6Ñ6ô 4ðB  ×%Ñ%€
ð -×2Ñ2Ð ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ðRð 
‡|�|ðRð 
‡|�|óRó %óðRð: ×ÒØÑ4ðó ð ‡‚ð ñ 0ôâ/ˆBÜ+ˆIð 	�‰�s˜B�i¨9ˆÓ5á+ñ 	6Ù/òð
 
ˆñð ñð ‡‚˜s˜eÑ$ð0Vð 
‡|�|ð0Vð 
‡|�|ð0Vð —‘ó0Vó %óóð0Vñf 
ˆ)Óñ�%—,‘,ð  5§<¡<ð ¸ð ÐX\ð Ðin×iuÑiuô ó ðñ( 
ˆ)Óð
 ØØ#ñ&Ø‡|�|ð&à‡|�|ð&ð ‡|�|ð&ð ð	&ð
 ð&ð ô&ó ð&ôR�U—^‘^×,Ñ,ô ô6:˜5Ÿ>™>×2Ñ2ô :ñD-ˆe�l‰lð -˜uŸ|™|ð -°Eð -ÀEÇLÁLõ -ñ 
ˆ)Óð ñAØ‡|�|ðAà‡|�|ðAð �L‰LðAð �,‰,ð	Að
 ðAð ‡\�\ôAó ðAð �F‰FØØØØØØØñ	
�ùó_"ùó8ùóhùó@ùó~ùó8ùóPùóBùóLùóBùóFùóTsH   Â;&q-Ä2&q3È8&q9Ê/&q?Ï&rÑ&rÙ&rÛ&rà*&râ5&r#è"&r)ê-&r/