ó
    KëgjõG  ã                   ó~  • S SK r S SKrS SKJr  S SKJs  Jr  S SKrS SKJ	r
  S SKJr  S SKJrJr  S,S jr\R"                  " S S S.5      \R$                  S\
R&                  S	\
R&                  S
\
R&                  S\
R&                  S\
R&                  4
S j5       5       r     S-S\R*                  S\R*                  S\R*                  S\S\R*                  S\R*                  S\S\S\4S jjr\R"                  " S S S S.5      \R$                  S\
R&                  S\
R&                  S	\
R&                  S
\
R&                  S\
R&                  S\
R&                  4S j5       5       r       S.S\R*                  S\R*                  S\R*                  S\R*                  S\S\R*                  S \R*                  S\R*                  S\S\S\S!\S"\R*                  S\R*                  4S# jjr " S$ S%\R8                  R:                  5      rS/S& jrS0S' jr  " S( S)\RB                  5      r" " S* S+\RB                  5      r#g)1é    N)Ú	rearrange)Úget_multiprocessor_countÚinput_guardc                 óÀ  • U R                   nUR                  5       nUb  UR                  5       OS nU(       a%  U R                  5       n Ub  UR                  5       OUnUb   U(       d  U [        R                  " U5      -  n UcM  S[        R
                  " U R                  5       R                  SSS9U-   5      -  n	Ub
  X	-  U-  U-   OX	-  U-  n
O[[        U SUS9nS[        R
                  " UR                  5       R                  SSS9U-   5      -  n	[        X¹-  S5      U-  n
Ub  X¢-   n
Ub   U(       a  U
[        R                  " U5      -  n
U
R                  U5      $ )Né   éÿÿÿÿT)ÚdimÚkeepdimz... (g d) -> ... g d)Údz... g d -> ... (g d))
ÚdtypeÚfloatÚFÚsiluÚtorchÚsqrtÚsquareÚmeanr   Úto)ÚxÚweightÚbiasÚzÚepsÚ
group_sizeÚnorm_before_gateÚupcastr   ÚrstdÚoutÚx_groups               Ú©C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\fla/modules/layernorm_gated.pyÚrms_norm_refr!      s:  € Ø�G‰G€EØ�\‰\‹^€FØÑ+ˆ4�:‰:Œ<°€DÞØ�G‰G‹IˆØ™ˆA�G‰GŒI¨AˆØ�}Ö-Ø”—’�q“	‰MˆØÑØ”5—:’:˜qŸx™x›z×/Ñ/°BÀÐ/ÐEÈÑKÓLÑLˆØ,0Ñ,<ˆq‰x˜&Ñ  DÒ(À1Á8ÈfÑCT‰ä˜AÐ5¸ÑDˆØ”5—:’:˜wŸ~™~Ó/×5Ñ5¸"ÀdÐ5ÐKÈcÑQÓRÑRˆÜ˜™Ð(>Ó?À&ÑHˆØÑØ‘*ˆCØ�}Ö)ØŒq�vŠv�a‹yÑˆØ�6‰6�%‹=Ðó    c                 ó   • U S   S L$ ©NÚB© ©Úargss    r    Ú<lambda>r)   3   ó   € ˜T #™Y¨dÑ2r"   c                 ó   • U S   S L$ ©NÚZr&   r'   s    r    r)   r)   4   ó   € ˜$˜s™)¨4Ñ/r"   )ÚHAS_BIASÚHAS_ZÚBLOCK_Nr/   r0   ÚNORM_BEFORE_GATEÚIS_RMS_NORMc                 óh  • [         R                  " S5      n[         R                  " S5      nU UU-  UU-  -   -  n UUU-  UU-  -   -  nU(       a  UUU	-  UU-  -   -  nU(       d  UUU
-  -  nUUU
-  -  nUUU-  -  nU(       a  UUU-  -  n[         R                  " SU5      n[         R                  " U U-   UU:  SS9R	                  [         R
                  5      nU(       a[  U(       dT  [         R                  " UU-   UU:  S9R	                  [         R
                  5      nUU[         R                  " U5      -  -  nU(       dl  [         R                  " USS9U-  n[         R                  " UU-   U5        [         R                  " UU:  UU-
  S5      n[         R                  " UU-  SS9U-  nO6[         R                  " UU:  US5      n[         R                  " UU-  SS9U-  nS[         R                  " UU-   5      -  n[         R                  " UU-   U5        UU:  n[         R                  " UU-   US9R	                  [         R
                  5      nU(       a5  [         R                  " UU-   US9R	                  [         R
                  5      nU(       d  UW-
  U-  OUU-  nU(       a  UU-  W-   OUU-  nU(       aX  U(       aQ  [         R                  " UU-   US9R	                  [         R
                  5      nUU[         R                  " U5      -  -  n[         R                  " UU-   UUS9  g )Nr   r   ç        ©ÚmaskÚother©r7   ©Úaxis)ÚtlÚ
program_idÚarangeÚloadr   Úfloat32ÚsigmoidÚsumÚstoreÚwherer   ) ÚXÚYÚWr%   r-   ÚMeanÚRstdÚstride_x_rowÚstride_y_rowÚstride_z_rowÚMÚNr   r1   r/   r0   r2   r3   ÚrowÚgroupÚcolsr   r   r   ÚxbarÚvarr   r7   ÚwÚbÚx_hatÚys                                    r    Úlayer_norm_fwd_kernelrX   2   sª  € ô4 �-Š-˜Ó
€CÜ�MŠM˜!Ó€EØˆˆ|Ñ	˜e a™iÑ	'Ñ'€AØˆˆ|Ñ	˜e a™iÑ	'Ñ'€AÞØ	ˆS�<Ñ %¨!¡)Ñ+Ñ+ˆÞØ�˜‘	ÑˆØˆE�A‰IÑ€DØˆ�‰�N€AÞØ	ˆU�Q‰Y‰ˆä�9Š9�Q˜Ó €DÜ
�Š��D‘˜t a™x¨rÑ2×5Ñ5´b·j±jÓA€AÞÖ%Ü�GŠG�A˜‘H 4¨!¡8Ñ,×/Ñ/´·
±
Ó;ˆØ	ˆQ”—’˜A“ÑÑˆÞÜ�vŠv�a˜aÑ  1Ñ$ˆÜ
�Š�˜‘˜TÔ"Ü�xŠx˜˜q™ ! d¡(¨BÓ/ˆÜ�fŠf�T˜D‘[ qÑ)¨AÑ-‰ä�xŠx˜˜q™ ! RÓ(ˆÜ�fŠf�T˜D‘[ qÑ)¨AÑ-ˆØŒr�wŠw�s˜S‘yÓ!Ñ!€DÜ‡H‚HˆT�C‰Z˜Ôà�!‰8€DÜ
�Š��D‘˜tÑ$×'Ñ'¬¯
©
Ó3€AÞÜ�GŠG�A˜‘H 4Ñ(×+Ñ+¬B¯J©JÓ7ˆÞ%0ˆQ�‰X˜Ò°a¸$±h€EÞ!ˆ�‰	�AŠ u¨q¡y€AÞÖ!Ü�GŠG�A˜‘H 4Ñ(×+Ñ+¬B¯J©JÓ7ˆØ	ˆQ”—’˜A“ÑÑˆä‡H‚HˆQ�‰X�q˜tÓ$r"   r   r   r   r   r   r   r   r   Úis_rms_normc	                 óN  • U R                   u  pšUc  U
nX¦-  S:X  d   eX¦-  nU R                  S5      S:X  d   eUb*  UR                  S5      S:X  d   eUR                   Xš4:X  d   eUR                   U
4:X  d   eUR                  S5      S:X  d   eUb*  UR                  S5      S:X  d   eUR                   U
4:X  d   eUb  UR                   U R                   :X  d   eO[        R                  " U 5      nUR                  S5      S:X  d   eU(       d1  [        R                  " X¹-  4[        R
                  U R                  S9OS n[        R                  " X¹-  4[        R
                  U R                  S9nSU R                  5       -  n[        U[        R                  " U5      5      nXo:”  a  [        S5      e[        [        US-  S5      S5      nX›4n[        U   " U UUUUUUU R                  S5      UR                  S5      Ub  UR                  S5      OSU	UUUUUUS	9  X\U4$ )
Nr   r   r   ©r   Údeviceé   ú4This layer norm doesn't support feature dim >= 64KB.é   é   ©r1   r2   r3   Ú	num_warps)ÚshapeÚstrider   Ú
empty_likeÚemptyr@   r\   Úelement_sizeÚminÚtritonÚnext_power_of_2ÚRuntimeErrorÚmaxrX   )r   r   r   r   r   r   r   r   rY   rM   rN   Úngroupsr   r   ÚMAX_FUSED_SIZEr1   rb   Úgrids                     r    Úlayer_norm_fwdrp   v   s  € ð �7‰7�D€AØÑØˆ
Ø‰>˜QÓÐÐØ‰o€GØ�8‰8�B‹<˜1ÓÐÐØ�}Ø�x‰x˜‹|˜qÓ Ð Ð Ø�w‰w˜1˜&Ó Ð Ð Ø�<‰<˜A˜4ÓÐÐØ�=‰=˜Ó Ó!Ð!Ð!ØÑØ�{‰{˜2‹ !Ó#Ð#Ð#Ø�z‰z˜a˜TÓ!Ð!Ð!à
�Ø�y‰y˜AŸG™GÓ#Ð#Ñ#ä×Ò˜qÓ!ˆØ�:‰:�b‹>˜QÓÐÐÞU`Œ5�;Š;˜™�¬e¯m©mÀAÇHÁHÒMÐfj€DÜ�;Š;˜™�¬e¯m©mÀAÇHÁHÑM€Dà˜aŸn™nÓ.Ñ.€NÜ�.¤&×"8Ò"8¸Ó"DÓE€GØÓÜÐQÓRÐRä”C˜ 3™¨Ó*¨AÓ.€IØˆ<€DÜ˜$ÒØ	ØØØØ	ØØØ	�‰�‹Ø�
‰
�1‹Ø‘}ˆ�‰�Œ¨!Ø	ØØØØ)ØØò#ð& �dˆ?Ðr"   c                 ó   • U S   S L$ r$   r&   r'   s    r    r)   r)   ¶   r*   r"   c                 ó   • U S   S L$ r,   r&   r'   s    r    r)   r)   ·   r.   r"   c                 ó   • U S   S L$ )NrF   r&   r'   s    r    r)   r)   ¸   s   €  T¨#¡Y°dÑ%:r"   )r/   r0   ÚRECOMPUTE_OUTPUTrt   c                 ó°
  • [         R                  " S5      n[         R                  " S5      nUU-  n [         R                  " SU5      n!U!U:  n"U U U-  UU-  -   -  n U(       a  UU U-  UU-  -   -  nU	U U-  UU-  -   -  n	UU U-  UU-  -   -  nUU U-  UU-  -   -  nU(       a  UU U-  UU-  -   -  nU(       d  U
UU-  -  n
UUU-  -  nUUU-  -  n[         R                  " UU!-   U"S9R	                  [         R
                  5      n#U(       d  U(       aE  U(       a>  UUU-  -  n[         R                  " UU!-   U"SS9R	                  [         R
                  5      n$[         R                  " U4[         R
                  S9n%U(       a$  [         R                  " U4[         R
                  S9n&[        US-   U-  U5      n'[        U U'5       GH<  n([         R                  " U U!-   U"SS9R	                  [         R
                  5      n)[         R                  " UU!-   U"SS9R	                  [         R
                  5      n*U(       d  [         R                  " U
U(-   5      n+U(       a[  U(       dT  [         R                  " UU!-   U"SS9R	                  [         R
                  5      n,U)n-U-U,-  [         R                  " U,5      -  n)[         R                  " UU(-   5      n.U(       d  U)W+-
  U.-  OU)U.-  n/[         R                  " U"U/S5      n/U(       aÃ  U(       a¼  [         R                  " UU!-   U"SS9R	                  [         R
                  5      n,[         R                  " U,5      n0U(       a  U/U#-  W$-   OU/U#-  n1U(       a  [         R                  " UU!-   U1U,-  U0-  U"S9  U*U1-  U0-  SU,SU0-
  -  -   -  n2[         R                  " U	U!-   U2U"S9  U*U,U0-  -  n*O4U(       a-  U(       a  U/U#-  W$-   OU/U#-  n1[         R                  " UU!-   U1U"S9  U#U*-  n3[         R                  " U/U3-  SS9U-  n4U(       d'  [         R                  " U3SS9U-  n5U3U/U4-  U5-   -
  U.-  n6OU3U/U4-  -
  U.-  n6U%U*U/-  -  n%U(       a  W&U*-  n&U(       aR  U(       dK  [         R                  " W,5      n0U6W--  U0-  SU,SU0-
  -  -   -  n2[         R                  " U	U!-   U2U"S9  U6U,U0-  -  n6[         R                  " UU!-   U6U"S9  X-  n U(       a	  X=-  nU	U-  n	U(       a  XN-  nX_-  nUU-  nGM?     [         R                  " UUU-  -   UU-  -   U!-   U%U"S9  U(       a&  [         R                  " UUU-  -   UU-  -   U!-   W&U"S9  g g )Nr   r   r9   r5   r6   )r   r:   )r<   r=   r>   r?   r   r@   Úzerosrh   ÚrangerA   rD   rC   rB   )7rE   rG   r%   r-   rF   ÚDYÚDXÚDWÚDBÚDZrH   rI   rJ   rL   rK   Ústride_dy_rowÚstride_dx_rowÚstride_dz_rowÚstride_dw_rowÚstride_db_rowrM   rN   r   Úrows_per_programr2   r3   r/   r0   rt   r1   Úrow_block_idrP   Ú	row_startrQ   r7   rT   rU   ÚdwÚdbÚrow_endrO   r   Údyr   r   Úx_ogr   ÚxhatÚ	z_sigmoidrW   ÚdzÚwdyÚc1Úc2Údxs7                                                          r    Úlayer_norm_bwd_kernelr‘   µ   sò  € ôN —=’= Ó#€LÜ�MŠM˜!Ó€EØÐ/Ñ/€IÜ�9Š9�Q˜Ó €DØ�!‰8€DØˆ�\Ñ	! E¨A¡IÑ	-Ñ-€AÞØ	ˆY˜Ñ%¨°©	Ñ1Ñ1ˆØ
ˆi˜-Ñ'¨%°!©)Ñ3Ñ3ˆØˆ)�mÑ
# e¨a¡iÑ
/Ñ/€BØˆ)�mÑ
# e¨a¡iÑ
/Ñ/€BÞØ	ˆY˜Ñ%¨°©	Ñ1Ñ1ˆÞØ�˜‘	ÑˆØˆE�A‰IÑ€DØˆ�‰�N€AÜ
�Š��D‘˜tÑ$×'Ñ'¬¯
©
Ó3€AÞžE¦xØ	ˆU�Q‰Y‰ˆÜ�GŠG�A˜‘H 4¨rÑ2×5Ñ5´b·j±jÓAˆÜ	�Š�7�*¤B§J¡JÑ	/€BÞÜ�XŠX�w�j¬¯
©
Ñ3ˆÜ�< !Ñ#Ð'7Ñ7¸Ó;€GÜ�Y ×(ˆä�GŠG�A˜‘H 4¨qÑ1×4Ñ4´R·Z±ZÓ@ˆÜ�WŠW�R˜$‘Y T°Ñ3×6Ñ6´r·z±zÓBˆÞÜ—7’7˜4 #™:Ó&ˆDÞÖ)Ü—’˜˜D™ t°2Ñ6×9Ñ9¼"¿*¹*ÓEˆAØˆDØ�q‘œ2Ÿ:š: a›=Ñ(ˆAÜ�wŠw�t˜c‘zÓ"ˆæ(3��D‘˜DÒ ¸¸T¹ˆÜ�xŠx˜˜d BÓ'ˆÞÖ%Ü—’˜˜D™ t°2Ñ6×9Ñ9¼"¿*¹*ÓEˆAÜŸ
š
 1›ˆIÞ (��q‘˜1’¨d°Q©hˆAÞÜ—’˜˜T™ 1 q¡5¨9Ñ#4¸4Ò@Ø�a‘˜)Ñ# q¨1°°I±Ñ+>Ñ'>Ñ?ˆBÜ�HŠH�R˜$‘Y ¨Ò.Ø�!�i‘-Ñ‰BæÞ$,�D˜1‘H˜q’L°$¸±(�Ü—’˜˜T™ 1¨4Ò0Ø�"‰fˆÜ�VŠV�D˜3‘J QÑ'¨!Ñ+ˆÞÜ—’˜ !Ñ$ qÑ(ˆBØ˜ ™ R™Ñ(¨DÑ0‰Bà˜˜r™	‘/ TÑ)ˆBØ
ˆb�4‰i‰ˆÞØ�"‰HˆBÞÖ)ÜŸ
š
 1›ˆIØ�d‘˜YÑ&¨!¨a°1°y±=Ñ.AÑ*AÑBˆBÜ�HŠH�R˜$‘Y ¨Ò.Ø�!�i‘-ÑˆBä
�Š��d‘˜B TÒ*à	ÑˆÞØÑˆAØ�-ÑˆBÞØÑˆAØ
ÑˆØ
ˆmÑ‹ñi )ôj ‡H‚HˆR�, Ñ.Ñ.°¸±Ñ:¸TÑAÀ2ÈDÒQÞÜ
�Š��l ]Ñ2Ñ2°U¸Q±YÑ>ÀÑEÀrÐPTÓUð r"   rˆ   r   r   Úrecompute_outputrŒ   c                 óÞ  • UR                   u  pïUc  UnXø-  S:X  d   eXø-  nUR                  S5      S:X  d   eU R                  S5      S:X  d   eU R                   Xï4:X  d   eUb*  UR                  S5      S:X  d   eUR                   Xï4:X  d   eUR                   U4:X  d   eUR                  S5      S:X  d   eUb*  UR                  S5      S:X  d   eUR                   U4:X  d   e[        R                  " U5      nUb9  Uc   eUR                   UR                   :X  d   eUR                  S5      S:X  d   eOUb  [        R                  " U5      OS nU(       a5  Uc  [        R                  " U5      nUR                   UR                   :X  d   eSUR	                  5       -  n[        U[        R                  " U5      5      nUU:”  a  [        S5      e[        [        US-  S5      S5      n[        UR                  R                  5      n[        R                  " U[        R                  " SU-  5      -  U-  5      n[        R                  " UU4[        R                   UR                  S	9nUb0  [        R                  " UU4[        R                   UR                  S	9OS n[        R                  " UU-  5      nUU4n["        U   " UUUUU(       a  UOS U UUUUUUUR                  S5      Ub  UR                  S5      OSU(       d  SOUR                  S5      U R                  S5      UR                  S5      Ub  UR                  S5      OSUR                  S5      Ub  UR                  S5      OSXèUU4UU	U
US
.6  UR%                  S5      R'                  UR(                  5      nUb*  UR%                  S5      R'                  UR(                  5      OS nU(       d  UUUU4$ UUUXÍ4$ )Nr   r   r   r]   r^   r_   r`   é   r[   ra   )rc   rd   r   re   rg   rh   ri   rj   rk   rl   r   r\   ÚindexÚmathÚceilrf   r@   r‘   rB   r   r   )rˆ   r   r   r   r   r   r   r   r   r   rY   r’   rŒ   r   rM   rN   rm   r�   rn   r1   rb   Úsm_countÚnrow_groupsÚ_dwÚ_dbr‚   ro   r…   r†   s                                r    Úlayer_norm_bwdrœ   /  s•  € ð  �7‰7�D€AØÑØˆ
Ø‰>˜QÓÐÐØ‰o€GØ�8‰8�B‹<˜1ÓÐÐØ�9‰9�R‹=˜AÓÐÐØ�8‰8˜�vÓÐÐØ�}Ø�x‰x˜‹|˜qÓ Ð Ð Ø�w‰w˜1˜&Ó Ð Ð Ø�<‰<˜A˜4ÓÐÐØ�=‰=˜Ó Ó!Ð!Ð!ØÑØ�{‰{˜2‹ !Ó#Ð#Ð#Ø�z‰z˜a˜TÓ!Ð!Ð!ä	×	Ò	˜!Ó	€BØ	�~Ø‰}Ðˆ}Ø�x‰x˜1Ÿ7™7Ó"Ð"Ð"Ø�y‰y˜‹} Ó!Ð!Ñ!à$%¡MŒU×Ò˜aÔ °tˆÞØ‰;Ü×"Ò" 1Ó%ˆCØ�y‰y˜AŸG™GÓ#Ð#Ð#ð ˜aŸn™nÓ.Ñ.€NÜ�.¤&×"8Ò"8¸Ó"DÓE€GØ�GÓÜÐQÓRÐRä”C˜ 3™¨Ó*¨AÓ.€IÜ'¨¯©¯©Ó7€Hô —)’)˜H¤t§y¢y°°Y±Ó'?Ñ?À'ÑIÓJ€KÜ
�+Š+�{ AÐ&¬e¯m©mÀFÇMÁMÑ
R€CØTXÑTdŒ%�+Š+�{ AÐ&¬e¯m©mÀDÇKÁKÒ
PÐjn€CÜ—y’y  [¡Ó1ÐØ˜Ð!€DÜ˜$ÒØ	ØØØ	Þ‰ TØ
Ø
ØØØ
ØØØ	�‰�‹Ø‘}ˆ�‰�Œ¨!Þ!‰ s§z¡z°!£}Ø
�	‰	�!‹Ø
�	‰	�!‹Ø™ˆ�	‰	�!Œ¨AØ�
‰
�1‹Ø™ˆ�
‰
�1Œ¨aØ	�sØð-ð. Ø)ØØó5ð8 
�‰�‹�‰�v—|‘|Ó	$€BØ&*Ñ&6ˆ�‰�‹�‰�t—z‘zÔ	"¸D€BÞ#3ˆB��B˜ÐÐN¸"¸bÀ"ÀbÐ9NÐNr"   c                   óP   • \ rS rSr\\  SS j5       5       r\\S 5       5       rSrg)ÚLayerNormFniŒ  Nc	                 óD  • UR                   n	UR                  SUR                   S   5      nUR                  S5      S:w  a  UR                  5       nUbV  UR                   U	:X  d   eUR                  SUR                   S   5      nUR                  S5      S:w  a  UR                  5       nUR                  5       nUb  UR                  5       n[	        UUUUUUUUS9u  p«nU R                  XX;XÄ5        X�l        XPl        X`l        Xpl	        X€l
        U
R                  U	5      $ )ú^If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))
        r   r   )r   r   r   rY   )rc   Úreshaperd   Ú
contiguousrp   Úsave_for_backwardÚ
x_shape_ogr   r   r   rY   )Úctxr   r   r   r   r   r   r   rY   r¤   rW   r   r   s                r    ÚforwardÚLayerNormFn.forwardŽ  s  € ð —W‘Wˆ
à�I‰I�b˜!Ÿ'™' "™+Ó&ˆØ�8‰8�B‹<˜1ÓØ—‘“ˆAØ‰=Ø—7‘7˜jÓ(Ð(Ð(Ø—	‘	˜"˜aŸg™g b™kÓ*ˆAØ�x‰x˜‹|˜qÓ Ø—L‘L“N�Ø×"Ñ"Ó$ˆØÑØ—?‘?Ó$ˆDÜ&ØØØØØØ!Ø-Ø#ñ	
‰ˆ�ð 	×Ñ˜a¨°TÔ=Ø#ŒØŒØ#ŒØ/ÔØ%ŒØ�y‰y˜Ó$Ð$r"   c                 óè  • U R                   u  p#pEpgUR                  SUR                  S   5      nUR                  S5      S:w  a  UR	                  5       nUR                  UR                  :X  d   e[        UUUUU R                  UUUU R                  U R                  U R                  5      u  p‰p«UR                  U R                  5      nUb  UR                  U R                  5      OS nX‰X«S S S S 4$ )Nr   r   )Úsaved_tensorsr¡   rc   rd   r¢   rœ   r   r   r   rY   r¤   )r¥   rˆ   r   r   r   r   r   r   r�   r…   r†   rŒ   s               r    ÚbackwardÚLayerNormFn.backward´  sß   € ð *-×):Ñ):Ñ&ˆ�4˜tØ�Z‰Z˜˜BŸH™H R™LÓ)ˆØ�9‰9�R‹=˜AÓØ—‘“ˆBØ�x‰x˜1Ÿ7™7Ó"Ð"Ð"Ü'ØØØØØ�G‰GØØØØ�N‰NØ× Ñ Ø�O‰Oó
‰ˆ�ð �Z‰Z˜Ÿ™Ó'ˆØ+-©>ˆR�Z‰Z˜Ÿ™Ô'¸tˆØ�r˜t T¨4°Ð5Ð5r"   r&   ©Nç�íµ ÷Æ°>NTF)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Ústaticmethodr¦   rª   Ú__static_attributes__r&   r"   r    rž   rž   Œ  s?   † àØØZ^Ø!ó"%ó ó ð"%ðH Øñ6ó ó ó6r"   rž   c           
      ó2   • [         R                  XX#XEXg5      $ ©N©rž   Úapply)r   r   r   r   r   r   r   rY   s           r    Úlayernorm_fnr¸   Î  s   € Ü×Ñ˜Q¨°ÐBRÓ`Ð`r"   c           
      ó4   • [         R                  XX#XEUS5      $ )NTr¶   )r   r   r   r   r   r   r   s          r    Ú
rmsnorm_fnrº   Ò  s   € Ü×Ñ˜Q¨°ÐBRÐTXÓYÐYr"   c                   ó˜   ^ • \ rS rSr     SS\S\S-  S\S\R                  S-  S\R                  S-  4
U 4S jjjr
S	 rSS
 jrSrU =r$ )ÚLayerNormGatediÖ  Nr   r   r   r\   r   c                 ó.  >• XVS.n[         TU ]  5         X l        [        R                  " [
        R                  " U40 UD65      U l        [        R                  " [
        R                  " U40 UD65      U l        X0l	        X@l
        U R                  5         g)ú¬If group_size is not None, we do GroupNorm with each group having group_size elements.
group_size=None is equivalent to group_size=hidden_size (i.e. there's only 1 group).
©r\   r   N)ÚsuperÚ__init__r   ÚnnÚ	Parameterr   rf   r   r   r   r   Úreset_parameters©	ÚselfÚhidden_sizer   r   r   r\   r   Úfactory_kwargsÚ	__class__s	           €r    rÁ   ÚLayerNormGated.__init__Ø  sp   ø€ ð %+Ñ;ˆÜ‰ÑÔØŒÜ—l’l¤5§;¢;¨{Ñ#M¸nÑ#MÓNˆŒÜ—L’L¤§¢¨[Ñ!K¸NÑ!KÓLˆŒ	Ø$ŒØ 0ÔØ×ÑÕr"   c                 óÐ   • [         R                  R                  R                  U R                  5        [         R                  R                  R                  U R                  5        g rµ   )r   rÂ   ÚinitÚones_r   Úzeros_r   ©rÆ   s    r    rÄ   ÚLayerNormGated.reset_parametersî  s6   € Ü�‰�‰×Ñ˜DŸK™KÔ(Ü�‰�‰×Ñ˜TŸY™YÕ'r"   c           
      ó€   • [        XR                  U R                  X R                  U R                  U R
                  S9$ )r    )r   r   r   r   )r¸   r   r   r   r   r   ©rÆ   r   r   s      r    r¦   ÚLayerNormGated.forwardò  s7   € ô ˜AŸ{™{¨D¯I©I¸ÇÁÐ\`×\dÑ\dØ-1×-BÑ-BñDð 	Dr"   )r   r   r   r   r   )çñhãˆµøä>NTNNrµ   ©r®   r¯   r°   r±   r   ÚintÚboolr   r\   r   rÁ   rÄ   r¦   r³   Ú__classcell__©rÉ   s   @r    r¼   r¼   Ö  sw   ø† ð
 Ø!%Ø!%Ø&*Ø$(ñ ð ð ð ˜$‘Jð	 ð
 ð ð —‘˜tÑ#ð ð �{‰{˜TÑ!÷ ð  ò,(÷Dò Dr"   r¼   c                   ó˜   ^ • \ rS rSr     SS\S\S-  S\S\R                  S-  S\R                  S-  4
U 4S jjjr
S	 rSS
 jrSrU =r$ )ÚRMSNormGatediù  Nr   r   r   r\   r   c                 óò   >• XVS.n[         TU ]  5         X l        [        R                  " [
        R                  " U40 UD65      U l        U R                  SS5        X0l	        X@l
        U R                  5         g)r¾   r¿   r   N)rÀ   rÁ   r   rÂ   rÃ   r   rf   r   Úregister_parameterr   r   rÄ   rÅ   s	           €r    rÁ   ÚRMSNormGated.__init__û  sb   ø€ ð %+Ñ;ˆÜ‰ÑÔØŒÜ—l’l¤5§;¢;¨{Ñ#M¸nÑ#MÓNˆŒØ×Ñ ¨Ô-Ø$ŒØ 0ÔØ×ÑÕr"   c                 ój   • [         R                  R                  R                  U R                  5        g rµ   )r   rÂ   rÌ   rÍ   r   rÏ   s    r    rÄ   ÚRMSNormGated.reset_parameters  s   € Ü�‰�‰×Ñ˜DŸK™KÕ(r"   c           
      ó€   • [        XR                  U R                  X R                  U R                  U R
                  S9$ )r    )r   r   r   r   )rº   r   r   r   r   r   rÒ   s      r    r¦   ÚRMSNormGated.forward  s7   € ô ˜!Ÿ[™[¨$¯)©)°q¿h¹hÐSW×SbÑSbØ+/×+@Ñ+@ñBð 	Br"   )r   r   r   r   )rÔ   NFNNrµ   rÕ   rÙ   s   @r    rÛ   rÛ   ù  sw   ø† ð
 Ø!%Ø!&Ø&*Ø$(ñ ð ð ð ˜$‘Jð	 ð
 ð ð —‘˜tÑ#ð ð �{‰{˜TÑ!÷ ð  ò*)÷Bò Br"   rÛ   )Nr­   NTT)NNNTF)NNTFFNNr¬   )Nr­   NT)$r–   r   Útorch.nnrÂ   Útorch.nn.functionalÚ
functionalr   ri   Útriton.languageÚlanguager<   Úeinopsr   Ú	fla.utilsr   r   r!   Ú
heuristicsÚjitÚ	constexprrX   ÚTensorr   rÖ   r×   rp   r‘   rœ   ÚautogradÚFunctionrž   r¸   rº   ÚModuler¼   rÛ   r&   r"   r    Ú<module>rñ      sö  ðó ã Ý ß Ð Û Ý Ý ç ;ôð. ×ÒÙ2Ù/ñó ð ‡�ð<%ð �\‰\ð<%ð �l‰lð<%ð  �<‰<ð!<%ð" —l‘lð#<%ð$ —‘ó%<%ó ó	ð
<%ðH ØØØ!Øñ<Ø‡|�|ð<à�L‰Lð<ð �,‰,ð<ð 
ð	<ð
 ‡|�|ð<ð 
�‰ð<ð ð<ð ð<ð õ<ð~ ×ÒÙ2Ù/Ù:ñó ð
 ‡�ðqVð2 —l‘lð3qVð4 —‘ð5qVð6 �l‰lð7qVð8 �<‰<ð9qVð: —l‘lð;qVð< �\‰\ó=qVó óðqVðx ØØ!ØØ"ØØñZOØ�‰ðZOà‡|�|ðZOð �L‰LðZOð �,‰,ð	ZOð
 
ðZOð �,‰,ðZOð �,‰,ðZOð ‡|�|ðZOð ðZOð ðZOð ðZOð ðZOð 	�‰ðZOð 
�‰õZOôz?6�%—.‘.×)Ñ)ô ?6ôDaôZô D�R—Y‘Yô  DôFB�2—9‘9õ Br"   