ó
    Këgju%  ã                  óò  • S SK Jr  S SKrS SKrS SKrS SKJs  Jr  S SKJr  S SK	J
r
JrJrJr  S SKJr  S SKJr  \R$                  S%S j5       r\S&S j5       r\S'S	 j5       rS(S
 jr " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r " S S\R.                  5      r  " S S\R.                  5      r! " S S \R.                  5      r" " S! S"\R.                  5      r# " S# S$\R.                  5      r$g))é    )ÚannotationsN)Únn)Úfast_gelu_implÚsigmoidÚsqreluÚswish)Ú
layer_norm)Ú
checkpointc                ó,   • [         R                  " X XS9$ )N©Údevice)ÚtorchÚtriu_indices)ÚnÚoffsetr   s      Ú¥C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\fla/modules/feature_map.pyÚ_triu_indicesr      s   € ô ×Ò˜a FÑ:Ð:ó    c                óœ   • [         R                  " SX5      nUR                  S5      n[        USUR                  5      nUSUS   US   4   $ )Nú...i,...j->...ijéÿÿÿÿr   .é   )r   ÚeinsumÚsizer   r   )ÚxÚyÚzÚNÚindicess        r   Úflatten_diag_outer_productr       sL   € ä�ŠÐ'¨Ó.€AØ	�‰ˆr‹
€AÜ˜A˜q !§(¡(Ó+€GØˆS�'˜!‘*˜g a™jÐ(Ñ)Ð)r   c                óæ   • [         R                  " SX5      nUR                  S5      n[        USUR                  5      n[         R
                  " X2R                  S9nUSUS   US   4   USXU4   4$ )Nr   r   r   r   .r   )r   r   r   r   r   Úarange)r   r   r   r   r   Údiags         r   Úflatten_diag_outer_product_off1r$   %   sl   € ä�ŠÐ'¨Ó.€AØ	�‰ˆr‹
€AÜ˜A˜q !§(¡(Ó+€GÜ�<Š<˜§(¡(Ñ+€DØˆS�'˜!‘*˜g a™jÐ(Ñ)¨1¨S°$¨_Ñ+=Ð=Ð=r   c                ó.   • X S-
  -  S:H  =(       a    U S:g  $ )Nr   r   © )r   s    r   Úis_power_of_2r'   .   s   € Ø�Q‘‰K˜1Ñ×( ! q¡&Ð(r   c                  óH   ^ • \ rS rSrSr    SU 4S jjrS rSS jrSrU =r	$ )	ÚHedgehogFeatureMapé2   z�
Hedgehog feature map as introduced in
`The Hedgehog & the Porcupine: Expressive Linear Attentions with Softmax Mimicry <https://arxiv.org/abs/2402.04347>`_
c                óx   >• [         TU ]  5         [        R                  " X5      U l        U R                  5         g ©N)ÚsuperÚ__init__r   ÚLinearÚlayerÚinit_weights_©ÚselfÚhead_dimÚ	__class__s     €r   r.   ÚHedgehogFeatureMap.__init__9   s,   ø€ ô 	‰ÑÔä—Y’Y˜xÓ2ˆŒ
Ø×ÑÕr   c                óÐ   • [         R                  R                  U R                  R                  5        [         R                  R                  U R                  R                  5        g)z$Initialize trainable map as identityN)r   ÚinitÚeye_r0   ÚweightÚzeros_Úbias©r3   s    r   r1   Ú HedgehogFeatureMap.init_weights_B   s4   € ä
�‰�‰�T—Z‘Z×&Ñ&Ô'Ü
�‰�‰�t—z‘z—‘Õ'r   c                óz   • U R                  U5      nSU-  n[        R                  " X"* /SS9R                  S5      $ )Né   r   ©Údim)r0   r   ÚcatÚsoftmax)r3   r   r   s      r   ÚforwardÚHedgehogFeatureMap.forwardG   s8   € Ø�J‰J�q‹MˆØ�‰EˆÜ�yŠy˜!˜R˜ bÑ)×1Ñ1°"Ó5Ð5r   )r0   ©r4   ÚintÚreturnÚNone©r   útorch.TensorrI   rL   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r.   r1   rE   Ú__static_attributes__Ú__classcell__©r5   s   @r   r)   r)   2   s.   ø† ñð
àðð 
÷ò(÷
6ò 6r   r)   c                  óZ   ^ • \ rS rSrSr  S       SU 4S jjjrS	S jrS
S jrSrU =r	$ )ÚT2RFeatureMapéM   z|
Simple linear mapping feature map as in
`Finetuning Pretrained Transformers into RNNs <https://arxiv.org/abs/2103.13076>`_
c                ó„   >• [         TU ]  5         Uc  UnXl        X l        X0l        [
        R                  " XUS9U l        g )N©r<   )r-   r.   r4   Údot_dimr<   r   r/   r0   )r3   r4   rZ   r<   r5   s       €r   r.   ÚT2RFeatureMap.__init__T   s;   ø€ ô 	‰ÑÔà‰?ØˆGà ŒØŒØŒ	ä—Y’Y˜x°tÑ<ˆ�
r   c                ó‚   • U R                   R                   SU R                   SU R                   SU R                   S3$ )Nz
(head_dim=z
, dot_dim=z, bias=Ú))r5   rM   r4   rZ   r<   r=   s    r   Ú__repr__ÚT2RFeatureMap.__repr__e   s>   € Ø—.‘.×)Ñ)Ð*¨*°T·]±]°OÀ:ÈdÏlÉlÈ^Ð[bÐcg×clÑclÐbmÐmnÐoÐor   c                ó@   • U R                  U5      R                  5       $ r,   )r0   Úrelu©r3   r   s     r   rE   ÚT2RFeatureMap.forwardh   s   € Ø�z‰z˜!‹}×!Ñ!Ó#Ð#r   )r<   rZ   r4   r0   )NF)r4   rH   rZ   ú
int | Noner<   úbool | NonerI   rJ   )rI   ÚstrrK   )
rM   rN   rO   rP   rQ   r.   r^   rE   rR   rS   rT   s   @r   rV   rV   M   sP   ø† ñð #Ø!ð	=àð=ð ð=ð ð	=ð
 
÷=ð =ô"p÷$ò $r   rV   c                  óJ   ^ • \ rS rSrSr S     SU 4S jjjrSS jrSrU =r$ )	ÚDPFPFeatureMapél   zž
Deterministic Parameter-Free Projection (DPFP) feature map in
`Linear Transformers Are Secretly Fast Weight Programmers <https://arxiv.org/abs/2102.11174>`_
c                ó.   >• [         TU ]  5         X l        g r,   )r-   r.   Únu)r3   r4   rk   r5   s      €r   r.   ÚDPFPFeatureMap.__init__s   s   ø€ ô
 	‰ÑÔØ�r   c           
     ó<  • UR                  5       n[        R                  " X"* /SS9n[        R                  " [        SU R                  S-   5       Vs/ s H  o1R                  USS9PM     snSS9n[        R                  " U/U R                  -  SS9nXT-  $ s  snf )Nr   rA   r   )ÚshiftsÚdims)ra   r   rC   Úrangerk   Úroll)r3   r   ra   ÚjÚx_rolledÚx_repeats         r   rE   ÚDPFPFeatureMap.forward{   s‡   € Ø�v‰v‹xˆÜ�IŠI�t˜U�m¨Ñ,ˆÜ—9’9ÄÀqÈ$Ï'É'ÐRSÉ)ÔATÓUÒAT¸AŸf™f¨A°B˜fÓ7ÑATÑUÐ[]Ñ^ˆÜ—9’9˜a˜S 4§7¡7™]°Ñ3ˆØÑ"Ð"ùò Vs   ÁB)rk   )é   )r4   rH   rk   rH   rI   rJ   rK   )	rM   rN   rO   rP   rQ   r.   rE   rR   rS   rT   s   @r   rh   rh   l   s=   ø† ñð ðàðð ðð 
÷	ð ÷#ò #r   rh   c                  ó>   ^ • \ rS rSr    SU 4S jjrSS jrSrU =r$ )ÚHadamardFeatureMapéƒ   c                óŽ   >• [         TU ]  5         [        R                  " X5      U l        [        R                  " X5      U l        g r,   )r-   r.   r   r/   Úlayer1Úlayer2r2   s     €r   r.   ÚHadamardFeatureMap.__init__„   s0   ø€ ô 	‰ÑÔä—i’i Ó3ˆŒÜ—i’i Ó3ˆ�r   c                óH   • U R                  U5      U R                  U5      -  $ r,   ©r{   r|   rb   s     r   rE   ÚHadamardFeatureMap.forward�   s   € Ø�{‰{˜1‹~ §¡¨A£Ñ.Ð.r   r   rG   rK   ©rM   rN   rO   rP   r.   rE   rR   rS   rT   s   @r   rx   rx   ƒ   s$   ø† ð4àð4ð 
÷4÷/ò /r   rx   c                  óB   ^ • \ rS rSr      SU 4S jjrSS jrSrU =r$ )ÚLearnableOuterProductFeatureMapé‘   c                óž   >• [         TU ]  5         [        R                  " XSS9U l        [        R                  " XSS9U l        US-  U l        g )NFrY   ç      à¿)r-   r.   r   r/   r{   r|   Ú
normalizer)r3   r4   Úfeature_dimr5   s      €r   r.   Ú(LearnableOuterProductFeatureMap.__init__’   s@   ø€ ô
 	‰ÑÔä—i’i ¸EÑBˆŒÜ—i’i ¸EÑBˆŒØ%¨Ñ-ˆ�r   c                óV   • [        U R                  U5      U R                  U5      5      $ r,   )r    r{   r|   rb   s     r   rE   Ú'LearnableOuterProductFeatureMap.forward�   s   € Ü)¨$¯+©+°a«.¸$¿+¹+Àa».ÓIÐIr   )r{   r|   r‡   )r4   rH   rˆ   rH   rI   rJ   rK   r�   rT   s   @r   rƒ   rƒ   ‘   s0   ø† ð	.àð	.ð ð	.ð 
÷		.÷Jò Jr   rƒ   c                  óL   ^ • \ rS rSr  S       SU 4S jjjrSS jrSrU =r$ )Ú(LearnablePolySketchNonNegativeFeatureMapé¡   c                ó‚  >• [         TU ]  5         [        U5      (       a  US:¼  d   SU S35       eUc  UnXl        X l        X0l        [        R                  " [        R                  " U5      5      U l
        [        R                  " [        R                  " U5      5      U l        [        R                  " [        R                  " XSS9/[        [!        ["        R$                  " U R
                  5      5      S-
  5       Vs/ s H  n[        R                  " X"SS9PM     snQ5      U l        [        R                  " [        R                  " XSS9/[        [!        ["        R$                  " U R
                  5      5      S-
  5       Vs/ s H  n[        R                  " X"SS9PM     snQ5      U l        g s  snf s  snf )Nr@   zThe degree z must be a power of 2FrY   )r-   r.   r'   r4   Úsketch_sizeÚdegreer   Ú	Parameterr   ÚonesÚgammaÚzerosÚbetaÚ
ModuleListr/   rp   rH   ÚmathÚlog2Ú	sketches1Ú	sketches2)r3   r4   r�   r‘   Ú_r5   s        €r   r.   Ú1LearnablePolySketchNonNegativeFeatureMap.__init__£   sg  ø€ ô 	‰ÑÔä˜V×$Ñ$¨°1«Ða¸ÀFÀ8ÐK`Ð6aÓaÐ4àÑØ"ˆKà ŒØ&ÔØŒä—\’\¤%§*¢*¨XÓ"6Ó7ˆŒ
Ü—L’L¤§¢¨XÓ!6Ó7ˆŒ	ô ŸšÜ�IŠI�h°%Ñ8ð(
äGLÌSÔQU×QZÒQZÐ[_×[fÑ[fÓQgÓMhÐklÑMlÔGmÓnÒGmÀ!Œb�iŠi˜°uÔ=ÑGmÑnð(
ó ˆŒô ŸšÜ�IŠI�h°%Ñ8ð(
äGLÌSÔQU×QZÒQZÐ[_×[fÑ[fÓQgÓMhÐklÑMlÔGmÓnÒGmÀ!Œb�iŠi˜°uÔ=ÑGmÑnð(
ó ˆ�ùò oùò os   ÄF7
Æ
F<
c                ó¾  • [        XR                  U R                  5      nU R                  S   " U5      U R                  S   " U5      -  U R
                  S-  -  n[        S[        [        R                  " U R                  5      5      S-
  5       H>  nU R                  U   " U5      U R                  U   " U5      -  U R
                  S-  -  nM@     [        X5      $ )Nr   r†   r   )r	   r”   r–   rš   r›   r4   rp   rH   r˜   r™   r‘   r    )r3   r   Úis      r   rE   Ú0LearnablePolySketchNonNegativeFeatureMap.forwardÁ   s³   € ä�qŸ*™* d§i¡iÓ0ˆà�N‰N˜1Ò˜aÓ  4§>¡>°!Ò#4°QÓ#7Ñ7¸$¿-¹-È4Ñ:OÑOˆÜ�qœ#œdŸiši¨¯©Ó4Ó5¸Ñ9Ö:ˆAØ—‘˜qÒ! !Ó$ t§~¡~°aÒ'8¸Ó';Ñ;¸d¿m¹mÈtÑ>SÑSŠAñ ;ô *¨!Ó/Ð/r   )r–   r‘   r”   r4   r�   rš   r›   ©Nr@   )r4   rH   r�   rd   r‘   rd   rI   rJ   rK   r�   rT   s   @r   r�   r�   ¡   sE   ø† ð
 #'Øð	àðð  ðð ð	ð
 
÷ð ÷<	0ò 	0r   r�   c                  ó>   ^ • \ rS rSr    SU 4S jjrSS jrSrU =r$ )ÚTaylorFeatureMapéÍ   c                óø   >• [         TU ]  5         Xl        [        R                  " S5      U l        [        R                  " U R                  5      U l        [        R                  " U R                  5      U l        g r¡   )r-   r.   r4   r˜   ÚsqrtÚr2ÚrdÚrrdr2   s     €r   r.   ÚTaylorFeatureMap.__init__Î   sL   ø€ ô 	‰ÑÔØ ŒÜ—)’)˜A“,ˆŒÜ—)’)˜DŸM™MÓ*ˆŒÜ—9’9˜TŸW™WÓ%ˆ�r   c                óæ   • [        X5      u  p#[        R                  " [        R                  " USSS24   5      XR                  -  X0R
                  U R                  -  -  X R
                  -  /SS9$ )N.r   r   r   rA   )r$   r   rC   Ú	ones_liker©   r¨   r§   )r3   r   Úx2_1Úx2_2s       r   rE   ÚTaylorFeatureMap.forwardØ   sa   € Ü4°QÓ:‰
ˆÜ�yŠyœ%Ÿ/š/¨!¨C°°1°¨H©+Ó6¸¿H¹H¹ÀdÏgÉgÐX\×X_ÑX_ÑN_ÑF`Ðbf×ipÑipÑbpÐqÐwyÑzÐzr   )r4   r§   r¨   r©   rG   rK   r�   rT   s   @r   r£   r£   Í   s&   ø† ð&àð&ð 
÷&÷{ò {r   r£   c                  óV   ^ • \ rS rSr   S         SU 4S jjjrSSS jjrSrU =r$ )	ÚRebasedFeatureMapéÝ   c                óH  >• [         TU ]  5         Xl        X l        X0l        X@l        S U l        S U l        U(       a/  [        R                  " [        R                  " U5      5      U l        U(       a0  [        R                  " [        R                  " U5      5      U l        g g r,   )r-   r.   r4   Ú	use_gammaÚuse_betaÚ	normalizer”   r–   r   r’   r   r“   r•   )r3   r4   r´   rµ   r¶   r5   s        €r   r.   ÚRebasedFeatureMap.__init__ß   so   ø€ ô 	‰ÑÔà ŒØ"ŒØ ŒØ"ŒàˆŒ
ØˆŒ	ÞÜŸš¤e§j¢j°Ó&:Ó;ˆDŒJÞÜŸš¤U§[¢[°Ó%:Ó;ˆD�Ið r   c           	     ó6  • U R                   (       aC  U R                  (       a2  U R                  (       a!  [        XR                  U R
                  5      nO÷U R                  (       a8  [        R                  " XR                  4U R                  U R
                  5      nO®U R                  (       a=  U R                   (       a,  [        R                  " U R
                  XR                  5      nO`U R                  (       a  UR                  U R                  5      nO3[        SU R                   SU R                    SU R                   S35      eU(       d  U$ [        X5      u  p4[        R                  " X@R                  S-  -  USU R                  -  S-  -  /SS9$ )	NzbNot supported combination of `use_gamma`, `use_beta` and `normalize`, which is currently set as (`z`, `z`)r†   r@   g      à?r   rA   )rµ   r´   r¶   r	   r”   r–   ÚFr4   r   ÚaddcmulÚmulÚRuntimeErrorr$   rC   )r3   r   Úflattenr­   r®   s        r   rE   ÚRebasedFeatureMap.forwardô   s  € Ø�=�=˜TŸ^Ÿ^°··Ü˜1Ÿj™j¨$¯)©)Ó4‰AØ�^�^Ü—’˜Q§¡Ð 0°$·*±*¸d¿i¹iÓH‰AØ�^�^ §§Ü—’˜dŸi™i¨¯J©JÓ7‰AØ�^�^Ø—‘�d—j‘jÓ!‰Aäð ">Ø>B¿n¹nÐ=MÈTÐRV×R_ÑR_ÐQ`Ð`dÐei×esÑesÐdtÐtvð xó yð yæØˆHÜ4°QÓ:‰
ˆä�yŠy˜$§¡°$Ñ!6Ñ6¸ÀÀDÇMÁMÑ@QÐVYÑ?YÑ8YÐZÐ`bÑcÐcr   )r–   r”   r4   r¶   rµ   r´   )TTT)
r4   rH   r´   re   rµ   re   r¶   re   rI   rJ   )T)r   rL   r½   re   rI   rL   r�   rT   s   @r   r±   r±   Ý   sT   ø† ð
 "&Ø $Ø!%ð<àð<ð ð<ð ð	<ð
 ð<ð 
÷<ð <÷*dô dr   r±   c                  ó:   ^ • \ rS rSr  SU 4S jjrSS jrSrU =r$ )ÚReLUFeatureMapi  c                ó"   >• [         TU ]  5         g r,   ©r-   r.   ©r3   r5   s    €r   r.   ÚReLUFeatureMap.__init__	  ó   ø€ ô 	‰ÑÕr   c                ó.   • [         R                  " U5      $ r,   )r¹   ra   rb   s     r   rE   ÚReLUFeatureMap.forward  s   € Ü�vŠv�a‹yÐr   r&   ©rI   rJ   rK   r�   rT   s   @r   rÀ   rÀ     ó   ø† ðà	÷÷
ò r   rÀ   c                  ó:   ^ • \ rS rSr  SU 4S jjrSS jrSrU =r$ )ÚSquaredReLUFeatureMapi  c                ó"   >• [         TU ]  5         g r,   rÂ   rÃ   s    €r   r.   ÚSquaredReLUFeatureMap.__init__  rÅ   r   c                ó   • [        U5      $ r,   )r   rb   s     r   rE   ÚSquaredReLUFeatureMap.forward  s   € Ü�a‹yÐr   r&   rÈ   rK   r�   rT   s   @r   rË   rË     rÉ   r   rË   c                  ó:   ^ • \ rS rSr  SU 4S jjrSS jrSrU =r$ )ÚGELUFeatureMapi  c                ó"   >• [         TU ]  5         g r,   rÂ   rÃ   s    €r   r.   ÚGELUFeatureMap.__init__  rÅ   r   c                ó   • [        U5      $ r,   )r   rb   s     r   rE   ÚGELUFeatureMap.forward$  s   € Ü˜aÓ Ð r   r&   rÈ   rK   r�   rT   s   @r   rÑ   rÑ     s   ø† ðà	÷÷
!ò !r   rÑ   c                  ó:   ^ • \ rS rSr  SU 4S jjrSS jrSrU =r$ )ÚSwishFeatureMapi(  c                ó"   >• [         TU ]  5         g r,   rÂ   rÃ   s    €r   r.   ÚSwishFeatureMap.__init__*  rÅ   r   c                ó   • [        U5      $ r,   )r   rb   s     r   rE   ÚSwishFeatureMap.forward/  s   € Ü�Q‹xˆr   r&   rÈ   rK   r�   rT   s   @r   r×   r×   (  s   ø† ðà	÷÷
ò r   r×   c                  ó:   ^ • \ rS rSr  SU 4S jjrSS jrSrU =r$ )ÚSigmoidFeatureMapi3  c                ó"   >• [         TU ]  5         g r,   rÂ   rÃ   s    €r   r.   ÚSigmoidFeatureMap.__init__5  rÅ   r   c                ó   • [        U5      $ r,   )r   rb   s     r   rE   ÚSigmoidFeatureMap.forward:  s   € Ü�q‹zÐr   r&   rÈ   rK   r�   rT   s   @r   rÝ   rÝ   3  s   ø† ðà	÷÷
ò r   rÝ   )r   rH   r   rH   r   ztorch.devicerI   rL   )r   rL   r   rL   rI   rL   )r   rL   r   rL   rI   z!tuple[torch.Tensor, torch.Tensor])r   rH   rI   Úbool)%Ú
__future__r   Ú	functoolsr˜   r   Útorch.nn.functionalr   Ú
functionalr¹   Úfla.modules.activationsr   r   r   r   Úfla.modules.layernormr	   Ú	fla.utilsr
   Úcacher   r    r$   r'   ÚModuler)   rV   rh   rx   rƒ   r�   r£   r±   rÀ   rË   rÑ   r×   rÝ   r&   r   r   Ú<module>rì      sE  ðõ #ã Û ã ß Ð Ý ç JÓ JÝ ,Ý  ð ‡�ó;ó ð;ð ó*ó ð*ð ó>ó ð>ô)ô6˜Ÿ™ô 6ô6$�B—I‘Iô $ô>#�R—Y‘Yô #ô./˜Ÿ™ô /ôJ b§i¡iô Jô )0¨r¯y©yô )0ôX{�r—y‘yô {ô 'd˜Ÿ	™	ô 'dôT�R—Y‘Yô ô˜BŸI™Iô ô!�R—Y‘Yô !ô�b—i‘iô ô˜Ÿ	™	õ r   