ó
    KëgjV  ã                  ó:  • S SK Jr  S SKJr  S SKJrJr  S SKrS SKJ	r	  S SK
Jr  S SKJrJrJrJr  S SKJr  S SKJrJrJrJr   S S	KJr  \(       a  S S
KJr   " S S\	R:                  5      r " S S\	R:                  5      r " S S\5      r g! \\4 a    Sr NOf = f)é    )Úannotations)Úpartial)ÚTYPE_CHECKINGÚAnyN)Ú
DeviceMesh)Ú	PlacementÚ	ReplicateÚShardÚdistribute_module)ÚParallelStyle)ÚpowgluÚpowglu_linearÚswigluÚswiglu_linear)ÚDTensor)ÚUnpackc                  ój   ^ • \ rS rSr     S             SU 4S jjjr      SS jrSrU =r$ )ÚGatedMLPé   c                óN  >• [         TU ]  5         Xl        Uc  SnUc!  [        X-  S-  S-  5      nSUS-   S-
  S-  -  nX l        X0l        X@l        XPl        X`l        US;  a  [        SU 35      e[        R                  " U R                  U R
                  SS	9U l        [        R                  " U R                  U R
                  SS	9U l        [        R                  " U R
                  U R                  SS	9U l        U R                  (       a  US
:X  a  [        5       U l        g g g )Né   é   é   é   é   )ÚswishÚpowluzUnsupported hidden_act: F)Úbiasr   )ÚsuperÚ__init__Úhidden_sizeÚintÚhidden_ratioÚintermediate_sizeÚ
hidden_actÚfuse_swigluÚpowglu_powerÚ
ValueErrorÚnnÚLinearÚ	gate_projÚup_projÚ	down_projÚSwiGLULinearr   )Úselfr!   r#   r$   r%   r&   r'   Ú	__class__s          €Ú�C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\fla/modules/mlp.pyr    ÚGatedMLP.__init__    s  ø€ ô 	‰ÑÔà&Ôð ÑØˆLØÑ$Ü # KÑ$>ÀÑ$BÀQÑ$FÓ GÐØ #Ð(9¸CÑ(?À!Ñ(CÈÑ'KÑ LÐØ(ÔØ!2ÔØ$ŒØ&ÔØ(ÔàÐ/Ó/ÜÐ7¸
°|ÐDÓEÐEäŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆŒØ×× 
¨gÓ 5Ü!-£ˆDÕð !6Ðó    c                ó  • U R                  U5      U R                  U5      pCU R                  S:X  av  U R                  (       a@  [	        X4U R
                  R                  U R
                  R                  U R                  5      $ U R                  [        X4U R                  5      5      $ U R                  (       a;  U R                  X4U R
                  R                  U R
                  R                  5      $ U R                  [        X45      5      $ )Nr   )r+   r,   r%   r&   r   r-   Úweightr   r'   r   r   r   )r/   ÚxÚkwargsÚgateÚys        r1   ÚforwardÚGatedMLP.forwardB   s¾   € ð
 —.‘. Ó# T§\¡\°!£_ˆaØ�?‰?˜gÓ%Ø××Ü$ T¨d¯n©n×.CÑ.CÀTÇ^Á^×EXÑEXÐZ^×ZkÑZkÓlÐlØ—>‘>¤&¨°$×2CÑ2CÓ"DÓEÐEØ××Ø×%Ñ% d¨t¯~©~×/DÑ/DÀdÇnÁn×FYÑFYÓZÐZØ�~‰~œf T›oÓ.Ð.r3   )
r-   r&   r+   r%   r#   r!   r$   r'   r   r,   )NNr   Tg      @)r!   r"   r#   ú
int | Noner$   r<   r%   Ústrr&   Úboolr'   ÚfloatÚreturnr   )r6   útorch.Tensorr7   zUnpack[Any]r@   rA   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r    r:   Ú__static_attributes__Ú__classcell__©r0   s   @r1   r   r      s‰   ø† ð
 $(Ø(,Ø!Ø Ø!ð 0àð 0ð !ð 0ð &ð	 0ð
 ð 0ð ð 0ð ð 0ð 
÷ 0ð  0ðD/àð/ð ð/ð 
÷	/ò /r3   r   c                  ó   • \ rS rSrS rSrg)r.   éQ   c                ó   • [        XX45      $ )N)r   )r/   r6   r9   r5   r   s        r1   r:   ÚSwiGLULinear.forwardS   s   € Ü˜Q 6Ó0Ð0r3   © N)rB   rC   rD   rE   r:   rF   rM   r3   r1   r.   r.   Q   s   † õ1r3   r.   c                  ól   ^ • \ rS rSrSSSS.     S
U 4S jjjr\S 5       r\S 5       rSS jrS	r	U =r
$ )ÚSwiGLULinearParalleléW   NT)Úinput_layoutsÚoutput_layoutsÚuse_local_outputc               ó¶   >• [         TU ]  5         U=(       d    [        S5      4U l        U=(       d
    [	        5       4U l        [        S5      4U l        X0l        g )Néÿÿÿÿ)r   r    r
   rQ   r	   rR   Údesired_input_layoutsrS   )r/   rQ   rR   rS   r0   s       €r1   r    ÚSwiGLULinearParallel.__init__X   sJ   ø€ ô 	‰ÑÔØ+×8¬u°R«yÐ:ˆÔØ-×<´³Ð>ˆÔÜ&+¨B£i \ˆÔ"Ø 0Õr3   c                ó  • Uu  pVpx[        U[        5      (       d  [        R                  " XTU SS9nUR                  U:w  a  UR	                  USS9n[        U[        5      (       d  [        R                  " XdU SS9nUR                  U:w  a  UR	                  USS9n[        U[        5      (       d!  [        R                  " Xt[        S5      45      nUb5  [        U[        5      (       d   [        R                  " X„[        5       45      nXVXx4$ )NF)Ú	run_checkT©Ú
placementsÚasync_opr   )Ú
isinstancer   Ú
from_localr[   Úredistributer
   r	   )	rQ   rV   ÚmodÚinputsÚdevice_meshr6   r9   r5   r   s	            r1   Ú_prepare_input_fnÚ&SwiGLULinearParallel._prepare_input_fne   sè   € ð $ÑˆˆfÜ˜!œW×%Ñ%Ü×"Ò" 1°=ÈEÑRˆAØ�<‰<Ð0Ó0Ø—‘Ð*?È$�ÐOˆAä˜!œW×%Ñ%Ü×"Ò" 1°=ÈEÑRˆAØ�<‰<Ð0Ó0Ø—‘Ð*?È$�ÐOˆAä˜&¤'×*Ñ*Ü×'Ò'¨¼eÀA»h¸[ÓIˆFàÑ¤J¨t´W×$=Ñ$=Ü×%Ò% d¼)»+¸ÓHˆDà�VÐ!Ð!r3   c                ót   • UR                   U :w  a  UR                  U SS9nU(       a  UR                  5       $ U$ )NTrZ   )r[   r_   Úto_local)rR   rS   r`   Úoutputsrb   s        r1   Ú_prepare_output_fnÚ'SwiGLULinearParallel._prepare_output_fn|   s=   € ð
 ×Ñ Ó/Ø×*Ñ*°nÈtÐ*ÐTˆGæ%5ˆw×ÑÓ!ÐB¸7ÐBr3   c                óÀ   • [        UUS [        U R                  U R                  U R                  5      [        U R
                  U R                  U R                  5      S9$ )N)Úpartition_fnÚinput_fnÚ	output_fn)r   r   rc   rQ   rV   rh   rR   rS   )r/   Úmodulerb   s      r1   Ú_applyÚSwiGLULinearParallel._apply†   sT   € Ü ØØØÜ˜T×3Ñ3°T×5GÑ5GÈ×IcÑIcÓdÜ˜d×5Ñ5°t×7JÑ7JÈD×LaÑLaÓbñ
ð 	
r3   )rV   rQ   rR   rS   )rQ   úPlacement | NonerR   rq   rS   r>   )rn   ú	nn.Modulerb   r   r@   rr   )rB   rC   rD   rE   r    Ústaticmethodrc   rh   ro   rF   rG   rH   s   @r1   rO   rO   W   sj   ø† ð +/Ø+/Ø!%ñ1ð (ð1ð )ð	1ð
 ÷1ð 1ð ñ"ó ð"ð, ñCó ðC÷
ò 
r3   rO   )!Ú
__future__r   Ú	functoolsr   Útypingr   r   ÚtorchÚtorch.nnr)   Útorch.distributedr   Útorch.distributed.tensorr   r	   r
   r   Ú!torch.distributed.tensor.parallelr   Úfla.modules.activationsr   r   r   r   r   ÚImportErrorÚAttributeErrorÚtransformers.processing_utilsr   ÚModuler   r.   rO   rM   r3   r1   Ú<module>r�      s�   ðõ #å ß %ã Ý Ý (ß SÓ SÝ ;ç PÓ PðÝ0ö Ý4ô0/ˆr�y‰yô 0/ôf1�2—9‘9ô 1ô6
˜=õ 6
øðA 	�^Ð$ó Ø‚Gðús   ÁB Â
BÂB