ó
    ‡Ž¡jH”  ã                  óT  • % S SK Jr  S SKrS SKrSSKJr  SSKJr  SSKJ	r	J
r
  \R                  " \5      r\	" 5       (       a  S SKr\
" 5       (       a  S SKJr  S SKJrJrJrJrJr  S SKJr  S5S	 jrS6S
 jrS7S8S jjr\R:                  S 5       r " S S5      r " S S\5      r  " S S\5      r! " S S\5      r" " S S\5      r# " S S\5      r$ " S S\5      r% " S S\5      r& " S S\5      r' " S S \5      r(\
" 5       (       a>   " S! S"\RR                  RT                  5      r+ " S# S$\RR                  RT                  5      r, " S% S&\5      r- " S' S(\5      r. " S) S*\5      r/ " S+ S,\/5      r0 " S- S.\-5      r1 " S/ S0\5      r2\2" 5       r3S0\4S1'   S9S2 jr5S3 r6        S:S4 jr7g);é    )ÚannotationsNé   )Úlogging)ÚGeneralInterface)Úis_torch_availableÚis_torch_distributed_available)ÚDTensorÚPartialÚ	ReplicateÚShardÚdistribute_tensor)Ú_StridedShardc                ó4   • [         R                  " SS U 5      $ )aS  
Replace the numbers in the `name` by wildcards, only if they are in-between dots (`.`) or if they are between
a dot (`.`) and the end of the string.
This matches how modules are named/numbered when using a nn.ModuleList or nn.Sequential, but will NOT match
numbers in a parameter name itself, e.g. if the param is named `"w1"` or `"w2"`.
z\.\d+(\.|$)c                ó*   • SU R                  S5      -   $ )Nz.*é   ©Úgroup)Úms    Ú¶C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/distributed/tensor_parallel.pyÚ<lambda>Ú2replace_layer_number_by_wildcard.<locals>.<lambda>*   s   € ¨D°1·7±7¸1³:Ò,=ó    )ÚreÚsub)Únames    r   Ú replace_layer_number_by_wildcardr   #   s   € ô �6Š6�.Ñ"=¸tÓDÐDr   c                óœ  • Uc  gU  Vs1 s H  n[        U5      iM     nn[        U5      nUR                  5       nU H©  nSU;   a  UR                  SS5      S   OUn[        R
                  " SSU5      nXq;   a%  UR                  US5        UR                  U5        Mb  SU;   d  Mj  UR                  SS5      S   =o�;   d  M†  UR                  US5        UR                  U5        M«     [        U5      S:”  a  [        R                  SU 35        [        U5      S:”  a(  [        R                  SS	R                  U5       35        ggs  snf )
zy
Verify the TP plan of the model, log a warning if the layers that were not sharded and the rules that were not applied.
NÚ.r   r   z\d+Ú*z>The following TP rules were not applied on any of the layers: z'The following layers were not sharded: z, )r   ÚsetÚcopyÚrsplitr   r   ÚpopÚdiscardÚlenÚloggerÚwarningÚjoin)	Úexpected_keysÚtp_planÚkeyÚgeneric_keysÚunsharded_layersÚunused_rulesÚ
param_nameÚgeneric_param_nameÚparent_param_names	            r   Úverify_tp_planr2   -   sA  € ð
 �ØáERÓSÂ]¸cÔ4°SÖ9Á]€LÐSÜ˜<Ó(ÐØ—<‘<“>€LãˆØ.1°S«j�S—Z‘Z  QÓ'¨Ò*¸cˆ
ÜŸVšV F¨C°Ó<ÐàÓ(Ø×ÑÐ/°Ô6Ø×$Ñ$ SÖ)ØÐ&Õ&ÐAS×AZÑAZÐ[^Ð`aÓAbÐcdÑAeÐ,eÐ,=Õ+qØ×ÑÐ.°Ô5Ø×$Ñ$ SÖ)ñ ô ˆ<Ó˜1ÓÜ�‰ÐWÐXdÐWeÐfÔgÜ
ÐÓ˜qÓ Ü�‰Ð@ÀÇÁÐK[ÓA\Ð@]Ð^Õ_ð !ùò# Ts   ‰E	c                ó‚   • [        U 5      nX1;   a  X   $ U(       a$  SU;   a  UR                  SS5      S   =oA;   a  X   $ g)a‹  
Get the TP style for a parameter from the TP plan.

The TP plan is a dictionary that maps parameter names to TP styles.
The parameter name can be a generic name with wildcards (e.g. "*.weight") or a specific name (e.g. "layer_1.weight").

The `is_weight` is important because for weights, we want to support `.weights` and `.bias` cases seamlessly! but
not parent classes for `post_init` calls
r   r   r   N)r   r"   )Úparameter_namer*   Ú	is_weightr0   Úmodule_names        r   Ú_get_parameter_tp_planr7   J   sU   € ô :¸.ÓIÐØÓ$ØÑ*Ð*Þ	�sÐ0Ó0ÐEW×E^ÑE^Ð_bÐdeÓEfÐghÑEiÐ6i°kÓ5uØÑ#Ð#Ør   c           
   #  ó2  #   • U R                  SS9 VVs0 s H  u  p[        U[        5      (       d  M  X_M      nnnUR                  5        VVs0 s H  u  pXR	                  5       _M     nnnU R
                  R                  U5         S v •  UR                  5        H§  u  pU R
                  U   nXdU   L a  XPR
                  U'   M+  Uc  M0  [        U[        5      (       a  MG  [        R                  " UUR                  UR                  SS9n[        R                  R                  UUR                  S9U R
                  U'   M©     g s  snnf s  snnf ! UR                  5        H§  u  pU R
                  U   nXdU   L a  XPR
                  U'   M+  Uc  M0  [        U[        5      (       a  MG  [        R                  " UUR                  UR                  SS9n[        R                  R                  UUR                  S9U R
                  U'   M©     f = f7f)NF©Úrecurse©Ú	run_check©Úrequires_grad)Únamed_parametersÚ
isinstancer	   ÚitemsÚto_localÚ_parametersÚupdateÚ
from_localÚdevice_meshÚ
placementsÚtorchÚnnÚ	Parameterr>   )Úmoduler   ÚparamÚ	originalsÚlocal_paramsÚoriginalÚcurrentÚreplacements           r   Ú_use_local_dtensor_paramsrR   \   sã  é € ð 17×0GÑ0GÐPUÐ0GÑ0VÔuÒ0V¡ ÔZdÐejÔls×Zt“�’Ñ0V€IÑuØ>G¿o¹oÔ>OÔPÒ>O©{¨t�DŸ.™.Ó*Ò*Ñ>O€LÑPØ
×Ñ×Ñ˜lÔ+ðÛà'Ÿo™oÖ/‰NˆDØ×(Ñ(¨Ñ.ˆGà tÑ,Ò,Ø+3×"Ñ" 4Ó(ð Ó$¬Z¸Ä×-IÓ-IÜ%×0Ò0ØØ×(Ñ(Ø×'Ñ'Ø#ñ	�ô ,1¯8©8×+=Ñ+=ØØ")×"7Ñ"7ð ,>ð ,�×"Ñ" 4Ó(ò 0ùó vùÛPøð
 (Ÿo™oÖ/‰NˆDØ×(Ñ(¨Ñ.ˆGà tÑ,Ò,Ø+3×"Ñ" 4Ó(ð Ó$¬Z¸Ä×-IÓ-IÜ%×0Ò0ØØ×(Ñ(Ø×'Ñ'Ø#ñ	�ô ,1¯8©8×+=Ñ+=ØØ")×"7Ñ"7ð ,>ð ,�×"Ñ" 4Ó(ò 0üsR   ‚H•E´EºHÁEÁ,HÂE Â<HÃHÃ&A1HÅ=HÆHÆ/A%HÈHc                  óB   • \ rS rSrS rSS jrS rS rS rS r	S	 r
S
rg)ÚTensorParallelLayeré{   c                ó   • g)zCWhether this module's forward requires local inputs and parameters.F© ©ÚselfrK   s     r   Úshould_use_local_tensorsÚ,TensorParallelLayer.should_use_local_tensors|   s   € àr   Nc                ó   • g)z3Validate a parameter before applying this TP style.NrW   )rY   rK   rL   Úmeshr4   s        r   Úvalidate_paramÚ"TensorParallelLayer.validate_param€   ó   € àr   c                ó   • g)z<Wrap ONE parameter as a DTensor placeholder. Default: no-op.NrW   )rY   rK   rL   r]   s       r   Úshard_paramÚTensorParallelLayer.shard_param„   r`   r   c                ó   • X#4$ ©NrW   )rY   rK   ÚargsÚkwargsr]   s        r   Útransform_inputs_pre_forwardÚ0TensorParallelLayer.transform_inputs_pre_forwardˆ   s
   € Øˆ|Ðr   c                ón   • U R                  U5      (       a  [        U5      $ [        R                  " 5       $ re   )rZ   rR   Ú
contextlibÚnullcontext©rY   rK   r]   s      r   Úcontext_around_forwardÚ*TensorParallelLayer.context_around_forward‹   s-   € Ø×(Ñ(¨×0Ñ0Ü,¨VÓ4Ð4Ü×%Ò%Ó'Ð'r   c                ó   • U$ re   rW   ©rY   rK   Úoutputr]   s       r   Útransform_output_post_forwardÚ1TensorParallelLayer.transform_output_post_forward�   ó   € Øˆr   c                óF   ^ ^^^• TR                   mUUUU 4S jnUTl         T$ )zCInstall pre / around / post transforms by replacing module.forward.c                 ó¼   >• TR                  TXT5      u  pTR                  TT5         T" U 0 UD6nS S S 5        TR                  TWT5      $ ! , (       d  f       N!= fre   ©rh   rn   rs   )rf   rg   rr   r]   rK   Úoriginal_forwardrY   s      €€€€r   Ú
tp_forwardÚ7TensorParallelLayer.install_forward.<locals>.tp_forward—   s\   ø€ Ø×<Ñ<¸VÀTÐSWÓX‰LˆDØ×,Ñ,¨V°TÕ:Ù)¨4Ð:°6Ñ:�÷ ;à×5Ñ5°f¸fÀdÓKÐK÷ ;Õ:ús   ©	AÁ
A)Úforward)rY   rK   r]   rz   ry   s   ``` @r   Úinstall_forwardÚ#TensorParallelLayer.install_forward“   s'   û€ à!Ÿ>™>Ð÷	Lð 	Lð $ˆŒØˆr   rW   re   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rZ   r^   rb   rh   rn   rs   r}   Ú__static_attributes__rW   r   r   rT   rT   {   s%   † òôòòò(ò
õr   rT   c                  óP   • \ rS rSrSrSSSS.SS jjrS rSS jrS	 rS
 r	S r
Srg)ÚColwiseParallelé¡   ub   Column-wise: weight & bias â†’ Shard(0) (Embedding: Shard(1)); input replicated, output Shard(-1).NT©Úinput_layoutsÚoutput_layoutsÚuse_local_outputc               ój   • U=(       d
    [        5       U l        Ub  UO
[        S5      U l        X0l        g ©Néÿÿÿÿ)r   rˆ   r   r‰   rŠ   ©rY   rˆ   r‰   rŠ   s       r   Ú__init__ÚColwiseParallel.__init__¤   s-   € Ø*×9¬i«kˆÔØ0>Ñ0J™nÔPUÐVXÓPYˆÔØ 0Õr   c                ó¼   • [        USS5      n[        U[        R                  R                  5      =(       a    [        R
                  " 5       (       + nU=(       d    U$ ©NÚ_hf_quantized_needs_local_tpF©Úgetattrr@   rH   rI   ÚLinearÚis_grad_enabled©rY   rK   Úuse_local_quantized_pathÚuses_local_inference_kernels       r   rZ   Ú(ColwiseParallel.should_use_local_tensors©   óF   € Ü#*¨6Ð3QÐSXÓ#YÐ Ü&0°¼¿¹¿¹Ó&I×&iÔRW×RgÒRgÓRiÔNiÐ#Ø'×FÐ+FÐFr   c           	     ó°  • UR                   R                  U5      n[        U R                  [        5      nUb  U(       d  g [        U[
        R                  R                  5      (       a  SOUR                  S-
  nUR                  U   nUR                  5       n	X‰-  S:w  a5  U=(       d    UnUR                  SS5      S   n
[        SU
 SU SU	 S35      eg )	Nr   r   r   r   zThe output size of `z` (z1) must be divisible by the tensor parallel size (z") when gathering a colwise output.)rC   Úgetr@   r‰   r   rH   rI   Ú	EmbeddingÚndimÚshapeÚsizer"   Ú
ValueError)rY   rK   rL   r]   r4   ÚmetaÚgathers_outputÚ	shard_dimÚoutput_sizeÚtp_sizeÚ
layer_names              r   r^   ÚColwiseParallel.validate_param®   sÒ   € Ø×!Ñ!×%Ñ% eÓ,ˆÜ# D×$7Ñ$7¼ÓCˆØ‰<ž~Øä# F¬E¯H©H×,>Ñ,>×?Ñ?‘AÀTÇYÁYÐQRÁ]ˆ	Ø—j‘j Ñ+ˆØ—)‘)“+ˆØÑ  AÓ%Ø+×4¨uˆNØ'×.Ñ.¨s°AÓ6°qÑ9ˆJÜØ& z l°#°k°]ð CØ�9Ð>ð@óð ð &r   c           	     óX  • UR                   R                  U5      nUc  g [        U[        R                  R
                  5      (       a  [        S5      O[        UR                  S-
  5      n[        R                  R                  [        XCU/S S9UR                  S9UR                   U'   g )Nr   r   ©Úsrc_data_rankr=   )rC   rž   r@   rH   rI   rŸ   r   r    rJ   r   r>   ©rY   rK   rL   r]   r¤   Ú	placements         r   rb   ÚColwiseParallel.shard_param¿   sŽ   € Ø×!Ñ!×%Ñ% eÓ,ˆØ‰<ØÜ *¨6´5·8±8×3EÑ3E× FÑ F”E˜!”HÌEÐRV×R[ÑR[Ð^_ÑR_ÓL`ˆ	Ü$)§H¡H×$6Ñ$6Ü˜d¨9¨+ÀTÑJØ×,Ñ,ð %7ð %
ˆ×Ñ˜5Ò!r   c                ó  • US   n[        U[        5      (       + nU(       aF  [        U[        R                  R                  5      (       a  [        R
                  " 5       (       d  X#4$ U(       aŽ  [        USS5      (       a|  [        R
                  " 5       (       aW  UR                  (       aF  UR                  S:X  a  UR                  5       OUR                  S5      n[        R                  XW5      nU4USS  -   U4$ U(       a!  [        R                  " XTU R                  /SS9nUR                  [        5       4:w  a  UR!                  [        5       /S9nU R#                  U5      (       a  UR%                  ['        5       /S9nU4USS  -   U4$ )	Nr   r“   Fr   Útpr;   ©rG   ©Úgrad_placements)r@   r	   rH   rI   r–   r—   r•   r>   r    Ú	get_groupÚ_AllReduceBackwardÚapplyrE   rˆ   rG   r   ÚredistributerZ   rB   r
   )rY   rK   rf   rg   r]   ÚxÚis_local_inputÚprocess_groups           r   rh   Ú,ColwiseParallel.transform_inputs_pre_forwardÉ   s=  € Ø�‰GˆÜ'¨¬7Ó3Ô3ˆö œj¨´·±·±×AÑAÌ%×J_ÒJ_×JaÑJaØ�<Ðö
 œg fÐ.LÈe×TÑTÜ×$Ò$×&Ñ&¨1¯?¯?Ø48·I±IÀ³N §¡Ô 0ÈÏÉÐW[ÓH\�Ü&×,Ñ,¨QÓ>�Ø�4˜$˜q˜r˜(‘? FÐ*Ð*ö Ü×"Ò" 1¨T×-?Ñ-?Ð,@ÈEÑRˆAØ�<‰<œI›K˜>Ó)Ø—‘¬9«;¨-�Ð8ˆAØ×(Ñ(¨×0Ñ0Ø—
‘
¬G«I¨;�
Ð7ˆAØˆt�d˜1˜2�h‰ Ð&Ð&r   c                óN  • [        U[        5      (       + =(       aM    [        U R                  [        5      =(       a,    U R                  R                  SUR	                  5       S-
  4;   nU R                  U5      (       a  U R                  (       a	  U(       a  U$ [        U[        5      (       d   [        R                  " X#[        S5      /SS9nUR                  U R                  4:w  a  UR                  U R                  /S9nU R                  (       a  UR                  5       $ U$ )Nr�   r   Fr;   r³   )r@   r	   r‰   r   ÚdimrZ   rŠ   rE   rG   r¹   rB   )rY   rK   rr   r]   Úoutput_is_local_shards        r   rs   Ú-ColwiseParallel.transform_output_post_forwardä   sí   € ô ˜6¤7Ó+Ô+÷ BÜ˜4×.Ñ.´Ó6÷Bà×#Ñ#×'Ñ'¨B°·
±
³¸qÑ0@Ð+AÑAð 	ð
 ×(Ñ(¨×0Ñ0°T×5J×5JÖOdØˆMÜ˜&¤'×*Ñ*Ü×'Ò'¨´u¸R³y°kÈUÑSˆFØ×Ñ ×!4Ñ!4Ð 6Ó6Ø×(Ñ(°T×5HÑ5HÐ4IÐ(ÐJˆFØ$(×$9×$9ˆv�‰Ó ÐE¸vÐEr   ©rŠ   Úboolre   )r   r€   r�   r‚   Ú__doc__r�   rZ   r^   rb   rh   rs   rƒ   rW   r   r   r…   r…   ¡   s-   † Ùlà(,¸TÐ\`÷ 1ò
Gô
ò"
ò'õ6Fr   r…   c                  ój   • \ rS rSrSrSSSS.SS jjrS rS rS	 r\	R                  S
 5       rS rSrg)ÚRowwiseParalleléô   u<  Row-wise: weight â†’ Shard(1), bias â†’ Replicate (Embedding: weight â†’ Shard(0)).

Linear input is sharded on the last dim; Embedding input is replicated. The module
forward produces a Partial output which the boundary redistribute reduces to
output_layouts (Replicate â†’ allreduce, Shard(1) â†’ reduce-scatter).
NTr‡   c               ór   • U=(       d    [        S5      U l        U=(       d
    [        5       U l        X0l        g rŒ   )r   rˆ   r   r‰   rŠ   rŽ   s       r   r�   ÚRowwiseParallel.__init__ü   s(   € Ø*×7¬e°B«iˆÔØ,×;´	³ˆÔØ 0Õr   c                ó¼   • [        USS5      n[        U[        R                  R                  5      =(       a    [        R
                  " 5       (       + nU=(       d    U$ r’   r”   r˜   s       r   rZ   Ú(RowwiseParallel.should_use_local_tensors  rœ   r   c           	     ó`  • UR                   R                  U5      nUc  g [        U[        R                  R
                  5      (       a  [        S5      nOUS:X  a
  [        5       O
[        S5      n[        R                  R                  [        XCU/S S9UR                  S9UR                   U'   g )Nr   Úbiasr�   r¬   r=   )rC   rž   r@   rH   rI   rŸ   r   r   rJ   r   r>   r®   s         r   rb   ÚRowwiseParallel.shard_param  s�   € Ø×!Ñ!×%Ñ% eÓ,ˆØ‰<ØÜ�fœeŸh™h×0Ñ0×1Ñ1Ü˜a›‰Ið (-°£œ	œ¼EÀ"»IˆIÜ$)§H¡H×$6Ñ$6Ü˜d¨9¨+ÀTÑJØ×,Ñ,ð %7ð %
ˆ×Ñ˜5Ò!r   c                ó   • [        U[        R                  R                  5      (       a
  [	        5       O
[        S5      nUS   nU R                  U:H  nU R                  U5      (       a  U(       a  [        U[        5      (       d  X#4$ [        U[        5      (       d!  [        R                  " XdU R                  /SS9nUR                  U4:w  a  UR                  U/S9nU R                  U5      (       a  UR                  5       nU4USS  -   U4$ )Nr�   r   Fr;   r³   r   )r@   rH   rI   rŸ   r   r   rˆ   rZ   r	   rE   rG   r¹   rB   )rY   rK   rf   rg   r]   Údesiredrº   Úinput_has_desired_layouts           r   rh   Ú,RowwiseParallel.transform_inputs_pre_forward  sê   € ä!+¨F´E·H±H×4FÑ4F×!GÑ!G”)”+ÌUÐSUËYˆØ�‰Gˆð $(×#5Ñ#5¸Ñ#@Ð Ø×(Ñ(¨×0Ñ0Ö5MÔV`ÐabÔdk×VlÑVlØ�<ÐÜ˜!œW×%Ñ%Ü×"Ò" 1¨T×-?Ñ-?Ð,@ÈEÑRˆAØ�<‰<˜G˜:Ó%Ø—‘¨7¨)�Ð4ˆAØ×(Ñ(¨×0Ñ0Ø—
‘
“ˆAØˆt�d˜1˜2�h‰ Ð&Ð&r   c              #  óB  #   • U R                  U5      (       d  S v •  g UR                  R                  S5      nUb  S UR                  S'    [        U5         S v •  S S S 5        Ub  X1R                  S'   g g ! , (       d  f       N!= f! Ub  X1R                  S'   f f = f7f)NrÍ   )rZ   rC   rž   rR   )rY   rK   r]   rÍ   s       r   rn   Ú&RowwiseParallel.context_around_forward%  s›   é € à×,Ñ,¨V×4Ñ4Üð
 ×%Ñ%×)Ñ)¨&Ó1ˆDØÑØ-1�×"Ñ" 6Ñ*ð6Ü.¨vÕ6Û÷ 7ð Ñ#Ø15×&Ñ& vÒ.ð $÷ 7Õ6ûð Ñ#Ø15×&Ñ& vÒ.ð $üs6   ‚A	BÁB ÁA7ÁB Á$BÁ7
BÂB ÂBÂBc                ó¨  • [        U[        R                  R                  5      =(       aN    [        U[        5      (       + =(       a2    [        U R
                  [        5      =(       a    UR                  (       + nU(       a�  UR                  S:X  a  UR                  5       OUR                  S5      n[        R                  " X%S9  UR                  R                  S5      =nb*  U[        U[        5      (       a  UR                  5       OU-   nU$ [        U[        5      (       d  [        R                  " X#[!        5       /SS9nUR"                  U R
                  4:w  a  UR%                  U R
                  /S9nU R'                  U5      (       a"  UR                  R                  S5      =nb  X&-   nU R(                  (       a  UR                  5       nU$ )Nr   r²   r   rÍ   Fr;   r³   )r@   rH   rI   r–   r	   r‰   r   r>   r    r¶   ÚdistÚ
all_reducerC   rž   rB   rE   r
   rG   r¹   rZ   rŠ   )rY   rK   rr   r]   Úuse_local_inference_pathr¼   rÍ   s          r   rs   Ú-RowwiseParallel.transform_output_post_forward7  sq  € ä�vœuŸx™xŸ™Ó/÷ )Ü˜v¤wÓ/Ô/÷)ä˜4×.Ñ.´	Ó:÷)ð ×(Ñ(Ô(ð	 	!ö $Ø04·	±	¸Q³˜DŸN™NÔ,ÀDÇNÁNÐSWÓDXˆMÜ�OŠO˜FÒ8Ø×*Ñ*×.Ñ.¨vÓ6Ð6�ÑCØ´jÀÄw×6OÑ6O 4§=¡=¤?ÐUYÑZ�ð ˆô ˜f¤g×.Ñ.Ü ×+Ò+¨F¼7»9¸+ÐQVÑW�Ø× Ñ  T×%8Ñ%8Ð$:Ó:Ø×,Ñ,¸×9LÑ9LÐ8MÐ,ÐN�Ø×,Ñ,¨V×4Ñ4À&×BTÑBT×BXÑBXÐY_ÓB`Ð:`¸$Ñ9mØ™�Ø×$×$ØŸ™Ó*�àˆr   rÂ   )r   r€   r�   r‚   rÄ   r�   rZ   rb   rh   rk   Úcontextmanagerrn   rs   rƒ   rW   r   r   rÆ   rÆ   ô   sE   † ñð )-¸TÐ\`÷ 1ò
Gò

ò'ð" ×Ññ6ó ð6õ"r   rÆ   c                  ó   • \ rS rSrSrS rSrg)ÚReplicatedWithGradAllReduceiQ  u±  Replicated parameter whose gradient is partial.

For norms that sit between a colwise and a rowwise layer and normalize along a sharded
axis â€” e.g. Qwen3's per-head ``q_norm``/``k_norm``, which only see this rank's heads. The
forward needs no collective (the param is replicated and the activation is already local),
but each rank's parameter gradient only covers its own heads, so the gradients have to be
summed across the mesh.
c                ó6   ^• U4S jnUR                  U5        U$ )Nc                óª   >• U R                  SS9 H?  nUR                  c  M  [        R                  " UR                  TR	                  5       S9  MA     g )NFr9   r   )Ú
parametersÚgradrÖ   r×   r¶   )ÚmodÚ
grad_inputÚgrad_outputrL   r]   s       €r   Ú_all_reduce_gradsÚFReplicatedWithGradAllReduce.install_forward.<locals>._all_reduce_grads^  s;   ø€ ØŸ™°˜Ó6�Ø—:‘:Ó)Ü—O’O E§J¡J°d·n±nÓ6FÔGò 7r   )Úregister_full_backward_hook)rY   rK   r]   rä   s     ` r   r}   Ú+ReplicatedWithGradAllReduce.install_forward[  s   ø€ õ	Hð
 	×*Ñ*Ð+<Ô=Øˆr   rW   N)r   r€   r�   r‚   rÄ   r}   rƒ   rW   r   r   rÜ   rÜ   Q  s   † ñõ	r   rÜ   c                  ó   • \ rS rSrSrS rSrg)ÚAllReduceParallelig  z@All-reduce a module's partial forward output across the TP mesh.c                óô   • Uc  g [        U[        5      (       d  [        R                  " X#[        5       /SS9nUR                  [        5       4:w  a  UR                  [        5       /S9nUR                  5       $ )NFr;   r³   )r@   r	   rE   r
   rG   r   r¹   rB   rq   s       r   rs   Ú/AllReduceParallel.transform_output_post_forwardj  se   € Ø‰>ØÜ˜&¤'×*Ñ*Ü×'Ò'¨´w³y°kÈUÑSˆFØ×Ñ¤£ Ó.Ø×(Ñ(´Y³[°MÐ(ÐBˆFØ�‰Ó Ð r   rW   N©r   r€   r�   r‚   rÄ   rs   rƒ   rW   r   r   ré   ré   g  s
   † ÙJõ!r   ré   c                  ó   • \ rS rSrSrS rSrg)ÚMlaKvAProjParallelit  aË  
For MLA attention used in DeepSeek-V2 style models (deepseek_v2, longcat_flash, glm_moe_dsa, glm4_moe_lite):
kv_a_proj_with_mqa output is [kv_lora_rank + qk_rope_head_dim] (can have different naming but important thing
to understand is that it is split)
Example below (from modeling_longcat_flash.py):

kv_a_proj_with_mqa
        |
        split
        /            k_pass    k_rot  <-- "bypasses kv_b_proj"
    |          |        (goes straight to attention,
kv_a_layernorm |         never touches kv_b_proj)
    |          |
kv_b_proj      |
(colwise)      |
    |          |
    k_pass     k_rot
        \      /
           cat
            |
        key_states

k_pass is passed to kv_b_proj (colwise) which has built-in all_reduce_backward so we don't have a partial gradient for it.
However, k_rot goes straight to attention, never touches kv_b_proj. So we need to average gradient across all ranks otherwise we only get gradient for one rank (partial gradient).
c                ób  • [        UR                  S5      (       d"  [        S[        U5      R                   S35      eUR                  R
                  nUR                  UR                  S   U-
  U/SS9u  pV[        R                  XcR                  5       5      n[        R                  " XV/SS9$ )NÚqk_rope_head_dimzConfig for z· does not have `qk_rope_head_dim`. MlaKvAProjParallel requires `qk_rope_head_dim` to be defined in the model config. Please add it to the model's config or update the TP plan mapping.r�   )r¿   )ÚhasattrÚconfigÚAttributeErrorÚtyper   rð   Úsplitr¡   r·   r¸   r¶   rH   Úcat)rY   rK   rr   r]   Úrope_dimÚpass_outputÚrope_outputs          r   rs   Ú0MlaKvAProjParallel.transform_output_post_forward�  s¤   € Ü�v—}‘}Ð&8×9Ñ9Ü Øœd 6›l×3Ñ3Ð4ð 5Uð Uóð ð
 —=‘=×1Ñ1ˆØ#)§<¡<°·±¸bÑ1AÀHÑ1LÈhÐ0WÐ]_ <Ð#`Ñ ˆÜ(×.Ñ.¨{¿N¹NÓ<LÓMˆÜ�yŠy˜+Ð3¸Ñ<Ð<r   rW   Nrì   rW   r   r   rî   rî   t  s   † ñõ6
=r   rî   c                  óH   ^ • \ rS rSrSSS.S	S jjrU 4S jrS rS rSrU =r	$ )
ÚSequenceParalleli�  r   T©Úsequence_dimrŠ   c               ó   • Xl         X l        g re   rý   )rY   rþ   rŠ   s      r   r�   ÚSequenceParallel.__init__ž  s   € Ø(ÔØ 0Õr   c                óü   >• [        UR                  SS95       HQ  u  p4UR                  U[        R                  R                  [        R                  " XB[        5       /SS95      5        MS     [        TU ])  X5      $ )NFr9   r;   )Úlistr?   Úregister_parameterrH   rI   rJ   r	   rE   r   Úsuperr}   )rY   rK   r]   Úp_nameÚpÚ	__class__s        €r   r}   Ú SequenceParallel.install_forward¢  sk   ø€ ä˜f×5Ñ5¸eÐ5ÐDÖE‰IˆFØ×%Ñ%ØœŸ™×*Ñ*¬7×+=Ò+=¸aÌ	ËÀ}Ð`eÑ+fÓgöñ Fô ‰wÑ& vÓ4Ð4r   c                óè   • [        U R                  5      nUS   n[        U[        5      (       d  [        R                  " XdU/SS9nO!UR
                  U4:w  a  UR                  U/S9nU4USS  -   U4$ )Nr   Fr;   r³   r   )r   rþ   r@   r	   rE   rG   r¹   )rY   rK   rf   rg   r]   Úseqrº   s          r   rh   Ú-SequenceParallel.transform_inputs_pre_forwardª  su   € Ü�D×%Ñ%Ó&ˆØ�‰GˆÜ˜!œW×%Ñ%Ü×"Ò" 1¨S¨E¸UÑC‰AØ�\‰\˜c˜VÓ#Ø—‘¨3¨%�Ð0ˆAØˆt�d˜1˜2�h‰ Ð&Ð&r   c                óv   • [        U[        5      (       a#  U R                  (       a  UR                  5       $ U$ U$ re   )r@   r	   rŠ   rB   rq   s       r   rs   Ú.SequenceParallel.transform_output_post_forward³  s.   € Ü�fœg×&Ñ&Ø(,×(=×(=�6—?‘?Ó$ÐIÀ6ÐIØˆr   )rþ   ÚintrŠ   rÃ   )
r   r€   r�   r‚   r�   r}   rh   rs   rƒ   Ú__classcell__)r  s   @r   rü   rü   �  s!   ø† Ø./È$÷ 1õ5ò'÷ð r   rü   c                  óT   • \ rS rSrSrSSS.   SS jjrS rSS jrS	 rS
 r	S r
Srg)ÚPackedColwiseParalleli¹  zVColumn-wise parallel style for fused linear weights packed along the output dimension.Tr   )rŠ   Úsplit_factorc               ó\   • [        5       4U l        Xl        X l        [	        5       /U l        g re   )r   rˆ   rŠ   r  r
   Úinput_grad_placements)rY   rŠ   r  s      r   r�   ÚPackedColwiseParallel.__init__¼  s*   € ô (›k˜^ˆÔØ 0ÔØ(Ôô '.£i [ˆÕ"r   c                ó   • g©NTrW   rX   s     r   rZ   Ú.PackedColwiseParallel.should_use_local_tensorsÉ  ó   € Ør   c                ó   • US:X  a  gUS-
  $ )zYDimension holding packed gate/up features: dim 0 for 2D Linear, dim 1 for 3D MoE experts.r   r�   r   rW   )rY   Ú
param_ndims     r   Ú_packed_output_shard_dimÚ.PackedColwiseParallel._packed_output_shard_dimÌ  s   € à˜‹?ØØ˜A‰~Ðr   c           	     óT  • UR                   R                  U5      nUc  g U R                  UR                  5      nUR                  S:X  a  [	        U5      nO[        XPR                  S9n[        R                  R                  [        XCU/S S9UR                  S9UR                   U'   g )Nr   ©r¿   r  r¬   r=   )rC   rž   r  r    r   r   r  rH   rI   rJ   r   r>   )rY   rK   rL   r]   r¤   r¦   r¯   s          r   rb   Ú!PackedColwiseParallel.shard_paramÒ  s”   € Ø×!Ñ!×%Ñ% eÓ,ˆØ‰<ØØ×1Ñ1°$·)±)Ó<ˆ	à�9‰9˜‹>Ü˜iÓ(‰Iä%¨)×BSÑBSÑTˆIÜ$)§H¡H×$6Ñ$6Ü˜d¨9¨+ÀTÑJØ×,Ñ,ð %7ð %
ˆ×Ñ˜5Ò!r   c                ó$  • US   n[        U[        5      (       d!  [        R                  " XTU R                  SS9nO3UR                  U R                  :w  a  UR                  U R                  S9nUR                  [        5       /S9nU4USS  -   U4$ )Nr   Fr;   r³   r´   r   )r@   r	   rE   rˆ   rG   r¹   rB   r
   ©rY   rK   rf   rg   r]   Úinput_tensors         r   rh   Ú2PackedColwiseParallel.transform_inputs_pre_forwardá  s‘   € Ø˜A‘wˆä˜,¬×0Ñ0Ü"×-Ò-¨lÀ$×BTÑBTÐ`eÑf‰LØ×$Ñ$¨×(:Ñ(:Ó:Ø'×4Ñ4À×@RÑ@RÐ4ÐSˆLð
 $×,Ñ,¼g»i¸[Ð,ÐIˆØˆ  a b Ñ)¨6Ð1Ð1r   c           	     ó€   • Ub  U R                   (       a  U$ [        R                  " X#[        SU R                  S94SS9$ )Nr�   r  Fr;   )rŠ   r	   rE   r   r  rq   s       r   rs   Ú3PackedColwiseParallel.transform_output_post_forwardï  s?   € Ø‰>˜T×2×2ØˆMÜ×!Ò!Øœ=¨R¸d×>OÑ>OÑPÐRÐ^cñ
ð 	
r   )r  rˆ   r  rŠ   N)rŠ   rÃ   r  r  )r  r  Úreturnr  )r   r€   r�   r‚   rÄ   r�   rZ   r  rb   rh   rs   rƒ   rW   r   r   r  r  ¹  s?   † Ù`ð
 "&Øñ	1ð ð1ð õ	1òôò
ò2õ
r   r  c                  ó0   • \ rS rSrSrSS.SS jjrS rSrg)	ÚPackedRowwiseParalleli÷  zCParameter style for fused weights packed along the final dimension.r   ©r  c               ó   • Xl         g re   r*  )rY   r  s     r   r�   ÚPackedRowwiseParallel.__init__ú  s   € Ø(Õr   c           	     ó  • UR                   R                  U5      nUc  g UR                  S:X  a
  [        5       O[	        SU R
                  S9n[        R                  R                  [        XCU/S S9UR                  S9UR                   U'   g )Nr   r�   r  r¬   r=   )rC   rž   r    r   r   r  rH   rI   rJ   r   r>   r®   s         r   rb   Ú!PackedRowwiseParallel.shard_paramý  s|   € Ø×!Ñ!×%Ñ% eÓ,ˆØ‰<ØØ#'§9¡9°£>”I”K´}ÈÐZ^×ZkÑZkÑ7lˆ	Ü$)§H¡H×$6Ñ$6Ü˜d¨9¨+ÀTÑJØ×,Ñ,ð %7ð %
ˆ×Ñ˜5Ò!r   N)r  r  ©r   r€   r�   r‚   rÄ   r�   rb   rƒ   rW   r   r   r)  r)  ÷  s   † ÙMà./÷ )õ
r   r)  c                  ó0   • \ rS rSrSrSS.S	S jjrS rSrg)
ÚMoEParamShardi  z½Param-only EP style for MoE expert weights (``grouped_gemm``).

Shards dim 0 and updates module.num_experts to the per-rank local count so the
experts forward and ep_router sentinel agree.
F©Úshards_expert_dimc               ó   • Xl         X l        g re   ©r¯   r3  )rY   r¯   r3  s      r   r�   ÚMoEParamShard.__init__  s   € Ø"ŒØ!2Õr   c           	     óœ  • UR                   R                  U5      nUc  g U R                  (       aS  [        US5      (       aB  UR                  S   nUR                  5       nXV-  S:w  a  [        SU SU S35      eXV-  Ul        [        R                  R                  [        XCU R                  /S S9UR                  S9UR                   U'   g )NÚnum_expertsr   zCannot evenly shard z experts across z expert-parallel ranks.r¬   r=   )rC   rž   r3  rñ   r¡   r¢   r£   r8  rH   rI   rJ   r   r¯   r>   )rY   rK   rL   r]   r¤   Úglobal_num_expertsÚexpert_parallel_sizes          r   rb   ÚMoEParamShard.shard_param  sÏ   € Ø×!Ñ!×%Ñ% eÓ,ˆØ‰<ØØ×!×!¤g¨f°m×&DÑ&DØ!%§¡¨A¡ÐØ#'§9¡9£;Ð Ø!Ñ8¸AÓ=Ü Ø*Ð+=Ð*>Ð>NØ+Ð,Ð,CðEóð ð "4Ñ!KˆFÔÜ$)§H¡H×$6Ñ$6Ü˜d¨4¯>©>Ð*:È$ÑOØ×,Ñ,ð %7ð %
ˆ×Ñ˜5Ò!r   r5  N)r3  rÃ   r/  rW   r   r   r1  r1    s   † ñð @E÷ 3õ
r   r1  c                  ó8   • \ rS rSrSr\S 5       r\S 5       rSrg)Ú_AllReduceForwardi(  z)Allreduce-sum forward, identity backward.c                ób   • [         R                  " U5      S:”  a  [         R                  " XS9  U$ )Nr   r   )rÖ   Úget_world_sizer×   ©Úctxrº   r¼   s      r   r|   Ú_AllReduceForward.forward+  s&   € ä×"Ò" =Ó1°AÓ5Ü—’ Ò7ØˆHr   c                ó
   • US 4$ re   rW   ©rA  rà   s     r   ÚbackwardÚ_AllReduceForward.backward1  s   € à˜�:Ðr   rW   N©	r   r€   r�   r‚   rÄ   Ústaticmethodr|   rE  rƒ   rW   r   r   r=  r=  (  s+   † Ù7à	ñ	ó 
ð	ð
 
ñ	ó 
ó	r   r=  c                  ó8   • \ rS rSrSr\S 5       r\S 5       rSrg)r·   i5  aV  Identity forward, allreduce-sum backward.

Used for MoE routing weights: the forward value is replicated (same on all
ranks), but the backward gradient is partial (each rank has 1/tp_size from
its expert shard). We need to sum the partial gradients without dividing by
world_size, which is what DTensor's Replicate backward does incorrectly.
c                ó   • X l         U$ re   )r¼   r@  s      r   r|   Ú_AllReduceBackward.forward>  s   € à -ÔØˆHr   c                óf   • UR                  5       n[        R                  " XR                  S9  US 4$ )Nr   )Ú
contiguousrÖ   r×   r¼   rD  s     r   rE  Ú_AllReduceBackward.backwardC  s*   € à—?‘?Ó$ˆDÜ�OŠO˜D×(9Ñ(9Ò:Ø˜�:Ðr   rW   NrG  rW   r   r   r·   r·   5  s/   † ñ	ð 
ñ	ó 
ð	ð 
ñ	ó 
ó	r   r·   c                  ó@   • \ rS rSrS rSS.S jrSS.S jrS
S jrS	rg)ÚMoeExpertsParalleliJ  c                ó   • gr  rW   rX   s     r   rZ   Ú+MoeExpertsParallel.should_use_local_tensorsK  r  r   F©Úis_expert_parallelc               ó   • UtpgUR                   S:X  a  UR                  5       OUR                  S5      n[        U[        5      (       a  UR	                  5       n[
        R                  Xh5      n[        U5      S:¼  aJ  Utpšn[        U
[        5      (       a  U
R	                  5       n
U(       d  [
        R                  X¨5      n
Xš/UQnU/UQ7U4$ )Nr   r²   r   )r    r¶   r@   r	   rB   r·   r¸   r%   )rY   rK   rf   rg   r]   rT  Úhidden_statesÚrouting_argsÚtp_groupÚtop_k_indexÚtop_k_weightsÚ
extra_argss               r   rh   Ú/MoeExpertsParallel.transform_inputs_pre_forwardN  s»   € Ø'+Ð$ˆØ'+§y¡y°A£~�4—>‘>Ô#¸4¿>¹>È$Ó;OˆÜ�m¤W×-Ñ-Ø)×2Ñ2Ó4ˆMÜ*×0Ñ0°ÓIˆäˆ|Ó Ó!Ø6BÐ3ˆK¨Ü˜-¬×1Ñ1Ø -× 6Ñ 6Ó 8�Þ%Ü 2× 8Ñ 8¸Ó Q�Ø'ÐD¸ÐDˆLàÐ- Ñ-¨vÐ5Ð5r   c               óÀ   ^ ^^^^^• TR                   m[        S TR                  5        5       5      (       a
  [        5       O	[	        5       mUUUUUU 4S jnUTl         T$ )zIInstall the transforms but pass `is_expert_parallel` in the forward call.c              3  óˆ   #   • U  H8  n[        U[        5      =(       a    [        S  UR                   5       5      v •  M:     g7f)c              3  óJ   #   • U  H  oR                  5       (       + v •  M     g 7fre   ©Úis_replicate©Ú.0r¯   s     r   Ú	<genexpr>Ú?MoeExpertsParallel.install_forward.<locals>.<genexpr>.<genexpr>e  s    é € Ð2rÒaqÐT]×7MÑ7MÓ7O×3OÑ3OÒaqùó   ‚!#N©r@   r	   ÚanyrG   ©rc  rL   s     r   rd  Ú5MoeExpertsParallel.install_forward.<locals>.<genexpr>d  s8   é € ð â0�Eô ˜5¤'Ó*×r¬sÑ2rÐaf×aqÒaqÓ2rÓ/rÔrÚ0ùó   ‚A Ac                 ó¸   >• TR                  TXTTS9u  pTR                  TT5         T" U 0 UD6nS S S 5        TR                  TWTTS9$ ! , (       d  f       N = f)NrS  )Úsourcerx   )	rf   rg   rr   rT  r]   rK   ry   Úoutput_sourcerY   s	      €€€€€€r   rz   Ú6MoeExpertsParallel.install_forward.<locals>.tp_forwardk  sn   ø€ Ø×<Ñ<Ø˜ dÐ?Qð =ð ‰LˆDð ×,Ñ,¨V°TÕ:Ù)¨4Ð:°6Ñ:�÷ ;à×5Ñ5°f¸fÀdÐS`Ð5ÐaÐa÷ ;Õ:ús   ¨	AÁ
A)r|   rh  rß   r
   r   )rY   rK   r]   rT  rz   ry   rn  s   ```` @@r   r}   Ú"MoeExpertsParallel.install_forward_  s_   ý€ à!Ÿ>™>Ðô ñ à#×.Ñ.Ô0ó÷ ñ ô ŒIô
 “ð 	÷	bò 	bð $ˆŒØˆr   Nc                óè   • Uc  g [        S UR                  5        5       5      nU(       d  U$ UR                  S:X  a  UR                  5       OUR                  S5      n[        R                  X&5      $ )Nc              3  óˆ   #   • U  H8  n[        U[        5      =(       a    [        S  UR                   5       5      v •  M:     g7f)c              3  óJ   #   • U  H  oR                  5       (       + v •  M     g 7fre   r`  rb  s     r   rd  ÚMMoeExpertsParallel.transform_output_post_forward.<locals>.<genexpr>.<genexpr>{  s    é € Ð.nÒ]mÐPY×3IÑ3IÓ3K×/KÑ/KÒ]mùrf  Nrg  ri  s     r   rd  ÚCMoeExpertsParallel.transform_output_post_forward.<locals>.<genexpr>z  s8   é € ð %
â,�ô �uœgÓ&×n¬3Ñ.nÐ]b×]mÒ]mÓ.nÓ+nÔnÚ,ùrk  r   r²   )rh  rß   r    r¶   r=  r¸   )rY   rK   rr   r]   rm  Úhas_sharded_parametersr¼   s          r   rs   Ú0MoeExpertsParallel.transform_output_post_forwardv  sg   € Ø‰>Øä!$ñ %
à×*Ñ*Ô,ó%
ó "
Ðö &ØˆMà,0¯I©I¸«N˜Ÿ™Ô(ÀÇÁÈtÓ@TˆÜ ×&Ñ& vÓ=Ð=r   rW   re   )	r   r€   r�   r‚   rZ   rh   r}   rs   rƒ   rW   r   r   rP  rP  J  s$   † òð ^cõ 6ð" CHõ ÷.>r   rP  c                  ó   • \ rS rSrSrS rSrg)ÚMoeIdentityParalleli…  zÝ
Used in longcat_flash zero_experts (nn.Identity) which return the same value on every GPU, but the
moe_tp_experts will sums across GPUs. Therefore we pre-divide the identity input by tp_size to cancel the extra scaling.
c                óB   • US   nXTR                  5       -  /USS  Q7U4$ )Nr   r   )r¢   r"  s         r   rh   Ú0MoeIdentityParallel.transform_inputs_pre_forward‹  s-   € Ø˜A‘wˆØŸy™y›{Ñ*Ð6¨T°!°"¨XÑ6¸Ð>Ð>r   rW   N)r   r€   r�   r‚   rÄ   rh   rƒ   rW   r   r   ry  ry  …  s   † ñõ
?r   ry  c                  ó   • \ rS rSrSrS rSrg)ÚEpRouterParalleli�  u-  Expert-parallel router: forward-only slicing of router outputs to local experts.

Expects the router to return `(router_logits, router_scores, router_indices, *extra)`.
`router_logits` and any trailing `extra` outputs are passed through unchanged.

The gate runs replicated on every rank and emits global expert IDs and scores.
Under EP each rank owns `num_experts // ep_size` experts, so this post-forward hook:

- zeroes scores for non-local experts
- remaps surviving global indices to local indices (`fmod` after masking non-local slots)
- sets dropped slots to sentinel `num_local_experts` (skipped by grouped_gemm experts forward)

Downstream `moe_tp_experts` allreduce-sums partial per-rank expert outputs.

Example: 4 tokens, top_k=4, 128 experts, EP=8 â†’ num_local_experts=16.

Router output (identical on all ranks):
    router_indices:
        [ 52,  42, 119,  67],
        [102,  89,  61,  40],
        [ 82, 103,   4,  34],
        [ 93,  23, 109,  11]

Owning rank (`index // 16`):
        [ 3,  2,  7,  4],
        [ 6,  5,  3,  2],
        [ 5,  6,  0,  2],
        [ 5,  1,  6,  0],

After slicing on rank 0 (owns experts 0-15):
    router_indices:          router_scores (illustrative):
        [16, 16, 16, 16],       [0.0, 0.0, 0.0, 0.0],
        [16, 16, 16, 16],       [0.0, 0.0, 0.0, 0.0],
        [16, 16,  4, 16],       [0.0, 0.0, 0.3, 0.0],
        [16, 16, 16, 11],       [0.0, 0.0, 0.0, 0.1],

On rank 1 (owns experts 16-31), global expert 23 remaps to local index 7 via `23 % 16`.

Scores and indices stay paired element-wise in `(seq, top_k)` shape.
c                ó  • UR                  5       UR                  5       pT[        USS 5      nUc  [        [        USS 5      SS 5      nUc"  [        S[	        U5      R
                   S35      eXe-  S:w  a  [        SU SU S35      eXe-  nUtp‰p«[        R                  " 5       (       aW  U	R                  (       aF  UR                  S	:X  a  UR                  5       OUR                  S
5      n[        R                  Xœ5      n	X§-  U:g  nU	R                  US5      n	U
R                  US5      n
US	:”  a  [        R                  " X§5      n
O(U
R                  U
S:„  S5      R                  U
S:  S5      n
U
R                  U
S:H  U5      n
X‰U
/UQ7$ )Nr8  rò   zRouter module z2 is missing `num_experts` and `config.num_experts`r   z*num_experts must be divisible by ep_size: z % z != 0r   r²   g        r�   )Úget_local_rankr¢   r•   ró   rô   r   r£   rH   r—   r>   r    r¶   r·   r¸   Úmasked_fillÚfmod)rY   rK   rr   r]   Úep_rankÚep_sizer8  Únum_local_expertsÚrouter_logitsÚrouter_scoresÚrouter_indicesÚextra_outputsr¼   Únon_local_masks                 r   rs   Ú.EpRouterParallel.transform_output_post_forwardº  s”  € Ø×.Ñ.Ó0°$·)±)³+�Ü˜f m°TÓ:ˆØÑÜ!¤'¨&°(¸DÓ"AÀ=ÐRVÓWˆKØÑÜ Ø ¤ f£×!6Ñ!6Ð 7Ð7iÐjóð ð Ñ  AÓ%ÜÐIÈ+ÈÐVYÐZaÐYbÐbgÐhÓiÐiØ'Ñ2ÐàGMÐDˆ nô × Ò ×"Ñ" }×'B×'BØ04·	±	¸Q³˜DŸN™NÔ,ÀDÇNÁNÐSWÓDXˆMÜ.×4Ñ4°]ÓRˆMØ(Ñ=À'ÑIˆØ%×1Ñ1°.À#ÓFˆØ'×3Ñ3°NÀBÓGˆØ˜qÓ Ü"ŸZšZ¨ÓJ‰Nà+×7Ñ7¸ÈÑ8JÈAÓN×ZÑZÐ[iÐlmÑ[mÐoqÓrˆNØ'×3Ñ3°NÀbÑ4HÐJ[Ó\ˆØ¨^ÐK¸mÑKÐKr   rW   Nrì   rW   r   r   r}  r}  �  s   † ñ'õRLr   r}  c                  ó   • \ rS rSrSrS rSrg)ÚRouterParallelMegaMoeiØ  zçRouter TP plan used with DeepGEMM Mega MoE.

Mega MoE handles EP dispatch inside the kernel and wants raw global expert ids
with unmasked routing weights, so the router doesn't pre-shard per EP rank like
``EpRouterParallel`` does.
c                ó   • U$ re   rW   rq   s       r   rs   Ú3RouterParallelMegaMoe.transform_output_post_forwardà  ru   r   rW   Nrì   rW   r   r   rŒ  rŒ  Ø  s   † ñõr   rŒ  c                  ó6   • \ rS rSrSrSS.S jrS rS
S jrS	rg)ÚMoeTensorParalellMegaMoeExpertsiä  uh  TP layer for DeepGEMM Mega MoE experts.

Mega MoE is inference-only (the kernel has no backward) and handles EP dispatch +
combine + per-rank token sharding internally â€” so we skip the gradient-sync hooks
that ``MoeExpertsParallel`` would apply, and we forward the EP ``process_group``
into the module so the symm-buffer rendezvous can run on first forward.
FrS  c               óF   • US   US   US   p‡nXgX„R                  5       4U4$ )Nr   r   r   )r¶   )	rY   rK   rf   rg   r]   rT  rV  rY  rZ  s	            r   rh   Ú<MoeTensorParalellMegaMoeExperts.transform_inputs_pre_forwardí  s0   € Ø48¸±G¸TÀ!¹WÀdÈ1Ág MˆØ¨M¿>¹>Ó;KÐLÈfÐTÐTr   c                ó   • [        U5      $ re   )rR   rm   s      r   rn   Ú6MoeTensorParalellMegaMoeExperts.context_around_forwardñ  s   € Ü(¨Ó0Ð0r   Nc                ó   • U$ re   rW   )rY   rK   rr   r]   rm  s        r   rs   Ú=MoeTensorParalellMegaMoeExperts.transform_output_post_forwardô  ru   r   rW   re   )	r   r€   r�   r‚   rÄ   rh   rn   rs   rƒ   rW   r   r   r�  r�  ä  s   † ñð ^cõ Uò1÷r   r�  c                  ó   • \ rS rSrSr\" 5       (       aæ  0 S\" \" 5       \" 5       S9_S\" \" 5       \" 5       S9_S\" \" 5       \" 5       S9_S\" \" 5       \	" S5      S9_S	\" \	" S5      \" 5       S9_S
\" \" 5       \" 5       S9_S\" \" 5       \" 5       S9_S\
" 5       _S\" 5       _S\" SS9_S\" \	" S5      SS9_S\" 5       _S\" 5       _S\" 5       _S\" 5       _S\" 5       _S\" 5       _\" 5       \" 5       S.EO0 rSrg)ÚParallelInterfaceiø  z4Registry of named TP styles for the DTensor backend.Úembedding_rowwise)rˆ   r‰   Úcolwise_gather_outputÚcolwise_repÚcolwiser�   ÚrowwiseÚrowwise_split_inputÚrowwise_repÚpacked_colwiseÚpacked_rowwiseÚsequence_parallelT)rŠ   Úgrouped_gemmr   r2  Ú	ep_routerÚmegamoe_routerÚmoe_tp_expertsÚmegamoe_expertsÚmoe_identity_expertÚreplicated_with_grad_allreduce)Úmla_kv_a_projr×   rW   N)r   r€   r�   r‚   rÄ   r   rÆ   r   r…   r   r  r)  rü   r1  r}  rŒ  rP  r�  ry  rÜ   rî   ré   Ú_global_mappingrƒ   rW   r   r   r˜  r˜  ø  s‰  † Ù>ñ0 *×+Ñ+ð+	
Ø¡¹y»{Ñ[dÓ[fÑ!gð	
à#¡_Á9Ã;Ñ_hÓ_jÑ%kð	
ð ™?¹»ÑU^ÓU`Ñað	
ð ‘±Y³[ÑQVÐWYÓQZÑ[ð		
ð
 ‘±U¸2³YÉyË{Ñ[ð	
ð "¡?ÁÃÑ]fÓ]hÑ#ið	
ð ™?¹»ÑU^ÓU`Ñað	
ð Ñ3Ó5ð	
ð Ñ3Ó5ð	
ð  Ñ!1À4Ñ!Hð	
ð ™M©%°«(ÀdÑKð	
ð Ñ)Ó+ð	
ð Ñ3Ó5ð	
ð Ñ0Ó2ð	
ð Ñ>Ó@ð	
ð  "Ñ#6Ó#8ð!	
ð" -Ñ.IÓ.Kð#	
ñ$ 0Ó1Ù+Ó-ó'	
ð, ó/ r   r˜  ÚALL_PARALLEL_STYLESc           	     óÜ   • U =(       d    0 R                  5        Vs1 s H  o[        ;  d  M  UiM     nnU(       a,  [        SU S[        [        R	                  5       5       35      eg s  snf )Nz$Unsupported tensor parallel styles: z. Supported styles are )Úvaluesr¬  r£   r  Úkeys)r*   ÚstyleÚunsupported_styless      r   Ú_validate_tp_plan_stylesr²    sq   € Ø.5¯m¸×-CÑ-CÔ-EÓjÒ-E EÔViÑIiŸ%Ñ-EÐÐjÞÜØ2Ð3EÐ2Fð G$Ü$(Ô)<×)AÑ)AÓ)CÓ$DÐ#EðGó
ð 	
ð ùò ks
   œA)­A)c           	     ó.  • [        U R                  5        U R                  5        Hë  u  p#[        UR	                  SS95       He  u  pEU(       a  U SU 3OUn[        X`R                  SS9nUc  M.  U[        ;   d  M:  [        U   nUR                  X4XS9  UR                  X4U5        Mg     [        X R                  SS9nUbG  U[        ;   a=  US:X  a  U R                  R                  5       Ul	        [        U   R                  X15        SUl        Mí     U $ )zKDTensor backend: shard params as placeholders and install TP forward hooks.Fr9   r   T)r4   r*   r5   )r4   rª  )r²  r*   Únamed_modulesr  r?   r7   r¬  r^   rb   rò   Úget_text_configr}   Ú
_is_hooked)	ÚmodelÚtp_meshr   rK   r  Ú_ÚfullÚ
style_namer°  s	            r   Úapply_tensor_parallelismr¼  "  s  € ô ˜UŸ]™]Ô+à×+Ñ+Ö-‰ˆä˜f×5Ñ5¸eÐ5ÐDÖE‰IˆFÞ)-�d�V˜1˜V˜HÑ%°6ˆDÜ/¸tÏ]É]ÐfjÑkˆJØÓ%¨*Ô8KÕ*KÜ+¨JÑ7�Ø×$Ñ$ V°WÐ$ÑRØ×!Ñ! &°'Ö:ñ Fô ,¸4ÏÉÐbgÑhˆ
ØÑ! jÔ4GÓ&GØ˜_Ó,ð !&§¡× <Ñ <Ó >�”Ü 
Ñ+×;Ñ;¸FÔLØ ˆÖñ% .ð( €Lr   c                ó,  • 0 nU R                  5        H}  u  pV[        U[        R                  5      (       aU  [        U[        5      (       a  UR                  5       nUR                  5       R                  5       R                  5       XE'   My  XdU'   M     U$ )z«Gather TP-sharded ``DTensor`` parameters to full CPU tensors for checkpoint saving.

Every rank must call this function so ``DTensor.full_tensor()`` collectives complete.
)	rA   r@   rH   ÚTensorr	   Úfull_tensorÚdetachÚcpurM  )Ú
state_dictÚ_tp_planÚ_device_meshÚ_tp_sizeÚgatheredr+   Útensors          r   Úgather_state_dict_for_saverÈ  >  sw   € ð €HØ!×'Ñ'Ö)‰ˆÜ�fœeŸl™l×+Ñ+Ü˜&¤'×*Ñ*Ø×+Ñ+Ó-�Ø"ŸM™M›O×/Ñ/Ó1×<Ñ<Ó>ˆH‹Mà"�S‹Mñ *ð €Or   )r   Ústrr'  rÉ  )r)   z	list[str]r*   údict[str, str] | None)T)r4   rÉ  r*   údict[str, str]r'  z
str | None)r*   rÊ  r'  ÚNone)rÂ  údict[str, torch.Tensor]rÃ  rË  rÅ  r  r'  rÍ  )8Ú
__future__r   rk   r   Úutilsr   Úutils.genericr   Úutils.import_utilsr   r   Ú
get_loggerr   r&   rH   Útorch.distributedÚdistributedrÖ   Útorch.distributed.tensorr	   r
   r   r   r   Ú(torch.distributed.tensor.placement_typesr   r   r2   r7   rÚ   rR   rT   r…   rÆ   rÜ   ré   rî   rü   r  r)  r1  ÚautogradÚFunctionr=  r·   rP  ry  r}  rŒ  r�  r˜  r¬  Ú__annotations__r²  r¼  rÈ  rW   r   r   Ú<module>rÚ     sÐ  ðö #ã Û 	å Ý ,ß Sð 
×	Ò	˜HÓ	%€á×ÑÛá!×#Ñ#Ý$ß^Õ^ÝFôEô`ö:ð$ ×Ññó ð÷<#ñ #ôLPFÐ)ô PFôfZÐ)ô ZôzÐ"5ô ô,
!Ð+ô 
!ô&=Ð,ô &=ôRÐ*ô ô8;
Ð/ô ;
ô|
Ð/ô 
ô"
Ð'ô 
ñ< "×#Ñ#ô˜EŸN™N×3Ñ3ô ô˜UŸ^™^×4Ñ4ô ô*8>Ð,ô 8>ôv?Ð-ô ?ôELÐ*ô ELôP	Ð,ô 	ôÐ&8ô ô(Ð(ô ñ< *;Ó)<Ð Ð&Ó <ô
òð8Ø'ðàðð ð	ð
 õr   