ó
    ‡Ž¡j0  ã                  ó.  • S SK Jr  S SKrS SKJr  S SKJr  SSKJr  SSK	J
r
Jr  \(       a  S SKJr  \
" 5       (       a
  S SKrS SKJr  \" 5       (       a  S SKJr  SS jr " S	 S
\R&                  5      r " S S5      rSS jrSS jrSS jr    SS jrg)é    )ÚannotationsN)Úwraps)ÚTYPE_CHECKINGé   )ÚCausalLMOutputWithPast)Úis_torch_availableÚis_torch_distributed_availablec                óp   • U R                   " U0 UD6nUR                  5         [        UR                  5      $ ©N)Úbind_partialÚapply_defaultsÚdictÚ	arguments)Úforward_signatureÚargsÚkwargsÚbounds       Ú¸C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/distributed/pipeline_parallel.pyÚ_bind_forward_kwargsr   #   s2   € Ø×*Ò*¨DÐ;°FÑ;€EØ	×ÑÔÜ�—‘Ó Ð ó    c                  ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )ÚPipelineIdentityLayeré)   zLAn identity layer replacing modules not owned by the current pipeline stage.c                ó"   >• [         TU ]  5         g r   )ÚsuperÚ__init__)Úselfr   r   Ú	__class__s      €r   r   ÚPipelineIdentityLayer.__init__,   s   ø€ Ü‰ÑÕr   c                ó^   • U(       a  US   $ [        [        UR                  5       5      5      $ )zQReturn the first arg from args or the first value from kwargs as nothing is used.r   )ÚnextÚiterÚvalues)r   r   r   s      r   ÚforwardÚPipelineIdentityLayer.forward/   s#   € æˆt�A‰wÐ?¤D¬¨f¯m©m«oÓ)>Ó$?Ð?r   © )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r$   Ú__static_attributes__Ú__classcell__)r   s   @r   r   r   )   s   ø† ÙVõ÷@ð @r   r   c                  óz   • \ rS rSrSrSS jrSSS.           SS jjrSS jrSS jr        SS	 jr	S
r
g)ÚPipelineStageé4   zCPipeline-parallel stage metadata derived from a 1-D PP device mesh.c                óî  • UR                  5       U l        UR                  5       U l        UR	                  5       U l        U R                  S:H  U l        U R                  U R                  S-
  :H  U l        U R                  S:”  a  U R                  S-
  OS U l        U R                  U R                  S-
  :  a  U R                  S-   OS U l	        [        R                  " U R
                  5      S:H  U l        g )Nr   é   Úgloo)Úget_local_rankÚpp_rankÚsizeÚpp_sizeÚ	get_groupÚpp_groupÚpp_is_first_stageÚpp_is_last_stageÚpp_prev_rankÚpp_next_rankÚdistÚget_backendÚcomm_on_cpu)r   Úpp_meshs     r   r   ÚPipelineStage.__init__7   s¼   € Ø×-Ñ-Ó/ˆŒØ—|‘|“~ˆŒØ×)Ñ)Ó+ˆŒØ!%§¡°Ñ!2ˆÔØ $§¡°·±¸qÑ0@Ñ @ˆÔØ04·±¸qÓ0@˜DŸL™L¨1Ò,ÀdˆÔØ04·±¸t¿|¹|ÈaÑ?OÓ0O˜DŸL™L¨1Ò,ÐUYˆÔÜ×+Ò+¨D¯M©MÓ:¸fÑDˆÕr   N)ÚtensorÚshapec               ó  • U R                   (       a  [        R                  " S5      OUnS=pxUS:X  a4  U R                  (       a  gU R                  n[        R
                  " XSUS9nOPUS:X  a<  U R                  (       a  gU R                  nUR                  XcS9R                  5       nO[        SU 35      eUR                  S5      n	U	(       a  UOUn
[        R                  " U	(       a  [        R                  O[        R                  XJU R                   S	9n[        R"                  " U/5       H  nUR%                  5         M     UR&                  S
:X  a  [        R(                  R+                  5         U	(       a  S$ UR                  X#S9$ )z>Point-to-point pipeline communication between adjacent stages.ÚcpuNÚrecv_forward©ÚdtypeÚdeviceÚsend_forward©rJ   rI   z.Unsupported pipeline communication operation: Úsend)ÚgroupÚcuda)r@   ÚtorchrJ   r:   r<   Úemptyr;   r=   ÚtoÚ
contiguousÚ
ValueErrorÚ
startswithr>   ÚP2POpÚisendÚirecvr9   Úbatch_isend_irecvÚwaitÚtyperO   Úsynchronize)r   Ú	operationrJ   rI   rC   rD   Úcomm_deviceÚsrcÚdestÚis_sendÚ	peer_rankÚopÚreqs                r   ÚcommunicateÚPipelineStage.communicateA   s6  € ð .2×-=×-=”e—l’l 5Ô)À6ˆØÐˆà˜Ó&Ø×%×%Øà×#Ñ#ˆCä—[’[ ¸KÑH‰Fà˜.Ó(Ø×$×$Øà×$Ñ$ˆDØ—Y‘Y k�YÐ?×JÑJÓL‰Fô ÐMÈiÈ[ÐYÓZÐZð ×&Ñ& vÓ.ˆÞ#‘D¨ˆ	Ü�ZŠZ¦gœŸ
š
´4·:±:¸vÐX\×XeÑXeÑfˆä×)Ò)¨2¨$Ö/ˆCØ�H‰HŽJñ 0à×Ñ˜vÓ%Ü�J‰J×"Ñ"Ô$æˆtÐI F§I¡I°V IÐ$IÐIr   c                ó\   • X R                   -  nX-  nXR                   S-
  :X  a  UOXC-   nXE4$ )zø
Return [start, end) owned by rank.

Example with pp_size=3 and num_layers=32:
    layers_per_rank = 32 // 3 = 10
    rank 0 -> [0, 10)   # layers 0-9
    rank 1 -> [10, 20)  # layers 10-19
    rank 2 -> [20, 32)  # layers 20-31 (10 + 2 remainder)
r2   )r7   )r   ÚrankÚ
num_layersÚlayers_per_rankÚstart_layerÚ	end_layers         r   Úlayer_range_for_rankÚ"PipelineStage.layer_range_for_rankl   s9   € ð %¯©Ñ4ˆØÑ,ˆà"&¯,©,¸Ñ*:Ó":‘JÀÑ@]ˆ	ØÐ%Ð%r   c                ó¶  • U S3nUR                  U S35      (       a  gUR                  U S35      (       d  UR                  S5      (       a  U R                  S-
  $ U S3nUR                  U5      (       d  g[        UR                  S5      S	   5      n[	        U R                  5       H*  nU R                  Xr5      u  p‰X†s=::  a  U	:  d  M$   Us  $   M,     g)
zPReturn the PP rank that owns a checkpoint parameter key, or ``None`` if unknown.Ú.zembed_tokens.r   znorm.zlm_head.r2   zlayers.Nr   )rU   r7   ÚintÚsplitÚrangerm   )
r   Úkeyri   Úbase_model_prefixÚbase_prefixÚlayers_prefixÚ	layer_idxrh   rk   rl   s
             r   Úfind_rank_for_keyÚPipelineStage.find_rank_for_key}   sÐ   € à*Ð+¨1Ð-ˆà�>‰>˜[˜M¨Ð7×8Ñ8Øà�>‰>˜[˜M¨Ð/×0Ñ0°C·N±NÀ:×4NÑ4NØ—<‘< !Ñ#Ð#à&˜- wÐ/ˆØ�~‰~˜m×,Ñ,Øä˜Ÿ	™	 #› qÑ)Ó*ˆ	Ü˜$Ÿ,™,Ö'ˆDØ%)×%>Ñ%>¸tÓ%PÑ"ˆKØÕ3¨)Ö3Ø’ò 4ñ (ð
 r   c               ó  • U R                   S::  a  U$ U R                   S-
  nU R                  (       a  [        R                  " S5      OUnSnU R                  (       a|  UR
                  U:X  d   SU SUR
                   35       eUR                  XRS9R                  5       n[        R                  " [        UR                  5      [        R                  US9nO$[        R                  " U[        R                  US9n[        R                  " X„U R                  S9  U R                  (       d,  [        R                  " [!        UR#                  5       5      X%S9n[        R                  " WX@R                  S9  UR                  X2S9$ )	z­Broadcast logits from the last PP rank to every rank. Only the last stage computes logits, but every rank must return them so `generate()` works without PP-specific changes.r2   rF   é   zExpected logits with z( dims (batch, seq_len, vocab_size), got rL   rH   )r_   rN   )r7   r@   rP   rJ   r;   ÚndimrR   rS   rC   ÚlistrD   ÚlongrQ   r>   Ú	broadcastr9   ÚtupleÚtolist)	r   rC   rI   rJ   Ú	last_rankr^   Úlogits_ndimÚlogitsÚ	shape_msgs	            r   Úbroadcast_from_lastÚ!PipelineStage.broadcast_from_last“   s,  € ð �<‰<˜1ÓØˆMà—L‘L 1Ñ$ˆ	Ø-1×-=×-=”e—l’l 5Ô)À6ˆàˆð × × Ø—;‘; +Ó-ð Ø'¨ }Ð4\Ð]c×]hÑ]hÐ\iÐjóÐ-ð —Y‘Y k�YÐ?×JÑJÓLˆFÜŸš¤T¨&¯,©,Ó%7¼u¿z¹zÐR]Ñ^‰IäŸš K´u·z±zÈ+ÑVˆIÜ�Š�y°t·}±}ÒEð ×$×$Ü—[’[¤ y×'7Ñ'7Ó'9Ó!:À%Ñ\ˆFÜ�Š�v 9·M±MÒBà�y‰y ˆyÐ4Ð4r   )r@   r9   r:   r;   r=   r<   r5   r7   )rA   ú(torch.distributed.device_mesh.DeviceMesh)r]   ÚstrrJ   útorch.devicerI   útorch.dtyperC   útorch.Tensor | NonerD   ztuple[int, ...] | NoneÚreturnr�   )rh   rq   ri   rq   rŽ   ztuple[int, int])rt   rŠ   ri   rq   ru   rŠ   rŽ   z
int | None)rC   r�   rI   rŒ   rJ   r‹   rŽ   útorch.Tensor)r'   r(   r)   r*   r+   r   re   rm   ry   r‡   r,   r&   r   r   r/   r/   4   s�   † ÙMôEð  '+Ø(,ñ)Jàð)Jð ð	)Jð
 ð)Jð $ð)Jð &ð)Jð 
õ)JôV&ô"ð, 5à#ð 5ð ð	 5ð
 ð 5ð 
÷ 5r   r/   c                ó  ^ ^^• [        U5      nUT l        [        T T R                  5      nUR                  n[        U5      nUR                  UR                  U5      u  pg[        T R                  SS5      nUR                  =(       d    U=(       a    UR                  n	U	(       d  [        5       Ul        [        U5       H  n
X¦:  d  X§:¼  d  M  [        5       XJ'   M     UR                  (       d  [        5       Ul        [        5       T l        U(       a  UR                  (       d  0 T l        [        T SS5      (       dF  T R"                  m[$        R&                  " T5      m[)        T5      UU U4S j5       nUT l        ST l        T $ )z8Naive even split of `base_model.layers` across PP ranks.Útie_word_embeddingsFÚ_pp_forward_wrappedc                 ó&   >• [        TTT/U Q70 UD6$ r   )Úpipeline_parallel_naive_forward)r   r   r   ÚmodelÚoriginal_forwards     €€€r   Úpp_naive_forwardÚ4apply_pipeline_parallelism.<locals>.pp_naive_forwardÙ   s    ø€ ä2°5Ð:JÐL]ÐoÐ`dÒoÐhnÑoÐor   T)r/   Ú	_pp_stageÚgetattrru   ÚlayersÚlenrm   r5   Úconfigr:   r;   r   Úembed_tokensrs   ÚnormÚlm_headÚall_tied_weights_keysr$   ÚinspectÚ	signaturer   r’   )r•   rA   ÚstageÚ
base_modelr›   ri   rk   rl   ÚtiedÚkeep_embed_tokensrx   r—   r   r–   s   `           @@r   Úapply_pipeline_parallelismr¨   ¶   sK  ú€ ô ˜'Ó"€EØ€E„Oä˜ × 7Ñ 7Ó8€JØ×Ñ€FÜ�V“€Jà"×7Ñ7¸¿¹ÀzÓRÑ€KÜ�5—<‘<Ð!6¸Ó>€Dð ×/Ñ/×T°D×4S¸U×=SÑ=SÐÞÜ"7Ó"9ˆ
Ôä˜:Ö&ˆ	ØÓ" iÕ&<Ü 5Ó 7ˆFÓñ 'ð ×!×!Ü/Ó1ˆ
ŒÜ-Ó/ˆŒö �E×*×*Ø&(ˆÔ#ô �5Ð/°×7Ñ7Ø Ÿ=™=ÐÜ#×-Ò-Ð.>Ó?Ðä	ÐÓ	 ö	pó 
!ð	pð )ˆŒØ$(ˆÔ!à€Lr   c                óº   • U R                  S5      =nb  [        UR                  5      $ U R                  S5      =nb  / UR                  S S QUP7$ [        S5      e)NÚinputs_embedsÚ	input_idsr   z>Cannot determine hidden states shape for pipeline recv_forward)Úgetr�   rD   rT   )Ú
fwd_kwargsÚhidden_sizerª   r«   s       r   Ú_hidden_states_shaper¯   ç   sb   € Ø#Ÿ™¨Ó8Ð8ˆÑEÜ�]×(Ñ(Ó)Ð)Ø—^‘^ KÓ0Ð0ˆ	Ñ=Ø2�—‘  !Ð$Ð2 kÑ2Ð2Ü
ÐUÓ
VÐVr   c                ó2   • U R                  SS5        XS'   U $ )zQStage 0 uses input_ids; later stages use received hidden states as inputs_embeds.r«   Nrª   )Úpop)r­   Úhidden_statess     r   Ú#_feed_hidden_states_as_input_embedsr³   ï   s   € à‡N�N�; Ô%Ø"/ˆÑØÐr   c                óà  • U R                   n[        U R                  5       5      R                  n[        U R                  5       5      R                  n[        X#U5      nUn	UR                  (       d;  [        X€R                  R                  5      n
UR                  SXgU
S9n[        X‹5      n	UR                  (       a   U" S0 U	D6nUR                  UR                  píOs[        X R                   5      nU	R#                  5        VVs0 s H  u  nnUS;  d  M  UU_M     nnnU" S0 UD6nUR                  SXgUR$                  S9  S UR                  píUR'                  X×US9n[)        XÞS9$ s  snnf )	NrG   )rJ   rI   rD   >   ÚlabelsÚlogits_to_keeprK   )rJ   rI   rC   rH   )r…   Úpast_key_valuesr&   )r™   r!   Ú
parametersrJ   rI   r   r:   r¯   r�   r®   re   r³   r;   r…   r·   rš   ru   ÚitemsÚlast_hidden_stater‡   r   )r•   r–   r   r   r   r¤   rJ   rI   Úcaller_kwargsr­   rD   r²   Úoutputsr…   r·   r¥   ÚkÚvÚbase_kwargsÚbase_outputss                       r   r”   r”   ö   s[  € ð �O‰O€Eä�%×"Ñ"Ó$Ó%×,Ñ,€FÜ�×!Ñ!Ó#Ó$×*Ñ*€Eô )Ð):À&ÓI€MØ€Jð ×"×"Ü$ ]·L±L×4LÑ4LÓMˆØ×)Ñ)¨.ÀÐ\aÐ)ÐbˆÜ8¸ÓVˆ
à××á"Ñ0 ZÑ0ˆØ")§.¡.°'×2IÑ2I‘ô ˜U×$;Ñ$;Ó<ˆ
Ø(2×(8Ñ(8Ô(:ÔdÒ(:¡  1¸aÐGcÑ>c“t�q˜!’tÑ(:ˆÑdÙ!Ñ0 KÑ0ˆØ×Ñ˜.°È\×MkÑMkÐÑlØ"&¨×(DÑ(D�ð
 ×&Ñ& vÀ6Ð&ÐJ€Fä!¨ÑQÐQùó es   ÄE*ÄE*)r   úinspect.Signaturer   r�   r   r   rŽ   r   )r•   ú	nn.ModulerA   r‰   rŽ   rÂ   )r­   r   r®   rq   rŽ   ztuple[int, ...])r­   r   r²   r�   rŽ   r   )r•   rÂ   r   rÁ   )Ú
__future__r   r¢   Ú	functoolsr   Útypingr   Úmodeling_outputsr   Úutilsr   r	   Útorch.nnÚnnrP   Útorch.distributedÚdistributedr>   r   ÚIdentityr   r/   r¨   r¯   r³   r”   r&   r   r   Ú<module>rÍ      s‰   ðõ #ã Ý Ý  å 5ß Fö Ýá×ÑÛÝá!×#Ñ#Ý$ô!ô@˜BŸK™Kô @÷5ñ 5ôD.ôbWôð$RØð$RØ;Lõ$Rr   