ó
    ‡Ž¡jÑ.  ã                  ó  • S SK Jr  S SKrS SKrS SKrS SKJr  SSKJrJ	r	  SSK
Jr  SSKJr  SS	KJrJr  SS
KJr  SSKJrJrJr  SSKJrJrJrJrJrJrJrJrJrJ r   \	RB                  " \"5      r#\(       a  S SK$J%r%   " S S5      r&g)é    )ÚannotationsN)ÚTYPE_CHECKINGé   )Úis_torch_greater_or_equalÚlogging)Úcreate_and_tag_model_cardé   )ÚDistributedConfig)Ú$apply_fully_sharded_data_parallelismÚis_fsdp_managed_module)Úapply_pipeline_parallelism)Ú_validate_tp_plan_stylesÚapply_tensor_parallelismÚgather_state_dict_for_save)
Ú_distributed_barrierÚ_ensure_torch_distributedÚ_get_torch_distributed_rankÚ!_get_torch_distributed_world_sizeÚ!_is_torch_distributed_initializedÚgather_full_state_dictÚ)initialize_fully_sharded_data_parallelismÚinitialize_pipeline_parallelismÚinitialize_tensor_parallelismÚ!save_model_checkpoint_distributedc                  ó¤  • \ rS rSr% SrSrSrS\S'   SrS\S'   Sr	Sr
S\S'   SrS\S	'   SS
 jr\SS j5       r\SS j5       r\SS j5       r\R"                  SS j5       r\R"                  S S j5       r\SSS.   S!S jj5       r\    S"S j5       rS#S jrSSSSSSSS.                 S$S jjrSS.       S%S jjrS&S jrSrg)'ÚDistributedMixiné4   zFDistributed orchestration and save/load hooks for [`PreTrainedModel`].Núdict[str, str] | NoneÚ_tp_planÚ_ep_planú!dict[str, tuple[str, str]] | NoneÚ_pp_planÚ
_fsdp_planc           
     ó´  • [        U 5      n[        [        USS5      =(       d    0 5      U l        [        [        USS5      =(       d    0 5      U l        [        [        USS5      =(       d    0 5      U l        [        [        USS5      =(       d    0 5      U l        U R                  U L aà  U R
                  R                  U R                  R                  =(       d    0 5        U R                  R                  U R                  R                  =(       d    0 5        U R                  R                  U R                  R                  =(       d    0 5        U R                  R                  U R                  R                  =(       d    0 5        U R                  5        GH¡  u  p#[        USS5      =n(       aR  U R                  R                  UR                  5       R!                  5        VVs0 s H  u  pVU SU 3U_M     snn5        [        USS5      =n(       aR  U R                  R                  UR                  5       R!                  5        VVs0 s H  u  pVU SU 3U_M     snn5        [        USS5      =n(       aR  U R
                  R                  UR                  5       R!                  5        VVs0 s H  u  pVU SU 3U_M     snn5        [        USS5      =n(       d  GMO  U R                  R                  UR                  5       R!                  5        VVs0 s H  u  pVU SU 3U_M     snn5        GM¤     gs  snnf s  snnf s  snnf s  snnf )zQCopy class-level plans onto the instance and merge config/children contributions.r   Nr    r"   r#   Ú.)ÚtypeÚdictÚgetattrr   r    r"   r#   Ú
base_modelÚupdateÚconfigÚbase_model_pp_planÚbase_model_tp_planÚbase_model_ep_planÚbase_model_fsdp_planÚnamed_childrenÚcopyÚitems)ÚselfÚ	model_clsÚnameÚmoduleÚplanÚkÚvs          Ú¬C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/distributed/mixin.pyÚinit_parallel_plansÚ$DistributedMixin.init_parallel_plans>   sƒ  € ä˜“Jˆ	ÜœW Y°
¸DÓA×GÀRÓHˆŒÜœW Y°
¸DÓA×GÀRÓHˆŒÜœW Y°
¸DÓA×GÀRÓHˆŒÜœw y°,ÀÓE×KÈÓLˆŒà�?‰?˜dÒ"Ø�M‰M× Ñ  §¡×!?Ñ!?×!EÀ2ÔFØ�M‰M× Ñ  §¡×!?Ñ!?×!EÀ2ÔFØ�M‰M× Ñ  §¡×!?Ñ!?×!EÀ2ÔFØ�O‰O×"Ñ" 4§;¡;×#CÑ#C×#IÀrÔJà ×/Ñ/×1‰LˆDÜ˜v z°4Ó8Ð8ˆtÕ8Ø—‘×$Ñ$À4Ç9Á9Ã;×CTÑCTÔCVÔ%WÒCV¹4¸1¨¨¨a°¨s m°QÒ&6ÑCVÒ%WÔXÜ˜v z°4Ó8Ð8ˆtÕ8Ø—‘×$Ñ$À4Ç9Á9Ã;×CTÑCTÔCVÔ%WÒCV¹4¸1¨¨¨a°¨s m°QÒ&6ÑCVÒ%WÔXÜ˜v z°4Ó8Ð8ˆtÕ8Ø—‘×$Ñ$À4Ç9Á9Ã;×CTÑCTÔCVÔ%WÒCV¹4¸1¨¨¨a°¨s m°QÒ&6ÑCVÒ%WÔXÜ˜v |°TÓ:Ð:ˆt×:Ð:Ø—‘×&Ñ&ÀTÇYÁYÃ[×EVÑEVÔEXÔ'YÒEX¹T¸Q¨4¨&°°!°¨°qÒ(8ÑEXÒ'Y×Zò 2ùã%Wùã%Wùã%Wùã'Ys   Ç+MÉMÊ7MÌ Mc                ó  • [        U R                  S5      (       ae  U R                  R                  R                  (       a@  U R                  (       d#  [        SU R                  R                   S35      eU R                  $ U R                  $ )z)The full tp plan for the model's modules.Údistributed_configzGExpert parallelism was requested (`enable_expert_parallel=True`), but `zs` does not define an expert-parallel plan. Add a `base_model_ep_plan` to its config, or disable expert parallelism.)	Úhasattrr+   r>   Úenable_expert_parallelr    Ú
ValueErrorÚ	__class__Ú__name__r   ©r3   s    r:   Útp_planÚDistributedMixin.tp_planV   ss   € ô �4—;‘;Ð 4×5Ñ5¸$¿+¹+×:XÑ:X×:o×:oØ—=—=Ü ðØŸ™×/Ñ/Ð0ð 1YðZóð ð
 —=‘=Ð Ø�}‰}Ðó    c                ó   • U R                   $ ©N)r#   rD   s    r:   Ú	fsdp_planÚDistributedMixin.fsdp_planc   s   € à�‰ÐrG   c                ó   • U R                   $ rI   )r"   rD   s    r:   Úpp_planÚDistributedMixin.pp_plang   s   € à�}‰}ÐrG   c                ó¼  • Uc  0 U l         g [        U[        5      (       d  [        S5      e[	        U5        U R                  5        VVs/ s H  u  p#UPM	     nnnUR                  5        Hb  nUR                  SS5      nSnU H"  n[        R                  " Xh5      (       d  M   Sn  O   U(       a  MH  [        R                  " SU S35        Md     Xl         g s  snnf )Nz&Can only set a dictionary as `tp_plan`Ú*z\d+FTzLayer pattern 'z�' does not match any parameters in the model. This rule may not be applied during tensor parallelization, or may lead to dimension mismatches)r   Ú
isinstancer'   rA   r   Únamed_parametersÚkeysÚreplaceÚreÚmatchÚwarningsÚwarn)	r3   r7   r5   Ú_Úmodel_param_namesÚlayer_patternÚregex_patternÚpattern_matchedÚ
param_names	            r:   rE   rF   k   sÏ   € à‰<ØˆDŒMØÜ˜$¤×%Ñ%ÜÐEÓFÐFä  Ô&à15×1FÑ1FÔ1HÔIÒ1H¡g d›TÑ1HÐÑIØ!ŸY™Yž[ˆMØ)×1Ñ1°#°vÓ>ˆMØ#ˆOÛ/�
Ü—8’8˜M×6Ó6Ø&*�OÙñ 0÷ #�?Ü—’Ø% m _ð 5dð döñ )ð �ùó Js   Á
Cc                óf   • Uc  0 U l         g [        U[        5      (       d  [        S5      eXl         g )Nz&Can only set a dictionary as `pp_plan`)r"   rQ   r'   rA   )r3   r7   s     r:   rM   rN   …   s/   € à‰<ØˆDŒMØÜ˜$¤×%Ñ%ÜÐEÓFÐFà�rG   )Údevice_meshÚ
device_mapc          	     óf  • Uc  S X24$ [        U[        5      (       a  [        R                  " U5      nUR                  S:”  d  UR
                  S:”  aZ  [        5         [        5       nUR                  UR
                  -  U:w  a)  [        SUR                   SUR
                   SU S35      eUR                  S:”  a7  UR                  c  SUl	        [        UR                  UR                  UUS9u  p2O;UR
                  S:”  a  [        U5      u  p2OUR                  S:”  a  [        U5      u  p2XU4$ )Nr	   z	tp_size (z) * fsdp_size (z) is not equal to world_size (Ú)Úauto)Útp_sizer`   ra   )rQ   r'   r
   Ú	from_dictre   Ú	fsdp_sizer   r   ÚRuntimeErrorrE   r   r   Úpp_sizer   )Úclsr>   r`   ra   Ú
world_sizes        r:   Úprepare_distribute_modelÚ)DistributedMixin.prepare_distribute_model�   sI  € ð Ñ%Ø˜Ð0Ð0äÐ(¬$×/Ñ/Ü!2×!<Ò!<Ð=OÓ!PÐà×%Ñ%¨Ó)Ð-?×-IÑ-IÈAÓ-MÜ%Ô'Ü:Ó<ˆJØ!×)Ñ)Ð,>×,HÑ,HÑHÈJÓVÜ"ØÐ 2× :Ñ :Ð;¸?ÐK]×KgÑKgÐJhð i3Ø3=°,¸aðAóð ð
 ×%Ñ%¨Ó)Ø!×)Ñ)Ñ1Ø-3Ð"Ô*Ü&CØ"×*Ñ*Ø*×2Ñ2Ø'Ø%ñ	'Ñ#ˆJ˜ð  ×)Ñ)¨AÓ-Ü&OÐPbÓ&cÑ#ˆJ˜Ø×'Ñ'¨!Ó+Ü&EÐFXÓ&YÑ#ˆJà!¨{Ð:Ð:rG   c                óî  • Ubñ  X!R                   l        X1l        UR                  Ul        UR                  S:”  aT  UR
                  S:”  a  US   OUn[        UR                  [        5      (       a  UR                  Ul        [        X5      nU$ UR                  S:”  a$  UR
                  S:”  a  US   OUn[        X5      nU$ UR                  S:”  a"  UR
                  S:”  a  US   OUn[        X5      nU$ )z:Apply TP or FSDP2 after model init, before weight loading.r	   ÚtpÚfsdpÚpp)r+   r>   Ú_device_meshre   Ú_tp_sizeÚndimrQ   rE   r'   r   rg   r   ri   r   )rj   Úmodelr>   r`   Útp_meshÚ	fsdp_meshÚpp_meshs          r:   Úmaybe_distribute_modelÚ'DistributedMixin.maybe_distribute_model¶   sò   € ð Ñ"Ø.@�L‰LÔ+Ø!,ÔØ/×7Ñ7ˆEŒNà!×)Ñ)¨AÓ-Ø/:×/?Ñ/?À!Ó/C˜+ dÒ+È�ÜÐ0×8Ñ8¼$×?Ñ?Ø$6×$>Ñ$>�E”MÜ0°Ó@�ð ˆð $×-Ñ-°Ó1Ø3>×3CÑ3CÀaÓ3G˜K¨Ò/È[�	Ü<¸UÓN�ð
 ˆð $×+Ñ+¨aÓ/Ø/:×/?Ñ/?À!Ó/C˜+ dÒ+È�Ü2°5ÓB�ØˆrG   c                óT   • Un[        5       (       a  U=(       a    [        5       S:H  nU$ )z7Return whether this rank should write checkpoint files.r   )r   r   )r3   Úis_main_processÚsave_on_this_ranks      r:   Úshould_save_on_this_rankÚ)DistributedMixin.should_save_on_this_rankÒ   s,   € à+ÐÜ,×.Ñ.Ø 1× XÔ6QÓ6SÐWXÑ6XÐØ Ð rG   FT)Úpush_to_hubr}   Úrepo_idÚfiles_timestampsÚcommit_messageÚtokenÚ	create_prc          	     ó†  • [        S5      (       d  [        S5      e[        U5      (       d  [        S5      e[	        USS5      c  [        S5      e[        X5        U(       a`  U(       aX  [        XPR                  US9n
U
R                  [        R                  R                  US5      5        U R                  UUUUUU	S	9  ggg)
zBSave an FSDP-wrapped model via DCP and optionally push to the Hub.z2.7zFsave_pretrained(..., distributed_checkpoint=True) requires torch>=2.7.z\save_pretrained(..., distributed_checkpoint=True) is only supported for FSDP-wrapped models.rr   Nz�save_pretrained(..., distributed_checkpoint=True) requires the model to have been initialized with a distributed_config (_device_mesh is None).)r„   z	README.md)rƒ   r„   r…   )r   ÚOSErrorr   rA   r(   r   r   Ú
model_tagsÚsaveÚosÚpathÚjoinÚ_upload_modified_files)r3   Úmodel_to_saveÚsave_directoryr€   r}   r�   r‚   rƒ   r„   r…   Ú
model_cards              r:   Úsave_distributed_checkpointÚ,DistributedMixin.save_distributed_checkpointÙ   sÂ   € ô )¨×/Ñ/ÜÐbÓcÐcÜ% m×4Ñ4ÜØnóð ô �= .°$Ó7Ñ?ÜðPóð ô 	*¨-ÔHæÖ,Ü2°7¿O¹OÐSXÑYˆJØ�O‰OœBŸG™GŸL™L¨¸ÓEÔFØ×'Ñ'ØØØ Ø-ØØ#ð (ò ð -ˆ;rG   )r}   c               ó  • Uc  U$ UR                   S:”  a6  [        X R                  U R                  UR                   5      nU(       d  0 nU$ UR                  S:”  a%  [        5       (       d  [        S5      e[        U5      $ U$ )zNGather TP- or FSDP-sharded weights to full CPU tensors for checkpoint writing.r	   z‰Saving an FSDP-wrapped model requires torch.distributed to be initialized. Call save_pretrained from every rank after init_process_group.)re   r   r   rr   rg   r   rA   r   )r3   rŽ   Ú
state_dictr>   r}   s        r:   Ú"gather_sharded_state_dict_for_saveÚ3DistributedMixin.gather_sharded_state_dict_for_save   s�   € ð Ñ%ØÐà×%Ñ%¨Ó)Ü3ØŸM™M¨4×+<Ñ+<Ð>P×>XÑ>XóˆJö %Ø�
ØÐà×'Ñ'¨!Ó+Ü4×6Ñ6Ü ðUóð ô *¨-Ó8Ð8àÐrG   c                ób   • Uc  gUR                   S:”  d  UR                  S:”  a  [        5         gg)zQBarrier so non-writer ranks wait for rank 0 to finish gathered checkpoint writes.Nr	   )re   rg   r   )r3   r>   s     r:   Ú&barrier_after_gathered_checkpoint_saveÚ7DistributedMixin.barrier_after_gathered_checkpoint_save  s3   € àÑ%ØØ×%Ñ%¨Ó)Ð-?×-IÑ-IÈAÓ-MÜ Õ"ð .NrG   )r    r#   r"   r   )ÚreturnÚNone)rš   zdict[str, str])rš   zdict[str, tuple[str, str]])r7   r   )r7   r!   )r>   zDistributedConfig | dict | Nonerš   z/tuple[DistributedConfig | None, object, object])ru   z	nn.Moduler>   úDistributedConfig | None)r|   Úboolrš   r�   )r�   zstr | os.PathLiker€   r�   r}   r�   r�   ú
str | Noner‚   zdict | Nonerƒ   rž   r„   zstr | bool | Noner…   r�   rš   r›   )r”   r'   r>   rœ   r}   r�   rš   r'   )r>   rœ   rš   r›   )rC   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rr   r   Ú__annotations__r    rs   r"   r#   r;   ÚpropertyrE   rJ   rM   ÚsetterÚclassmethodrl   ry   r~   r‘   r•   r˜   Ú__static_attributes__© rG   r:   r   r   4   sµ  ‡ ÙPà€LØ&*€HÐ#Ó*Ø&*€HÐ#Ó*Ø€HØ26€HÐ/Ó6Ø(,€JÐ%Ó,ô[ð0 ó
ó ð
ð óó ðð óó ðð ‡^�^óó ðð2 ‡^�^óó ðð ð
 Øñ$;à;ð$;ð 
9ô$;ó ð$;ðL ðàðð 5óó ðô6!ð "Ø"&Ø"Ø(,Ø%)Ø#'Øñ%ð *ð%ð
 ð%ð  ð%ð ð%ð &ð%ð #ð%ð !ð%ð ð%ð 
õ%ðZ #'ñð ðð 5ð	ð  ðð 
õ÷<#rG   r   )'Ú
__future__r   rŠ   rU   rW   Útypingr   Úutilsr   r   Ú	utils.hubr   Úconfiguration_utilsr
   rp   r   r   Úpipeline_parallelr   Útensor_parallelr   r   r   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerrC   ÚloggerÚtorch.nnÚnnr   r¨   rG   r:   Ú<module>r´      sd   ðõ #ã 	Û 	Û Ý  ç 6Ý 1Ý 2ß NÝ 9÷ñ ÷
÷ ÷ ð 
×	Ò	˜HÓ	%€ö Ý÷o#ò o#rG   