ó
    ‡Ž¡j.*  ã                  ó²  • S SK Jr  S SKrS SKrS SKJrJr  SSKJrJ	r	J
r
JrJr  SSKJr  SSKJr  SS	KJr  \(       a  S SKJr  SS
KJr  \" 5       (       a  S SKr\	" 5       (       a  \
" S5      (       a  S SKJr  S SKJrJr  \R8                  " \5      rSS jrSS jr SS jr!SS jr"SS jr#      SS jr$      SS jr%S S jr&      S!S jr'S r(S r)g)"é    )ÚannotationsN)ÚTYPE_CHECKINGÚAnyé   )Úis_torch_availableÚis_torch_distributed_availableÚis_torch_greater_or_equalÚloggingÚ	strtobool)ÚQuantizationMethodé   ©Ú replace_layer_number_by_wildcard)Ú!_is_torch_distributed_initialized)ÚDistributedConfigz2.6)Úfully_shard)ÚCPUOffloadPolicyÚMixedPrecisionPolicyc                 óâ   • [        5       =(       a_    [        [        R                  R	                  SS5      5      S:H  =(       a,    [        [        R                  R	                  SS5      5      S:H  $ )uM   Check if FSDP is active via Accelerate (env var based) â€” covers FSDP1 only.ÚACCELERATE_USE_FSDPÚFalser   ÚFSDP_CPU_RAM_EFFICIENT_LOADING)r   r   ÚosÚenvironÚget© ó    Ú«C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/distributed/fsdp.pyÚis_fsdp_enabledr   )   sV   € ô 	*Ó+÷ 	VÜ”b—j‘j—n‘nÐ%:¸GÓDÓEÈÑJ÷	Vä”b—j‘j—n‘nÐ%EÀwÓOÓPÐTUÑUðr   c                ój   • [        5       (       d  g[        U SS5      (       a  gSSKJn  [	        X5      $ )z.Check if a module is managed by FSDP (1 or 2).FÚ_is_fsdp_managed_moduleTr   )ÚFullyShardedDataParallel)r   ÚgetattrÚtorch.distributed.fsdpr"   Ú
isinstance)Úmoduler"   s     r   Úis_fsdp_managed_moduler'   2   s1   € ä)×+Ñ+Øô ˆvÐ0°%×8Ñ8Øå?ä�fÓ7Ð7r   c                óÆ   • U c  0 $ 0 nU R                   (       a  [        5       US'   U R                  (       a*  [        [        R
                  [        R                  SS9US'   U$ )zMBuild ``fully_shard`` policy kwargs from ``DistributedConfig`` runtime flags.NÚoffload_policy)Úparam_dtypeÚreduce_dtypeÚoutput_dtypeÚ	mp_policy)Úfsdp_cpu_offloadr   Úfsdp_mixed_precisionr   ÚtorchÚbfloat16Úfloat32)Údistributed_configÚfsdp_policy_kwargss     r   Ú_get_fsdp_policy_kwargsr5   @   s]   € àÑ!Øˆ	àÐØ×*×*Ü/?Ó/AÐÐ+Ñ,Ø×.×.Ü*>ÜŸ™ÜŸ™Øñ+
Ð˜;Ñ'ð
 Ðr   c                ó”   • S nS n[        U S5      (       a  U R                  5       n[        U S5      (       a  U R                  5       nX4$ )NÚget_input_embeddingsÚget_output_embeddings)Úhasattrr7   r8   )ÚmodelÚinput_embedÚoutput_heads      r   Ú_get_input_output_embeddingsr=   Q   sM   € Ø€KØ€KÜˆuÐ,×-Ñ-Ø×0Ñ0Ó2ˆÜˆuÐ-×.Ñ.Ø×1Ñ1Ó3ˆØÐ#Ð#r   c                ó<  ^
• [        U 5      S:w  a  g[        U5      u  p#X#4 Vs1 s H	  oDc  M  UiM     snm
/ / peU  H'  u  ptUR                  U5        UR                  U5        M)     [        S U 5       5      n[        U
4S jU 5       5      n	U=(       a    U	$ s  snf )Nr   Fc              3  óZ   #   • U  H!  oS :H  =(       d    UR                  S5      v •  M#     g7f)Únormz.normN)Úendswith)Ú.0Únames     r   Ú	<genexpr>Ú(is_norm_and_head_pair.<locals>.<genexpr>f   s#   é € ÐTÊeÀd ™×A¨4¯=©=¸Ó+AÔAÊeùs   ‚)+c              3  ó,   >#   • U  H	  oT;   v •  M     g 7f©Nr   )rB   r&   Úhead_moduless     €r   rD   rE   g   s   øé € ÐGºw°V LÖ0ºwùs   ƒ)Úlenr=   ÚappendÚany)Úno_reshard_targetsr:   r;   r<   r&   ÚnamesÚmodulesrC   Úhas_final_normÚhas_output_headrH   s             @r   Úis_norm_and_head_pairrQ   [   s˜   ø€ Ü
ÐÓ !Ó#ØÜ;¸EÓBÑ€KØ*5Ñ)CÓZÒ)C˜v—FÑ)CÑZ€Là˜ˆ7Û*‰ˆØ�‰�TÔØ�‰�vÖñ +ô ÑTÉeÓTÓT€NÜÔG¹wÓGÓG€OØ×-˜oÐ-ùò [s
   ¤B®Bc                ó   • [        USS5      =(       d    0 nU(       d  U $ [        U5      u  p4UR                  5        VVs0 s H  u  pVXe_M	     nnnUR                  U5      nUR                  U5      n	Ub  U	c  U $ U R	                  5       n
U
R                  US5        U R                  U	5      S:X  a  U
R                  U	S5        SX¨'   U
$ s  snnf )ao  
Rewrite the plan so tied embed/lm_head weights are wrapped once.
Example:
    {"model.embed_tokens": "free_full_weight",
    "model.layers.*": "free_full_weight",
    "model.norm": "keep_full_weight",
    "lm_head": "keep_full_weight"}
->
    {"model.layers.*": "free_full_weight",
    "model.norm": "keep_full_weight",
    "model.embed_tokens": "keep_full_weight"}
Úall_tied_weights_keysNÚkeep_full_weight)r#   r=   Únamed_modulesr   ÚcopyÚpop)Ú	fsdp_planr:   Ú	tied_keysr;   r<   rC   r&   Úname_by_moduleÚembed_moduleÚhead_moduleÚadapted_plans              r   Ú _resolve_tied_embed_lm_head_planr^   k   sÔ   € ô  ˜Ð6¸Ó=×CÀ€IÞØÐä;¸EÓBÑ€KØ7<×7JÑ7JÔ7LÔMÒ7L¡| t�f’lÑ7L€NÑMØ!×%Ñ% kÓ2€LØ ×$Ñ$ [Ó1€KàÑ˜{Ñ2ØÐà—>‘>Ó#€LØ×Ñ�\ 4Ô(à‡}�}�[Ó!Ð%7Ó7Ø×Ñ˜ dÔ+Ø%7ˆÑ"àÐùó Ns   Á C
c                óÐ   • / n/ nU R                  5        HL  u  pEXA;   a  UO
[        U5      nXa;   d  M  X   S:X  a  UR                  XE45        M:  UR                  XE45        MN     X#4$ )zUExpand plan keys into reshard and no-reshard ``(module_name, module)`` shard targets.rT   )rU   r   rJ   )r:   rX   Úreshard_targetsrL   Úmodule_namer&   Úplan_keys          r   Úexpand_fsdp_planrc   ‘   sw   € ð
 46€OØ68Ðà$×2Ñ2Ö4ÑˆØ"-Ó":‘;Ô@`ÐalÓ@mˆØÕ ØÑ"Ð&8Ó8Ø"×)Ñ)¨;Ð*?Ö@à×&Ñ&¨Ð'<Ö=ñ  5ð Ð.Ð.r   c                ób  ^• U(       d  g[         R                  U 5      n0 n0 nUR                  5        H:  u  mnUS;  a  XTT'   M  TU;  d  M  [        U4S jU 5       5      (       a  M6  XST'   M<     U(       a  [        R                  SU 35        U(       a  [        R                  SU 35        gg)zk
Verify the FSDP plan of the model, log a warning if plan keys were not applied or strategies are invalid.
N>   Úfree_full_weightrT   c              3  ó@   >#   • U  H  n[        U5      T:H  v •  M     g 7frG   r   )rB   rC   Úkeys     €r   rD   Ú#verify_fsdp_plan.<locals>.<genexpr>²   s!   øé € Ð/vÒjuÐbfÔ0PÐQUÓ0VÐZ]Ö0]Òjuùs   ƒz4The following FSDP entries have unknown strategies: z9The following FSDP rules were not applied to any module: )ÚdictÚfromkeysÚitemsrK   ÚloggerÚwarning)Úmodule_namesrX   Úname_lookupÚunused_rulesÚinvalid_strategiesÚstrategyrg   s         @r   Úverify_fsdp_planrs   ¤   s¤   ø€ ö Øä—-‘- Ó-€KØ#%€LØ)+Ðà"Ÿ™Ö*‰ˆˆXØÐCÓCØ&.˜sÓ#Ø˜Õ#¬CÔ/vÑjuÓ/v×,vÓ,vØ (˜Óñ	 +ö Ü�‰ÐMÐN`ÐMaÐbÔcÞÜ�‰ÐRÐS_ÐR`ÐaÕbð r   c                ó  • [        [        U SS5      =(       d    0 5      nU(       d!  [        [        U 5      R                   S35      e[        U R
                  SS5      n[        U5      n[        X 5      n[        X5      u  pgU H-  u  p‰[        U	4USS.UD6  [        R                  SU S35        M/     [        Xp5      (       aY  / / pºU H'  u  pÉU
R                  U5        UR                  U	5        M)     [        U4US	S.UD6  [        R                  S
U
 S35        O3U H-  u  pÉ[        U	4US	S.UD6  [        R                  SU S35        M/     [        U 4SU0UD6  [        R                  S[        U5       S35        SU l        U $ )z¹
Apply FSDP2 (fully_shard) to a model.

Torch availability, distributed initialization and the version requirement
are asserted upstream by `initialize_fully_sharded_data_parallelism`.
Ú
_fsdp_planNzr does not have a FSDP2 plan declared. Set `base_model_fsdp_plan` on the config and `_fsdp_plan` on the head class.r3   T)ÚmeshÚreshard_after_forwardzApplied fully_shard to z (reshard=True)FzGrouped tail z (reshard=False)rv   z'FSDP2 applied to model via _fsdp_plan: z entries)ri   r#   Ú
ValueErrorÚtypeÚ__name__Úconfigr5   r^   rc   r   rl   ÚdebugrQ   rJ   ÚinforI   r!   )r:   Ú	fsdp_meshrX   r3   r4   Úadapted_fsdp_planr`   rL   ra   r&   rM   rN   rC   s                r   Ú$apply_fully_sharded_data_parallelismr€   »   s™  € ô ”W˜U L°$Ó7×=¸2Ó>€IÞÜÜ�E‹{×#Ñ#Ð$ð %Wð Wó
ð 	
ô
 ! §¡Ð/CÀTÓJÐÜ0Ð1CÓDÐä8¸ÓJÐÜ*:¸5Ó*TÑ'€Oã.ÑˆÜ�FÐ] À$Ñ]ÐJ\Ò]Ü�‰Ð.¨{¨m¸?ÐKÖLñ  /ô Ð/×7Ñ7Ø˜RˆwÛ.‰LˆDØ�L‰L˜ÔØ�N‰N˜6Ö"ñ /ô 	�GÐ_ )À5Ñ_ÐL^Ò_Ü�‰�} U GÐ+;Ð<Õ=ã.‰LˆDÜ˜Ðb YÀeÑbÐOaÒbÜ�L‰LÐ2°4°&Ð8HÐIÖJñ /ô
 �Ñ<˜IÐ<Ð);Ò<ä
‡K�KÐ9¼#¸i».Ð9IÈÐRÔSð %)€EÔ!ð €Lr   c                 ót   • SSK Jn   S[        [        R                  " U 5      R
                  5      ;   a  SS0$ 0 $ )z°
Returns checkpoint kwargs for FSDP model saving.

Checks if the `adapter_only` parameter is supported by `save_fsdp_model` from accelerate
and returns the appropriate kwargs.
r   ©Úsave_fsdp_modelÚadapter_onlyT)Úaccelerate.utilsrƒ   ÚlistÚinspectÚ	signatureÚ
parametersr‚   s    r   Úget_fsdp_ckpt_kwargsrŠ   ò   s5   € õ 1àœœg×/Ò/°Ó@×KÑKÓLÓLØ Ð%Ð%àˆ	r   c                ó¸  • SSK Jn  SSKJn  [	        U R
                  U5      (       a!  U" U 5      UR                  R                  l        [        U SS5      [        R                  :X  as  U R                  R                  R                  R                  (       aC  UR                  R                  R!                  U R                  R                  R                  SS9  ggg)a3  
Updates the FSDP plugin for PEFT LoRA/QLoRA compatibility.

When using FSDP with PEFT LoRA, the auto wrap policy needs to be updated to additionally wrap
LoRA trainable layers separately. When using FSDP with QLoRA, the mixed precision policy needs
to be updated to use the quantization storage data type.
r   )Ú
PeftConfig)Úfsdp_auto_wrap_policyÚquantization_methodNT)Úoverride)ÚpeftrŒ   Úpeft.utils.otherr�   r%   Úactive_peft_configÚstateÚfsdp_pluginÚauto_wrap_policyr#   r   ÚBITS_AND_BYTESÚhf_quantizerÚquantization_configÚbnb_4bit_quant_storageÚis_floating_pointÚset_mixed_precision)r:   ÚacceleratorrŒ   r�   s       r   Úupdate_fsdp_plugin_peftr�     s­   € õ  Ý6ä�%×*Ñ*¨J×7Ñ7Ù9NÈuÓ9Uˆ×Ñ×%Ñ%Ô6ä�Ð,¨dÓ3Ô7I×7XÑ7XÓXØ×Ñ×2Ñ2×IÑI×[×[à×Ñ×%Ñ%×9Ñ9Ø×Ñ×2Ñ2×IÑIÐTXð 	:ò 	
ð \ð 	Yr   )ÚreturnÚbool)r&   ú	nn.Modulerž   rŸ   )r3   zDistributedConfig | Nonerž   zdict[str, Any])r:   r    rž   z)tuple[nn.Module | None, nn.Module | None])rL   zlist[tuple[str, nn.Module]]r:   r    rž   rŸ   )rX   údict[str, str]r:   r    rž   r¡   )r:   r    rX   r¡   rž   z?tuple[list[tuple[str, nn.Module]], list[tuple[str, nn.Module]]])rn   z	list[str]rX   zdict[str, str] | Nonerž   ÚNone)r:   r    r~   z(torch.distributed.device_mesh.DeviceMeshrž   r    )*Ú
__future__r   r‡   r   Útypingr   r   Úutilsr   r   r	   r
   r   Úutils.quantization_configr   Útensor_parallelr   r   Útorch.nnÚnnÚconfiguration_utilsr   r0   Ú"torch.distributed._composable.fsdpr   r$   r   r   Ú
get_loggerrz   rl   r   r'   r5   r=   rQ   r^   rc   rs   r€   rŠ   r�   r   r   r   Ú<module>r­      sê   ðõ #ã Û 	ß %ç uÕ uÝ :Ý =Ý 4ö Ýå6á×ÑÛá!×#Ñ#Ñ(AÀ%×(HÑ(HÝ>ßMà	×	Ò	˜HÓ	%€ôô8ôô"$ô.ð #Øð#àð#ð ô#ðL/Øð/àð/ð Eô/ô&cð.2Øð2Ø!Ið2àô2ònó
r   