ó
    ÊÙ¢jŸ  ã                   ó   • S SK Jr  S SKJrJrJrJr  S SKrS SKJ	r	  S SKJ
r
  S SKJr  S SKJrJr  S SKrS SKJr  S S	KJr  S S
KJrJr  S SKJr  S SKJr  S SKJr  S SKJ r   \(       a  S SK!J"r#  S SK$J%r%   " S S\5      r&g)é    )ÚAbstractContextManager)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalN)Úapply_to_collection)ÚTensor)ÚModule)Úget_argsÚoverride)Ú_optimizer_handles_unscaling)Ú_PRECISION_INPUT)Ú_convert_fp_tensorÚ_DtypeContextManager)Úrank_zero_warn)ÚOptimizable)Ú	Precision)ÚMisconfigurationException©ÚMixedPrecision©ÚShardedGradScalerc                   ó¶  ^ • \ rS rSrSrS!S\S\S   SS4S jjr\S	\	S\	4S
 j5       r
\S\S\SS4S j5       r\S"S j5       r\S\4S j5       r\S\4S j5       r\S\4S j5       r\S\S\4S j5       r\S\S\4S j5       r\S\S	SS\4U 4S jj5       r\S\SSS\/ \4   S\S\4
U 4S jj5       r\S\\\4   4S j5       r\S\\\4   SS4S j5       rS rU =r$ )#ÚFSDPPrecisioné%   aÕ  Precision plugin for training with Fully Sharded Data Parallel (FSDP).

.. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

Args:
    precision: Full precision (32-true), half precision (16-true, bf16-true) or
        mixed precision (16-mixed, bf16-mixed).
    scaler: An optional :class:`torch.distributed.fsdp.sharded_grad_scaler.ShardedGradScaler` to use.

Raises:
    ValueError:
        If unsupported ``precision`` is provided.

NÚ	precisionÚscalerr   Úreturnc                 ó¦  • [        [        5      nX;  a  [        SU< SU S35      eSSKJn  Ub#  U R
                  S:w  a  [        SU< SU S35      eUc  US:X  a  U" 5       OS U l        Xl        [        R                  [        R                  [        R                  [        R                  [        R                  S.nXPR
                     U l        g )	Nz`precision=z9)` is not supported in FSDP. `precision` must be one of: Ú.r   r   ú16-mixedz` does not use a scaler, found )ú
bf16-mixedr!   ú	bf16-trueú16-trueú32-true)r   r   Ú
ValueErrorÚ*torch.distributed.fsdp.sharded_grad_scalerr   r   r   ÚtorchÚfloat32Úbfloat16Úfloat16Ú_desired_input_dtype)Úselfr   r   Úsupported_precisionr   Úprecision_to_types         Ú¶C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\pytorch_lightning/plugins/precision/fsdp.pyÚ__init__ÚFSDPPrecision.__init__5   sÌ   € Ü&Ô'7Ó8ÐØÓ/ÜØ˜i™]ð +0Ø0CÐ/DÀAðGóð õ
 	QàÑ $§.¡.°JÓ">Ü˜{¨9©-Ð7VÐW]ÐV^Ð^_Ð`ÓaÐaà-3©^À	ÈZÓ@WÑ'Ô)Ð]aˆŒØ"Œô  Ÿ-™-ÜŸ™ÜŸ™Ü—}‘}Ü—}‘}ñ
Ðð %6·n±nÑ$EˆÕ!ó    Úmodulec                 óX   • SU R                   ;   a  UR                  U R                  S9$ U$ )NÚtrue©Údtype)r   Útor,   )r-   r4   s     r0   Úconvert_moduleÚFSDPPrecision.convert_moduleN   s*   € à�T—^‘^Ó#Ø—9‘9 4×#<Ñ#<�9Ð=Ð=Øˆr3   Ú_Ú__c                 óH   • [        SU R                  R                   S35      e)NzA`gradient_clip_algorithm='norm'` is currently not supported for `Ú`)r   Ú	__class__Ú__name__)r-   r<   r=   s      r0   Úclip_grad_by_normÚFSDPPrecision.clip_grad_by_normT   s,   € ô (ØOÐPT×P^ÑP^×PgÑPgÐOhÐhiÐjó
ð 	
r3   c                 ó|  • SSK Jn  U R                  S;   a  [        SU R                   S35        U R                  S;   a  [        R
                  =n=p4ObU R                  S;   a  [        R                  =n=p4O>U R                  S:X  a  [        R                  =n=p4O[        S	U R                  < S
35      eU" UUUS9$ )Nr   r   )r$   r#   zFSDP with `z}` enables computation in lower precision. FSDP will always retain a full-precision copy of the model parameters for sharding.)r$   r!   )r#   r"   r%   z-Was unable to infer precision type, received r    )Úparam_dtypeÚreduce_dtypeÚbuffer_dtype)	Ú2torch.distributed.fsdp.fully_sharded_data_parallelr   r   r   r(   r+   r*   r)   r   )r-   ÚTorchMixedPrecisionrE   rF   rG   s        r0   Úmixed_precision_configÚ$FSDPPrecision.mixed_precision_config^   sÇ   € ålà�>‰>Ð5Ó5ÜØ˜dŸn™nÐ-ð .fð fôð
 �>‰>Ð4Ó4Ü8=¿¹ÐEˆKÐE˜,¨Ø�^‰^Ð:Ó:Ü8=¿¹ÐFˆKÐF˜,¨Ø�^‰^˜yÓ(Ü8=¿¹ÐEˆKÐE˜,¨ä+Ð.[Ð\`×\jÑ\jÑ[mÐmnÐ,oÓpÐpá"Ø#Ø%Ø%ñ
ð 	
r3   c                 ó,   • [        U R                  5      $ ©N©r   r,   ©r-   s    r0   Útensor_init_contextÚ!FSDPPrecision.tensor_init_contextw   ó   € ä# D×$=Ñ$=Ó>Ð>r3   c                 ó,   • [        U R                  5      $ rM   rN   rO   s    r0   Úmodule_init_contextÚ!FSDPPrecision.module_init_context{   rR   r3   c                 óÖ   • SU R                   ;   aE  [        R                  " SU R                   S:X  a  [        R                  S9$ [        R                  S9$ [        U R                  5      $ )NÚmixedÚcudar"   r7   )r   r(   Úautocastr*   r+   r   r,   rO   s    r0   Úforward_contextÚFSDPPrecision.forward_context   sO   € à�d—n‘nÓ$Ü—>’> &À4Ç>Á>ÐUaÓCa´·±ÑvÐvÔgl×gtÑgtÑvÐvÜ# D×$=Ñ$=Ó>Ð>r3   Údatac                 ó>   • [        U[        [        U R                  S9$ ©N)Úfunctionr8   Údst_type)r   r   r	   r,   ©r-   r\   s     r0   Úconvert_inputÚFSDPPrecision.convert_input…   s   € ä" 4Ô2DÌFÐ]a×]vÑ]vÑwÐwr3   c                 óP   • [        U[        [        [        R                  " 5       S9$ r^   )r   r   r	   r(   Úget_default_dtypera   s     r0   Úconvert_outputÚFSDPPrecision.convert_output‰   s   € ä" 4Ô2DÌFÔ]b×]tÒ]tÓ]vÑwÐwr3   Útensorzpl.LightningModulec                 ór   >• U R                   b  U R                   R                  U5      n[        TU ]  X5      $ rM   )r   ÚscaleÚsuperÚpre_backward)r-   rh   r4   r@   s      €r0   rl   ÚFSDPPrecision.pre_backward�   s1   ø€ à�;‰;Ñ"Ø—[‘[×&Ñ& vÓ.ˆFÜ‰wÑ# FÓ3Ð3r3   Ú	optimizerÚmodelÚclosureÚkwargsc                 óx  >• U R                   c  [        TU ]  " U4X#S.UD6$ U" 5       n[        U5      (       d  U R                   R	                  U5        U R                  X!5        US L nUR                  (       a  U(       d9  U R                   R                  " U40 UD6nU R                   R                  5         U$ U$ )N)ro   rp   )	r   rk   Úoptimizer_stepr   Úunscale_Ú_after_closureÚautomatic_optimizationÚstepÚupdate)	r-   rn   ro   rp   rq   Úclosure_resultÚskipped_backwardÚstep_outputr@   s	           €r0   rs   ÚFSDPPrecision.optimizer_step“   sª   ø€ ð �;‰;Ñä‘7Ò)¨)Ð\¸5Ñ\ÐU[Ñ\Ð\Ù ›ˆä+¨I×6Ñ6ð �K‰K× Ñ  Ô+à×Ñ˜EÔ-Ø)¨TÐ1Ðà×+×+Ö3CàŸ+™+×*Ò*¨9Ñ?¸Ñ?ˆKØ�K‰K×ÑÔ ØÐØÐr3   c                 óT   • U R                   b  U R                   R                  5       $ 0 $ rM   )r   Ú
state_dictrO   s    r0   r~   ÚFSDPPrecision.state_dict°   s$   € à�;‰;Ñ"Ø—;‘;×)Ñ)Ó+Ð+Øˆ	r3   r~   c                 óV   • U R                   b  U R                   R                  U5        g g rM   )r   Úload_state_dict)r-   r~   s     r0   r�   ÚFSDPPrecision.load_state_dict¶   s#   € à�;‰;Ñ"Ø�K‰K×'Ñ'¨
Õ3ð #r3   )r,   r   r   rM   )r   rI   ) rA   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r1   r   r
   r:   r   rB   ÚpropertyrJ   r   rP   rT   rZ   rb   rf   r	   rl   r   r   rs   ÚdictÚstrr~   r�   Ú__static_attributes__Ú__classcell__)r@   s   @r0   r   r   %   sñ  ø† ññFÐ"2ð F¸HÐEXÑ<Yð FÐeiõ Fð2 ð Vð °ó ó ðð
 ð
 Cð 
¨sð 
°tó 
ó ð
ð ó
ó ð
ð0 ð?Ð%;ó ?ó ð?ð ð?Ð%;ó ?ó ð?ð ð?Ð!7ó ?ó ð?ð
 ðx #ð x¨#ó xó ðxð ðx 3ð x¨3ó xó ðxð ð4 6ð 4Ð3Gð 4ÈFö 4ó ð4ð
 ðàðð $ðð ˜"˜c˜'Ñ"ð	ð
 ðð 
öó ðð8 ð˜D  c ™Nó ó ðð
 ð4¨$¨s°C¨x©.ð 4¸Tó 4ó ö4r3   r   )'Ú
contextlibr   Útypingr   r   r   r   r(   Úlightning_utilitiesr   r	   Útorch.nnr
   Útyping_extensionsr   r   Úpytorch_lightningÚplÚ&lightning_fabric.plugins.precision.ampr   Ú'lightning_fabric.plugins.precision.fsdpr   Ú(lightning_fabric.plugins.precision.utilsr   r   Úlightning_fabric.utilitiesr   Ú lightning_fabric.utilities.typesr   Ú-pytorch_lightning.plugins.precision.precisionr   Ú&pytorch_lightning.utilities.exceptionsr   rH   r   rI   r'   r   r   © r3   r0   Ú<module>r›      sN   ðõ .ß 9Ó 9ã Ý 3Ý Ý ß 0ã Ý OÝ Dß ]Ý 5Ý 8Ý CÝ LæÝhÝLôT4�Iõ T4r3   