ó
    ‡Ž¡jš  ã                   óŽ   • S SK r S SKrS SKrSSKJr  SSKJr  SSKJr  \R                  " \
5      r " S S5      r " S S	\5      rg)
é    Né   )ÚTrainerCallback)ÚPREFIX_CHECKPOINT_DIR)Úloggingc                   ó>   • \ rS rSrS
S\4S jjrS rS rS rS r	Sr
g	)ÚCheckpointManageré   Ú	kill_waitc                 ó8   • Xl         SU l        SU l        X l        g)a  
Initialize the CheckpointManager for Just-In-Time checkpoint handling.

Args:
    trainer: The Trainer instance that will be used to save checkpoints when SIGTERM is received.
    kill_wait (`int`, *optional*, defaults to 3): Grace period to distinguish between SIGTERM and SIGKILL.
FN)ÚtrainerÚis_checkpoint_requestedÚ_original_sigterm_handlerr
   )Úselfr   r
   s      Ú±C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/trainer_jit_checkpoint.pyÚ__init__ÚCheckpointManager.__init__   s   € ð ŒØ',ˆÔ$Ø)-ˆÔ&Ø"�ó    c                 ó–   • [         R                   " [         R                  U R                  5      U l        [        R                  S5        g )Nz4JIT checkpoint signal handler registered for SIGTERM)ÚsignalÚSIGTERMÚ_sigterm_handlerr   ÚloggerÚinfo©r   s    r   Úsetup_signal_handlerÚ&CheckpointManager.setup_signal_handler   s,   € Ü)/¯ª´v·~±~Àt×G\ÑG\Ó)]ˆÔ&Ü�‰ÐJÕKr   c                 óà   • U R                   (       a  g [        R                  SU R                   S35        [        R
                  " U R                  U R                  5      R                  5         g )Nz4SIGTERM received, will request JIT checkpoint after Ús)r   r   r   r
   Ú	threadingÚTimerÚ_enable_checkpointÚstart)r   ÚsignumÚframes      r   r   Ú"CheckpointManager._sigterm_handler   sK   € Ø×'×'Øä�‰ÐJÈ4Ï>É>ÐJZÐZ[Ð\Ô]Ü�Š˜Ÿ™¨×(?Ñ(?Ó@×FÑFÕHr   c                 ó<   • [         R                  S5        SU l        g )Nz/Kill wait period elapsed, requesting checkpointT)r   r   r   r   s    r   r!   Ú$CheckpointManager._enable_checkpoint&   s   € Ü�‰ÐEÔFØ'+ˆÕ$r   c                 ó¾  •  SU l         [        R                  S5        U R                  R                  R
                  n[        R                  SU 35        U R                  R                  S S9n[         SU 3n[        R                  R                  X#5      n[        R                  " USS9  [        R                  R                  X#S5      n[        US	5       nUR                  S
U S35        S S S 5        [        R                  SU 35        U R                  R                  U R                  R                  S S9  [        R                  R!                  U5      (       a+  [        R"                  " U5        [        R                  S5        [        R                  S5        g ! , (       d  f       N¹= f! [$         a  n[        R'                  SU 35        e S nAff = f)NFzStarting JIT checkpointing...zSaving JIT checkpoint at step )ÚtrialÚ-T)Úexist_okzcheckpoint-is-incomplete.txtÚwzCheckpoint started at step z and in progress...z2Created checkpoint progress sentinel marker file: zSentinel marker file removedz/Immediate JIT checkpoint completed successfullyzFailed to save JIT checkpoint: )r   r   r   r   ÚstateÚglobal_stepÚ_get_output_dirr   ÚosÚpathÚjoinÚmakedirsÚopenÚwriteÚ_save_checkpointÚmodelÚexistsÚremoveÚ	ExceptionÚerror)r   Úcurrent_stepÚ
output_dirÚcheckpoint_folderÚcheckpoint_pathÚsentinel_fileÚfÚes           r   Úexecute_jit_checkpointÚ(CheckpointManager.execute_jit_checkpoint*   sr  € ð!	à+0ˆDÔ(ä�K‰KÐ7Ô8ØŸ<™<×-Ñ-×9Ñ9ˆLÜ�K‰KÐ8¸¸ÐGÔHàŸ™×5Ñ5¸DÐ5ÐAˆJÜ#8Ð"9¸¸<¸.Ð IÐÜ Ÿg™gŸl™l¨:ÓIˆOô �KŠK˜°$Ò7ô ŸG™GŸL™L¨ÐHfÓgˆMÜ�m SÔ)¨QØ—‘Ð5°l°^ÐCVÐWÔX÷ *ä�K‰KÐLÈ]ÈOÐ\Ô]ð �L‰L×)Ñ)¨$¯,©,×*<Ñ*<ÀDÐ)ÑIô �w‰w�~‰~˜m×,Ñ,Ü—	’	˜-Ô(Ü—‘Ð:Ô;ä�K‰KÐIÕJ÷ *Õ)ûô ó 	Ü�L‰LÐ:¸1¸#Ð>Ô?Øûð	ús1   ‚CF4 ÃF#Ã0B2F4 Æ#
F1Æ-F4 Æ4
GÆ>GÇG)r   r   r
   r   N)é   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úintr   r   r   r!   rC   Ú__static_attributes__© r   r   r   r      s$   † ñ#¨3õ #òLòIò,õ"r   r   c                   óB   • \ rS rSrSrS rS rS rS rS r	S r
S	 rS
rg)ÚJITCheckpointCallbackéO   a:  
Callback for Just-In-Time checkpointing on SIGTERM signals.

When SIGTERM is received, the checkpoint manager sets `is_checkpoint_requested=True`.
The callbacks detect this flag and set `control.should_training_stop=True`, which signals
the Trainer's training loop to exit gracefully after saving the checkpoint.
c                 ó    • S U l         S U l        g )N)r   Újit_managerr   s    r   r   ÚJITCheckpointCallback.__init__X   s   € ØˆŒØ59ˆÕr   c                 óÂ   • Xl         UR                  R                  (       a>  [        US9U l        U R                  R                  5         [        R                  S5        g g )N)r   zJIT checkpointing enabled)r   ÚargsÚenable_jit_checkpointr   rQ   r   r   r   )r   r   s     r   Úset_trainerÚ!JITCheckpointCallback.set_trainer\   sE   € ØŒØ�<‰<×-×-Ü0¸ÑAˆDÔØ×Ñ×1Ñ1Ô3Ü�K‰KÐ3Õ4ð .r   c                 ó¢   • U R                   (       a>  U R                   R                  (       a"  SUl        U R                   R                  5         g g g ©NT©rQ   r   Úshould_training_stoprC   ©r   rT   r-   ÚcontrolÚkwargss        r   Úon_pre_optimizer_stepÚ+JITCheckpointCallback.on_pre_optimizer_stepc   ó<   € Ø×× × 0Ñ 0× H× HØ+/ˆGÔ(Ø×Ñ×3Ñ3Õ5ð !IÐr   c                 ó¢   • U R                   (       a>  U R                   R                  (       a"  SUl        U R                   R                  5         g g g rY   rZ   r\   s        r   Úon_step_beginÚ#JITCheckpointCallback.on_step_beginh   ra   r   c                 ó°   • U R                   (       aE  U R                   R                  (       a)  SUl        SUl        U R                   R	                  5         g g g ©NFT©rQ   r   Úshould_saver[   rC   r\   s        r   Úon_step_endÚ!JITCheckpointCallback.on_step_endm   óD   € Ø×× × 0Ñ 0× H× HØ"'ˆGÔØ+/ˆGÔ(Ø×Ñ×3Ñ3Õ5ð !IÐr   c                 ó°   • U R                   (       aE  U R                   R                  (       a)  SUl        SUl        U R                   R	                  5         g g g rf   rg   r\   s        r   Úon_epoch_endÚ"JITCheckpointCallback.on_epoch_ends   rk   r   c                 óô   • U R                   (       ag  U R                   R                  bO  [        R                  " [        R                  U R                   R                  5        [        R                  S5        g g g )Nz;Restored original SIGTERM handler after training completion)rQ   r   r   r   r   r   r\   s        r   Úon_train_endÚ"JITCheckpointCallback.on_train_endy   sP   € à×× × 0Ñ 0× JÑ JÑ VÜ�MŠMœ&Ÿ.™.¨$×*:Ñ*:×*TÑ*TÔUÜ�K‰KÐUÕVð !WÐr   )rQ   r   N)rF   rG   rH   rI   Ú__doc__r   rV   r_   rc   ri   rm   rp   rK   rL   r   r   rN   rN   O   s+   † ñò:ò5ò6ò
6ò
6ò6õWr   rN   )r0   r   r   Útrainer_callbackr   Útrainer_utilsr   Úutilsr   Ú
get_loggerrF   r   r   rN   rL   r   r   Ú<module>rw      sB   ðÛ 	Û Û å -Ý 0Ý ð 
×	Ò	˜HÓ	%€÷?ñ ?ôD.W˜Oõ .Wr   