ó
    ˆŽ¡j@2  ã                   óÚ  • S SK Jr  S SKrS SKJr  S SKJr  SSKJr	  SSK
JrJr  SSKJrJr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  SSKJr  SSKJrJr  SSKJr  SSK J!r!  SSK"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+  \" SS9\ " S S\!5      5       5       r, " S S\)5      r- " S S\#5      r. " S S\$5      r/ " S S\5      r0 " S S\(5      r1 " S  S!\'5      r2 " S" S#\%5      r3 " S$ S%\&5      r4/ S&Qr5g)'é    )ÚCallableN)Ústricté   )Úinitialization)ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstring)ÚTransformersKwargsÚmaybe_autocasté   )ÚGemma3RotaryEmbedding)ÚOlmo2Config)	ÚOlmo2AttentionÚOlmo2DecoderLayerÚOlmo2ForCausalLMÚOlmo2ForSequenceClassificationÚ
Olmo2ModelÚOlmo2PreTrainedModelÚOlmo2RMSNormÚapply_rotary_pos_embÚeager_attention_forwardzallenai/Olmo-3-7B-Instruct)Ú
checkpointc                   óª   ^ • \ rS rSr% SrSrSrS/rSSSSSSS	S
.rS/S/4SS/S/4S/S/4S.r	Sr
\S-  \S'   Sr\\   S-  \S'   U 4S jrS rSrU =r$ )ÚOlmo3Configé-   aU  
Example:

```python
>>> from transformers import Olmo3Model, Olmo3Config

>>> # Initializing a Olmo3 7B style configuration
>>> configuration = Olmo3Config()

>>> # Initializing a model from the Olmo3 7B style configuration
>>> model = Olmo3Model(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```
Úolmo3g    €„AÚpast_key_valuesÚcolwise_gather_outputÚrowwise_split_inputÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi   NÚsliding_windowÚlayer_typesc                 óö   >• U R                   c  U R                  U l         U R                  c8  [        U R                  5       Vs/ s H  o"S-   S-  S:w  a  SOSPM     snU l        [
        TU ]  " S0 UD6  g s  snf )Né   é   r   Úsliding_attentionÚfull_attention© )Únum_key_value_headsÚnum_attention_headsr1   ÚrangeÚnum_hidden_layersÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s      €ÚµC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/olmo3/modular_olmo3.pyr=   ÚOlmo3Config.__post_init__V   s~   ø€ Ø×#Ñ#Ñ+Ø'+×'?Ñ'?ˆDÔ$à×ÑÑ#äW\Ð]a×]sÑ]sÔWtó ÚWtÐRS¨A©°¡{°aÓ'7Ñ#Ð=MÒMÑWtñ ˆDÔô 	‰ÒÑ' Ó'ùò	 s   ÁA6c                 óv  • UR                  SS 5      nSS0SS0S.nU R                  b  U R                  OUU l        Ub  U R                  S   R                  U5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  SU R
                  5      5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  SU R
                  5      5        U R                  5         U$ )NÚrope_scalingÚ	rope_typeÚdefault)r5   r6   r6   Ú
rope_thetar5   )ÚpopÚrope_parametersÚupdateÚgetÚ
setdefaultÚdefault_thetaÚstandardize_rope_params)r>   r?   rE   Údefault_rope_paramss       rB   Úconvert_rope_params_to_dictÚ'Olmo3Config.convert_rope_params_to_dicta   s:  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÑ7W˜t×3Ò3Ð]pˆÔØÑ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGð ×Ñ×#Ñ#Ð$4Ó5Ñ=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9¸,ÈÏ
É
ÐS_Ðae×asÑasÓHtÔuØ×Ñ×#Ñ#Ð$7Ó8Ñ@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™* \°4×3EÑ3EÓFô	
ð
 	×$Ñ$Ô&Øˆó    )r1   r8   rJ   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typerN   Úkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr0   ÚintÚ__annotations__r1   ÚlistÚstrr=   rQ   Ú__static_attributes__Ú__classcell__©rA   s   @rB   r!   r!   -   s¢   ø‡ ñð" €JØ€MØ#4Ð"5Ðà%<Ø%<Ø%<Ø%:Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð "&€N�C˜$‘JÓ%Ø$(€K��c‘˜TÑ!Ó(õ	(÷ð rS   r!   c                   ó   • \ rS rSrSrg)ÚOlmo3RMSNormé}   r7   N©rT   rU   rV   rW   ra   r7   rS   rB   re   re   }   ó   † ÚrS   re   c                   ó   ^ • \ rS rSrS\S\4U 4S jjr SS\R                  S\	\R                  \R                  4   S\R                  S-  S	\
S-  S
\\   S\	\R                  \R                  S-  4   4S jjrSrU =r$ )ÚOlmo3Attentionéƒ   ÚconfigÚ	layer_idxc                 óš   >• [         TU ]  XS9  UR                  U   U l        U R                  S:X  a  UR                  U l        g S U l        g )N)rm   r5   )r<   Ú__init__r1   Úattention_typer0   ©r>   rl   rm   rA   s      €rB   ro   ÚOlmo3Attention.__init__„   sJ   ø€ Ü‰Ñ˜ÐÑ5Ø$×0Ñ0°Ñ;ˆÔØ7;×7JÑ7JÐNaÓ7a˜f×3Ñ3ˆÕÐgkˆÕrS   Nr+   Úposition_embeddingsr,   r$   r?   Úreturnc                 óf  • UR                   S S n/ UQSPU R                  P7nU R                  U R                  U5      5      nU R	                  U R                  U5      5      n	U R                  U5      n
UR                  U5      R                  SS5      nU	R                  U5      R                  SS5      n	U
R                  U5      R                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [         5      nU" U UU	U
U4U R"                  (       d  SOU R$                  U R&                  U R(                  S.UD6u  pïUR*                  " / UQSP76 R-                  5       nU R/                  U5      nXï4$ )Néÿÿÿÿr3   r   g        )ÚdropoutÚscalingr0   )ÚshapeÚhead_dimÚq_normÚq_projÚk_normÚk_projÚv_projÚviewÚ	transposer   rK   rm   r   Úget_interfacerl   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrx   r0   ÚreshapeÚ
contiguousÚo_proj)r>   r+   rs   r,   r$   r?   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   rB   ÚforwardÚOlmo3Attention.forward‰   s£  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>ˆØ—[‘[ §¡¨]Ó!;Ó<ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(rS   )rp   r0   ©N)rT   rU   rV   rW   r!   r]   ro   ÚtorchÚTensorÚtupler   r   r   r“   ra   rb   rc   s   @rB   rj   rj   ƒ   s¢   ø† ðl˜{ð l°s÷ lð )-ñ+)à—|‘|ð+)ð # 5§<¡<°·±Ð#=Ñ>ð+)ð Ÿ™ tÑ+ð	+)ð
  ™ð+)ð Ð+Ñ,ð+)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷+)ó +)rS   rj   c                   ó   • \ rS rSrSrg)ÚOlmo3DecoderLayeré·   r7   Nrg   r7   rS   rB   rš   rš   ·   rh   rS   rš   c            	       ó€   ^ • \ rS rSrS
S\4U 4S jjjr SS\S\S-  S\\R                  \
4   4U 4S jjjrS rS	rU =r$ )ÚOlmo3RotaryEmbeddingé»   Nrl   c                 ó$   >• [         TU ]  U5        g r•   )r<   ro   )r>   rl   ÚdevicerA   s      €rB   ro   ÚOlmo3RotaryEmbedding.__init__¼   s   ø€ Ü‰Ñ˜Õ rS   Ú
layer_typert   c                 ó"   >• [         TU ]  X5      $ r•   )r<   Úcompute_default_rope_parameters)rl   r    r¢   r?   rA   s       €rB   r¤   Ú4Olmo3RotaryEmbedding.compute_default_rope_parameters¿   s   ø€ ô ‰wÑ6°vÓJÐJrS   c                 ó´  • [        X S35      n[        X S35      nUS S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        US	S
9   Xg-  R                  SS5      n	[        R                  " X™4SS9n
U
R                  5       U-  nU
R                  5       U-  nS S S 5        X¼4$ ! , (       d  f       WW4$ = f)NÚ	_inv_freqÚ_attention_scalingr   rv   r3   )Údtyper    ÚmpsÚcpuF)Údevice_typeÚenabledr   )Údim)ÚgetattrÚexpandry   Útor–   Úfloatr    Ú
isinstanceÚtyper`   r   r�   ÚcatrŽ   r�   )r>   ÚxÚposition_idsr¢   Úinv_freqÚattention_scalingÚinv_freq_expandedÚposition_ids_expandedr¬   ÚfreqsÚembrŽ   r�   s                rB   r“   ÚOlmo3RotaryEmbedding.forwardÄ   s:  € ä˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �Tš1˜d�]Ñ#×*Ñ*¨<×+=Ñ+=¸aÑ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÐoð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆÜ¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷	 Dð ˆxˆ÷ DÔCð �Cˆxˆús   Ã*AEÅ
Er7   r•   )NN)rT   rU   rV   rW   r!   ro   r`   r˜   r–   r—   r²   r¤   r“   ra   rb   rc   s   @rB   r�   r�   »   s^   ø† ñ!˜{÷ !ð !ð DHñKØðKØ69¸D±jðKà	ˆu�|‰|˜UÐ"Ñ	#÷Kð K÷
ð rS   r�   c                   ó   • \ rS rSrS rSrg)ÚOlmo3PreTrainedModeléØ   c                 óž  • [         R                  " U5        [        U[        5      (       a¢  UR                   H‘  nUR
                  nUR                  U   S:w  a  [        UR                  U      nU" UR                  US9u  pE[        R                  " [        X S35      U5        [        R                  " [        X S35      U5        M“     g g )NrG   )r¢   r§   Ú_original_inv_freq)r   Ú_init_weightsr³   r�   r1   r¤   rF   r   rl   ÚinitÚcopy_r¯   )r>   Úmoduler¢   Úrope_init_fnÚcurr_inv_freqÚ_s         rB   rÄ   Ú"Olmo3PreTrainedModel._init_weightsÙ   s®   € Ü×%Ò% fÔ-Ü�fÔ2×3Ñ3Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ó<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ñ#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ò 1ð 4rS   r7   N)rT   rU   rV   rW   rÄ   ra   r7   rS   rB   rÀ   rÀ   Ø   s   † õ	^rS   rÀ   c                   óä   ^ • \ rS rSrS\4U 4S jjr      SS\R                  S-  S\R                  S-  S\R                  S-  S\	S-  S	\R                  S-  S
\S-  S\\   S\4S jjrSrU =r$ )Ú
Olmo3Modeléè   rl   c           	      ó   >• [         TU ]  U5        [        UR                  UR                  S9U l        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        US9U l        g s  snf )N)Úeps©rl   )r<   ro   re   Úhidden_sizeÚrms_norm_epsr/   ÚnnÚ
ModuleListr:   r;   rš   r.   r�   Ú
rotary_embrq   s      €rB   ro   ÚOlmo3Model.__init__é   ss   ø€ Ü‰Ñ˜Ô Ü  ×!3Ñ!3¸×9LÑ9LÑMˆŒ	Ü—m’mÜCHÈ×IaÑIaÔCbÓcÒCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô /°fÑ=ˆ�ùò ds   ÁBNr)   r,   r·   r$   r*   Ú	use_cacher?   rt   c           
      ób  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U=n	[        5      (       d)  U R                  UUUUS.n
[        S
0 U
D6[        S
0 U
D6S.n	Un0 n[        U R                  R                  5       H  nU R!                  X³U5      XÍ'   M     [#        U R$                  S U R                  R&                   5       HE  u  pïU" U4X�R                  R                  U      UUXÀR                  R                  U      S.UD6nMG     U R)                  U5      n[+        UUS	9$ )Nz:You must specify exactly one of input_ids or inputs_embedsrÑ   r   r3   )r    )rl   r*   r,   r$   r·   )r6   r5   )r,   r·   r$   rs   )Úlast_hidden_stater$   r7   )Ú
ValueErrorr-   r   rl   Úget_seq_lengthr–   Úarangery   r    Ú	unsqueezer³   Údictr	   r
   Úsetr1   rÖ   Ú	enumerater.   r;   r/   r   )r>   r)   r,   r·   r$   r*   rØ   r?   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr+   rs   r¢   r@   Údecoder_layers                   rB   r“   ÚOlmo3Model.forwardñ   sÉ  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø*.×*;Ñ*;¸IÓ*FˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÑFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð
 &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øðà2·;±;×3JÑ3JÈ1Ñ3MÑNØ)Ø /Ø$7¿¹×8OÑ8OÐPQÑ8RÑ$Sñð ñŠMñ !Xð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ñ
ð 	
rS   )r.   r/   rÖ   )NNNNNN)rT   rU   rV   rW   r!   ro   r–   Ú
LongTensorr—   r   ÚFloatTensorÚboolr   r   r   r“   ra   rb   rc   s   @rB   rÍ   rÍ   è   s³   ø† ð>˜{÷ >ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð ˜$‘;ð;
ð Ð+Ñ,ð;
ð 
!÷;
ó ;
rS   rÍ   c                   ó   • \ rS rSrSrg)ÚOlmo3ForCausalLMi/  r7   Nrg   r7   rS   rB   rë   rë   /  rh   rS   rë   c                   ó   • \ rS rSrSrg)ÚOlmo3ForSequenceClassificationi3  r7   Nrg   r7   rS   rB   rí   rí   3  rh   rS   rí   )r!   rë   rí   rÍ   rÀ   )6Úcollections.abcr   r–   Útorch.nnrÔ   Úhuggingface_hub.dataclassesr   Ú r   rÅ   Úcache_utilsr   r   Úmasking_utilsr	   r
   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   Úutils.genericr   r   Úgemma3.modeling_gemma3r   Úolmo2.configuration_olmo2r   Úolmo2.modeling_olmo2r   r   r   r   r   r   r   r   r   r!   re   rj   rš   r�   rÀ   rÍ   rë   rí   Ú__all__r7   rS   rB   Ú<module>rþ      sâ   ðõ %ã Ý Ý .å &ß .ß RÝ 7Ý 6ß FÝ &Ý #ß ?Ý :Ý 3÷
÷ 
õ 
ñ Ð7Ñ8ØôK�+ó Kó ó 9ðKô\	�<ô 	ô1)�^ô 1)ôh	Ð)ô 	ôÐ0ô ô:
^Ð/ô 
^ô D
�ô D
ôN	Ð'ô 	ô	Ð%Cô 	ò�rS   