ó
    ‡Ž¡j‹%  ã                   ó2  • S SK r S SKJr  S SK Jr  SSKJr  SSKJrJ	r	  SSK
Jr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJrJrJr  SSKJrJrJrJrJrJ r J!r!J"r"  SS0r#Sr$\RJ                  " \&5      r'\" SS9\ " S S\5      5       5       r( " S S\RR                  5      r* " S S\RV                  5      r, " S S\5      r- " S S\"5      r. " S S \5      r/ " S! S"\!5      r0 " S# S$\ 5      r1 " S% S&\5      r2 " S' S(\5      r3 " S) S*\5      r4/ S+Qr5g),é    N)Ústrict)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_mask)ÚBaseModelOutputWithPast)ÚRopeParameters)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )ÚLlamaAttentionÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚ
vocab_fileztokenizer.modelu   â–�zgoogle/gemma-7b)Ú
checkpointc                   ó¬  • \ rS rSr% SrSrS/rSSSSSSSS.rS/S	/4S
S/S
/4S
/S
/4S.rSr	\
\S'   Sr\
\S'   Sr\
\S'   Sr\
\S'   Sr\
\S'   Sr\
\S'   Sr\
\S'   Sr\\S'   Sr\
\S'   Sr\\S'   S r\\S!'   S"r\\S#'   S$r\
S%-  \S&'   S'r\
\\
   -  S%-  \S('   S)r\
S%-  \S*'   S"r\\S+'   S%r\ \!-  S%-  \S,'   S-r"\\S.'   S/r#\\
-  \S0'   S%r$\S%-  \S1'   S2r%g%)3ÚGemmaConfigé1   aÖ  
use_bidirectional_attention (`bool`, *optional*):
    If True, the model will attend to all text tokens instead of using a causal mask.

```python
>>> from transformers import GemmaModel, GemmaConfig
>>> # Initializing a Gemma gemma-7b style configuration
>>> configuration = GemmaConfig()
>>> # Initializing a model from the gemma-7b style configuration
>>> model = GemmaModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```ÚgemmaÚpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi è Ú
vocab_sizei   Úhidden_sizei `  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚnum_key_value_headsé   Úhead_dimÚgelu_pytorch_tanhÚ
hidden_acti    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeç�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacher   NÚpad_token_idé   Úeos_token_idr   Úbos_token_idÚtie_word_embeddingsÚrope_parametersFÚattention_biasg        Úattention_dropoutÚuse_bidirectional_attention© )&Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr+   ÚintÚ__annotations__r,   r-   r/   r1   r2   r4   r6   Ústrr7   r8   Úfloatr:   r;   Úboolr<   r>   Úlistr?   r@   rA   r   ÚdictrB   rC   rD   Ú__static_attributes__rE   ó    ÚµC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/gemma/modular_gemma.pyr   r   1   sd  ‡ ñð €JØ#4Ð"5Ðà%.Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø!Ð˜Ó!Ø€HˆcÓØ)€J�Ó)Ø#'Ð˜SÓ'Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø €L�#˜‘*Ó Ø $Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø/3Ð ¨¡Ö3rW   r   c            	       ól   ^ • \ rS rSrSrSS\S\S\S\4U 4S jjjrS\R                  4U 4S	 jjr
S
rU =r$ )ÚGemmaTextScaledWordEmbeddingéi   zT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 óŽ   >• [         TU ]  XU5        X@l        [        R                  " [
        R                  " U5      SS9U l        g )NF)Ú
persistent)ÚsuperÚ__init__Úscalar_embed_scaler   ÚBufferÚtorchÚtensorr_   )Úselfr\   r]   r^   r_   Ú	__class__s        €rX   rc   Ú%GemmaTextScaledWordEmbedding.__init__n   s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÑQˆÕrW   r$   c                 ó‚   >• [         TU ]  U5      U R                  R                  U R                  R
                  5      -  $ ©N)rb   Úforwardr_   ÚtoÚweightÚdtype)rh   r$   ri   s     €rX   rm   Ú$GemmaTextScaledWordEmbedding.forwards   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRrW   )r_   rd   )ç      ð?)rF   rG   rH   rI   rJ   rO   rR   rc   rf   ÚTensorrm   rV   Ú__classcell__©ri   s   @rX   rZ   rZ   i   sM   ø† ññR sð R¸3ð RÈSð RÐ_d÷ Rð Rð
S §¡÷ Sõ SrW   rZ   c                   óJ   ^ • \ rS rSrS	S\S\4U 4S jjjrS rS rS r	Sr
U =r$ )
ÚGemmaRMSNorméw   ÚdimÚepsc                 óŒ   >• [         TU ]  5         X l        [        R                  " [
        R                  " U5      5      U l        g rl   )rb   rc   rz   r   Ú	Parameterrf   Úzerosro   )rh   ry   rz   ri   s      €rX   rc   ÚGemmaRMSNorm.__init__x   s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆ�rW   c                 óˆ   • U[         R                  " UR                  S5      R                  SSS9U R                  -   5      -  $ )Nr   éÿÿÿÿT)Úkeepdim)rf   ÚrsqrtÚpowÚmeanrz   )rh   Úxs     rX   Ú_normÚGemmaRMSNorm._norm}   s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ð>ÀÇÁÑIÓJÑJÐJrW   c                 ó¢   • U R                  UR                  5       5      nUSU R                  R                  5       -   -  nUR                  U5      $ )Nrr   )r†   rR   ro   Útype_as)rh   r…   Úoutputs      rX   rm   ÚGemmaRMSNorm.forward€   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Ñ4Ñ5ˆØ�~‰~˜aÓ Ð rW   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)Útuplero   Úshaperz   )rh   s    rX   Ú
extra_reprÚGemmaRMSNorm.extra_repr‡   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<rW   )rz   ro   )r9   )rF   rG   rH   rI   rO   rR   rc   r†   rm   r�   rV   rt   ru   s   @rX   rw   rw   w   s0   ø† ñ5˜Cð 5 e÷ 5ð 5ò
Kò!÷=ð =rW   rw   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚGemmaMLPé‹   c                 ó>  >• [         TU ]  U5        [        R                  " U R                  U R
                  SS9U l        [        R                  " U R                  U R
                  SS9U l        [        R                  " U R
                  U R                  SS9U l        g )NF)Úbias)	rb   rc   r   ÚLinearr,   r-   Ú	gate_projÚup_projÚ	down_proj©rh   Úconfigri   s     €rX   rc   ÚGemmaMLP.__init__Œ   ss   ø€ Ü‰Ñ˜Ô ÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆ�rW   )r™   r—   r˜   )rF   rG   rH   rI   rc   rV   rt   ru   s   @rX   r’   r’   ‹   s   ø† ÷Yó YrW   r’   c                   ó   • \ rS rSrSrg)ÚGemmaRotaryEmbeddingé“   rE   N©rF   rG   rH   rI   rV   rE   rW   rX   rž   rž   “   ó   † ÚrW   rž   c                   ó8   ^ • \ rS rSrSrS\S\4U 4S jjrSrU =r	$ )ÚGemmaAttentioné—   z=Multi-headed attention from 'Attention Is All You Need' paperr›   Ú	layer_idxc                 óP   >• [         TU ]  5         [        USS5      (       + U l        g )NrD   F)rb   rc   ÚgetattrÚ	is_causal)rh   r›   r¥   ri   s      €rX   rc   ÚGemmaAttention.__init__š   s"   ø€ Ü‰ÑÔÜ$ VÐ-JÈEÓRÔRˆ�rW   )r¨   )
rF   rG   rH   rI   rJ   r   rO   rc   rV   rt   ru   s   @rX   r£   r£   —   s"   ø† ÙGðS˜{ð S°s÷ Sõ SrW   r£   c                   óB   • \ rS rSr\R
                  " 5       S 5       rSrg)ÚGemmaPreTrainedModeléŸ   c                 ó(  • [         R                  " X5        SUR                  R                  ;   a!  [        R
                  " UR                  5        g [        U[        5      (       a,  [        R                  " UR                  UR                  5        g g )NÚRMSNorm)r   Ú_init_weightsri   rF   ÚinitÚzeros_ro   Ú
isinstancerZ   Ú	constant_r_   rd   )rh   Úmodules     rX   r¯   Ú"GemmaPreTrainedModel._init_weights    sa   € ä×%Ò% dÔ3à˜×(Ñ(×1Ñ1Ó1Ü�KŠK˜Ÿ™Õ&Ü˜Ô <×=Ñ=Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÕIð >rW   rE   N)rF   rG   rH   rI   rf   Úno_gradr¯   rV   rE   rW   rX   r«   r«   Ÿ   s   † Ø
‡]‚]ƒ_ñJó óJrW   r«   c                   óä   ^ • \ rS rSrS\4U 4S jjr      SS\R                  S-  S\R                  S-  S\R                  S-  S\	S-  S	\R                  S-  S
\S-  S\\   S\4S jjrSrU =r$ )Ú
GemmaModeléª   r›   c                 ó°   >• [         TU ]  U5        [        UR                  UR                  U R
                  U R                  R                  S-  S9U l        g )Ng      à?)r_   )rb   rc   rZ   r+   r,   r^   r›   r(   rš   s     €rX   rc   ÚGemmaModel.__init__«   sK   ø€ Ü‰Ñ˜Ô ä8Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmñ
ˆÕrW   Nr$   r'   Úposition_idsr!   r%   r;   ÚkwargsÚreturnc           
      óT  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U R                  UUUUS9n	Un
U R                  X£S9nU R                  S U R                  R                    H  nU" U
4U	UUUUS.UD6n
M     U R                  U
5      n
[        U
U(       a  US	9$ S S	9$ )
Nz:You must specify exactly one of input_ids or inputs_embeds)r›   r   r=   )Údevice)r›   r%   r'   r!   r¼   )r¼   )r'   r¼   r!   r;   Úposition_embeddings)Úlast_hidden_stater!   )Ú
ValueErrorr(   r   r›   Úget_seq_lengthrf   ÚarangerŽ   rÀ   Ú	unsqueezer
   Ú
rotary_embr)   r/   r*   r   )rh   r$   r'   r¼   r!   r%   r;   r½   Úpast_seen_tokensÚcausal_maskr&   rÁ   Údecoder_layers                rX   rm   ÚGemmaModel.forward²   sU  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Þ/8˜Oñ
ð 	
à>Bñ
ð 	
rW   )r(   )NNNNNN)rF   rG   rH   rI   r   rc   rf   Ú
LongTensorrs   r   ÚFloatTensorrS   r   r   r   rm   rV   rt   ru   s   @rX   r¸   r¸   ª   s³   ø† ð
˜{÷ 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!÷2
ó 2
rW   r¸   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚGemmaForCausalLMéç   c                  ó8   >• [        5       R                  " S0 U D6$ )a  
Example:

```python
>>> from transformers import AutoTokenizer, GemmaForCausalLM

>>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```rE   )rb   rm   )Úsuper_kwargsri   s    €rX   rm   ÚGemmaForCausalLM.forwardè   s   ø€ ô$ ‹w�ŠÑ. Ñ.Ð.rW   rE   )rF   rG   rH   rI   rm   rV   rt   ru   s   @rX   rÏ   rÏ   ç   s   ø† ÷/ó /rW   rÏ   c                   ó   • \ rS rSrSrg)ÚGemmaForSequenceClassificationéý   rE   Nr    rE   rW   rX   rÕ   rÕ   ý   r¡   rW   rÕ   c                   ó   • \ rS rSrSrg)ÚGemmaForTokenClassificationi  rE   Nr    rE   rW   rX   rØ   rØ     r¡   rW   rØ   )r   r¸   rÏ   rÕ   rØ   r«   )6rf   Úhuggingface_hub.dataclassesr   r   Ú r   r°   Úcache_utilsr   r   Úconfiguration_utilsr	   Úmasking_utilsr
   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   ÚVOCAB_FILES_NAMESÚSPIECE_UNDERLINEÚ
get_loggerrF   Úloggerr   Ú	EmbeddingrZ   ÚModulerw   r’   rž   r£   r«   r¸   rÏ   rÕ   rØ   Ú__all__rE   rW   rX   Ú<module>rë      s!  ðó" Ý .Ý å &ß .Ý 3Ý /Ý 7Ý 1Ý -Ý &ß @Ñ @÷	÷ 	ó 	ð "Ð#4Ð5Ð àÐ à	×	Ò	˜HÓ	%€ñ Ð,Ñ-Øô34Ð"ó 34ó ó .ð34ôlS 2§<¡<ô Sô=�2—9‘9ô =ô(Yˆxô Yô	Ð/ô 	ôS�^ô SôJÐ/ô Jô:
�ô :
ôz/Ð'ô /ô,	Ð%Cô 	ô	Ð"=ô 	ò�rW   