ó
    ‡Ž¡jè]  ã                   óþ  • S SK Jr  S SKrS SKJr  SSKJr  SSKJ	r	  SSK
JrJr  SSKJr  SSKJrJr  SS	KJrJr  SS
KJr  SSKJrJrJr  SSKJrJr  SSKJrJ r   SSK!J"r"J#r#  SSK$J%r%  SSK&J'r'J(r(J)r)  SSK*J+r+  SSK,J-r-J.r.  SSK/J0r0  SSK1J2r2   " S S\Rf                  5      r4 " S S\Rf                  5      r5 " S S\Rf                  5      r6S r7\" S5      S=S j5       r8S\Rr                  S \:S!\Rr                  4S" jr;   S>S#\Rf                  S$\Rr                  S%\Rr                  S&\Rr                  S'\Rr                  S-  S(\<\:-  S)\<S-  S*\<S-  S!\=\Rr                  \Rr                  4   4S+ jjr>\" \85       " S, S-\Rf                  5      5       r? " S. S/\5      r@ " S0 S1\R‚                  5      rB\( " S2 S3\#5      5       rC\( " S4 S5\C5      5       rD\( " S6 S7\C\5      5       rE " S8 S9\\C5      rF " S: S;\\C5      rG/ S<QrHg)?é    )ÚCallableNé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚGemma2Configc                   óJ   ^ • \ rS rSrS	S\S\4U 4S jjjrS rS rS r	Sr
U =r$ )
ÚGemma2RMSNormé1   ÚdimÚepsc                 óŒ   >• [         TU ]  5         X l        [        R                  " [
        R                  " U5      5      U l        g ©N)ÚsuperÚ__init__r&   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr%   r&   Ú	__class__s      €Ú¸C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/gemma2/modeling_gemma2.pyr*   ÚGemma2RMSNorm.__init__2   s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆ�ó    c                 óˆ   • U[         R                  " UR                  S5      R                  SSS9U R                  -   5      -  $ )Né   éÿÿÿÿT)Úkeepdim)r-   ÚrsqrtÚpowÚmeanr&   )r0   Úxs     r2   Ú_normÚGemma2RMSNorm._norm7   s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ð>ÀÇÁÑIÓJÑJÐJr4   c                 ó¢   • U R                  UR                  5       5      nUSU R                  R                  5       -   -  nUR                  U5      $ )Nç      ð?)r=   Úfloatr/   Útype_as)r0   r<   Úoutputs      r2   ÚforwardÚGemma2RMSNorm.forward:   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Ñ4Ñ5ˆØ�~‰~˜aÓ Ð r4   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)Útupler/   Úshaper&   )r0   s    r2   Ú
extra_reprÚGemma2RMSNorm.extra_reprA   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r4   )r&   r/   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚintrA   r*   r=   rD   rI   Ú__static_attributes__Ú__classcell__©r1   s   @r2   r#   r#   1   s0   ø† ñ5˜Cð 5 e÷ 5ð 5ò
Kò!÷=ð =r4   r#   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )Ú	Gemma2MLPéE   c                 ó¼  >• [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l	        [        UR                     U l        g ©NF©Úbias)r)   r*   ÚconfigÚhidden_sizeÚintermediate_sizer+   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fn©r0   rZ   r1   s     €r2   r*   ÚGemma2MLP.__init__F   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆŒÜ˜V×5Ñ5Ñ6ˆ�r4   c                 óˆ   • U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ r(   )r`   rb   r^   r_   )r0   r<   r`   s      r2   rD   ÚGemma2MLP.forwardP   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr4   )rb   rZ   r`   r^   r[   r\   r_   )rK   rL   rM   rN   r*   rD   rP   rQ   rR   s   @r2   rT   rT   E   s   ø† õ7÷ð r4   rT   c            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚGemma2RotaryEmbeddingéU   Údevicez5.18)ÚversionrZ   c                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF©Ú
persistent)r)   r*   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrZ   Úrope_parametersrm   Úcompute_default_rope_parametersr   Úattention_scalingr+   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r0   rZ   rj   Úrope_init_fnrx   r1   s        €r2   r*   ÚGemma2RotaryEmbedding.__init__V   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr4   Úreturnc           	      ó   • U R                   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNr@   r   r6   ©Údtype)rt   Úgetattrr[   Únum_attention_headsr-   ÚarangerA   Úto)rZ   rj   ÚkwargsÚbaser%   Úattention_factorrx   s          r2   ru   Ú5Gemma2RotaryEmbedding.compute_default_rope_parametersg   s{   € ð ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r4   c                 óL  • U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      R	                  UR
                  5      nUS S 2S S S 24   R                  5       n[        UR
                  R                  [        5      (       a0  UR
                  R                  S:w  a  UR
                  R                  OSn[        USS9   UR                  5       UR                  5       -  R                  SS5      n[        R                  " Xf4SS	9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR	                  UR                   S
9W	R	                  UR                   S
94$ ! , (       d  f       N@= f)Nr   r7   r    ÚmpsÚcpuF)Údevice_typeÚenabledr6   ©r%   r�   )rx   rA   ÚexpandrH   r†   rj   Ú
isinstanceÚtypeÚstrr   Ú	transposer-   ÚcatÚcosrv   Úsinr‚   )
r0   r<   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrŽ   ÚfreqsÚembr—   r˜   s
             r2   rD   ÚGemma2RotaryEmbedding.forward{   sN  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆÜ¨¸UÓCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   ÃBFÆ
F#)rv   rZ   rx   rr   rz   rs   rm   r(   )rK   rL   rM   rN   r   r!   r*   ÚstaticmethodrG   r-   ÚTensorrA   ru   Úno_gradr   rD   rP   rQ   rR   s   @r2   rh   rh   U   sŒ   ø† Ù�X vÑ.ñO˜|÷ Oó /ðOð  Ù�X vÑ.ñ5°ð 5ÐX]Ð^c×^jÑ^jÐlqÐ^qÑXrô 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ö<r4   rh   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nr7   r6   r�   )rH   r-   r–   )r<   Úx1Úx2s      r2   Úrotate_halfr¥   ‹   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r4   Úrotary_pos_embc                 ó˜   • UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXV4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer¥   )ÚqÚkr—   r˜   Úunsqueeze_dimÚq_embedÚk_embeds          r2   Úapply_rotary_pos_embr®   ’   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØÐÐr4   Úhidden_statesÚn_repr}   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r    N)rH   r‘   Úreshape)r¯   r°   ÚbatchÚnum_key_value_headsÚslenr€   s         r2   Ú	repeat_kvr¶   ¬   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr4   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚsoftcapc                 ój  • Uc  U R                   S-  n[        X R                  5      n	[        X0R                  5      n
[        R                  " XR                  SS5      5      U-  nUb  X·-  n[        R                  " U5      nX·-  nUb  X´-   n[        R                  R                  US[        R                  S9R                  UR                  5      n[        R                  R                  XµU R                  S9n[        R                  " Xº5      nUR                  SS5      R                  5       nXË4$ )Nç      à¿r6   r   r7   )r%   r‚   )ÚpÚtrainingr    )r€   r¶   Únum_key_value_groupsr-   Úmatmulr•   Útanhr+   Ú
functionalÚsoftmaxÚfloat32r†   r‚   r¼   rÂ   Ú
contiguous)r·   r¸   r¹   rº   r»   r¼   r½   r¾   r‡   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r2   Úeager_attention_forwardrÎ   ¸   s  € ð �Ø—/‘/ 4Ñ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÑØ#Ñ-ˆÜ—z’z ,Ó/ˆØ#Ñ-ˆØÑ!Ø#Ñ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r4   c                   ó0  ^ • \ rS rSrSrS\S\4U 4S jjr   SS\R                  S\
\R                  \R                  4   S-  S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  S-  \
\R                     S-  4   4S jjrSrU =r$ )ÚGemma2AttentionéÚ   z=Multi-headed attention from 'Attention Is All You Need' paperrZ   Ú	layer_idxc                 óZ  >• [         TU ]  5         [        US5      (       a  UR                  U   OS U l        Xl        X l        [        USUR                  UR                  -  5      U l
        UR                  UR                  -  U l        UR                  S-  U l        U R
                  R                  U l        [        USS5      (       + U l        ["        R$                  " UR                  UR                  U R                  -  UR&                  S9U l        ["        R$                  " UR                  UR                  U R                  -  UR&                  S9U l        ["        R$                  " UR                  UR                  U R                  -  UR&                  S9U l        ["        R$                  " UR                  U R                  -  UR                  UR&                  S9U l        U R
                  R0                  U l        U R                  S:X  a  UR2                  U l        g S U l        g )NÚlayer_typesr€   rÀ   Úuse_bidirectional_attentionFrX   Úsliding_attention)r)   r*   ÚhasattrrÔ   Ú
layer_typerZ   rÒ   rƒ   r[   r„   r€   r´   rÃ   Úquery_pre_attn_scalarr½   Úattention_dropoutÚ	is_causalr+   r]   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_window©r0   rZ   rÒ   r1   s      €r2   r*   ÚGemma2Attention.__init__Þ   s³  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÑ;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×3Ñ3°TÑ9ˆŒØ!%§¡×!>Ñ!>ˆÔÜ$ VÐ-JÈEÓRÔRˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfñ
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ó7]˜f×3Ñ3ˆÕÐcgˆÕr4   Nr¯   Úposition_embeddingsr»   Úpast_key_valuesr‡   r}   c                 ó4  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [        5      nU" U UU	U
U4U R                  (       a  U R                   OSU R"                  U R$                  U R&                  S.UD6u  pïUR(                  " / UQSP76 R+                  5       nU R-                  U5      nXï4$ )Nr7   r    r6   ç        )r¼   r½   râ   r¾   )rH   r€   rÝ   Úviewr•   rÞ   rß   r®   ÚupdaterÒ   r   Úget_interfacerZ   Ú_attn_implementationrÎ   rÂ   rÚ   r½   râ   rá   r²   rÉ   rà   )r0   r¯   rå   r»   ræ   r‡   Úinput_shapeÚhidden_shapeÚquery_statesrÊ   rË   r—   r˜   Úattention_interfacerÍ   rÌ   s                   r2   rD   ÚGemma2Attention.forwardø   s�  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯m¯m�D×*Ò*ÀØ—L‘LØ×.Ñ.Ø×/Ñ/ñ%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r4   )rÚ   rá   rZ   r€   rÛ   rÞ   rÒ   rØ   rÃ   rà   rÝ   r½   râ   rß   )NNN)rK   rL   rM   rN   Ú__doc__r!   rO   r*   r-   r    rG   r   r   r   rD   rP   rQ   rR   s   @r2   rÐ   rÐ   Ú   sÂ   ø† áGðh˜|ð h¸÷ hð: IMØ.2Ø(,ñ()à—|‘|ð()ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð()ð Ÿ™ tÑ+ð	()ð
  ™ð()ð Ð-Ñ.ð()ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷()ó ()r4   rÐ   c                   óü   ^ • \ rS rSrS\S\4U 4S jjr    SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\R                  S-  S
\S-  S\R                  4S jjrSrU =r$ )ÚGemma2DecoderLayeri#  rZ   rÒ   c                 ó¤  >• [         TU ]  5         UR                  U l        Xl        [	        XS9U l        [        U5      U l        [        UR                  UR                  S9U l
        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        g )N)rZ   rÒ   ©r&   )r)   r*   r[   rZ   rÐ   Ú	self_attnrT   Úmlpr#   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrã   s      €r2   r*   ÚGemma2DecoderLayer.__init__$  s¦   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØŒÜ(°ÑLˆŒÜ˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÑYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÑ(bˆÔ%ä)6°v×7IÑ7IÈv×ObÑObÑ)cˆÔ&Ü*7¸×8JÑ8JÐPV×PcÑPcÑ*dˆÕ'r4   Nr¯   rå   r»   r™   ræ   r}   c           	      óü   • UnU R                  U5      nU R                  " SUUUUUS.UD6u  pU R                  U5      nXq-   nUnU R                  U5      nU R	                  U5      nU R                  U5      nXq-   nU$ )N)r¯   rå   r»   r™   ræ   © )rú   r÷   rû   rü   rø   rý   )	r0   r¯   rå   r»   r™   ræ   r‡   ÚresidualÚ_s	            r2   rD   ÚGemma2DecoderLayer.forward0  s¥   € ð !ˆà×,Ñ,¨]Ó;ˆð  Ÿ>š>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Ñ0ˆàÐr4   )rZ   r[   rú   rø   rû   rý   rü   r÷   )NNNN)rK   rL   rM   rN   r!   rO   r*   r-   r    rG   Ú
LongTensorr   rD   rP   rQ   rR   s   @r2   rô   rô   #  s¤   ø† ð
e˜|ð 
e¸÷ 
eð IMØ.2Ø04Ø(,ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð 
�‰÷ó r4   rô   c            	       ól   ^ • \ rS rSrSrSS\S\S\S\4U 4S jjjrS\R                  4U 4S	 jjr
S
rU =r$ )ÚGemma2TextScaledWordEmbeddingiR  zT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 óŽ   >• [         TU ]  XU5        X@l        [        R                  " [
        R                  " U5      SS9U l        g )NFro   )r)   r*   Úscalar_embed_scaler+   rw   r-   Útensorr
  )r0   r  r  r	  r
  r1   s        €r2   r*   Ú&Gemma2TextScaledWordEmbedding.__init__W  s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÑQˆÕr4   Ú	input_idsc                 ó‚   >• [         TU ]  U5      U R                  R                  U R                  R
                  5      -  $ r(   )r)   rD   r
  r†   r/   r‚   )r0   r  r1   s     €r2   rD   Ú%Gemma2TextScaledWordEmbedding.forward\  s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRr4   )r
  r  )r@   )rK   rL   rM   rN   rò   rO   rA   r*   r-   r    rD   rP   rQ   rR   s   @r2   r  r  R  sM   ø† ññR sð R¸3ð RÈSð RÐ_d÷ Rð Rð
S §¡÷ Sõ Sr4   r  c                   óŽ   ^ • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.r\R&                  " 5       U 4S j5       rS	rU =r$ )
ÚGemma2PreTrainedModeli`  rZ   ÚmodelTrô   ræ   )r¯   Ú
attentionsc                 ó  >• [         TU ]  U5        SUR                  R                  ;   a!  [        R
                  " UR                  5        g [        U[        5      (       a,  [        R                  " UR                  UR                  5        g g )NÚRMSNorm)r)   Ú_init_weightsr1   rK   ÚinitÚzeros_r/   r’   r  Ú	constant_r
  r  )r0   r·   r1   s     €r2   r  Ú#Gemma2PreTrainedModel._init_weightsr  sa   ø€ ä‰Ñ˜fÔ%à˜×(Ñ(×1Ñ1Ó1Ü�KŠK˜Ÿ™Õ&Ü˜Ô =×>Ñ>Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÕIð ?r4   r   )rK   rL   rM   rN   r!   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrô   rÐ   Ú_can_record_outputsr-   r¡   r  rP   rQ   rR   s   @r2   r  r  `  sn   ø‡ àÓØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà+Ø%ñÐð
 ‡]‚]ƒ_ôJó öJr4   r  c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚGemma2Modeli|  rZ   c           	      ó"  >• [         TU ]  U5        UR                  U l        UR                  U l        [        UR                  UR                  U R                  U R                  R                  S-  S9U l        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                   S9U l        [%        U5      U l        SU l        U R+                  5         g s  snf )Ng      à?)r
  rö   F)r)   r*   Úpad_token_idr	  Ú
vocab_sizer  r[   rZ   Úembed_tokensr+   Ú
ModuleListÚrangeÚnum_hidden_layersrô   Úlayersr#   rù   Únormrh   Ú
rotary_embÚgradient_checkpointingÚ	post_initrã   s      €r2   r*   ÚGemma2Model.__init__~  sß   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒä9Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmñ
ˆÔô —m’mÜDIÈ&×JbÑJbÔDcÓdÒDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÑNˆŒ	Ü/°Ó7ˆŒØ&+ˆÔ#ð 	�‰Õùò es   Â DNr  r»   r™   ræ   Úinputs_embedsÚ	use_cacher‡   r}   c           	      óÚ  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U=n	[        5      (       d)  U R                  UUUUS.n
[        S
0 U
D6[        S
0 U
D6S.n	UnU R                  X³5      n[        U R                   S U R                  R"                   5       H,  u  pÞU" U4X�R                  R$                  U      UUUS.UD6nM.     U R'                  U5      n[)        UUS	9$ )Nz:You must specify exactly one of input_ids or inputs_embeds)rZ   r   r    )rj   )rZ   r7  r»   ræ   r™   )Úfull_attentionrÖ   )r»   rå   r™   ræ   )Úlast_hidden_stateræ   r   )Ú
ValueErrorr-  r   rZ   Úget_seq_lengthr-   r…   rH   rj   r¨   r’   Údictr   r   r3  Ú	enumerater1  r0  rÔ   r2  r   )r0   r  r»   r™   ræ   r7  r8  r‡   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr¯   rå   ÚiÚdecoder_layers                  r2   rD   ÚGemma2Model.forward�  s�  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø*.×*;Ñ*;¸IÓ*FˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÑFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð &ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øðà2·;±;×3JÑ3JÈ1Ñ3MÑNØ$7Ø)Ø /ñð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä&Ø+Ø+ñ
ð 	
r4   )r-  r4  r1  r2  r	  r3  r,  )NNNNNN)rK   rL   rM   rN   r!   r*   r   r   r   r-   r  r    r   ÚFloatTensorÚboolr   r   r   rD   rP   rQ   rR   s   @r2   r)  r)  |  sÍ   ø† ð˜|÷ ð$  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð ˜$‘;ð;
ð Ð+Ñ,ð;
ð 
!ô;
ó ó ó  ö;
r4   r)  c                   óX  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
        SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚGemma2ForCausalLMiÑ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr¯   ÚlogitsÚkeep_full_weightc                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g rW   )
r)   r*   r)  r  r,  r+   r]   r[   rJ  r5  rc   s     €r2   r*   ÚGemma2ForCausalLM.__init__Ø  sU   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒð 	�‰Õr4   Nr  r»   r™   ræ   r7  Úlabelsr8  Úlogits_to_keepr‡   r}   c	           
      ó   • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nU R                  R                  bF  XÐR                  R                  -  n[        R                  " U5      nXÐR                  R                  -  nSnUb  U R                  " XÖU R                  40 U	D6n[        UUU
R                  U
R                  U
R                  S9$ )a"  
Example:

```python
>>> from transformers import AutoTokenizer, Gemma2ForCausalLM

>>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```)r  r»   r™   ræ   r7  r8  N)ÚlossrL  ræ   r¯   r  r   )r  r;  r’   rO   ÚslicerJ  rZ   Úfinal_logit_softcappingr-   rÅ   Úloss_functionr,  r   ræ   r¯   r  )r0   r  r»   r™   ræ   r7  rP  r8  rQ  r‡   Úoutputsr¯   Úslice_indicesrL  rS  s                  r2   rD   ÚGemma2ForCausalLM.forwardá  s  € ð@ ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ�;‰;×.Ñ.Ñ:ØŸk™k×AÑAÑAˆFÜ—Z’Z Ó'ˆFØŸk™k×AÑAÑAˆFàˆØÑØ×%Ò% f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r4   )rJ  r  r,  )NNNNNNNr   )rK   rL   rM   rN   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr*   r   r   r-   r  r    r   rF  rG  rO   r   r   r   rD   rP   rQ   rR   s   @r2   rI  rI  Ñ  s"  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð × Ñ  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eŸl™lÑ*ð;
ð Ð+Ñ,ð;
ð 
 ô;
ó ó ö;
r4   rI  c                   ó   • \ rS rSrSrg)ÚGemma2ForSequenceClassificationi!  r   N©rK   rL   rM   rN   rP   r   r4   r2   r_  r_  !  ó   † Úr4   r_  c                   ó   • \ rS rSrSrg)ÚGemma2ForTokenClassificationi%  r   Nr`  r   r4   r2   rc  rc  %  ra  r4   rc  )rI  r)  r  r_  rc  )r    )rè   NN)IÚcollections.abcr   r-   Útorch.nnr+   Ú r   r  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_gemma2r!   ÚModuler#   rT   rh   r¥   r®   r    rO   r¶   rA   rG   rÎ   rÐ   rô   Ú	Embeddingr  r  r)  rI  r_  rc  Ú__all__r   r4   r2   Ú<module>rz     s4  ðõ* %ã Ý å &Ý !ß .Ý )ß Lß RÝ B÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý .ô=�B—I‘Iô =ô(�—	‘	ô ô 3<˜BŸI™Iô 3<òl(ñ Ð-Ó.óó /ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð$ Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&õ%ñD Ð)Ó*ôE)�b—i‘ió E)ó +ðE)ôP,Ð3ô ,ô^S B§L¡Lô Sð ôJ˜Oó Jó ðJð6 ôQ
Ð'ó Q
ó ðQ
ðh ôL
Ð-¨ó L
ó ðL
ô^	Ð&FÐH]ô 	ô	Ð#@ÐBWô 	ò�r4   