ó
    ‡Ž¡j…À  ã                   ó4  • S SK Jr  S SKJr  S SKrS SKJr  SSKJr	  SSK
Jr  SSKJrJr  SSKJr  SS	KJr  SS
KJrJr  SSKJrJrJrJrJrJrJr  SSKJrJ r   SSK!J"r"J#r#J$r$J%r%  SSK&J'r'J(r(  SSK)J*r*J+r+  SSK,J-r-  SSK.J/r/J0r0J1r1J2r2J3r3  SSK4J5r5  SSK6J7r7J8r8  SSK9J:r:  SSK;J<r<  SSK=J>r>J?r?  \1" SS9\ " S S\"5      5       5       r@\1" SS9\ " S S\/5      5       5       rA " S  S!\R„                  5      rC " S" S#\Rˆ                  5      rE " S$ S%\Rˆ                  5      rF " S& S'\Rˆ                  5      rGS( rH\" S)5      SZS* j5       rIS+\R”                  S,\KS-\R”                  4S. jrL   S[S/\Rˆ                  S0\R”                  S1\R”                  S2\R”                  S3\R”                  S-  S4\M\K-  S5\MS-  S6\MS-  S-\N\R”                  \R”                  4   4S7 jjrO\" \I5       " S8 S9\Rˆ                  5      5       rP " S: S;\ 5      rQ\1 " S< S=\+5      5       rRS>\KS-\\K\K\K\K/\S4   4S? jrT\1 " S@ SA\R5      5       rU\1 " SB SC\R\5      5       rV " SD SE\Rˆ                  5      rWS\SF\R”                  SG\R°                  S-  S-\R”                  4SH jjrYSI\SJ\R”                  S3\R”                  S-  SK\S-  SL\R”                  S-  SM\R”                  S-\Z4SN jr[\1" SOS9 " SP SQ\R5      5       r\\1" SOS9 " SR SS\R\5      5       r]\1" STS9 " SU SV\\R5      5       r^ " SW SX\\R5      r_/ SYQr`g)]é    )ÚCallable)Ú	dataclassNé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚGemma3ConfigÚGemma3TextConfigzK
    Base class for Gemma3 outputs, with hidden states and attentions.
    ©Úcustom_introc                   óB   • \ rS rSr% SrSr\R                  S-  \S'   Sr	g)ÚGemma3ModelOutputWithPasté<   a  
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
NÚimage_hidden_states© )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r3   ÚtorchÚFloatTensorÚ__annotations__Ú__static_attributes__r4   ó    Ú¸C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/gemma3/modeling_gemma3.pyr1   r1   <   s   ‡ ñð 59Ð˜×*Ñ*¨TÑ1Ö8r>   r1   zR
    Base class for Gemma3 causal language model (or autoregressive) outputs.
    c                   ó  • \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\S-  \S'   Sr\\R                     S-  \S'   Sr\\R                     S-  \S'   Sr\R                  S-  \S	'   S
rg)ÚGemma3CausalLMOutputWithPastéL   a  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr3   r4   )r5   r6   r7   r8   r9   rC   r:   r;   r<   rD   rE   r   rF   ÚtuplerG   r3   r=   r4   r>   r?   rA   rA   L   s‹   ‡ ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø48Ð˜×*Ñ*¨TÑ1Ö8r>   rA   c            	       ól   ^ • \ rS rSrSrSS\S\S\S\4U 4S jjjrS\R                  4U 4S	 jjr
S
rU =r$ )ÚGemma3TextScaledWordEmbeddingéj   zT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 óŽ   >• [         TU ]  XU5        X@l        [        R                  " [
        R                  " U5      SS9U l        g )NF©Ú
persistent)ÚsuperÚ__init__Úscalar_embed_scaleÚnnÚBufferr:   ÚtensorrO   )ÚselfrL   rM   rN   rO   Ú	__class__s        €r?   rT   Ú&Gemma3TextScaledWordEmbedding.__init__o   s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÑQˆÕr>   Ú	input_idsc                 ó‚   >• [         TU ]  U5      U R                  R                  U R                  R
                  5      -  $ ©N)rS   ÚforwardrO   ÚtoÚweightÚdtype)rY   r\   rZ   s     €r?   r_   Ú%Gemma3TextScaledWordEmbedding.forwardt   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRr>   )rO   rU   )ç      ð?)r5   r6   r7   r8   r9   ÚintÚfloatrT   r:   ÚTensorr_   r=   Ú__classcell__©rZ   s   @r?   rJ   rJ   j   sM   ø† ññR sð R¸3ð RÈSð RÐ_d÷ Rð Rð
S §¡÷ Sõ Sr>   rJ   c                   ó6   ^ • \ rS rSrS\4U 4S jjrS rSrU =r$ )Ú	Gemma3MLPéx   Úconfigc                 ó¼  >• [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l	        [        UR                     U l        g ©NF©Úbias)rS   rT   rm   Úhidden_sizeÚintermediate_sizerV   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fn©rY   rm   rZ   s     €r?   rT   ÚGemma3MLP.__init__y   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆŒÜ˜V×5Ñ5Ñ6ˆ�r>   c                 óˆ   • U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ r^   )rw   ry   ru   rv   )rY   Úxrw   s      r?   r_   ÚGemma3MLP.forwardƒ   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr>   )ry   rm   rw   ru   rr   rs   rv   )	r5   r6   r7   r8   r-   rT   r_   r=   rh   ri   s   @r?   rk   rk   x   s   ø† ð7Ð/÷ 7÷ð r>   rk   c                   óJ   ^ • \ rS rSrS	S\S\4U 4S jjjrS rS rS r	Sr
U =r$ )
ÚGemma3RMSNorméˆ   ÚdimÚepsc                 óŒ   >• [         TU ]  5         X l        [        R                  " [
        R                  " U5      5      U l        g r^   )rS   rT   rƒ   rV   Ú	Parameterr:   Úzerosra   )rY   r‚   rƒ   rZ   s      €r?   rT   ÚGemma3RMSNorm.__init__‰   s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆ�r>   c                 óˆ   • U[         R                  " UR                  S5      R                  SSS9U R                  -   5      -  $ )Nr)   éÿÿÿÿT)Úkeepdim)r:   ÚrsqrtÚpowÚmeanrƒ   )rY   r}   s     r?   Ú_normÚGemma3RMSNorm._normŽ   s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ð>ÀÇÁÑIÓJÑJÐJr>   c                 ó¢   • U R                  UR                  5       5      nUSU R                  R                  5       -   -  nUR                  U5      $ )Nrd   )rŽ   rf   ra   Útype_as)rY   r}   Úoutputs      r?   r_   ÚGemma3RMSNorm.forward‘   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Ñ4Ñ5ˆØ�~‰~˜aÓ Ð r>   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)rH   ra   Úshaperƒ   )rY   s    r?   Ú
extra_reprÚGemma3RMSNorm.extra_repr˜   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r>   )rƒ   ra   )g�íµ ÷Æ°>)r5   r6   r7   r8   re   rf   rT   rŽ   r_   r–   r=   rh   ri   s   @r?   r€   r€   ˆ   s0   ø† ñ5˜Cð 5 e÷ 5ð 5ò
Kò!÷=ð =r>   r€   c                   óÞ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9 SS\S\S-  S	\	\
R                  \4   4S
 jj5       5       r\
R                  " 5       \S 5       5       rSrU =r$ )ÚGemma3RotaryEmbeddingéœ   Údevicez5.18)ÚversionNrm   c           
      óž  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        [        [        UR                  5      5      U l        0 U l	        U R                   HÜ  nU R
                  R                  U   nUc  M!  US   U R                  U'   U R                  nU R                  U   S:w  a  [        U R                  U      nU" U R
                  X#S9u  pg[        X S3[        R                  " USS95        [        X S3[        R                  " UR!                  5       SS95        [        X S3U5        MÞ     g )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqFrQ   Ú_original_inv_freqÚ_attention_scaling)rS   rT   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrm   ÚsortedÚsetÚlayer_typesrž   Úrope_parametersÚcompute_default_rope_parametersr   ÚsetattrrV   rW   Úclone)	rY   rm   r›   r¡   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingrZ   s	           €r?   rT   ÚGemma3RotaryEmbedding.__init__�   s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÑAˆKØÑ"Ùà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÓ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÈfÑ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ñ4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÑ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTò +r>   r¡   Úreturnc           	      ó  • U R                   U   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNrd   r   r)   ©rb   )r«   Úgetattrrr   Únum_attention_headsr:   Úarangerf   r`   )rm   r›   r¡   ÚkwargsÚbaser‚   Úattention_factorÚinv_freqs           r?   r¬   Ú5Gemma3RotaryEmbedding.compute_default_rope_parameters³   s€   € ð& ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r>   c                 ó  • [        X S35      n[        X S35      nUS S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        US	S
9   Xg-  R                  SS5      n	[        R                  " X™4SS9n
U
R                  5       U-  nU
R                  5       U-  nS S S 5        WR                  UR                  S9WR                  UR                  S94$ ! , (       d  f       N@= f)Nr¢   r¤   r   r‰   r+   ©rb   r›   ÚmpsÚcpuF)Údevice_typeÚenabledr)   ©r‚   r¸   )r¹   Úexpandr•   r`   r:   rf   r›   Ú
isinstanceÚtypeÚstrr&   Ú	transposeÚcatÚcosÚsinrb   )rY   r}   Úposition_idsr¡   r¿   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrÅ   ÚfreqsÚembrÎ   rÏ   s                r?   r_   ÚGemma3RotaryEmbedding.forwardÎ   sS  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �Tš1˜d�]Ñ#×*Ñ*¨<×+=Ñ+=¸aÑ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÐoð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   Ã*AE5Å5
F)rm   rª   r¦   r§   rž   r^   )NN)r5   r6   r7   r8   r%   r-   rT   ÚstaticmethodrË   rH   r:   rg   rf   r¬   Úno_gradr   r_   r=   rh   ri   s   @r?   r™   r™   œ   s™   ø† Ù�X vÑ.ñUÐ/÷ Uó /ðUð* Ù�X vÑ.àHLñ5Ø ð5Ø;>À¹:ð5à	ˆu�|‰|˜UÐ"Ñ	#ô5ó /ó ð5ð2 ‡]‚]ƒ_Øñ<ó ó ö<r>   r™   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nr‰   r)   rÇ   )r•   r:   rÍ   )r}   Úx1Úx2s      r?   Úrotate_halfrÜ   ä   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r>   Úrotary_pos_embc                 ó˜   • UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXV4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerÜ   )ÚqÚkrÎ   rÏ   Úunsqueeze_dimÚq_embedÚk_embeds          r?   Úapply_rotary_pos_embrå   ë   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØÐÐr>   rF   Ún_repr´   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r+   N)r•   rÈ   Úreshape)rF   ræ   ÚbatchÚnum_key_value_headsÚslenr·   s         r?   Ú	repeat_kvrì     s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr>   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚsoftcapc                 ój  • Uc  U R                   S-  n[        X R                  5      n	[        X0R                  5      n
[        R                  " XR                  SS5      5      U-  nUb  X·-  n[        R                  " U5      nX·-  nUb  X´-   n[        R                  R                  US[        R                  S9R                  UR                  5      n[        R                  R                  XµU R                  S9n[        R                  " Xº5      nUR                  SS5      R                  5       nXË4$ )Nç      à¿r)   r   r‰   )r‚   rb   )ÚpÚtrainingr+   )r·   rì   Únum_key_value_groupsr:   ÚmatmulrÌ   ÚtanhrV   Ú
functionalÚsoftmaxÚfloat32r`   rb   rò   rø   Ú
contiguous)rí   rî   rï   rð   rñ   rò   ró   rô   r¼   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r?   Úeager_attention_forwardr    s  € ð �Ø—/‘/ 4Ñ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÑØ#Ñ-ˆÜ—z’z ,Ó/ˆØ#Ñ-ˆØÑ!Ø#Ñ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r>   c                   ó  ^ • \ rS rSrSrS\S\4U 4S jjr   SS\R                  S\R                  S	\R                  S-  S
\
S-  S\\   S\\R                  \R                  S-  \\R                     S-  4   4S jjrSrU =r$ )ÚGemma3Attentioni3  z=Multi-headed attention from 'Attention Is All You Need' paperrm   Ú	layer_idxc                 ó  >• [         TU ]  5         [        US5      (       a  UR                  U   OS U l        Xl        X l        [        USUR                  UR                  -  5      U l
        UR                  UR                  -  U l        UR                  S-  U l        U R
                  R                  U l        U R
                  R                   (       + U l        [$        R&                  " UR                  UR                  U R                  -  UR(                  S9U l        [$        R&                  " UR                  UR                  U R                  -  UR(                  S9U l        [$        R&                  " UR                  UR                  U R                  -  UR(                  S9U l        [$        R&                  " UR                  U R                  -  UR                  UR(                  S9U l        U R
                  R2                  U l        U R                  S:X  a  UR4                  OS U l        U R                  S:H  U l        [9        UR                  UR:                  S9U l        [9        UR                  UR:                  S9U l        g )Nrª   r·   rö   rp   Úsliding_attention)r‚   rƒ   ) rS   rT   Úhasattrrª   r¡   rm   r  r¹   rr   rº   r·   rê   rù   Úquery_pre_attn_scalarró   Úattention_dropoutÚuse_bidirectional_attentionÚ	is_causalrV   rt   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_windowÚ
is_slidingr€   Úrms_norm_epsÚq_normÚk_norm©rY   rm   r  rZ   s      €r?   rT   ÚGemma3Attention.__init__7  sï  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÑ;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×3Ñ3°TÑ9ˆŒØ!%§¡×!>Ñ!>ˆÔØ!Ÿ[™[×DÑDÔDˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfñ
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ó7]˜f×3Ò3ÐcgˆÔØŸ/™/Ð-@Ñ@ˆŒä#¨¯©¸V×=PÑ=PÑQˆŒÜ#¨¯©¸V×=PÑ=PÑQˆ�r>   NrF   Úposition_embeddingsrñ   rE   r¼   r´   c                 ób  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  U5      nU R                  U	5      n	Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [         5      nU" U UU	U
U4U R"                  (       a  U R$                  OSU R&                  U R(                  S.UD6u  pïUR*                  " / UQSP76 R-                  5       nU R/                  U5      nXï4$ )Nr‰   r+   r)   ç        )rò   ró   r  )r•   r·   r  ÚviewrÌ   r  r  r  r  rå   Úupdater  r   Úget_interfacerm   Ú_attn_implementationr  rø   r  ró   r  rè   rÿ   r  )rY   rF   r  rñ   rE   r¼   Úinput_shapeÚhidden_shapeÚquery_statesr   r  rÎ   rÏ   Úattention_interfacer  r  s                   r?   r_   ÚGemma3Attention.forwardU  s£  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð /3¯m¯m�D×*Ò*ÀØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r>   )r  r  rm   r·   r  r  r  r  r  r¡   rù   r  r  r  ró   r  r  )NNN)r5   r6   r7   r8   r9   r-   re   rT   r:   rg   r   r   r!   rH   r_   r=   rh   ri   s   @r?   r  r  3  s¯   ø† áGðRÐ/ð R¸C÷ RðB -1Ø.2Ø(,ñ*)à—|‘|ð*)ð #Ÿ\™\ð*)ð Ÿ™ tÑ+ð	*)ð
  ™ð*)ð Ð+Ñ,ð*)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷*)ó *)r>   r  c                   ó$  ^ • \ rS rSrS\S\4U 4S jjr    SS\R                  S\R                  S\R                  S-  S	\R                  S-  S
\
S-  S\\   S\\R                  \\R                  \R                  4   S-  4   4S jjrSrU =r$ )ÚGemma3DecoderLayeri‚  rm   r  c                 ó°  >• [         TU ]  5         Xl        UR                  U l        X l        [        XS9U l        [        U5      U l        [        U R                  UR                  S9U l        [        U R                  UR                  S9U l        [        U R                  UR                  S9U l        [        U R                  UR                  S9U l        g )N)rm   r  ©rƒ   )rS   rT   rm   rr   r  r  Ú	self_attnrk   Úmlpr€   r  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr  s      €r?   rT   ÚGemma3DecoderLayer.__init__ƒ  sª   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ"ŒÜ(°ÑLˆŒÜ˜VÓ$ˆŒÜ,¨T×-=Ñ-=À6×CVÑCVÑWˆÔÜ(5°d×6FÑ6FÈF×L_ÑL_Ñ(`ˆÔ%Ü)6°t×7GÑ7GÈV×M`ÑM`Ñ)aˆÔ&Ü*7¸×8HÑ8HÈf×NaÑNaÑ*bˆÕ'r>   NrF   r  rñ   rÐ   rE   r¼   r´   c           	      óü   • UnU R                  U5      nU R                  " SUUUUUS.UD6u  pU R                  U5      nXq-   nUnU R                  U5      nU R	                  U5      nU R                  U5      nXq-   nU$ )N)rF   r  rñ   rÐ   rE   r4   )r.  r,  r/  r0  r-  r1  )	rY   rF   r  rñ   rÐ   rE   r¼   ÚresidualÚ_s	            r?   r_   ÚGemma3DecoderLayer.forward�  s£   € ð !ˆà×,Ñ,¨]Ó;ˆàŸ>š>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Ñ0ˆàÐr>   )	rm   rr   r.  r  r-  r/  r1  r0  r,  )NNNN)r5   r6   r7   r8   r-   re   rT   r:   rg   Ú
LongTensorr   r   r!   rH   r;   r_   r=   rh   ri   s   @r?   r)  r)  ‚  sÈ   ø† ð
cÐ/ð 
c¸C÷ 
cð -1Ø.2Ø04Ø(,ñà—|‘|ðð #Ÿ\™\ðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð Ð+Ñ,ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷ó r>   r)  c                   ó’   ^ • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.rSr\R(                  " 5       U 4S	 j5       rS
rU =r$ )ÚGemma3PreTrainedModeli°  rm   ÚmodelTr)  rE   )rF   rG   )ÚimageÚtextc                 óö  >• [         TU ]  U5        [        U[        5      (       a!  [        R
                  " UR                  5        g SUR                  R                  ;   a!  [        R
                  " UR                  5        g [        U[        5      (       a,  [        R                  " UR                  UR                  5        g [        U[        5      (       a¢  UR                   H‘  nUR                   nUR"                  U   S:w  a  [$        UR"                  U      nU" UR&                  US9u  pE[        R(                  " [+        X S35      U5        [        R(                  " [+        X S35      U5        M“     g g )NÚRMSNormrŸ   r    r¢   r£   )rS   Ú_init_weightsrÉ   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightrZ   r5   ra   rJ   Ú	constant_rO   rU   r™   rª   r¬   rž   r   rm   Úcopy_r¹   )rY   rí   r¡   r°   r±   r5  rZ   s         €r?   r?  Ú#Gemma3PreTrainedModel._init_weightsÃ  s  ø€ ä‰Ñ˜fÔ%Ü�fÔ7×8Ñ8Ü�KŠK˜×9Ñ9Õ:à˜&×*Ñ*×3Ñ3Ó3Ü�KŠK˜Ÿ™Õ&Ü˜Ô =×>Ñ>Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÕIÜ˜Ô 5×6Ñ6Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ó<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ñ#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ò 1ð 7r>   r4   )r5   r6   r7   r8   r,   r<   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr)  r  Ú_can_record_outputsÚinput_modalitiesr:   rØ   r?  r=   rh   ri   s   @r?   r9  r9  °  st   ø‡ àÓØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà+Ø%ñÐð )Ðà
‡]‚]ƒ_ô^ó ö^r>   r9  r  c           
      óT   ^ • S[         S[         S[         S[         S[        4
U 4S jjnU$ )z9
Enables a bidirectional mask within the sliding window.
Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr´   c                 ó$   >• [        X#-
  5      T:  $ )z‹A token can attend to any other token if their absolute distance is within
the (exclusive) sliding window size (distance < sliding_window).)Úabs)rS  rT  rU  rV  r  s       €r?   Ú
inner_maskÚ1_bidirectional_window_overlay.<locals>.inner_maskÜ  s   ø€ ô �5‘>Ó" ^Ñ3Ð3r>   )re   Úbool)r  rY  s   ` r?   Ú_bidirectional_window_overlayr\  ×  s3   ø€ ð
4œcð 4¬Sð 4¼ð 4Äcð 4Ìd÷ 4ð
 Ðr>   c                   ó  ^ • \ rS rSr% \\S'   SrS\4U 4S jjr\\	\
      SS\R                  S-  S\R                  S-  S\R                  S-  S	\S-  S
\R                  S-  S\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚGemma3TextModeliä  rm   ©r<  c           	      ó"  >• [         TU ]  U5        UR                  U l        UR                  U l        [        UR                  UR                  U R                  U R                  R                  S-  S9U l        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                   S9U l        [%        U5      U l        SU l        U R+                  5         g s  snf )Nç      à?)rO   r+  F)rS   rT   Úpad_token_idrN   Ú
vocab_sizerJ   rr   rm   Úembed_tokensrV   Ú
ModuleListÚrangeÚnum_hidden_layersr)  Úlayersr€   r  Únormr™   Ú
rotary_embÚgradient_checkpointingÚ	post_initr  s      €r?   rT   ÚGemma3TextModel.__init__é  sá   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒô :Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmñ
ˆÔô —m’mÜDIÈ&×JbÑJbÔDcÓdÒDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÑNˆŒ	Ü/°Ó7ˆŒØ&+ˆÔ#ð 	�‰Õùò es   Â DNr\   rñ   rÐ   rE   Úinputs_embedsÚ	use_cacher¼   r´   c           	      ó
  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U=n	[        5      (       d|  U R                  UUUUS.n
U
R                  5       nU R                  R                  (       a(  S U
S'   [        U R                  R                  5      US'   [!        S0 U
D6[#        S0 UD6S	.n	Un0 n[%        U R                  R&                  5       H  nU R)                  XÃU5      XÞ'   M     [+        U R,                  S U R                  R.                   5       HF  u  nnU" U4X�R                  R&                  U      XÐR                  R&                  U      UUS
.UD6nMH     U R1                  U5      n[3        UUS9$ )Nú:You must specify exactly one of input_ids or inputs_embeds©rm   r   r+   ©r›   ©rm   rn  rñ   rE   rÐ   c                  óH   • [         R                  " S[         R                  S9$ )NTr¸   )r:   rX   r[  )Úargss    r?   Ú<lambda>Ú)Gemma3TextModel.forward.<locals>.<lambda>$  s   € ÄÇÂÈTÔY^×YcÑYcÒ@dr>   Úor_mask_function©Úfull_attentionr	  )rñ   r  rÐ   rE   )Úlast_hidden_staterE   r4   )Ú
ValueErrorrd  r	   rm   Úget_seq_lengthr:   r»   r•   r›   rß   rÉ   ÚdictÚcopyr  r\  r  r   r   r©   rª   rj  Ú	enumeraterh  rg  ri  r   )rY   r\   rñ   rÐ   rE   rn  ro  r¼   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrF   r  r¡   ÚiÚdecoder_layers                    r?   r_   ÚGemma3TextModel.forwardü  s  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÑFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKð #.×"2Ñ"2Ó"4Ðà�{‰{×6×6Ù2d�Ð.Ñ/Ü:WÐX\×XcÑXc×XrÑXrÓ:sÐ#Ð$6Ñ7ô #5Ñ"C°{Ñ"CÜ%FÑ%]ÐI\Ñ%]ñ#Ðð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAˆ}Ù)Øðà2·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä&Ø+Ø+ñ
ð 	
r>   )rd  rk  rh  ri  rN   rj  rc  )NNNNNN)r5   r6   r7   r8   r-   r<   rQ  rT   r'   r(   r"   r:   r7  rg   r   r;   r[  r   r!   r   r_   r=   rh   ri   s   @r?   r^  r^  ä  sä   ø‡ àÓØ ÐðÐ/÷ ð&  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð ×&Ñ&¨Ñ-ð	C
ð
  ™ðC
ð ×(Ñ(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ñ,ðC
ð 
!ôC
ó ó ó  öC
r>   r^  c                   ól  ^ • \ rS rSr% SS0rSS0rSS/S/40rSS0r\\	S	'   S	\4U 4S
 jjr
\\        SS\R                  S-  S\R                  S-  S\R                  S-  S\S-  S\R"                  S-  S\R                  S-  S\S-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚGemma3ForCausalLMiE  úlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrF   rD   Úkeep_full_weightrm   c                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g ro   )
rS   rT   r^  r:  rc  rV   rt   rr   rŒ  rl  rz   s     €r?   rT   ÚGemma3ForCausalLM.__init__M  sU   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒð 	�‰Õr>   Nr\   rñ   rÐ   rE   rn  Úlabelsro  Úlogits_to_keepr¼   r´   c	           
      ó   • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nU R                  R                  bF  XÐR                  R                  -  n[        R                  " U5      nXÐR                  R                  -  nSnUb  U R                  " XÖU R                  40 U	D6n[        UUU
R                  U
R                  U
R                  S9$ )a"  
Example:

```python
>>> from transformers import AutoTokenizer, Gemma3ForCausalLM

>>> model = Gemma3ForCausalLM.from_pretrained("google/gemma-2-9b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```)r\   rñ   rÐ   rE   rn  ro  N)rC   rD   rE   rF   rG   r4   )r:  r|  rÉ   re   ÚslicerŒ  rm   Úfinal_logit_softcappingr:   rû   Úloss_functionrc  r   rE   rF   rG   )rY   r\   rñ   rÐ   rE   rn  r‘  ro  r’  r¼   ÚoutputsrF   Úslice_indicesrD   rC   s                  r?   r_   ÚGemma3ForCausalLM.forwardV  s  € ð@ ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ�;‰;×.Ñ.Ñ:ØŸk™k×AÑAÑAˆFÜ—Z’Z Ó'ˆFØŸk™k×AÑAÑAˆFàˆØÑØ×%Ò% f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r>   )rŒ  r:  rc  )NNNNNNNr   )r5   r6   r7   r8   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr-   r<   rT   r#   r"   r:   r7  rg   r   r;   r[  re   r   r!   r   r_   r=   rh   ri   s   @r?   rŠ  rŠ  E  s0  ø‡ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€JØÓðÐ/÷ ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð × Ñ  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eŸl™lÑ*ð;
ð Ð+Ñ,ð;
ð 
 ô;
ó ó ö;
r>   rŠ  c                   óR   ^ • \ rS rSrS\4U 4S jjrS\R                  4S jrSr	U =r
$ )r@  i–  rm   c                 óˆ  >• [         TU ]  5         [        R                  " [        R
                  " UR                  R                  UR                  R                  5      5      U l	        [        UR                  R                  UR                  R                  S9U l        [        UR                  R                  UR                  R                  -  5      U l        [        UR"                  S-  5      U l        U R                   U R$                  -  U l        [        R(                  " U R&                  U R&                  S9U l        g )Nr+  ra  )Úkernel_sizeÚstride)rS   rT   rV   r…   r:   r†   Úvision_configrr   Útext_configrC  r€   Úlayer_norm_epsÚmm_soft_emb_normre   Ú
image_sizeÚ
patch_sizeÚpatches_per_imageÚmm_tokens_per_imageÚtokens_per_sider   Ú	AvgPool2dÚavg_poolrz   s     €r?   rT   Ú"Gemma3MultiModalProjector.__init__—  sì   ø€ Ü‰ÑÔä*,¯,ª,Ü�KŠK˜×,Ñ,×8Ñ8¸&×:LÑ:L×:XÑ:XÓYó+
ˆÔ'ô !.Ø× Ñ ×,Ñ,°&×2FÑ2F×2UÑ2Uñ!
ˆÔô "% V×%9Ñ%9×%DÑ%DÈ×H\ÑH\×HgÑHgÑ%gÓ!hˆÔÜ" 6×#=Ñ#=¸sÑ#BÓCˆÔØ×1Ñ1°T×5IÑ5IÑIˆÔÜŸš°×1AÑ1AÈ$×JZÑJZÑ[ˆ�r>   Úvision_outputsc                 óž  • UR                   u  p#nUR                  SS5      nUR                  X$U R                  U R                  5      nUR	                  5       nU R                  U5      nUR                  S5      nUR                  SS5      nU R                  U5      n[        R                  " XpR                  5      nUR                  U5      $ )Nr+   r)   )r•   rÌ   rè   r¨  rÿ   r¬  Úflattenr¥  r:   rú   rC  r‘   )	rY   r®  Ú
batch_sizer5  rr   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            r?   r_   Ú!Gemma3MultiModalProjector.forward§  sÇ   € Ø%3×%9Ñ%9Ñ"ˆ
�{à"0×":Ñ":¸1¸aÓ"@ÐØ"9×"AÑ"AØ T×%;Ñ%;¸T×=SÑ=Só#
Ðð #:×"DÑ"DÓ"FÐà $§¡Ð.EÓ FÐØ 5× =Ñ =¸aÓ @ÐØ 5× ?Ñ ?ÀÀ1Ó EÐà $× 5Ñ 5Ð6KÓ LÐä#(§<¢<Ð0E×GfÑGfÓ#gÐ Ø'×/Ñ/°Ó?Ð?r>   )r¬  r   rC  r¥  r¨  rª  )r5   r6   r7   r8   r,   rT   r:   rg   r_   r=   rh   ri   s   @r?   r@  r@  –  s)   ø† ð\˜|÷ \ð @ e§l¡l÷ @ò @r>   r@  Útoken_type_idsr›   c                 óþ   • U S:H  R                  US9n[        R                  R                  USSS9S S 2S S24   nX#) -  n[        R
                  " UR                  5       SS9S-
  n[        R                  " X%S5      nU$ )Nr+   rs  )r+   r   r   )rð   r‰   rÇ   )r`   rV   rü   Úpadr:   Úcumsumre   Úwhere)r·  r›   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          r?   Úget_block_sequence_ids_for_maskrÁ  º  s…   € ð  !Ñ#×'Ñ'¨vÐ'Ð6€HÜŸ™×)Ñ)¨(°FÀ!Ð)ÐDÂQÈÈÈÀVÑLÐØÐ!3Ñ3€OÜ—’˜_×0Ñ0Ó2¸Ñ:¸QÑ>€IÜŸš X¸"Ó=ÐØÐr>   rm   rn  rE   rÐ   rÀ  c                 ój  • U UUUUS.n[        S
0 UDSU0D6n[        USS5      (       a,  SUR                  ;   a  UR                  R                  S5      nOSn[	        S
0 UDSU0D6u  n	    p«p¬U	(       a  UnO[        XRX¼5      n[        S
0 UD[        U5      [        U R                  5      US.D6nUUS	.$ )z×Create full_attention and sliding_attention masks with correct composition.

For global (full attention) layers:  OR(causal, blockwise)
For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
rt  rÀ  r  NTr   r  )ry  Úand_mask_functionr  rz  r4   )	r   r¹   r  Úindexr   r   r   r   r  )rm   rn  rñ   rE   rÐ   rÀ  r„  Ú	full_maskÚsliding_layer_idxÚ
early_exitr5  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                  r?   Úcreate_masks_for_vision_modelrÌ  Å  sò   € ð Ø&Ø(Ø*Ø$ñ€Kô #ÑX [ÑXÐEWÒX€Iô ˆ ¨d×3Ñ3¸À×@ZÑ@ZÓ8ZØ+×6Ñ6×<Ñ<¸TÓBÑàÐô 4Nñ 4Ø
ñ4à#ò4Ñ0€J��1�a Aö Ø$6Ñ!ä$@Ø°	ó%
Ð!ô &ñ Ø
ðä*Ð+DÓEÜ0°×1FÑ1FÓGØ#ó	€Lð $Ø)ñð r>   zy
    The Base Gemma3 model which consists of a vision backbone and a language model without language modeling head.,
    c                   ó  ^ • \ rS rSrSrS\4U 4S jjr\\" SS9S\	R                  S\\   S	\\-  4S
 j5       5       rS\	R                   S\	R                  S\	R                  4S jr\\         SS\	R                   S-  S\	R                  S-  S\	R$                  S-  S\	R                   S-  S\S-  S\	R                   S-  S\	R                  S-  S\	R                   S-  S\S-  S\\   S	\\-  4S jj5       5       rSrU =r$ )ÚGemma3Modeli  Frm   c                 ó(  >• [         TU ]  U5        [        R                  " UR                  S9U l        [        U5      U l        UR                  R                  U l	        [        R                  " UR                  S9nX l
        U R                  5         g )Nrr  )rS   rT   r*   Úfrom_configr¢  Úvision_towerr@  Úmulti_modal_projectorr£  rc  Úlanguage_modelrl  )rY   rm   rÓ  rZ   s      €r?   rT   ÚGemma3Model.__init__
  so   ø€ Ü‰Ñ˜Ô Ü%×1Ò1¸×9MÑ9MÑNˆÔÜ%>¸vÓ%FˆÔ"Ø ×,Ñ,×7Ñ7ˆŒä"×.Ò.°f×6HÑ6HÑIˆØ,ÔØ�‰Õr>   zOProjects the last hidden state from the vision model into language model space.r.   Úpixel_valuesr¼   r´   c                 ót   • U R                   " SUSS.UD6nUR                  nU R                  U5      Ul        U$ )NT)rÕ  Úreturn_dictr4   )rÑ  r|  rÒ  Úpooler_output)rY   rÕ  r¼   r®  r|  s        r?   Úget_image_featuresÚGemma3Model.get_image_features  sF   € ð
 ×*Ò*Ða¸ÐRVÑaÐZ`ÑaˆØ*×<Ñ<ÐØ'+×'AÑ'AÐBSÓ'TˆÔ$àÐr>   r\   rn  Úimage_featuresc           
      ó(  • Uck  X R                  5       " [        R                  " SU R                  R                  [        R
                  UR                  S95      :H  nUR                  S5      nOXR                  R                  :H  nUR                  5       nUR                  S   UR                  S   -  nUR                  S5      R                  UR                  5      n[        XRR                  S   -  UR                  5       :H  SU SU 35        U$ )z×
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
equal to the length of multimodal features. If the lengths are different, an error is raised.
r4   rÂ   r‰   r   r+   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr:   Úfullrm   Úimage_token_idÚlongr›   ÚallÚsumr•   rß   r`   r$   Únumel)rY   r\   rn  rÛ  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r?   Úget_placeholder_maskÚ Gemma3Model.get_placeholder_mask  s  € ð ÑØ!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÑió3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`ÐaqÐ`rÐsô	
ð "Ð!r>   Nrñ   rÐ   rE   r·  r‘  ro  Ú	lm_kwargsc
           
      óP  • USL USL-  (       a  [        S5      eUbQ  U R                  R                  U R                  :¼  a-  XR                  R                  :H  nUR	                  5       nSXË'   OUnUc  U R                  5       " U5      nUba  U R                  USS9R                  nUR                  UR                  UR                  5      nU R                  XUS9nUR                  X½5      n[        U=n[        5      (       dO  U R                  R                  5       UUUUS.nUb!  [!        XgR                  S9n[#        SS	U0UD6nO[%        S0 UD6nU R&                  " SUUUUU	SS
.U
D6n[)        UR*                  UR,                  UR.                  UR0                  Ub  WS9$ SS9$ )aþ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

>>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma32-3b-mix-224")
>>> processor = AutoProcessor.from_pretrained("google/gemma32-3b-mix-224")

>>> prompt = "Where is the cat standing?"
>>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(**inputs,)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Where is the cat standing?\nsnow"
```Nrq  r   T)r×  )rn  rÛ  rt  rs  rÀ  )rñ   rÐ   rE   rn  ro  r×  )r|  rE   rF   rG   r3   r4   )r}  rm   rß  rc  r®   rÝ  rÙ  rØ  r`   r›   rb   rç  Úmasked_scatterrÉ   r  Úget_text_configrÁ  rÌ  r   rÓ  r1   r|  rE   rF   rG   )rY   r\   rÕ  rñ   rÐ   rE   r·  rn  r‘  ro  ré  rä  Úllm_input_idsrÛ  rƒ  r„  rÀ  r—  s                     r?   r_   ÚGemma3Model.forward7  sî  € ðX ˜Ð -°tÐ";×<ÜÐYÓZÐZð Ñ  T§[¡[×%?Ñ%?À4Ç?Á?Ó%RØ!*¯k©k×.HÑ.HÑ!HÐØ%ŸO™OÓ-ˆMØ01ˆMÒ-à%ˆMàÑ Ø ×5Ñ5Ô7¸ÓFˆMð Ñ#Ø!×4Ñ4°\ÈtÐ4ÐT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:ØÀ~ð ";ð "Ðð *×8Ñ8Ð9KÓ\ˆMô °Ð?Ð-Ä×FÑFàŸ+™+×5Ñ5Ó7Ø!.Ø"0Ø#2Ø ,ñˆKð Ñ)Ü%DÀ^×\pÑ\pÑ%qÐ"Ü&Cñ 'Ø'9ð'à!ñ'Ñ#ô
 '@Ñ&NÀ+Ñ&NÐ#à×%Ò%ð 
Ø.Ø%Ø+Ø'ØØñ
ð ñ
ˆô )Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ñ2J ñ
ð 	
ð
 QUñ
ð 	
r>   )rÓ  rÒ  rÑ  rc  )	NNNNNNNNN)r5   r6   r7   r8   Úaccepts_loss_kwargsr,   rT   r#   r"   r:   r;   r   r!   rH   r   rÙ  r7  rç  rg   r   r[  r1   r_   r=   rh   ri   s   @r?   rÎ  rÎ    s¤  ø† ð  Ðð˜|÷ ð ÙÐ!rÑsðØ!×-Ñ-ðØ9?Ð@RÑ9Sðà	Ð+Ñ	+óó tó ðð"Ø×)Ñ)ð"Ø:?×:KÑ:Kð"Ø]b×]nÑ]nô"ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ñd
à×#Ñ# dÑ*ðd
ð ×'Ñ'¨$Ñ.ðd
ð Ÿ™ tÑ+ð	d
ð
 ×&Ñ&¨Ñ-ðd
ð  ™ðd
ð ×(Ñ(¨4Ñ/ðd
ð ×(Ñ(¨4Ñ/ðd
ð × Ñ  4Ñ'ðd
ð ˜$‘;ðd
ð Ð.Ñ/ðd
ð 
Ð*Ñ	*ôd
ó ó öd
r>   rÎ  c                   óv  ^ • \ rS rSrSS0rS\4U 4S jjr\S\R                  S\
\   4S j5       r\\          SS
\R                  S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\S	-  S\\R                  -  S\
\   S\\-  4S jj5       5       rSU 4S jjr\  SS\S\R                  S\R                  S	-  S\S	-  S\R                  S	-  S\R                  S	-  S\S	-  S\4S jj5       rSrU =r$ )ÚGemma3ForConditionalGenerationi   r‹  z(model.language_model.embed_tokens.weightrm   c                 óê   >• [         TU ]  U5        [        U5      U l        [        R
                  " UR                  R                  UR                  R                  SS9U l	        U R                  5         g ro   )rS   rT   rÎ  r:  rV   rt   r£  rr   rc  rŒ  rl  rz   s     €r?   rT   Ú'Gemma3ForConditionalGeneration.__init__¨  sS   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ü—y’y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÑkˆŒØ�‰Õr>   rÕ  r¼   c                 ó<   • U R                   R                  " U40 UD6$ r^   )r:  rÙ  )rY   rÕ  r¼   s      r?   rÙ  Ú1Gemma3ForConditionalGeneration.get_image_features®  s   € à�z‰z×,Ò,¨\ÑD¸VÑDÐDr>   Nr\   rñ   rÐ   rE   r·  rn  r‘  ro  r’  ré  r´   c                 ó   • U R                   " SUUUUUUUU	USS.
UD6nUS   n[        U
[        5      (       a  [        U
* S5      OU
nU R	                  USS2USS24   5      nSnUb3  U R
                  " SXøU R                  R                  R                  S.UD6n[        UUUR                  UR                  UR                  UR                  S9$ )aÉ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

>>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
>>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

>>> messages = [
...     {
...         "role": "system",
...         "content": [
...             {"type": "text", "text": "You are a helpful assistant."}
...         ]
...     },
...     {
...         "role": "user", "content": [
...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
...             {"type": "text", "text": "Where is the cat standing?"},
...         ]
...     },
... ]

>>> inputs = processor.apply_chat_template(
...     messages,
...     tokenize=True,
...     return_dict=True,
...     return_tensors="pt",
...     add_generation_prompt=True
... )
>>> # Generate
>>> generate_ids = model.generate(**inputs)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
```
T)
r\   rÕ  r·  rñ   rÐ   rE   rn  ro  r‘  r×  r   N)rD   r‘  rc  )rC   rD   rE   rF   rG   r3   r4   )r:  rÉ   re   r”  rŒ  r–  rm   r£  rc  rA   rE   rF   rG   r3   )rY   r\   rÕ  rñ   rÐ   rE   r·  rn  r‘  ro  r’  ré  r—  rF   r˜  rD   rC   s                    r?   r_   Ú&Gemma3ForConditionalGeneration.forward²  sù   € ðz —*’*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØñ
ð ñ
ˆð   ™
ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%ð Ø¸¿¹×9PÑ9P×9[Ñ9[ñØ_hñˆDô ,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ñ
ð 	
r>   c                 óT   >• [         TU ]  " U4X#S.UD6nU(       d  U(       a  S US'   U$ )N)ro  Úis_first_iterationr·  )rS   Úprepare_inputs_for_generation)rY   r\   ro  rù  r¼   Úmodel_inputsrZ   s         €r?   rú  Ú<Gemma3ForConditionalGeneration.prepare_inputs_for_generation  s=   ø€ Ü‘wÒ<Øð
Ø!*ñ
ØU[ñ
ˆö #®)à-1ˆLÐ)Ñ*àÐr>   rù  c                 óŠ   • U R                  5       UUUUS.nUb   [        XQR                  S9n	[        SSU	0UD6$ [	        S0 UD6$ )Nrt  rs  rÀ  r4   )rì  rÁ  r›   rÌ  r   )
rm   rn  rñ   rE   rÐ   r·  rù  r¼   r„  rÀ  s
             r?   r   Ú8Gemma3ForConditionalGeneration.create_masks_for_generate  si   € ð ×,Ñ,Ó.Ø*Ø,Ø.Ø(ñ
ˆð Ñ%Ü!@À×XlÑXlÑ!mÐÜ0ñ Ø#5ðàñð ô
 )Ñ7¨;Ñ7Ð7r>   )rŒ  r:  )
NNNNNNNNNr   )TF)NF)r5   r6   r7   r8   rš  r,   rT   r"   r:   r;   r   r!   rÙ  r#   r7  rg   r   r[  re   rH   rA   r_   rú  r×   r
   r  r   r=   rh   ri   s   @r?   rñ  rñ     s  ø† ð +Ð,VÐWÐð˜|÷ ð ðE¨u×/@Ñ/@ð EÈFÐSeÑLfó Eó ðEð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ñ[
à×#Ñ# dÑ*ð[
ð ×'Ñ'¨$Ñ.ð[
ð Ÿ™ tÑ+ð	[
ð
 ×&Ñ&¨Ñ-ð[
ð  ™ð[
ð ×(Ñ(¨4Ñ/ð[
ð ×(Ñ(¨4Ñ/ð[
ð × Ñ  4Ñ'ð[
ð ˜$‘;ð[
ð ˜eŸl™lÑ*ð[
ð Ð.Ñ/ð[
ð 
Ð-Ñ	-ô[
ó ó ð[
÷zð ð /3Ø*/ñ8Ø ð8à—|‘|ð8ð Ÿ™ tÑ+ð8ð  ™ð	8ð
 —l‘l TÑ)ð8ð Ÿ™ tÑ+ð8ð ! 4™Kð8ð 
ô8ó ö8r>   rñ  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.c                   ó$   • \ rS rSr% \\S'   SrSrg)Ú#Gemma3TextForSequenceClassificationi8  rm   r_  r4   N)r5   r6   r7   r8   r-   r<   rQ  r=   r4   r>   r?   r   r   8  s   ‡ ð ÓØ Ór>   r   c                   ó*  ^ • \ rS rSr        SS\R
                  S-  S\R                  S-  S\R                  S-  S\R
                  S-  S\S-  S\R
                  S-  S	\R                  S-  S
\R
                  S-  S\	\
   S\4U 4S jjjrSrU =r$ )ÚGemma3ForSequenceClassificationiB  Nr\   rÕ  rñ   rÐ   rE   r·  rn  r‘  r¼   r´   c	                 ó6   >• [         T
U ]  " SUUUUUUUUS.U	D6$ )N)r\   rñ   rÐ   rE   rn  rÕ  r·  r‘  r4   )rS   r_   )rY   r\   rÕ  rñ   rÐ   rE   r·  rn  r‘  r¼   rZ   s             €r?   r_   Ú'Gemma3ForSequenceClassification.forwardC  s=   ø€ ô ‰wŠð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øñ

ð ñ

ð 
	
r>   r4   )NNNNNNNN)r5   r6   r7   r8   r:   r7  r;   rg   r   r   r!   r   r_   r=   rh   ri   s   @r?   r  r  B  sÞ   ø† ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ñ
à×#Ñ# dÑ*ð
ð ×'Ñ'¨$Ñ.ð
ð Ÿ™ tÑ+ð	
ð
 ×&Ñ&¨Ñ-ð
ð  ™ð
ð ×(Ñ(¨4Ñ/ð
ð ×(Ñ(¨4Ñ/ð
ð × Ñ  4Ñ'ð
ð Ð+Ñ,ð
ð 
*÷
ö 
r>   r  )r9  r^  rŠ  rñ  rÎ  r  r   )r+   )r  NNr^   )aÚcollections.abcr   Údataclassesr   r:   Útorch.nnrV   Ú r   rA  Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr    r!   r"   r#   r$   Úutils.deprecationr%   Úutils.genericr&   r'   Úutils.output_capturingr(   Úautor*   Úconfiguration_gemma3r,   r-   r1   rA   Ú	EmbeddingrJ   ÚModulerk   r€   r™   rÜ   rå   rg   re   rì   rf   rH   r  r  r)  r9  r[  r\  r^  rŠ  r@  r›   rÁ  r  rÌ  rÎ  rñ  r   r  Ú__all__r4   r>   r?   Ú<module>r     së  ðõ* %Ý !ã Ý å &Ý !ß .Ý 3Ý )ß L÷÷ ñ ÷ \÷ó ÷ Lß FÝ &ß nÕ nÝ 0ß GÝ 5Ý ß @ñ ðñð
 ô9Ð 7ó 9ó óð9ñ ðñð
 ô9 ;ó 9ó óð9ô0S B§L¡Lô Sô�—	‘	ô ô =�B—I‘Iô =ô(E<˜BŸI™Iô E<òP(ñ Ð-Ó.óó /ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð$ Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&õ%ñD Ð)Ó*ôK)�b—i‘ió K)ó +ðK)ô\+Ð3ô +ð\ ô#^˜Oó #^ó ð#^ðL
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ñ:^ô 
ð ô]
Ð+ó ]
ó ð]
ð@ ôM
Ð-¨ó M
ó ðM
ô`!@ §	¡	ô !@ñH°E·L±Lð È%Ï,É,ÐY]ÑJ]ð Ðin×iuÑiuõ ð9Øð9à—<‘<ð9ð —L‘L 4Ñ'ð9ð ˜T‘\ð	9ð
 —,‘, Ñ%ð9ð Ÿ™ð9ð 
ô9ñx ðñô
W
Ð'ó W
óð
W
ñt ðñô
P8Ð%:¸Oó P8óð
P8ñf ð^ñô
!Ð*JÐLaó !óð
!ô

Ð&FÐH]ô 
ò4�r>   