ó
    ‡Ž¡jÎœ  ã                   ó”  • S SK Jr  S SKJr  S SKrS SKJr  S SKJr  SSK	J
r  SSKJrJr  SSKJr  SS	KJrJrJrJrJrJrJr  SS
KJrJr  SSKJrJrJr  SSK J!r!J"r"  SSK#J$r$J%r%  SSK&J'r'  SSK(J)r)J*r*J+r+J,r,  SSK-J.r.  SSK/J0r0  SSK1J2r2J3r3J4r4J5r5J6r6J7r7J8r8J9r9J:r:  SSK;J<r<J=r=J>r>J?r?  SSK@JArA  \,R„                  " \C5      rD\*" SS9\ " S S\0\5      5       5       rE\*" SS9\ " S S\5      5       5       rF " S S\?5      rG " S S\<5      rH " S  S!\R’                  5      rJ " S" S#\45      rK " S$ S%\75      rL " S& S'\85      rM " S( S)\25      rN " S* S+\5      rOSrP " S, S-\65      rQS.\RS/\\R\R\R\R/\S4   4S0 jrT " S1 S2\55      rU " S3 S4\35      rV " S5 S6\R®                  5      rXSLS7\R²                  S8\R´                  S-  S/\R²                  4S9 jjr[S:\S;\R²                  S<\R²                  S-  S=\S-  S>\R²                  S-  S?\R²                  S/\\4S@ jr] " SA SB\>5      r^ " SC SD\=5      r_\*" SESF9 " SG SH\\Q5      5       r` " SI SJ\\Q5      ra/ SKQrbg)Mé    )ÚCallable)ÚAnyN)Ústricté   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocasté   )ÚGemma2Config)	ÚGemma2AttentionÚGemma2ForCausalLMÚ	Gemma2MLPÚGemma2ModelÚGemma2PreTrainedModelÚGemma2RMSNormÚGemma2RotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚPaliGemmaCausalLMOutputWithPastÚ!PaliGemmaForConditionalGenerationÚPaliGemmaModelÚPaligemmaModelOutputWithPast)ÚSiglipVisionConfigzgoogle/gemma-3-4b-it)Ú
checkpointc            
       óø   ^ • \ rS rSr% SrSrSSSSSSSSSS.	rSS	S
.rSr\	\
S'   Sr\	\
S'   Sr\\   S-  \
S'   Sr\S-  \
S'   Sr\S-  \
S'   Sr\S-  \
S'   Sr\S-  \
S'   S rS\\\4   4U 4S jjrS rSrU =r$ )ÚGemma3TextConfigéD   a�  
query_pre_attn_scalar (`float`, *optional*, defaults to 256):
    scaling factor used on the attention scores
final_logit_softcapping (`float`, *optional*):
    Scaling factor when applying tanh softcapping on the logits.
attn_logit_softcapping (`float`, *optional*):
    Scaling factor when applying tanh softcapping on the attention scores.
use_bidirectional_attention (`bool`, *optional*, defaults to `False`):
    If True, the model will attend to all text tokens instead of using a causal mask. This does not change
    behavior for vision tokens.

```python
>>> from transformers import Gemma3TextModel, Gemma3TextConfig
>>> # Initializing a Gemma3Text gemma3_text-7b style configuration
>>> configuration = Gemma3TextConfig()
>>> # Initializing a model from the gemma3_text-7b style configuration
>>> model = Gemma3TextModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
Úgemma3_textÚcolwiseÚreplicated_with_grad_allreduceÚrowwise)	zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projg    €„.Ag     ˆÃ@)ÚglobalÚlocali@  Ú
vocab_sizei   Úmax_position_embeddingsNÚlayer_typesÚfinal_logit_softcappingÚattn_logit_softcappingÚrope_parametersFÚuse_bidirectional_attentionc                 ól  • U R                   (       a  U R                  S-  S-   U l        UR                  SS5      U l        U R                  cM  [        U R                  5       Vs/ s H'  n[        US-   U R                  -  5      (       a  SOSPM)     snU l        [        R                  " S0 UD6  g s  snf )Nr!   é   Úsliding_window_patterné   Úsliding_attentionÚfull_attention© )
rA   Úsliding_windowÚgetÚ_sliding_window_patternr=   ÚrangeÚnum_hidden_layersÚboolr
   Ú__post_init__)ÚselfÚkwargsÚis      Ú·C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/gemma3/modular_gemma3.pyrO   ÚGemma3TextConfig.__post_init__s   sª   € Ø×+×+Ø#'×#6Ñ#6¸!Ñ#;¸qÑ"@ˆDÔð (.§z¡zÐ2JÈAÓ'NˆÔ$à×ÑÑ#ô ˜t×5Ñ5Ô6ó â6�Aô (,¨Q°©U°d×6RÑ6RÑ,R×'SÑ'SÑ#ÐYiÒiÙ6ñ ˆDÔô
 	×&Ò&Ñ0¨Ó0ùò s   Á$.B1Úreturnc                 ór   >• [         TU ]  5       n[        U SS5      (       a  U R                  S-
  S-  US'   U$ )NrA   FrC   r!   rI   )ÚsuperÚto_dictÚgetattrrI   )rP   ÚoutputÚ	__class__s     €rS   rX   ÚGemma3TextConfig.to_dict‚   sA   ø€ Ü‘‘Ó"ˆô �4Ð6¸×>Ñ>Ø(,×(;Ñ(;¸aÑ(?À1Ñ'DˆFÐ#Ñ$Øˆó    c                 ó‚  • UR                  SS 5      nSS0SS0S.nU R                  b  U R                  OUU l        Ub  U R                  S   R                  U5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  SU R
                  S   5      5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  S	U R
                  S
   5      5        U R                  5         U$ )NÚrope_scalingÚ	rope_typeÚdefault)rF   rG   rG   Ú
rope_thetar9   rF   Úrope_local_base_freqr:   )Úpopr@   ÚupdaterJ   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rP   rQ   r_   Údefault_rope_paramss       rS   Úconvert_rope_params_to_dictÚ,Gemma3TextConfig.convert_rope_params_to_dict‹   sI  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÑ7W˜t×3Ò3Ð]pˆÔØÑ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGð ×Ñ×#Ñ#Ð$4Ó5Ñ=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9Ø˜&Ÿ*™* \°4×3EÑ3EÀhÑ3OÓPô	
ð ×Ñ×#Ñ#Ð$7Ó8Ñ@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™*Ð%;¸T×=OÑ=OÐPWÑ=XÓYô	
ð
 	×$Ñ$Ô&Øˆr]   )rK   r=   r@   rI   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚbase_model_tp_planrg   r;   ÚintÚ__annotations__r<   r=   ÚlistÚstrr>   Úfloatr?   r@   ÚdictrA   rN   rO   r   rX   rj   Ú__static_attributes__Ú__classcell__©r[   s   @rS   r3   r3   D   sÄ   ø‡ ñð, €Jà%.Ø%.Ø%.Ø%EØ%EØ%.Ø"+Ø )Ø"+ñ
Ðð  +°XÑ>€Mà€J�ÓØ#*Ð˜SÓ*Ø$(€K��c‘˜TÑ!Ó(Ø,0Ð˜U T™\Ó0Ø+/Ð˜E D™LÓ/Ø#'€O�T˜D‘[Ó'Ø/4Ð ¨¡Ó4ò1ð˜˜c 3˜h™÷ ÷ð r]   r3   c                   ó  ^ • \ rS rSr% SrSrSSSS.r\\S.r	S	r
\\\\4   -  S	-  \S
'   S	r\\\\4   -  S	-  \S'   Sr\S	-  \S'   Sr\S	-  \S'   Sr\S	-  \S'   Sr\S	-  \S'   Sr\S	-  \S'   Sr\S	-  \S'   U 4S jrSrU =r$ )ÚGemma3Configé©   aÃ  
mm_tokens_per_image (`int`, *optional*, defaults to 256):
    The number of tokens per image embedding.
boi_token_index (`int`, *optional*, defaults to 255999):
    The begin-of-image token index to wrap the image prompt.
eoi_token_index (`int`, *optional*, defaults to 256000):
    The end-of-image token index to wrap the image prompt.

Example:

```python
>>> from transformers import Gemma3ForConditionalGeneration, Gemma3Config, SiglipVisionConfig, Gemma3TextConfig

>>> # Initializing a Siglip-like vision config
>>> vision_config = SiglipVisionConfig()

>>> # Initializing a Gemma3 Text config
>>> text_config = Gemma3TextConfig()

>>> # Initializing a Gemma3 gemma-3-4b style configuration
>>> configuration = Gemma3Config(vision_config, text_config)

>>> # Initializing a model from the gemma-3-4b style configuration
>>> model = Gemma3ForConditionalGeneration(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```Úgemma3Úimage_token_indexÚboi_token_indexÚeoi_token_index)Úimage_token_idÚboi_token_idÚeoi_token_id)Útext_configÚvision_configNr†   r‡   é   Úmm_tokens_per_imageiÿç i è i   g{®Gáz”?Úinitializer_rangeTÚtie_word_embeddingsc                 óÒ  >• U R                   c%  [        5       U l         [        R                  S5        O9[	        U R                   [
        5      (       a  [        S0 U R                   D6U l         [	        U R                  [
        5      (       a  [        S0 U R                  D6U l        O1U R                  c$  [        5       U l        [        R                  S5        [        TU ]$  " S0 UD6  g )Nz@text_config is None, using default Gemma3TextConfig text config.zFvision_config is None, using default SiglipVisionConfig vision config.rH   )
r†   r3   ÚloggerÚinfoÚ
isinstancerx   r‡   r0   rW   rO   )rP   rQ   r[   s     €rS   rO   ÚGemma3Config.__post_init__Ý   s­   ø€ Ø×ÑÑ#Ü/Ó1ˆDÔÜ�K‰KÐZÕ[Ü˜×(Ñ(¬$×/Ñ/Ü/ÑC°$×2BÑ2BÑCˆDÔä�d×(Ñ(¬$×/Ñ/Ü!3Ñ!I°d×6HÑ6HÑ!IˆDÕØ×ÑÑ'Ü!3Ó!5ˆDÔÜ�K‰KÐ`Ôaä‰ÒÑ' Ó'r]   )rl   rm   rn   ro   rp   rq   Úattribute_mapr3   r0   Úsub_configsr†   rx   rv   r   rt   r‡   r‰   rs   r�   r‚   r€   rŠ   rw   r‹   rN   rO   ry   rz   r{   s   @rS   r}   r}   ©   sÍ   ø‡ ñð: €Jà-Ø)Ø)ñ€Mð (Ø+ñ€Kð
 =A€KÐ! D¨¨c¨¡NÑ2°TÑ9Ó@Ø@D€MÐ%¨¨S°#¨X©Ñ6¸Ñ=ÓDØ&)Ð˜˜t™Ó)Ø")€O�S˜4‘ZÓ)Ø")€O�S˜4‘ZÓ)Ø$+Ð�s˜T‘zÓ+Ø&*Ð�u˜t‘|Ó*Ø'+Ð˜ ™Ó+÷(ó (r]   r}   c                   ó   • \ rS rSrSrg)ÚGemma3ModelOutputWithPastéí   rH   N©rl   rm   rn   ro   ry   rH   r]   rS   r”   r”   í   ó   † Úr]   r”   c                   ó   • \ rS rSrSrg)ÚGemma3CausalLMOutputWithPastéñ   rH   Nr–   rH   r]   rS   r™   r™   ñ   r—   r]   r™   c            	       ól   ^ • \ rS rSrSrSS\S\S\S\4U 4S jjjrS\R                  4U 4S	 jjr
S
rU =r$ )ÚGemma3TextScaledWordEmbeddingéõ   zT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 óŽ   >• [         TU ]  XU5        X@l        [        R                  " [
        R                  " U5      SS9U l        g )NF©Ú
persistent)rW   Ú__init__Úscalar_embed_scaleÚnnÚBufferÚtorchÚtensorr¡   )rP   rž   rŸ   r    r¡   r[   s        €rS   r¥   Ú&Gemma3TextScaledWordEmbedding.__init__ú   s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÑQˆÕr]   Ú	input_idsc                 ó‚   >• [         TU ]  U5      U R                  R                  U R                  R
                  5      -  $ ©N)rW   Úforwardr¡   ÚtoÚweightÚdtype)rP   r¬   r[   s     €rS   r¯   Ú%Gemma3TextScaledWordEmbedding.forwardÿ   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRr]   )r¡   r¦   )ç      ð?)rl   rm   rn   ro   rp   rs   rw   r¥   r©   ÚTensorr¯   ry   rz   r{   s   @rS   rœ   rœ   õ   sM   ø† ññR sð R¸3ð RÈSð RÐ_d÷ Rð Rð
S §¡÷ Sõ Sr]   rœ   c                   ó0   ^ • \ rS rSrS\4U 4S jjrSrU =r$ )Ú	Gemma3MLPi  Úconfigc                 ó$   >• [         TU ]  U5        g r®   ©rW   r¥   ©rP   r¸   r[   s     €rS   r¥   ÚGemma3MLP.__init__  s   ø€ Ü‰Ñ˜Õ r]   rH   )rl   rm   rn   ro   r3   r¥   ry   rz   r{   s   @rS   r·   r·     s   ø† ð!Ð/÷ !õ !r]   r·   c                   ó8   ^ • \ rS rSrSS\S\4U 4S jjjrSrU =r$ )ÚGemma3RMSNormi  ÚdimÚepsc                 ó    >• [         TU ]  XS9  g )N©r¿   rÀ   rº   )rP   r¿   rÀ   r[   s      €rS   r¥   ÚGemma3RMSNorm.__init__	  s   ø€ Ü‰Ñ˜SÐÒ*r]   rH   )g�íµ ÷Æ°>)	rl   rm   rn   ro   rs   rw   r¥   ry   rz   r{   s   @rS   r¾   r¾     s   ø† ñ+˜Cð + e÷ +ö +r]   r¾   c            	       óž   • \ rS rSrS
S\4S jjr SS\S\S-  S\\R                  \
4   4S jjr\R                  " 5       \S 5       5       rS	rg)ÚGemma3RotaryEmbeddingi  Nr¸   c           
      ó¾  • [         R                  R                  U 5        UR                  U l        UR                  U l        Xl        [        [        UR                  5      5      U l	        0 U l
        U R                   HÜ  nU R                  R                  U   nUc  M!  US   U R                  U'   U R                  nU R                  U   S:w  a  [        U R                  U      nU" U R                  X#S9u  pg[        X S3[         R                  " USS95        [        X S3[         R                  " UR!                  5       SS95        [        X S3U5        MÞ     g )	Nr`   ra   ©Ú
layer_typeÚ	_inv_freqFr£   Ú_original_inv_freqÚ_attention_scaling)r§   ÚModuler¥   r<   Úmax_seq_len_cachedÚoriginal_max_seq_lenr¸   ÚsortedÚsetr=   r`   r@   Úcompute_default_rope_parametersr   Úsetattrr¨   Úclone)rP   r¸   ÚdevicerÈ   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalings           rS   r¥   ÚGemma3RotaryEmbedding.__init__  s-  € Ü
�	‰	×Ñ˜4Ô Ø"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÑAˆKØÑ"Ùà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÓ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÈfÑ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ñ4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÑ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTò +r]   rÈ   rU   c           	      ó  • U R                   U   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
rb   Úhead_dimNr´   r   r!   ©r²   )r@   rY   Úhidden_sizeÚnum_attention_headsr©   Úarangerw   r°   )r¸   rÔ   rÈ   rQ   Úbaser¿   Úattention_factorÚinv_freqs           rS   rÑ   Ú5Gemma3RotaryEmbedding.compute_default_rope_parameters#  s€   € ð" ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r]   c                 ó  • [        X S35      n[        X S35      nUS S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        US	S
9   Xg-  R                  SS5      n	[        R                  " X™4SS9n
U
R                  5       U-  nU
R                  5       U-  nS S S 5        WR                  UR                  S9WR                  UR                  S94$ ! , (       d  f       N@= f)NrÉ   rË   r   éÿÿÿÿrC   )r²   rÔ   ÚmpsÚcpuF)Údevice_typeÚenabledr!   ©r¿   rÜ   )rY   ÚexpandÚshaper°   r©   rw   rÔ   r�   Útyperv   r    Ú	transposeÚcatÚcosÚsinr²   )rP   ÚxÚposition_idsrÈ   râ   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrè   ÚfreqsÚembrð   rñ   s                rS   r¯   ÚGemma3RotaryEmbedding.forward<  sS  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �Tš1˜d�]Ñ#×*Ñ*¨<×+=Ñ+=¸aÑ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÐoð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   Ã*AE5Å5
F)r¸   r=   rÍ   rÎ   r`   r®   )NN)rl   rm   rn   ro   r3   r¥   rv   Útupler©   rµ   rw   rÑ   Úno_gradr   r¯   ry   rH   r]   rS   rÅ   rÅ     si   † ñUÐ/õ Uð, IMñ5Ø ð5Ø;>À¹:ð5à	ˆu�|‰|˜UÐ"Ñ	#õ5ð2 ‡]‚]ƒ_Øñ<ó ó ó<r]   rÅ   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr   SS\R                  S\R                  S\R                  S-  S	\	S-  S
\
\   S\\R                  \R                  S-  \\R                     S-  4   4S jjrSrU =r$ )ÚGemma3AttentioniS  r¸   Ú	layer_idxc                 ó^  >• [         TU ]  X5        U R                  S:X  a  UR                  OS U l        U R                  S:H  U l        U R
                  R                  (       + U l        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        g )NrF   rÂ   )rW   r¥   rÈ   rI   Ú
is_slidingr¸   rA   Ú	is_causalr¾   rÛ   Úrms_norm_epsÚq_normÚk_norm©rP   r¸   rþ   r[   s      €rS   r¥   ÚGemma3Attention.__init__T  s‚   ø€ Ü‰Ñ˜Ô+Ø7;·±ÐJ]Ó7]˜f×3Ò3ÐcgˆÔØŸ/™/Ð-@Ñ@ˆŒØ!Ÿ[™[×DÑDÔDˆŒä#¨¯©¸V×=PÑ=PÑQˆŒÜ#¨¯©¸V×=PÑ=PÑQˆ�r]   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesrQ   rU   c                 ób  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  U5      nU R                  U	5      n	Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [         5      nU" U UU	U
U4U R"                  (       a  U R$                  OSU R&                  U R(                  S.UD6u  pïUR*                  " / UQSP76 R-                  5       nU R/                  U5      nXï4$ )Nrå   rC   r!   g        )ÚdropoutÚscalingrI   )rì   rÛ   Úq_projÚviewrî   Úk_projÚv_projr  r  r*   re   rþ   r   Úget_interfacer¸   Ú_attn_implementationr+   ÚtrainingÚattention_dropoutr  rI   ÚreshapeÚ
contiguousÚo_proj)rP   r  r  r	  r
  rQ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrð   rñ   Úattention_interfaceÚattn_outputÚattn_weightss                   rS   r¯   ÚGemma3Attention.forward]  s£  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð /3¯m¯m�D×*Ò*ÀØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r]   )r  r   r  r  rI   )NNN)rl   rm   rn   ro   r3   rs   r¥   r©   rµ   r   r   r   rú   r¯   ry   rz   r{   s   @rS   rý   rý   S  s«   ø† ðRÐ/ð R¸C÷ Rð -1Ø.2Ø(,ñ*)à—|‘|ð*)ð #Ÿ\™\ð*)ð Ÿ™ tÑ+ð	*)ð
  ™ð*)ð Ð+Ñ,ð*)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷*)ó *)r]   rý   c                   ó$  ^ • \ rS rSrS\S\4U 4S jjr    SS\R                  S\R                  S\R                  S-  S	\R                  S-  S
\
S-  S\\   S\\R                  \\R                  \R                  4   S-  4   4S jjrSrU =r$ )ÚGemma3DecoderLayeriŠ  r¸   rþ   c                 ó°  >• [         TU ]  5         Xl        UR                  U l        X l        [        XS9U l        [        U5      U l        [        U R                  UR                  S9U l        [        U R                  UR                  S9U l        [        U R                  UR                  S9U l        [        U R                  UR                  S9U l        g )N)r¸   rþ   ©rÀ   )rW   r¥   r¸   rÝ   rþ   rý   Ú	self_attnr·   Úmlpr¾   r  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr  s      €rS   r¥   ÚGemma3DecoderLayer.__init__‹  sª   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ"ŒÜ(°ÑLˆŒÜ˜VÓ$ˆŒÜ,¨T×-=Ñ-=À6×CVÑCVÑWˆÔÜ(5°d×6FÑ6FÈF×L_ÑL_Ñ(`ˆÔ%Ü)6°t×7GÑ7GÈV×M`ÑM`Ñ)aˆÔ&Ü*7¸×8HÑ8HÈf×NaÑNaÑ*bˆÕ'r]   Nr  r  r	  ró   r
  rQ   rU   c           	      óü   • UnU R                  U5      nU R                  " SUUUUUS.UD6u  pU R                  U5      nXq-   nUnU R                  U5      nU R	                  U5      nU R                  U5      nXq-   nU$ )N)r  r  r	  ró   r
  rH   )r(  r&  r)  r*  r'  r+  )	rP   r  r  r	  ró   r
  rQ   ÚresidualÚ_s	            rS   r¯   ÚGemma3DecoderLayer.forward—  s£   € ð !ˆà×,Ñ,¨]Ó;ˆàŸ>š>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Ñ0ˆàÐr]   )	r¸   rÝ   r(  rþ   r'  r)  r+  r*  r&  )NNNN)rl   rm   rn   ro   r3   rs   r¥   r©   rµ   Ú
LongTensorr   r   r   rú   ÚFloatTensorr¯   ry   rz   r{   s   @rS   r#  r#  Š  sÈ   ø† ð
cÐ/ð 
c¸C÷ 
cð -1Ø.2Ø04Ø(,ñà—|‘|ðð #Ÿ\™\ðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð Ð+Ñ,ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷ó r]   r#  c                   óP   • \ rS rSrSrSrS/r\R                  " 5       S 5       r	Sr
g)ÚGemma3PreTrainedModeli»  Úmodel)ÚimageÚtextr#  c                 ó  • [         R                  " X5        [        U[        5      (       a!  [        R
                  " UR                  5        g SUR                  R                  ;   a!  [        R
                  " UR                  5        g [        U[        5      (       a,  [        R                  " UR                  UR                  5        g [        U[        5      (       a¢  UR                   H‘  nUR                   nUR"                  U   S:w  a  [$        UR"                  U      nU" UR&                  US9u  pE[        R(                  " [+        X S35      U5        [        R(                  " [+        X S35      U5        M“     g g )NÚRMSNormra   rÇ   rÉ   rÊ   )r   Ú_init_weightsr�   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightr[   rl   r±   rœ   Ú	constant_r¡   r¦   rÅ   r=   rÑ   r`   r   r¸   Úcopy_rY   )rP   ÚmodulerÈ   rÖ   r×   r/  s         rS   r:  Ú#Gemma3PreTrainedModel._init_weightsÀ  s  € ä×%Ò% dÔ3Ü�fÔ7×8Ñ8Ü�KŠK˜×9Ñ9Õ:à˜&×*Ñ*×3Ñ3Ó3Ü�KŠK˜Ÿ™Õ&Ü˜Ô =×>Ñ>Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÕIÜ˜Ô 5×6Ñ6Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ó<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ñ#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ò 1ð 7r]   rH   N)rl   rm   rn   ro   Úbase_model_prefixÚinput_modalitiesÚ_no_split_modulesr©   rû   r:  ry   rH   r]   rS   r4  r4  »  s1   † ØÐØ(ÐØ-Ð.Ðà
‡]‚]ƒ_ñ^ó ó^r]   r4  rI   rU   c           
      óT   ^ • S[         S[         S[         S[         S[        4
U 4S jjnU$ )z9
Enables a bidirectional mask within the sliding window.
Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrU   c                 ó$   >• [        X#-
  5      T:  $ )z‹A token can attend to any other token if their absolute distance is within
the (exclusive) sliding window size (distance < sliding_window).)Úabs)rG  rH  rI  rJ  rI   s       €rS   Ú
inner_maskÚ1_bidirectional_window_overlay.<locals>.inner_maskÙ  s   ø€ ô �5‘>Ó" ^Ñ3Ð3r]   )rs   rN   )rI   rM  s   ` rS   Ú_bidirectional_window_overlayrO  Ô  s3   ø€ ð
4œcð 4¬Sð 4¼ð 4Äcð 4Ìd÷ 4ð
 Ðr]   c                   óô   ^ • \ rS rSr% \\S'   SrS\4U 4S jjr      SS\R                  S-  S\R                  S-  S\R                  S-  S	\S-  S
\R                  S-  S\S-  S\\   S\4S jjrSrU =r$ )ÚGemma3TextModeliá  r¸   ©r7  c                 ó°   >• [         TU ]  U5        [        UR                  UR                  U R
                  U R                  R                  S-  S9U l        g )Nç      à?)r¡   )rW   r¥   rœ   r;   rÝ   r    r¸   Úembed_tokensr»   s     €rS   r¥   ÚGemma3TextModel.__init__å  sM   ø€ Ü‰Ñ˜Ô ô :Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmñ
ˆÕr]   Nr¬   r	  ró   r
  Úinputs_embedsÚ	use_cacherQ   rU   c           	      ó
  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U=n	[        5      (       d|  U R                  UUUUS.n
U
R                  5       nU R                  R                  (       a(  S U
S'   [        U R                  R                  5      US'   [!        S0 U
D6[#        S0 UD6S	.n	Un0 n[%        U R                  R&                  5       H  nU R)                  XÃU5      XÞ'   M     [+        U R,                  S U R                  R.                   5       HF  u  nnU" U4X�R                  R&                  U      XÐR                  R&                  U      UUS
.UD6nMH     U R1                  U5      n[3        UUS9$ )Nú:You must specify exactly one of input_ids or inputs_embeds)r¸   r   rC   ©rÔ   ©r¸   rW  r	  r
  ró   c                  óH   • [         R                  " S[         R                  S9$ )NTrÜ   )r©   rª   rN   )Úargss    rS   Ú<lambda>Ú)Gemma3TextModel.forward.<locals>.<lambda>  s   € ÄÇÂÈTÔY^×YcÑYcÒ@dr]   Úor_mask_function©rG   rF   )r	  r  ró   r
  )Úlast_hidden_stater
  rH   )Ú
ValueErrorrU  r	   r¸   Úget_seq_lengthr©   rß   rì   rÔ   Ú	unsqueezer�   rx   ÚcopyrA   rO  rI   r   r   rÐ   r=   Ú
rotary_embÚ	enumerateÚlayersrM   Únormr   )rP   r¬   r	  ró   r
  rW  rX  rQ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsr  r  rÈ   rR   Údecoder_layers                    rS   r¯   ÚGemma3TextModel.forwardí  s  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÑFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKð #.×"2Ñ"2Ó"4Ðà�{‰{×6×6Ù2d�Ð.Ñ/Ü:WÐX\×XcÑXc×XrÑXrÓ:sÐ#Ð$6Ñ7ô #5Ñ"C°{Ñ"CÜ%FÑ%]ÐI\Ñ%]ñ#Ðð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAˆ}Ù)Øðà2·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä&Ø+Ø+ñ
ð 	
r]   )rU  )NNNNNN)rl   rm   rn   ro   r3   rt   rD  r¥   r©   r1  rµ   r   r2  rN   r   r   r   r¯   ry   rz   r{   s   @rS   rQ  rQ  á  sÊ   ø‡ ØÓØ Ðð
Ð/÷ 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð ×&Ñ&¨Ñ-ð	C
ð
  ™ðC
ð ×(Ñ(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ñ,ðC
ð 
!÷C
ó C
r]   rQ  c                   ó<   ^ • \ rS rSr% \\S'   S\4U 4S jjrSrU =r$ )ÚGemma3ForCausalLMi3  r¸   c                 óD   >• [         TU ]  U5        [        U5      U l        g r®   )rW   r¥   rQ  r5  r»   s     €rS   r¥   ÚGemma3ForCausalLM.__init__6  s   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆ�
r]   )r5  )	rl   rm   rn   ro   r3   rt   r¥   ry   rz   r{   s   @rS   rs  rs  3  s   ø‡ ØÓð-Ð/÷ -õ -r]   rs  c                   óR   ^ • \ rS rSrS\4U 4S jjrS\R                  4S jrSr	U =r
$ )r;  i;  r¸   c                 óˆ  >• [         TU ]  5         [        R                  " [        R
                  " UR                  R                  UR                  R                  5      5      U l	        [        UR                  R                  UR                  R                  S9U l        [        UR                  R                  UR                  R                  -  5      U l        [        UR"                  S-  5      U l        U R                   U R$                  -  U l        [        R(                  " U R&                  U R&                  S9U l        g )Nr%  rT  )Úkernel_sizeÚstride)rW   r¥   r§   Ú	Parameterr©   Úzerosr‡   rÝ   r†   r>  r¾   Úlayer_norm_epsÚmm_soft_emb_normrs   Ú
image_sizeÚ
patch_sizeÚpatches_per_imager‰   Útokens_per_siderx  Ú	AvgPool2dÚavg_poolr»   s     €rS   r¥   Ú"Gemma3MultiModalProjector.__init__<  sì   ø€ Ü‰ÑÔä*,¯,ª,Ü�KŠK˜×,Ñ,×8Ñ8¸&×:LÑ:L×:XÑ:XÓYó+
ˆÔ'ô !.Ø× Ñ ×,Ñ,°&×2FÑ2F×2UÑ2Uñ!
ˆÔô "% V×%9Ñ%9×%DÑ%DÈ×H\ÑH\×HgÑHgÑ%gÓ!hˆÔÜ" 6×#=Ñ#=¸sÑ#BÓCˆÔØ×1Ñ1°T×5IÑ5IÑIˆÔÜŸš°×1AÑ1AÈ$×JZÑJZÑ[ˆ�r]   Úvision_outputsc                 óž  • UR                   u  p#nUR                  SS5      nUR                  X$U R                  U R                  5      nUR	                  5       nU R                  U5      nUR                  S5      nUR                  SS5      nU R                  U5      n[        R                  " XpR                  5      nUR                  U5      $ )NrC   r!   )rì   rî   r  r€  r  rƒ  Úflattenr}  r©   Úmatmulr>  Útype_as)	rP   r…  Ú
batch_sizer/  rÝ   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            rS   r¯   Ú!Gemma3MultiModalProjector.forwardL  sÇ   € Ø%3×%9Ñ%9Ñ"ˆ
�{à"0×":Ñ":¸1¸aÓ"@ÐØ"9×"AÑ"AØ T×%;Ñ%;¸T×=SÑ=Só#
Ðð #:×"DÑ"DÓ"FÐà $§¡Ð.EÓ FÐØ 5× =Ñ =¸aÓ @ÐØ 5× ?Ñ ?ÀÀ1Ó EÐà $× 5Ñ 5Ð6KÓ LÐä#(§<¢<Ð0E×GfÑGfÓ#gÐ Ø'×/Ñ/°Ó?Ð?r]   )rƒ  rx  r>  r}  r€  r�  )rl   rm   rn   ro   r}   r¥   r©   rµ   r¯   ry   rz   r{   s   @rS   r;  r;  ;  s)   ø† ð\˜|÷ \ð @ e§l¡l÷ @ò @r]   r;  Útoken_type_idsrÔ   c                 óþ   • U S:H  R                  US9n[        R                  R                  USSS9S S 2S S24   nX#) -  n[        R
                  " UR                  5       SS9S-
  n[        R                  " X%S5      nU$ )NrC   r[  )rC   r   r   )Úvaluerå   rê   )r°   r§   Ú
functionalÚpadr©   Úcumsumrs   Úwhere)r�  rÔ   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          rS   Úget_block_sequence_ids_for_maskrœ  _  s…   € ð  !Ñ#×'Ñ'¨vÐ'Ð6€HÜŸ™×)Ñ)¨(°FÀ!Ð)ÐDÂQÈÈÈÀVÑLÐØÐ!3Ñ3€OÜ—’˜_×0Ñ0Ó2¸Ñ:¸QÑ>€IÜŸš X¸"Ó=ÐØÐr]   r¸   rW  r	  r
  ró   r›  c                 ój  • U UUUUS.n[        S
0 UDSU0D6n[        USS5      (       a,  SUR                  ;   a  UR                  R                  S5      nOSn[	        S
0 UDSU0D6u  n	    p«p¬U	(       a  UnO[        XRX¼5      n[        S
0 UD[        U5      [        U R                  5      US.D6nUUS	.$ )z×Create full_attention and sliding_attention masks with correct composition.

For global (full attention) layers:  OR(causal, blockwise)
For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
r\  r›  r   NTr   rþ   )ra  Úand_mask_functionrþ   rb  rH   )	r   rY   r   Úindexr   r   r   r   rI   )r¸   rW  r	  r
  ró   r›  rn  Ú	full_maskÚsliding_layer_idxÚ
early_exitr/  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                  rS   Úcreate_masks_for_vision_modelr§  j  sò   € ð Ø&Ø(Ø*Ø$ñ€Kô #ÑX [ÑXÐEWÒX€Iô ˆ ¨d×3Ñ3¸À×@ZÑ@ZÓ8ZØ+×6Ñ6×<Ñ<¸TÓBÑàÐô 4Nñ 4Ø
ñ4à#ò4Ñ0€J��1�a Aö Ø$6Ñ!ä$@Ø°	ó%
Ð!ô &ñ Ø
ðä*Ð+DÓEÜ0°×1FÑ1FÓGØ#ó	€Lð $Ø)ñð r]   c                   ó´  ^ • \ rS rSrSrS\4U 4S jjr\\" SS9S\	R                  S\\   S	\\-  4S
 j5       5       r\\         SS\	R                   S-  S\	R                  S-  S\	R"                  S-  S\	R                   S-  S\S-  S\	R                   S-  S\	R                  S-  S\	R                   S-  S\S-  S\\   S	\\-  4S jj5       5       rSrU =r$ )ÚGemma3Modeli¦  Fr¸   c                 ó(   >• [         TU ]  U5        U ?g r®   )rW   r¥   Útext_config_dtyper»   s     €rS   r¥   ÚGemma3Model.__init__ª  s   ø€ Ü‰Ñ˜Ô ØÑ"r]   zOProjects the last hidden state from the vision model into language model space.©Úcustom_introÚpixel_valuesrQ   rU   c                 ót   • U R                   " SUSS.UD6nUR                  nU R                  U5      Ul        U$ )NT)r¯  Úreturn_dictrH   )Úvision_towerrc  Úmulti_modal_projectorÚpooler_output)rP   r¯  rQ   r…  rc  s        rS   Úget_image_featuresÚGemma3Model.get_image_features®  sF   € ð
 ×*Ò*Ða¸ÐRVÑaÐZ`ÑaˆØ*×<Ñ<ÐØ'+×'AÑ'AÐBSÓ'TˆÔ$àÐr]   Nr¬   r	  ró   r
  r�  rW  ÚlabelsrX  Ú	lm_kwargsc
           
      óP  • US L US L-  (       a  [        S5      eUbQ  U R                  R                  U R                  :¼  a-  XR                  R                  :H  nUR	                  5       nSXË'   OUnUc  U R                  5       " U5      nUba  U R                  USS9R                  nUR                  UR                  UR                  5      nU R                  XUS9nUR                  X½5      n[        U=n[        5      (       dO  U R                  R                  5       UUUUS.nUb!  [!        XgR                  S9n[#        SSU0UD6nO[%        S0 UD6nU R&                  " SUUUUU	SS	.U
D6n[)        UR*                  UR,                  UR.                  UR0                  Ub  WS
9$ S S
9$ )NrZ  r   T)r±  )rW  Úimage_featuresr\  r[  r›  )r	  ró   r
  rW  rX  r±  )rc  r
  r  Ú
attentionsÚimage_hidden_statesrH   )rd  r¸   rƒ   r;   rÓ   Úget_input_embeddingsrµ  r´  r°   rÔ   r²   Úget_placeholder_maskÚmasked_scatterr�   rx   Úget_text_configrœ  r§  r   Úlanguage_modelr”   rc  r
  r  r»  )rP   r¬   r¯  r	  ró   r
  r�  rW  r·  rX  r¸  Úspecial_image_maskÚllm_input_idsrº  rm  rn  r›  Úoutputss                     rS   r¯   ÚGemma3Model.forward¹  sí  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZð Ñ  T§[¡[×%?Ñ%?À4Ç?Á?Ó%RØ!*¯k©k×.HÑ.HÑ!HÐØ%ŸO™OÓ-ˆMØ01ˆMÒ-à%ˆMàÑ Ø ×5Ñ5Ô7¸ÓFˆMð Ñ#Ø!×4Ñ4°\ÈtÐ4ÐT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:ØÀ~ð ";ð "Ðð *×8Ñ8Ð9KÓ\ˆMô °Ð?Ð-Ä×FÑFàŸ+™+×5Ñ5Ó7Ø!.Ø"0Ø#2Ø ,ñˆKð Ñ)Ü%DÀ^×\pÑ\pÑ%qÐ"Ü&Cñ 'Ø'9ð'à!ñ'Ñ#ô
 '@Ñ&NÀ+Ñ&NÐ#à×%Ò%ð 
Ø.Ø%Ø+Ø'ØØñ
ð ñ
ˆô )Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ñ2J ñ
ð 	
ð
 QUñ
ð 	
r]   rH   )	NNNNNNNNN)rl   rm   rn   ro   Úaccepts_loss_kwargsr}   r¥   r   r   r©   r2  r   r   rú   r   rµ  r1  rµ   r   rN   r”   r¯   ry   rz   r{   s   @rS   r©  r©  ¦  ss  ø† àÐð#˜|÷ #ð ÙÐ!rÑsðØ!×-Ñ-ðØ9?Ð@RÑ9Sðà	Ð+Ñ	+óó tó ðð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ñG
à×#Ñ# dÑ*ðG
ð ×'Ñ'¨$Ñ.ðG
ð Ÿ™ tÑ+ð	G
ð
 ×&Ñ&¨Ñ-ðG
ð  ™ðG
ð ×(Ñ(¨4Ñ/ðG
ð ×(Ñ(¨4Ñ/ðG
ð × Ñ  4Ñ'ðG
ð ˜$‘;ðG
ð Ð.Ñ/ðG
ð 
Ð*Ñ	*ôG
ó ó öG
r]   r©  c                   ó  ^ • \ rS rSr\\          SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\
S-  S\R                  S-  S	\R                  S-  S
\R                  S-  S\S-  S\\R                  -  S\\   S\\-  4S jj5       5       rSU 4S jjr  SS\S	\R                  S\R                  S-  S\
S-  S\R                  S-  S\R                  S-  S\S-  S\4S jjrSrU =r$ )ÚGemma3ForConditionalGenerationi  Nr¬   r¯  r	  ró   r
  r�  rW  r·  rX  Úlogits_to_keepr¸  rU   c                 ó   • U R                   " SUUUUUUUU	USS.
UD6nUS   n[        U
[        5      (       a  [        U
* S5      OU
nU R	                  USS2USS24   5      nSnUb3  U R
                  " SXøU R                  R                  R                  S.UD6n[        UUUR                  UR                  UR                  UR                  S9$ )aÉ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

>>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
>>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

>>> messages = [
...     {
...         "role": "system",
...         "content": [
...             {"type": "text", "text": "You are a helpful assistant."}
...         ]
...     },
...     {
...         "role": "user", "content": [
...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
...             {"type": "text", "text": "Where is the cat standing?"},
...         ]
...     },
... ]

>>> inputs = processor.apply_chat_template(
...     messages,
...     tokenize=True,
...     return_dict=True,
...     return_tensors="pt",
...     add_generation_prompt=True
... )
>>> # Generate
>>> generate_ids = model.generate(**inputs)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
```
T)
r¬   r¯  r�  r	  ró   r
  rW  rX  r·  r±  r   N)Úlogitsr·  r;   )ÚlossrË  r
  r  r»  r¼  rH   )r5  r�   rs   ÚsliceÚlm_headÚloss_functionr¸   r†   r;   r™   r
  r  r»  r¼  )rP   r¬   r¯  r	  ró   r
  r�  rW  r·  rX  rÉ  r¸  rÄ  r  Úslice_indicesrË  rÌ  s                    rS   r¯   Ú&Gemma3ForConditionalGeneration.forward  sù   € ðz —*’*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØñ
ð ñ
ˆð   ™
ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%ð Ø¸¿¹×9PÑ9P×9[Ñ9[ñØ_hñˆDô ,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ñ
ð 	
r]   c                 óT   >• [         TU ]  " U4X#S.UD6nU(       d  U(       a  S US'   U$ )N)rX  Úis_first_iterationr�  )rW   Úprepare_inputs_for_generation)rP   r¬   rX  rÓ  rQ   Úmodel_inputsr[   s         €rS   rÔ  Ú<Gemma3ForConditionalGeneration.prepare_inputs_for_generatione  s=   ø€ Ü‘wÒ<Øð
Ø!*ñ
ØU[ñ
ˆö #®)à-1ˆLÐ)Ñ*àÐr]   r¸   rÓ  c                 óŠ   • U R                  5       UUUUS.nUb   [        XQR                  S9n	[        SSU	0UD6$ [	        S0 UD6$ )Nr\  r[  r›  rH   )rÀ  rœ  rÔ   r§  r   )
r¸   rW  r	  r
  ró   r�  rÓ  rQ   rn  r›  s
             rS   r   Ú8Gemma3ForConditionalGeneration.create_masks_for_generateo  si   € ð ×,Ñ,Ó.Ø*Ø,Ø.Ø(ñ
ˆð Ñ%Ü!@À×XlÑXlÑ!mÐÜ0ñ Ø#5ðàñð ô
 )Ñ7¨;Ñ7Ð7r]   rH   )
NNNNNNNNNr   )TF)NF)rl   rm   rn   ro   r   r   r©   r1  r2  rµ   r   rN   rs   r   r   rú   r™   r¯   rÔ  r
   rx   r   ry   rz   r{   s   @rS   rÈ  rÈ    s´  ø† ØØð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ñ[
à×#Ñ# dÑ*ð[
ð ×'Ñ'¨$Ñ.ð[
ð Ÿ™ tÑ+ð	[
ð
 ×&Ñ&¨Ñ-ð[
ð  ™ð[
ð ×(Ñ(¨4Ñ/ð[
ð ×(Ñ(¨4Ñ/ð[
ð × Ñ  4Ñ'ð[
ð ˜$‘;ð[
ð ˜eŸl™lÑ*ð[
ð Ð.Ñ/ð[
ð 
Ð-Ñ	-ô[
ó ó ð[
÷zð  /3Ø*/ñ8Ø ð8à—|‘|ð8ð Ÿ™ tÑ+ð8ð  ™ð	8ð
 —l‘l TÑ)ð8ð Ÿ™ tÑ+ð8ð ! 4™Kð8ð 
÷8ó 8r]   rÈ  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.r­  c                   ó$   • \ rS rSr% \\S'   SrSrg)Ú#Gemma3TextForSequenceClassificationi‹  r¸   rR  rH   N)rl   rm   rn   ro   r3   rt   rD  ry   rH   r]   rS   rÚ  rÚ  ‹  s   ‡ ð ÓØ Ór]   rÚ  c                   ó*  ^ • \ rS rSr        SS\R
                  S-  S\R                  S-  S\R                  S-  S\R
                  S-  S\S-  S\R
                  S-  S	\R                  S-  S
\R
                  S-  S\	\
   S\4U 4S jjjrSrU =r$ )ÚGemma3ForSequenceClassificationi•  Nr¬   r¯  r	  ró   r
  r�  rW  r·  rQ   rU   c	                 ó6   >• [         T
U ]  " SUUUUUUUUS.U	D6$ )N)r¬   r	  ró   r
  rW  r¯  r�  r·  rH   )rW   r¯   )rP   r¬   r¯  r	  ró   r
  r�  rW  r·  rQ   r[   s             €rS   r¯   Ú'Gemma3ForSequenceClassification.forward–  s=   ø€ ô ‰wŠð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øñ

ð ñ

ð 
	
r]   rH   )NNNNNNNN)rl   rm   rn   ro   r©   r1  r2  rµ   r   r   r   r   r¯   ry   rz   r{   s   @rS   rÜ  rÜ  •  sÞ   ø† ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ñ
à×#Ñ# dÑ*ð
ð ×'Ñ'¨$Ñ.ð
ð Ÿ™ tÑ+ð	
ð
 ×&Ñ&¨Ñ-ð
ð  ™ð
ð ×(Ñ(¨4Ñ/ð
ð ×(Ñ(¨4Ñ/ð
ð × Ñ  4Ñ'ð
ð Ð+Ñ,ð
ð 
*÷
ö 
r]   rÜ  )	r}   r3   r4  rQ  rs  rÈ  r©  rÜ  rÚ  r®   )cÚcollections.abcr   Útypingr   r©   Útorch.nnr§   Úhuggingface_hub.dataclassesr   Ú r   r<  Úcache_utilsr   r	   Úconfiguration_utilsr
   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr    Úgemma2.configuration_gemma2r"   Úgemma2.modeling_gemma2r#   r$   r%   r&   r'   r(   r)   r*   r+   Úpaligemma.modeling_paligemmar,   r-   r.   r/   Úsiglipr0   Ú
get_loggerrl   r�   r3   r}   r”   r™   Ú	Embeddingrœ   r·   r¾   rÅ   rý   r#  ÚGEMMA3_START_DOCSTRINGr4  rs   rN   rO  rQ  rs  rÌ   r;  rµ   rÔ   rœ  rx   r§  r©  rÈ  rÚ  rÜ  Ú__all__rH   r]   rS   Ú<module>rö     s±  ðõ %Ý ã Ý Ý .å &ß .Ý 3÷÷ ñ ÷ \ß uÑ u÷÷ GÝ &ß RÓ RÝ +Ý 6÷
÷ 
õ 
÷ó õ (ð 
×	Ò	˜HÓ	%€ñ Ð1Ñ2Øô`�|Ð%5ó `ó ó 3ð`ñF Ð1Ñ2Øô?(Ð#ó ?(ó ó 3ð?(ôD	Ð <ô 	ô	Ð#Bô 	ôS B§L¡Lô Sô!�	ô !ô
+�Mô +ô
B<Ð1ô B<ôL4)�oô 4)ôn+Ð3ô +ð\ Ð ô^Ð1ô ^ð2
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ñ:^ô 
ôO
�kô O
ôd-Ð)ô -ô!@ §	¡	ô !@ñH°E·L±Lð È%Ï,É,ÐY]ÑJ]ð Ðin×iuÑiuõ ð9Øð9à—<‘<ð9ð —L‘L 4Ñ'ð9ð ˜T‘\ð	9ð
 —,‘, Ñ%ð9ð Ÿ™ð9ð 
ô9ôx\
�.ô \
ô~C8Ð%Fô C8ñL ð^ñô
!Ð*JÐLaó !óð
!ô

Ð&FÐH]ô 
ò4
�r]   