ó
    ˆŽ¡jÒo  ã                   ó†  • S SK r S SKJr  S SKJr  S SKrS SKJr  S SKJ	r	  S SKJ
r
JrJr  SSKJr  SSKJr  SS	KJrJrJrJr  SS
KJr  SSKJr  SSKJrJrJr  SSKJ r   SSK!J"r"J#r#J$r$  SSK%J&r&  SSK'J(r(J)r)  \RT                  " \+5      r,\" SS9\	 " S S\5      5       5       r-\ " S S\5      5       r.\ " S S\5      5       r/ " S S\R`                  5      r1\ " S S\)5      5       r2\" SS9 " S  S!\(5      5       r3 " S" S#\&5      r4\ " S$ S%\25      5       r5\" S&S9 " S' S(\25      5       r6\" S)S9 " S* S+\25      5       r7/ S,Qr8g)-é    N)Ú	dataclass)ÚLiteral)Ústrict)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚPreTrainedConfig)ÚBaseModelOutputÚMaskedLMOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleé   )ÚCONFIG_MAPPINGÚ
AutoConfigÚ	AutoModel)ÚModernBertPredictionHead)ÚSmolVLMModelÚSmolVLMPreTrainedModelúModernVBERT/modernvbert)Ú
checkpointc                   óì   ^ • \ rS rSr% SrSr\\S.rSr\	\
-  S-  \S'   Sr\	\
-  S-  \S'   Sr\\S	'   S
r\\S'   Sr\\S'   Sr\\S'   Sr\S   \S'   Sr\\-  \S'   Sr\\S'   Sr\\S'   U 4S jrSrU =r$ )ÚModernVBertConfigé,   aï  
pixel_shuffle_factor (`int | None`, *optional*, defaults to 4):
    Scale factor used by any pixel-shuffle / upsampling operations in the vision head.
initializer_cutoff_factor (`float | None`, *optional*, defaults to 2.0):
    The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
classifier_pooling (`Literal["cls", "mean"]`, *optional*, defaults to `"cls"`):
    The pooling strategy to use for classification tasks.
classifier_bias (`bool | None`, *optional*, defaults to `False`):
    Whether to add a bias term to the classification head

Example:
```python
>>> from transformers import ModernVBertConfig

>>> # Initializing configuration
>>> configuration = ModernVBertConfig()

>>> # Initializing a model from the configuration (model class is implemented in
>>> # `modernvbert.modeling_modernvbert`)

>>> from transformers import ModernVBertModel
>>> model = ModernVBertModel(configuration)

>>> # Accessing the model configuration
>>> cfg = model.config
```Úmodernvbert)Útext_configÚvision_configNr#   r$   içÄ  Úimage_token_idé   Úpixel_shuffle_factorg{®Gáz”?Úinitializer_rangeç       @Úinitializer_cutoff_factorÚcls)r+   ÚmeanÚclassifier_poolingç        Úclassifier_dropoutFÚclassifier_biasÚtie_word_embeddingsc                 óž  >• U R                   c  [        S   " 5       U l         O=[        U R                   [        5      (       a  [        S   " S0 U R                   D6U l         U R                  c  [        S   " 5       U l        O=[        U R                  [        5      (       a  [        S   " S0 U R                  D6U l        [
        TU ]  " S0 UD6  g )NÚ
modernbertÚsiglip_vision_model© )r#   r   Ú
isinstanceÚdictr$   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €ÚÁC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/modernvbert/modular_modernvbert.pyr9   ÚModernVBertConfig.__post_init__X   s©   ø€ Ø×ÑÑ#Ü-¨lÒ;Ó=ˆDÕÜ˜×(Ñ(¬$×/Ñ/Ü-¨lÒ;ÑO¸d×>NÑ>NÑOˆDÔà×ÑÑ%Ü!/Ð0EÒ!FÓ!HˆDÕÜ˜×*Ñ*¬D×1Ñ1Ü!/Ð0EÒ!FÑ!\È×I[ÑI[Ñ!\ˆDÔä‰ÒÑ' Ó'ó    )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typer   Úsub_configsr#   r   r7   Ú__annotations__r$   r%   Úintr'   r(   Úfloatr*   r-   r   r/   r0   Úboolr1   r9   Ú__static_attributes__Ú__classcell__©r<   s   @r=   r    r    ,   s¬   ø‡ ñð6 €JØ",¸zÑJ€Kà26€KÐ! DÑ(¨4Ñ/Ó6Ø48€MÐ# dÑ*¨TÑ1Ó8Ø€N�CÓØ !Ð˜#Ó!Ø#Ð�uÓ#Ø'*Ð˜uÓ*Ø16Ð˜ Ñ.Ó6Ø&)Ð˜ ™Ó)Ø!€O�TÓ!Ø %Ð˜Ó%÷(ó (r?   r    c                   óÆ   • \ rS rSr% SrSr\R                  \S'   Sr	\
\R                     S-  \S'   Sr\
\R                     S-  \S'   Sr\
\R                     S-  \S'   Srg)	ÚModernVBertBaseModelOutputéf   a	  
Base class for ModernVBERT model's outputs.
Args:
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.
        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
        Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.
        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder
NÚlast_hidden_stateÚhidden_statesÚ
attentionsÚimage_hidden_statesr5   )r@   rA   rB   rC   rD   rQ   ÚtorchÚFloatTensorrG   rR   ÚtuplerS   rT   rK   r5   r?   r=   rO   rO   f   si   ‡ ñð, ,0Ð�u×(Ñ(Ó/Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø;?Ð˜˜u×0Ñ0Ñ1°DÑ8Ö?r?   rO   c                   óð   • \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  \S'   Sr
\\R                  S4   S-  \S'   Sr\\R                  S4   S-  \S'   Sr\R                  S-  \S	'   S
rg)ÚModernVBertMaskedLMOutputé„   a÷  
Base class for ModernVBERT model's outputs with masked language modeling loss.
Args:
    loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided):
        Masked language modeling (MLM) loss.
    logits (`torch.FloatTensor`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
        Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.
        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder
NÚlossÚlogits.rR   rS   rT   r5   )r@   rA   rB   rC   rD   r[   rU   rV   rG   r\   rR   rW   rS   rT   rK   r5   r?   r=   rY   rY   „   s‚   ‡ ñð, &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø $€FˆE×ÑÓ$Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ó;Ø48Ð˜×*Ñ*¨TÑ1Ö8r?   rY   c                   ó8   ^ • \ rS rSrSrU 4S jrS rS rSrU =r	$ )ÚModernVBertConnectoré£   zÞ
Connector module for ModernVBERT. It performs a pixel shuffle operation followed by a linear projection to match the text model's hidden size.
Based on https://pytorch.org/docs/stable/generated/torch.nn.PixelShuffle.html
c                 óê   >• [         TU ]  5         UR                  U l        [        R                  " UR
                  R                  UR                  S-  -  UR                  R                  SS9U l        g )Nr   F©Úbias)	r8   Ú__init__r'   ÚnnÚLinearr$   Úhidden_sizer#   Úmodality_projection©r:   Úconfigr<   s     €r=   rc   ÚModernVBertConnector.__init__©   s^   ø€ Ü‰ÑÔØ$*×$?Ñ$?ˆÔ!Ü#%§9¢9Ø× Ñ ×,Ñ,°×0KÑ0KÈQÑ0NÑOØ×Ñ×*Ñ*Øñ$
ˆÕ r?   c                 ó¤  • UR                  5       u  p4n[        US-  5      =pgUR                  X6Xu5      nUR                  X6[        Xr-  5      XR-  5      nUR                  SSSS5      nUR	                  U[        Xr-  5      [        Xb-  5      XRS-  -  5      nUR                  SSSS5      nUR	                  U[        XBS-  -  5      XRS-  -  5      $ )Ng      à?r   r   é   r	   )ÚsizerH   ÚviewÚpermuteÚreshape)r:   rT   r'   Ú
batch_sizeÚ
seq_lengthÚ	embed_dimÚheightÚwidths           r=   Úpixel_shuffleÚ"ModernVBertConnector.pixel_shuffle²   só   € Ø,?×,DÑ,DÓ,FÑ)ˆ
 	Ü˜Z¨™_Ó-Ð-ˆØ1×6Ñ6°zÈ5Ó\ÐØ1×6Ñ6Ø¤ EÑ$@Ó AÀ9ÑCcó
Ðð 2×9Ñ9¸!¸QÀÀ1ÓEÐØ1×9Ñ9ØÜ�Ñ,Ó-Ü�Ñ-Ó.Ø¨qÑ0Ñ1ó	
Ðð 2×9Ñ9¸!¸QÀÀ1ÓEÐØ"×*Ñ*Øœ˜JÀÑ*AÑBÓCÀYÐhiÑRiÑEjó
ð 	
r?   c                 óZ   • U R                  XR                  5      nU R                  U5      $ ©N)rv   r'   rg   )r:   rT   s     r=   ÚforwardÚModernVBertConnector.forwardÅ   s,   € Ø"×0Ñ0Ð1D×F_ÑF_Ó`ÐØ×'Ñ'Ð(;Ó<Ð<r?   )rg   r'   )
r@   rA   rB   rC   rD   rc   rv   rz   rK   rL   rM   s   @r=   r^   r^   £   s   ø† ñõ

ò
÷&=ð =r?   r^   c                   óJ   • \ rS rSr\r/ r\R                  " 5       S 5       r	Sr
g)ÚModernVBertPreTrainedModeléÊ   c                 ó@  ^ • [         R                  " T U5        S[        R                  S[        4U 4S jjn[        U[        5      (       ab  T R                  R                  [        R                  " ST R                  R                  R                  -  5      -  nU" UR                  U5        g [        U[        5      (       ab  T R                  R                  [        R                  " ST R                  R                  R                  -  5      -  nU" UR                  U5        g [        U[         ["        45      (       a_  T R                  R                  [        R                  " T R                  R                  R$                  5      -  nU" UR&                  U5        g g )NÚmoduleÚstdc                 ó@  >• [        TR                  SS5      n[        R                  " U R                  SUU* U-  X!-  S9  [        U [        R                  [        R                  45      (       a/  U R                  b!  [        R                  " U R                  5        g g g )Nr*   r)   r.   )r,   r�   ÚaÚb)Úgetattrri   ÚinitÚtrunc_normal_Úweightr6   rd   re   ÚConv2drb   Úzeros_)r€   r�   Úcutoff_factorr:   s      €r=   Úinit_weightÚ=ModernVBertPreTrainedModel._init_weights.<locals>.init_weightÓ   s�   ø€ Ü# D§K¡KÐ1LÈcÓRˆMÜ×ÒØ—‘ØØØ �. 3Ñ&ØÑ%òô ˜&¤2§9¡9¬b¯i©iÐ"8×9Ñ9Ø—;‘;Ñ*Ü—K’K §¡Õ,ð +ð :r?   r)   )r   Ú_init_weightsrd   ÚModulerI   r6   r^   ri   r(   ÚmathÚsqrtr#   Únum_hidden_layersrg   ÚModernVBertForMaskedLMÚlm_headÚ$ModernVBertForSequenceClassificationÚ!ModernVBertForTokenClassificationrf   Ú
classifier)r:   r€   rŒ   Úout_stdÚfinal_out_stds   `    r=   rŽ   Ú(ModernVBertPreTrainedModel._init_weightsÏ   s)  ø€ ä×%Ò% d¨FÔ3ð	-¤§	¡	ð 	-´÷ 	-ô �fÔ2×3Ñ3Ø—k‘k×3Ñ3´d·i²iÀÀdÇkÁk×F]ÑF]×FoÑFoÑ@oÓ6pÑpˆGÙ˜×2Ñ2°GÕ<Ü˜Ô 6×7Ñ7Ø—k‘k×3Ñ3´d·i²iÀÀdÇkÁk×F]ÑF]×FoÑFoÑ@oÓ6pÑpˆGÙ˜Ÿ™¨Õ0ÜØä4Ü1ð÷
ñ 
ð !ŸK™K×9Ñ9¼D¿IºIÀdÇkÁk×F]ÑF]×FiÑFiÓ<jÑjˆMÙ˜×)Ñ)¨=Õ9ð
r?   r5   N)r@   rA   rB   rC   r    Úconfig_classÚ_no_split_modulesrU   Úno_gradrŽ   rK   r5   r?   r=   r}   r}   Ê   s%   † à$€LØÐà
‡]‚]ƒ_ñ:ó ó:r?   r}   aG  
    ModernVBertModel is a model that combines a vision encoder (SigLIP) and a text encoder (ModernBert).

    ModernVBert is the base model of the visual retriever ColModernVBert, and was introduced in the following paper:
    [*ModernVBERT: Towards Smaller Visual Document Retrievers*](https://arxiv.org/abs/2510.01149).
    )Úcustom_introc                   óJ  ^ • \ rS rSrS\4U 4S jjr\\" SSS9       SS\R                  S	\R                  S-  S
\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\\   S\\-  4S jj5       5       rSrU =r$ )ÚModernVBertModeléò   ri   c                 ó”  >• [         TU ]  U5        [        U5      U l        [        R
                  " UR                  5      U l        [        R
                  " UR                  5      U l	        [        UR                  R                  UR                  R                  -  S-  UR                  S-  -  5      U l        U R                  5         g )Nr   )r8   rc   r^   Ú	connectorr   Úfrom_configr#   Ú
text_modelr$   Úvision_modelrH   Ú
image_sizeÚ
patch_sizer'   Úimage_seq_lenÚ	post_initrh   s     €r=   rc   ÚModernVBertModel.__init__û   s¡   ø€ Ü‰Ñ˜Ô ô .¨fÓ5ˆŒÜ#×/Ò/°×0BÑ0BÓCˆŒÜ%×1Ò1°&×2FÑ2FÓGˆÔä Ø×"Ñ"×-Ñ-°×1EÑ1E×1PÑ1PÑPÐUVÑVØ×*Ñ*¨AÑ-ñ/ó
ˆÔð 	�‰Õr?   áØ  
        Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to
        the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where
        max_num_images is the maximum number of images among the batch_size samples in the batch.
        Padding images are not needed beyond padding the pixel_values at the entrance of the model.
        For efficiency, we only pass through the vision_model's forward the real images by
        discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where
        image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3.
        r   ©rž   r   NÚ	input_idsÚattention_maskÚposition_idsÚinputs_embedsÚpixel_valuesÚpixel_attention_maskrT   r;   Úreturnc                 ó   • Uc9  U R                   R                  5       " U5      R                  UR                  5      nUb  U R	                  XVS9R
                  nUb4  UR                  UR                  UR                  S9nU R                  XUS9nU R                   " SUUUS.UD6n	[        U	R                  U	R                  U	R                  US9$ )aT  
pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
    Mask to avoid performing attention on padding pixel indices.
image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
    The hidden states of the image encoder after modality projection.
)r²   r³   )ÚdtypeÚdevice)r®   r±   rT   )r±   r¯   r°   )rQ   rR   rS   rT   r5   )r¥   Úget_input_embeddingsÚtor·   Úget_image_featuresÚpooler_outputr¶   Úinputs_mergerrO   rQ   rR   rS   )
r:   r®   r¯   r°   r±   r²   r³   rT   r;   Úoutputss
             r=   rz   ÚModernVBertModel.forward  sö   € ð> Ñ Ø ŸO™O×@Ñ@ÔBÀ9ÓM×PÑPÐQZ×QaÑQaÓbˆMð Ñ#Ø"&×"9Ñ"9Ø)ð #:ð #ç‰mð  ð
 Ñ*Ø"5×"8Ñ"8¸}×?RÑ?RÐ[h×[oÑ[oÐ"8Ð"pÐØ ×.Ñ.Ø#ÐVið /ð ˆMð
 —/’/ð 
Ø'Ø)Ø%ñ
ð ñ	
ˆô *Ø%×7Ñ7Ø!×/Ñ/Ø×)Ñ)Ø 3ñ	
ð 	
r?   )r£   r©   r¥   r¦   )NNNNNNN)r@   rA   rB   rC   r    rc   r   r   rU   Ú
LongTensorÚTensorrV   Ú
BoolTensorr   r   rW   rO   rz   rK   rL   rM   s   @r=   r    r    ò   sü   ø† ðÐ0÷ ð  Ùðð -ñð '+Ø.2Ø04Ø26Ø15Ø8<Ø8<ñ/
à×#Ñ#ð/
ð Ÿ™ tÑ+ð/
ð ×&Ñ&¨Ñ-ð	/
ð
 ×(Ñ(¨4Ñ/ð/
ð ×'Ñ'¨$Ñ.ð/
ð $×.Ñ.°Ñ5ð/
ð #×.Ñ.°Ñ5ð/
ð Ð+Ñ,ð/
ð 
Ð+Ñ	+ô/
óó ö/
r?   r    c                   ó   • \ rS rSrSrg)ÚModernVBertPredictionHeadiJ  r5   N)r@   rA   rB   rC   rK   r5   r?   r=   rÃ   rÃ   J  s   † Úr?   rÃ   c                   ó~  ^ • \ rS rSrSS0rS\4U 4S jjrS rS r\	\
" SS	S
9        SS\R                  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\\   S\\-  4S jj5       5       rSrU =r$ )r“   iN  zlm_head.weightz1model.text_model.embeddings.tok_embeddings.weightri   c                 óh  >• [         TU ]  U5        UR                  R                  U l        [	        U5      U l        [        UR                  5      U l        [        R                  " UR                  R                  U R                  UR                  R                  S9U l        U R                  5         g )Nra   )r8   rc   r#   Ú
vocab_sizer    ÚmodelrÃ   Úprojection_headrd   re   rf   Údecoder_biasr”   rª   rh   s     €r=   rc   ÚModernVBertForMaskedLM.__init__R  s   ø€ Ü‰Ñ˜Ô à ×,Ñ,×7Ñ7ˆŒä% fÓ-ˆŒ
Ü8¸×9KÑ9KÓLˆÔÜ—y’y ×!3Ñ!3×!?Ñ!?ÀÇÁÐW]×WiÑWi×WvÑWvÑwˆŒð 	�‰Õr?   c                 ó   • U R                   $ ry   ©r”   )r:   s    r=   Úget_output_embeddingsÚ,ModernVBertForMaskedLM.get_output_embeddings^  s   € Ø�|‰|Ðr?   c                 ó   • Xl         g ry   rÌ   )r:   Únew_embeddingss     r=   Úset_output_embeddingsÚ,ModernVBertForMaskedLM.set_output_embeddingsa  s   € Ø%�r?   r¬   r   r­   Nr®   r¯   r°   r±   r²   r³   rT   Úlabelsr;   r´   c	                 óX  • U R                   " SUUUUUUUS.U	D6n
U
S   nU R                  U R                  U5      5      nSnUb<  [        5       nU" UR	                  SU R
                  5      UR	                  S5      5      n[        UUU
R                  U
R                  U
R                  S9$ )áÔ  
pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
    Mask to avoid performing attention on padding pixel indices.
image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
    The hidden states of the image encoder after modality projection.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    text_config.]` or `model.image_token_id`. Tokens with indices set to `model.image_token_id` are
    ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., text_config.]`.
©r®   r¯   r°   r±   r²   r³   rT   r   Néÿÿÿÿ)r[   r\   rR   rS   rT   r5   )
rÇ   r”   rÈ   r   rn   rÆ   rY   rR   rS   rT   )r:   r®   r¯   r°   r±   r²   r³   rT   rÓ   r;   r½   rR   r\   r[   Ú	criterions                  r=   rz   ÚModernVBertForMaskedLM.forwardd  s¿   € ðH —*’*ð 	
ØØ)Ø%Ø'Ø%Ø!5Ø 3ñ	
ð ñ	
ˆð   ™
ˆà—‘˜d×2Ñ2°=ÓAÓBˆàˆØÑÜ(Ó*ˆIÙ˜VŸ[™[¨¨T¯_©_Ó=¸v¿{¹{È2»ÓOˆDä(ØØØ!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ñ
ð 	
r?   )r”   rÇ   rÈ   rÆ   ©NNNNNNNN)r@   rA   rB   rC   Ú_tied_weights_keysr    rc   rÍ   rÑ   r   r   rU   r¿   rÀ   rV   rÁ   r   r   rW   rY   rz   rK   rL   rM   s   @r=   r“   r“   N  s*  ø† à*Ð,_Ð`Ðð
Ð0÷ 
òò&ð Ùðð -ñð '+Ø.2Ø04Ø26Ø15Ø8<Ø8<Ø*.ñ0
à×#Ñ#ð0
ð Ÿ™ tÑ+ð0
ð ×&Ñ&¨Ñ-ð	0
ð
 ×(Ñ(¨4Ñ/ð0
ð ×'Ñ'¨$Ñ.ð0
ð $×.Ñ.°Ñ5ð0
ð #×.Ñ.°Ñ5ð0
ð × Ñ  4Ñ'ð0
ð Ð+Ñ,ð0
ð 
Ð*Ñ	*ô0
óó ö0
r?   r“   za
    The ModernVBert Model with a sequence classification head on top that performs pooling.
    c                   ój  ^ • \ rS rSrS\4U 4S jjr\\" SSS9        SS\R                  S	\R                  S-  S
\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\\   S\\-  4S jj5       5       rSrU =r$ )r•   i¤  ri   c                 ó„  >• [         TU ]  U5        UR                  U l        Xl        [	        U5      U l        [        UR                  5      U l        [        R                  " UR                  5      U l        [        R                  " UR                  R                  UR                  5      U l        U R!                  5         g ry   )r8   rc   Ú
num_labelsri   r    rÇ   rÃ   r#   Úheadrd   ÚDropoutr/   Údropre   rf   r—   rª   rh   s     €r=   rc   Ú-ModernVBertForSequenceClassification.__init__ª  s†   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä% fÓ-ˆŒ
Ü-¨f×.@Ñ.@ÓAˆŒ	Ü—J’J˜v×8Ñ8Ó9ˆŒ	ÜŸ)š) F×$6Ñ$6×$BÑ$BÀF×DUÑDUÓVˆŒð 	�‰Õr?   r¬   r   r­   Nr®   r¯   r°   r±   r²   r³   rT   rÓ   r;   r´   c	                 óÂ  • U R                   " SUUUUUUUS.U	D6n
U
S   nU R                  R                  S:X  a
  USS2S4   nO´U R                  R                  S:X  aš  Ub  UR                  SS u  pÍOUR                  SS u  pÍUb  UR                  OUR                  nUc%  [
        R                  " XÍ4U[
        R                  S9nX²R                  S5      -  R                  S	S
9UR                  S	SS9-  nU R                  U5      nU R                  U5      nU R                  U5      nSnUGb�  U R                  R                  c‘  U R                  S	:X  a  SU R                  l        OoU R                  S	:”  aN  UR                  [
        R                   :X  d  UR                  [
        R"                  :X  a  SU R                  l        OSU R                  l        U R                  R                  S:X  aJ  [%        5       nU R                  S	:X  a&  U" UR'                  5       UR'                  5       5      nOŽU" UU5      nO„U R                  R                  S:X  a=  [)        5       nU" UR+                  SU R                  5      UR+                  S5      5      nO-U R                  R                  S:X  a  [-        5       nU" UU5      n[/        UUU
R0                  U
R2                  S9$ )rÕ   rÖ   r   r+   Nr,   r   )r·   r¶   r×   rl   )ÚdimT)rä   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classification©r[   r\   rR   rS   r5   )rÇ   ri   r-   Úshaper·   rU   ÚonesrJ   Ú	unsqueezeÚsumrß   rá   r—   Úproblem_typerÞ   r¶   ÚlongrH   r   Úsqueezer   rn   r   r   rR   rS   )r:   r®   r¯   r°   r±   r²   r³   rT   rÓ   r;   r½   rQ   rq   Úseq_lenr·   Úpooled_outputr\   r[   Úloss_fcts                      r=   rz   Ú,ModernVBertForSequenceClassification.forward·  s�  € ðF —*’*ð 	
ØØ)Ø%Ø'Ø%Ø!5Ø 3ñ	
ð ñ	
ˆð $ A™JÐà�;‰;×)Ñ)¨UÓ2Ø 1²!°Q°$Ñ 7ÑØ�[‰[×+Ñ+¨vÓ5ØÑ(Ø&3×&9Ñ&9¸"¸1Ð&=Ñ#�
˜Gà&/§o¡o°b°qÐ&9Ñ#�
Ø)2Ñ)>�Y×%Ò%ÀM×DXÑDXˆFàÑ%Ü!&§¢¨ZÐ,AÈ&ÔX]×XbÑXbÑ!c�Ø!2×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÐ ]Ð`n×`rÑ`rØ˜tð asð añ !Ðð Ÿ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆàˆØÒØ�{‰{×'Ñ'Ñ/Ø—?‘? aÓ'Ø/;�D—K‘KÕ,Ø—_‘_ qÓ(¨f¯l©l¼e¿j¹jÓ.HÈFÏLÉLÔ\a×\eÑ\eÓLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ó7Ü"›9�Ø—?‘? aÓ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÓJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÓIÜ,Ó.�Ù ¨Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
r?   )r—   ri   rá   rß   rÇ   rÞ   rÚ   )r@   rA   rB   rC   r    rc   r   r   rU   r¿   rÀ   rV   rÁ   r   r   rW   r   rz   rK   rL   rM   s   @r=   r•   r•   ¤  s   ø† ðÐ0÷ ð Ùðð -ñð '+Ø.2Ø04Ø26Ø15Ø8<Ø8<Ø*.ñQ
à×#Ñ#ðQ
ð Ÿ™ tÑ+ðQ
ð ×&Ñ&¨Ñ-ð	Q
ð
 ×(Ñ(¨4Ñ/ðQ
ð ×'Ñ'¨$Ñ.ðQ
ð $×.Ñ.°Ñ5ðQ
ð #×.Ñ.°Ñ5ðQ
ð × Ñ  4Ñ'ðQ
ð Ð+Ñ,ðQ
ð 
Ð)Ñ	)ôQ
óó öQ
r?   r•   zw
    The ModernVBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   ój  ^ • \ rS rSrS\4U 4S jjr\\" SSS9        SS\R                  S	\R                  S-  S
\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\\   S\\-  4S jj5       5       rSrU =r$ )r–   i  ri   c                 óx  >• [         TU ]  U5        UR                  U l        [        U5      U l        [        UR                  5      U l        [        R                  " UR                  5      U l        [        R                  " UR                  R                  UR                  5      U l        U R                  5         g ry   )r8   rc   rÞ   r    rÇ   rÃ   r#   rß   rd   rà   r/   rá   re   rf   r—   rª   rh   s     €r=   rc   Ú*ModernVBertForTokenClassification.__init__  s�   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä% fÓ-ˆŒ
Ü-¨f×.@Ñ.@ÓAˆŒ	Ü—J’J˜v×8Ñ8Ó9ˆŒ	ÜŸ)š) F×$6Ñ$6×$BÑ$BÀF×DUÑDUÓVˆŒð 	�‰Õr?   r¬   r   r­   Nr®   r¯   r°   r±   r²   r³   rT   rÓ   r;   r´   c	                 óh  • U R                   " SUUUUUUUS.U	D6n
U
S   nU R                  U5      nU R                  U5      nU R                  U5      nSnUb<  [	        5       nU" UR                  SU R                  5      UR                  S5      5      n[        UUU
R                  U
R                  S9$ )rÕ   rÖ   r   Nr×   ré   r5   )
rÇ   rß   rá   r—   r   rn   rÞ   r   rR   rS   )r:   r®   r¯   r°   r±   r²   r³   rT   rÓ   r;   r½   rQ   r\   r[   ró   s                  r=   rz   Ú)ModernVBertForTokenClassification.forward*  sÍ   € ðH —*’*ð 	
ØØ)Ø%Ø'Ø%Ø!5Ø 3ñ	
ð ñ	
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆàˆØÑÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
r?   )r—   rá   rß   rÇ   rÞ   rÚ   )r@   rA   rB   rC   r    rc   r   r   rU   r¿   rÀ   rV   rÁ   r   r   rW   r   rz   rK   rL   rM   s   @r=   r–   r–     s  ø† ð
Ð0÷ 
ð Ùðð -ñð '+Ø.2Ø04Ø26Ø15Ø8<Ø8<Ø*.ñ1
à×#Ñ#ð1
ð Ÿ™ tÑ+ð1
ð ×&Ñ&¨Ñ-ð	1
ð
 ×(Ñ(¨4Ñ/ð1
ð ×'Ñ'¨$Ñ.ð1
ð $×.Ñ.°Ñ5ð1
ð #×.Ñ.°Ñ5ð1
ð × Ñ  4Ñ'ð1
ð Ð+Ñ,ð1
ð 
Ð&Ñ	&ô1
óó ö1
r?   r–   )r    r}   r    r“   r•   r–   )9r�   Údataclassesr   Útypingr   rU   Útorch.nnrd   Úhuggingface_hub.dataclassesr   r   r   r   Ú r
   r†   Úconfiguration_utilsr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úautor   r   r   Úmodernbert.modeling_modernbertr   Úsmolvlm.modeling_smolvlmr   r   Ú
get_loggerr@   Úloggerr    rO   rY   r�   r^   r}   r    rÃ   r“   r•   r–   Ú__all__r5   r?   r=   Ú<module>r     s¡  ðó Ý !Ý ã Ý Ý .ß AÑ Aå &Ý 3÷ó õ .Ý &ß @Ñ @Ý -ß 8Ñ 8Ý Eß Kð 
×	Ò	˜HÓ	%€ñ Ð4Ñ5Øô5(Ð(ó 5(ó ó 6ð5(ðp ô@ ó @ó ð@ð: ô9 ó 9ó ð9ô<$=˜2Ÿ9™9ô $=ðN ô$:Ð!7ó $:ó ð$:ñN ðñôM
�|ó M
óðM
ô`	Ð 8ô 	ð ôR
Ð7ó R
ó ðR
ñj ðñô
l
Ð+Eó l
óð
l
ñ^ ðñô
K
Ð(Bó K
óð
K
ò\�r?   