ó
    ˆŽ¡j“  ã                   ó¤  • S SK r S SKJr  S SKJr  S SKrS SKJr  S SKJr  S SK	J
r
JrJr  SSKJr  SS	KJr  SS
KJr  SSKJrJr  SSKJrJr  SSKJr  SSKJrJrJrJ r J!r!J"r"  SSK#J$r$  SSK%J&r&J'r'  SSK(J)r)  SSK*J+r+J,r,J-r-  SSK.J/r/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6J7r7  \-Rp                  " \95      r:\," SS9\ " S S\5      5       5       r; " S S\Rx                  5      r= " S S\Rx                  5      r> " S  S!\65      r?\" S"5      S>S# j5       r@\" \@5       " S$ S%\Rx                  5      5       rA " S& S'\5      rB\, " S( S)\'5      5       rC\, " S* S+\C5      5       rD " S, S-\Rx                  5      rE\," S.S/9 " S0 S1\C5      5       rF\," S2S/9 " S3 S4\C5      5       rG\," S5S/9 " S6 S7\C5      5       rH\, " S8 S9\C5      5       rI\," S:S/9 " S; S<\C5      5       rJ/ S=QrKg)?é    N)ÚCallable)ÚLiteral)Ústrict)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚPreTrainedConfig)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Úeager_attention_forward)ÚGemma3RotaryEmbeddingÚrotate_halfzanswerdotai/ModernBERT-base)Ú
checkpointc                   óÀ  ^ • \ rS rSr% SrSrS/rSSS.rSr\	\
S	'   S
r\	\
S'   Sr\	\
S'   Sr\	\
S'   Sr\	\
S'   Sr\\
S'   Sr\	\
S'   Sr\\
S'   Sr\\
S'   Sr\\
S'   Sr\\
S'   Sr\	S-  \
S '   S!r\	\\	   -  S-  \
S"'   S#r\	S-  \
S$'   S#r\	S-  \
S%'   S!r\	S-  \
S&'   Sr\\
S''   S(r\\	-  \
S)'   Sr \\   S-  \
S*'   Sr!\"\#S+   \"4   S-  \
S,'   S-r$\	\
S.'   S(r%\\	-  \
S/'   Sr&\\
S0'   S(r'\\	-  \
S1'   S2r(\\
S3'   S4r)\#S5   \
S6'   S(r*\\	-  \
S7'   Sr+\\
S8'   Sr,\\
S9'   Sr-\\
S:'   Sr.\\
S;'   S<r/\	\
S='   S2r0\\
S>'   U 4S? jr1S@ r2U 4SA jr3\4SB 5       r5\5Rl                  SC 5       r5SDr7U =r8$ )EÚModernBertConfigé4   aŸ  
initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
    The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
norm_eps (`float`, *optional*, defaults to 1e-05):
    The epsilon used by the rms normalization layers.
norm_bias (`bool`, *optional*, defaults to `False`):
    Whether to use bias in the normalization layers.
local_attention (`int`, *optional*, defaults to 128):
    The window size for local attention.
mlp_dropout (`float`, *optional*, defaults to 0.0):
    The dropout ratio for the MLP layers.
decoder_bias (`bool`, *optional*, defaults to `True`):
    Whether to use bias in the decoder layers.
classifier_pooling (`str`, *optional*, defaults to `"cls"`):
    The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
    CLS token doesn't attend to all tokens on long sequences.
classifier_bias (`bool`, *optional*, defaults to `False`):
    Whether to use bias in the classifier.
classifier_activation (`str`, *optional*, defaults to `"gelu"`):
    The activation function for the classifier.
deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
    Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
sparse_prediction (`bool`, *optional*, defaults to `False`):
    Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
    The index to ignore for the sparse prediction.

Examples:

```python
>>> from transformers import ModernBertModel, ModernBertConfig

>>> # Initializing a ModernBert style configuration
>>> configuration = ModernBertConfig()

>>> # Initializing a model from the modernbert-base style configuration
>>> model = ModernBertModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```Ú
modernbertÚpast_key_valuesg     ˆAg     ˆÃ@)ÚglobalÚlocaliÀÄ  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚgeluÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeç       @Úinitializer_cutoff_factorgñhãˆµøä>Únorm_epsFÚ	norm_biasikÄ  NÚpad_token_idijÄ  Úeos_token_idiiÄ  Úbos_token_idÚcls_token_idÚsep_token_idÚattention_biasç        Úattention_dropoutÚlayer_types©Úfull_attentionÚsliding_attentionÚrope_parametersé€   Úlocal_attentionÚembedding_dropoutÚmlp_biasÚmlp_dropoutTÚdecoder_biasÚcls)rQ   ÚmeanÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictioniœÿÿÿÚsparse_pred_ignore_indexÚtie_word_embeddingsc                 óì   >• UR                  SS5      nU R                  c?  [        U R                  5       Vs/ s H  n[	        X2-  5      (       a  SOSPM     snU l        [
        TU ]  " S0 UD6  g s  snf )NÚglobal_attn_every_n_layersr
   rI   rH   © )ÚgetrF   Úranger3   ÚboolÚsuperÚ__post_init__)ÚselfÚkwargsr\   ÚiÚ	__class__s       €Ú¿C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/modernbert/modular_modernbert.pyrb   ÚModernBertConfig.__post_init__‡   sy   ø€ à%+§Z¡ZÐ0LÈaÓ%PÐ"Ø×ÑÑ#ô ˜t×5Ñ5Ô6ó â6�Aô (,¨AÑ,J×'KÑ'KÑ#ÐQaÒaÙ6ñ ˆDÔô
 	‰ÒÑ' Ó'ùò s   ¸ A1c                 ó¾  • UR                  SS 5      nSS0SS0S.nU R                  b  U R                  OUU l        Ub<  U R                  S   R                  U5        U R                  S   R                  U5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  SU R
                  S	   5      5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  S
U R
                  S   5      5        U R                  5         U$ )NÚrope_scalingÚ	rope_typeÚdefault)rI   rH   rH   rI   Ú
rope_thetaÚglobal_rope_thetar-   Úlocal_rope_thetar.   )ÚpoprJ   Úupdater^   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rc   rd   rj   Údefault_rope_paramss       rg   Úconvert_rope_params_to_dictÚ,ModernBertConfig.convert_rope_params_to_dict’   sc  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÑ7W˜t×3Ò3Ð]pˆÔØÑ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGØ× Ñ Ð!4Ñ5×<Ñ<¸\ÔJð ×Ñ×#Ñ#Ð$4Ó5Ñ=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9Ø˜&Ÿ*™*Ð%8¸$×:LÑ:LÈXÑ:VÓWô	
ð ×Ñ×#Ñ#Ð$7Ó8Ñ@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™*Ð%7¸×9KÑ9KÈGÑ9TÓUô	
ð
 	×$Ñ$Ô&Øˆó    c                 óH   >• [         TU ]  5       nUR                  SS 5        U$ )NÚreference_compile)ra   Úto_dictrp   )rc   Úoutputrf   s     €rg   r{   ÚModernBertConfig.to_dict°   s#   ø€ Ü‘‘Ó"ˆØ�
‰
Ð&¨Ô-Øˆrx   c                 ó    • U R                   S-  $ )zKHalf-window size: `local_attention` is the total window, so we divide by 2.r#   ©rL   ©rc   s    rg   Úsliding_windowÚModernBertConfig.sliding_windowµ   s   € ð ×#Ñ# qÑ(Ð(rx   c                 ó   • US-  U l         g)z<Set sliding_window by updating local_attention to 2 * value.r#   Nr   ©rc   Úvalues     rg   r�   r‚   º   s   € ð  % q™yˆÕrx   )rF   rL   rJ   )9Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencers   r/   ÚintÚ__annotations__r0   r1   r3   r5   r7   Ústrr8   r9   Úfloatr;   r<   r=   r`   r>   r?   Úlistr@   rA   rB   rC   rE   rF   rJ   Údictr   rL   rM   rN   rO   rP   rS   rT   rU   rV   rW   rX   rY   rZ   rb   rv   r{   Úpropertyr�   ÚsetterÚ__static_attributes__Ú__classcell__©rf   s   @rg   r)   r)   4   s  ø‡ ñ(ðT €JØ#4Ð"5ÐØ(°8Ñ<€Mà€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø#Ð�sÓ#Ø#'Ð˜SÓ'Ø#Ð�uÓ#Ø'*Ð˜uÓ*Ø€HˆeÓØ€IˆtÓØ$€L�#˜‘*Ó$Ø+0€L�#˜˜S™	‘/ DÑ(Ó0Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø$(€K��c‘˜TÑ!Ó(ØY]€O�T˜'Ð"GÑHÈ$ÐNÑOÐRVÑVÓ]Ø€O�SÓØ%(Ð�u˜s‘{Ó(Ø€HˆdÓØ"€K�˜‘Ó"Ø€L�$ÓØ16Ð˜ Ñ.Ó6Ø&)Ð˜ ™Ó)Ø!€O�TÓ!Ø!'Ð˜3Ó'Ø%*Ð˜dÓ*Ø#Ð�tÓ#Ø$(Ð˜cÓ(Ø $Ð˜Ó$õ	(òõ<ð
 ñ)ó ð)ð ×Ññ)ó ö)rx   r)   c                   ó˜   ^ • \ rS rSrSrS\4U 4S jjr SS\R                  S-  S\R                  S-  S\R                  4S	 jjr
S
rU =r$ )ÚModernBertEmbeddingséÀ   zN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
Úconfigc                 ó\  >• [         TU ]  5         Xl        [        R                  " UR
                  UR                  UR                  S9U l        [        R                  " UR                  UR                  UR                  S9U l        [        R                  " UR                  5      U l        g )N)Úpadding_idx©ÚepsÚbias)ra   Ú__init__r›   r   Ú	Embeddingr/   r0   r>   Útok_embeddingsÚ	LayerNormr<   r=   ÚnormÚDropoutrM   Údrop©rc   r›   rf   s     €rg   r¡   ÚModernBertEmbeddings.__init__Å   su   ø€ Ü‰ÑÔØŒÜ Ÿlšl¨6×+<Ñ+<¸f×>PÑ>PÐ^d×^qÑ^qÑrˆÔÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ñ`ˆŒ	Ü—J’J˜v×7Ñ7Ó8ˆ�	rx   NÚ	input_idsÚinputs_embedsÚreturnc                 ó®   • Ub"  U R                  U R                  U5      5      nU$ U R                  U R                  U R                  U5      5      5      nU$ ©N)r§   r¥   r£   )rc   rª   r«   Úhidden_statess       rg   ÚforwardÚModernBertEmbeddings.forwardÌ   sS   € ð Ñ$Ø ŸI™I d§i¡i°Ó&>Ó?ˆMð Ðð !ŸI™I d§i¡i°×0CÑ0CÀIÓ0NÓ&OÓPˆMØÐrx   )r›   r§   r¥   r£   ©NN)r†   r‡   rˆ   r‰   rŠ   r)   r¡   ÚtorchÚ
LongTensorÚTensorr°   r•   r–   r—   s   @rg   r™   r™   À   sW   ø† ñð9Ð/÷ 9ð _cñØ×)Ñ)¨DÑ0ðØHMÏÉÐW[ÑH[ðà	�‰÷ó rx   r™   c                   ón   ^ • \ rS rSrSrS\4U 4S jjrS\R                  S\R                  4S jr	Sr
U =r$ )	ÚModernBertMLPéÖ   a*  Applies the GLU at the end of each ModernBERT layer.

Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
r›   c                 ó¤  >• [         TU ]  5         Xl        [        R                  " UR
                  [        UR                  5      S-  UR                  S9U l	        [        UR                     U l        [        R                  " UR                  5      U l        [        R                  " UR                  UR
                  UR                  S9U l        g )Nr#   ©r    )ra   r¡   r›   r   ÚLinearr0   r�   r1   rN   ÚWir   r7   Úactr¦   rO   r§   ÚWor¨   s     €rg   r¡   ÚModernBertMLP.__init__Ý   s‘   ø€ Ü‰ÑÔØŒÜ—)’)˜F×.Ñ.´°F×4LÑ4LÓ0MÐPQÑ0QÐX^×XgÑXgÑhˆŒÜ˜&×2Ñ2Ñ3ˆŒÜ—J’J˜v×1Ñ1Ó2ˆŒ	Ü—)’)˜F×4Ñ4°f×6HÑ6HÈvÏÉÑ_ˆ�rx   r¯   r¬   c                 ó¨   • U R                  U5      R                  SSS9u  p#U R                  U R                  U R	                  U5      U-  5      5      $ )Nr#   éÿÿÿÿ©Údim)r¼   Úchunkr¾   r§   r½   )rc   r¯   ÚinputÚgates       rg   r°   ÚModernBertMLP.forwardå   sG   € Ø—g‘g˜mÓ,×2Ñ2°1¸"Ð2Ð=‰ˆØ�w‰w�t—y‘y §¡¨%£°4Ñ!7Ó8Ó9Ð9rx   )r¼   r¾   r½   r›   r§   )r†   r‡   rˆ   r‰   rŠ   r)   r¡   r³   rµ   r°   r•   r–   r—   s   @rg   r·   r·   Ö   s7   ø† ñð`Ð/÷ `ð: U§\¡\ð :°e·l±l÷ :ò :rx   r·   c            	       óz   ^ • \ rS rSrS	S\4U 4S jjjr S
S\S\S-  S\\R                  \
4   4U 4S jjjrSrU =r$ )ÚModernBertRotaryEmbeddingéê   Nr›   c                 ó$   >• [         TU ]  U5        g r®   )ra   r¡   )rc   r›   Údevicerf   s      €rg   r¡   Ú"ModernBertRotaryEmbedding.__init__ë   s   ø€ Ü‰Ñ˜Õ rx   Ú
layer_typer¬   c                 ó"   >• [         TU ]  X5      $ r®   )ra   Úcompute_default_rope_parameters)r›   rÌ   rÎ   rd   rf   s       €rg   rÐ   Ú9ModernBertRotaryEmbedding.compute_default_rope_parametersî   s   ø€ ô ‰wÑ6°vÓJÐJrx   r]   r®   r²   )r†   r‡   rˆ   r‰   r)   r¡   r�   Útupler³   rµ   r�   rÐ   r•   r–   r—   s   @rg   rÉ   rÉ   ê   sU   ø† ñ!Ð/÷ !ð !ð IMñKØ ðKØ;>À¹:ðKà	ˆu�|‰|˜UÐ"Ñ	#÷Kö Krx   rÉ   Úrotary_pos_embc                 ób  • U R                   nUR                  U5      nUR                  U5      nU R                  5       U-  [        U R                  5       5      U-  -   nUR                  5       U-  [        UR                  5       5      U-  -   nUR	                  U5      UR	                  U5      4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)ÚdtypeÚ	unsqueezer�   r&   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           rg   Úapply_rotary_pos_embrà   ô   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐArx   c                   ó  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr  SS\R                  S\
\R                  \R                  4   S-  S	\R                  S-  S
\\   S\
\R                  \R                  S-  4   4
S jjrSrU =r$ )ÚModernBertAttentioni  an  Performs multi-headed self attention on a batch of unpadded sequences.

If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
which requires padding and unpadding inputs, adding some overhead.

See `forward` method for additional details.
Nr›   Ú	layer_idxc                 óP  >• [         TU ]  5         Xl        X l        UR                  UR
                  -  S:w  a&  [        SUR                   SUR
                   S35      eUR                  U l        UR                  U l        UR                  UR
                  -  U l	        [        R                  " UR                  SU R                  -  UR
                  -  UR                  S9U l        UR                  U   S:X  a  UR                  S-   U l        OS U l        S	U l        [        R                  " UR                  UR                  UR                  S9U l        UR                  S
:”  a&  [        R$                  " UR                  5      U l        g [        R&                  " 5       U l        g )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (Ú)r
   rº   rI   é   FrD   )ra   r¡   r›   rã   r0   r5   Ú
ValueErrorrE   rW   Úhead_dimr   r»   rC   ÚWqkvrF   r�   Ú	is_causalr¾   r¦   ÚIdentityÚout_drop©rc   r›   rã   rf   s      €rg   r¡   ÚModernBertAttention.__init__  sv  ø€ Ü‰ÑÔØŒØ"Œà×Ñ × :Ñ :Ñ:¸aÓ?ÜØ# F×$6Ñ$6Ð#7Ð7mÐnt÷  oIñ  oIð  nJð  JKð  Lóð ð "(×!9Ñ!9ˆÔØ(.×(GÑ(GˆÔ%Ø×*Ñ*¨f×.HÑ.HÑHˆŒÜ—I’IØ×Ñ  D§M¡MÑ 1°F×4NÑ4NÑ NÐU[×UjÑUjñ
ˆŒ	ð ×Ñ˜iÑ(Ð,?Ó?ð #)×"7Ñ"7¸!Ñ";ˆDÕà"&ˆDÔàˆŒä—)’)˜F×.Ñ.°×0BÑ0BÈ×I^ÑI^Ñ_ˆŒØ@F×@XÑ@XÐ[^Ó@^œŸ
š
 6×#;Ñ#;Ó<ˆ�Ôdf×doÒdoÓdqˆ�rx   r¯   Úposition_embeddingsÚattention_maskrd   r¬   c                 óÄ  • UR                   S S nU R                  U5      nUR                  " / UQSPSPU R                  P76 nUR	                  SS9u  pxn	UR                  SS5      nUR                  SS5      nU	R                  SS5      n	Uu  p«[        XxX«SS9u  px[        R                  " U R                  R                  [        5      nU" U UUU	U4U R                  (       a  U R                  OSU R                  S	-  U R                  U R                  S
.UD6u  pÞUR                   " / UQSP76 R#                  5       nU R%                  U R'                  U5      5      nXÞ4$ )NrÁ   r
   éýÿÿÿrÂ   ræ   r#   )rÜ   rD   ç      à¿)ÚdropoutÚscalingr�   Údeterministic)Úshaperé   Úviewrè   ÚunbindÚ	transposerà   r   Úget_interfacer›   Ú_attn_implementationr$   ÚtrainingrE   r�   rW   ÚreshapeÚ
contiguousrì   r¾   )rc   r¯   rï   rð   rd   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesrÚ   rÛ   Úattention_interfaceÚattn_outputÚattn_weightss                  rg   r°   ÚModernBertAttention.forward7  so  € ð $×)Ñ)¨#¨2Ð.ˆà�i‰i˜Ó&ˆØ�hŠhÐ:˜Ð: QÐ:¨Ð:¨D¯M©MÒ:ˆØ14·±À°Ð1CÑ.ˆ ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÐjkÑ#lÑ ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯m¯m�D×*Ò*ÀØ—M‘M 4Ñ'Ø×.Ñ.Ø×7Ñ7ñ%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m D§G¡G¨KÓ$8Ó9ˆØÐ(Ð(rx   )
r¾   ré   rE   r›   rW   rè   rê   rã   rì   r�   r®   r²   )r†   r‡   rˆ   r‰   rŠ   r)   r�   r¡   r³   rµ   rÒ   r   r   r°   r•   r–   r—   s   @rg   râ   râ     s¯   ø† ññrÐ/ð r¸CÀ$¹J÷ rð rð@ IMØ.2ñ	')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð')ð Ÿ™ tÑ+ð	')ð
 Ð+Ñ,ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')ó ')rx   râ   c                   óÆ   ^ • \ rS rSrSS\S\S-  4U 4S jjjr  SS\R                  S\R                  S-  S\R                  S-  S	\	\
   S
\R                  4
S jjrSrU =r$ )ÚModernBertEncoderLayeria  Nr›   rã   c                 óÄ  >• [         TU ]  5         Xl        X l        US:X  a  [        R
                  " 5       U l        O9[        R                  " UR                  UR                  UR                  S9U l        [        XS9U l        [        R                  " UR                  UR                  UR                  S9U l        [        U5      U l        UR                   U   U l        g )Nr   rž   )r›   rã   )ra   r¡   r›   rã   r   rë   Ú	attn_normr¤   r0   r<   r=   râ   ÚattnÚmlp_normr·   ÚmlprF   Úattention_typerí   s      €rg   r¡   ÚModernBertEncoderLayer.__init__b  s¡   ø€ Ü‰ÑÔØŒØ"ŒØ˜‹>ÜŸ[š[›]ˆD�NäŸ\š\¨&×*<Ñ*<À&Ç/Á/ÐX^×XhÑXhÑiˆDŒNÜ'¨vÑKˆŒ	ÜŸš V×%7Ñ%7¸V¿_¹_ÐSY×ScÑScÑdˆŒÜ  Ó(ˆŒØ$×0Ñ0°Ñ;ˆÕrx   r¯   rð   rï   rd   r¬   c                 ó    • U R                   " U R                  U5      4UUS.UD6u  pVX-   nXR                  U R                  U5      5      -   nU$ )N)rï   rð   )r  r  r  r  )rc   r¯   rð   rï   rd   r  Ú_s          rg   r°   ÚModernBertEncoderLayer.forwardo  s_   € ð ŸšØ�N‰N˜=Ó)ð
à 3Ø)ñ
ð ñ	
‰ˆð &Ñ3ˆØ%¯©°·±¸}Ó1MÓ(NÑNˆØÐrx   )r  r  r  r›   rã   r  r  r®   r²   )r†   r‡   rˆ   r‰   r)   r�   r¡   r³   rµ   r   r   r°   r•   r–   r—   s   @rg   r
  r
  a  s‚   ø† ñ<Ð/ð <¸CÀ$¹J÷ <ð <ð  /3Ø37ñ	à—|‘|ðð Ÿ™ tÑ+ðð #Ÿ\™\¨DÑ0ð	ð
 Ð+Ñ,ðð 
�‰÷ó rx   r
  c                   ó¢   ^ • \ rS rSr% \\S'   SrSrSS/rSr	Sr
SrSr\\S.r\R"                  " 5       S\R&                  4U 4S	 jj5       rS
rU =r$ )ÚModernBertPreTrainedModeli�  r›   ÚmodelTr™   r
  )r¯   Ú
attentionsÚmodulec                 óž  >^• [         T	U ]  U5        U R                  R                  mTc  SmS[        R
                  S[        4U4S jjnU R                  R                  U R                  R                  [        R                  " SU R                  R                  -  5      -  U R                  R                  U R                  R                  S-  S.n[        U[        5      (       a  U" UR                  US   5        g [        U[        5      (       a-  U" UR                   US	   5        U" UR"                  US
   5        g [        U[$        5      (       a-  U" UR&                  US	   5        U" UR"                  US
   5        g [        U[(        5      (       a  U" UR*                  US
   5        g [        U[,        5      (       a  U" UR.                  US
   5        g [        U[0        [2        [4        [6        45      (       a  U" UR8                  US   5        g [        U[:        5      (       a¢  UR<                   H‘  nUR>                  nUR@                  U   S:w  a  [B        UR@                  U      nU" UR                  US9u  pg[D        RF                  " [I        X S35      U5        [D        RF                  " [I        X S35      U5        M“     g g )Nr
   r  Ústdc                 óô   >• [         R                  " U R                  SUT* U-  TU-  S9  [        U [        R
                  5      (       a/  U R                  b!  [         R                  " U R                  5        g g g )NrD   )rR   r  ÚaÚb)ÚinitÚtrunc_normal_ÚweightÚ
isinstancer   r»   r    Úzeros_)r  r  Úcutoff_factors     €rg   Úinit_weightÚ<ModernBertPreTrainedModel._init_weights.<locals>.init_weight˜  sg   ø€ Ü×ÒØ—‘ØØØ �. 3Ñ&Ø #Ñ%òô ˜&¤"§)¡)×,Ñ,Ø—;‘;Ñ*Ü—K’K §¡Õ,ð +ð -rx   r:   ró   )ÚinÚoutÚ	embeddingÚ	final_outr)  r'  r(  r*  rl   )rÎ   Ú	_inv_freqÚ_original_inv_freq)%ra   Ú_init_weightsr›   r;   r   ÚModuler�   r9   ÚmathÚsqrtr3   r0   r"  r™   r£   r·   r¼   r¾   râ   ré   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierrÉ   rF   rÐ   rk   r   r  Úcopy_Úgetattr)
rc   r  r%  ÚstdsrÎ   Úrope_init_fnÚcurr_inv_freqr  r$  rf   s
           @€rg   r-  Ú'ModernBertPreTrainedModel._init_weights‘  s>  ù€ ä‰Ñ˜fÔ%ØŸ™×=Ñ=ˆØÑ ØˆMð	-¤§	¡	ð 	-´÷ 	-ð —+‘+×/Ñ/Ø—;‘;×0Ñ0´4·9²9¸SÀ4Ç;Á;×C`ÑC`Ñ=`Ó3aÑaØŸ™×6Ñ6ØŸ™×0Ñ0°$Ñ6ñ	
ˆô �fÔ2×3Ñ3Ù˜×-Ñ-¨t°KÑ/@ÕAÜ˜¤×.Ñ.Ù˜Ÿ	™	 4¨¡:Ô.Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 3×4Ñ4Ù˜Ÿ™ T¨$¡ZÔ0Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 8×9Ñ9Ù˜Ÿ™ d¨5¡kÕ2Ü˜Ô 5×6Ñ6Ù˜Ÿ™¨¨U©Õ4ÜØä3Ü+Ü0Ü.ð	÷
ñ 
ñ ˜×)Ñ)¨4°Ñ+<Õ=Ü˜Ô 9×:Ñ:Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ó<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ñ#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ò 1ð ;rx   r]   )r†   r‡   rˆ   r‰   r)   rŽ   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr
  râ   Ú_can_record_outputsr³   Úno_gradr   r.  r-  r•   r–   r—   s   @rg   r  r  �  sp   ø‡ àÓØÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 0Ø)ñÐð
 ‡]‚]ƒ_ð7^ B§I¡Iö 7^ó ö7^rx   r  c                   óö   ^ • \ rS rSrS\4U 4S jjrS rS r\\	\
    SS\R                  S-  S\R                  S-  S	\R                  S-  S
\R                  S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚModernBertModeliÌ  r›   c           	      ó¦  >• [         TU ]  U5        Xl        [        U5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l
        [
        R                  " UR                  UR                  UR                  S9U l        [!        US9U l        SU l        U R'                  5         g s  snf )Nrž   )r›   F)ra   r¡   r›   r™   Ú
embeddingsr   Ú
ModuleListr_   r3   r
  Úlayersr¤   r0   r<   r=   Ú
final_normrÉ   Ú
rotary_embÚgradient_checkpointingÚ	post_initrí   s      €rg   r¡   ÚModernBertModel.__init__Î  s¡   ø€ Ü‰Ñ˜Ô ØŒÜ.¨vÓ6ˆŒÜ—m’mÜHMÈf×NfÑNfÔHgÓhÒHg¸9Ô# FÖ6ÑHgÑhó
ˆŒô Ÿ,š, v×'9Ñ'9¸v¿¹ÐU[×UeÑUeÑfˆŒÜ3¸6ÑBˆŒØ&+ˆÔ#Ø�‰Õùò is   ÁCc                 ó.   • U R                   R                  $ r®   ©rL  r£   r€   s    rg   Úget_input_embeddingsÚ$ModernBertModel.get_input_embeddingsÚ  s   € Ø�‰×-Ñ-Ð-rx   c                 ó$   • XR                   l        g r®   rU  r„   s     rg   Úset_input_embeddingsÚ$ModernBertModel.set_input_embeddingsÝ  s   € Ø).�‰Õ&rx   Nrª   rð   Úposition_idsr«   rd   r¬   c                 ó¬  • US L US L-  (       a  [        S5      eUb  UR                  S   OUR                  S   nUb  UR                  OUR                  nUc#  [        R                  " XgS9R                  S5      nU R                  XS9n[        U=n	[        5      (       d'  U R                  UUS.n
[        S
0 U
D6[        S
0 U
D6S.n	0 n[        U R                  R                  5       H  nU R                  XƒU5      X¼'   M     U R                   H'  nU" U4X�R                      X½R                      S.UD6nM)     U R#                  U5      n[%        US	9$ )Nz:You must specify exactly one of input_ids or inputs_embedsræ   ©rÌ   r   )rª   r«   )r›   r«   rð   rG   )rð   rï   )Úlast_hidden_stater]   )rç   r÷   rÌ   r³   ÚarangerÖ   rL  r"  r’   r›   r   r   ÚsetrF   rP  rN  r  rO  r   )rc   rª   rð   r[  r«   rd   Úseq_lenrÌ   r¯   Úattention_mask_mappingÚmask_kwargsrï   rÎ   Úencoder_layers                 rg   r°   ÚModernBertModel.forwardà  sg  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZà,9Ñ,E�-×%Ñ% aÒ(È9Ï?É?Ð[\ÑK]ˆØ%.Ñ%:�×!Ò!À×@TÑ@TˆàÑÜ Ÿ<š<¨Ñ?×IÑIÈ!ÓLˆLàŸ™°)˜ÐYˆä°NÐBÐ0ÄD×IÑIàŸ+™+Ø!.Ø"0ñˆKô #<Ñ"J¸kÑ"JÜ%MÑ%\ÐP[Ñ%\ñ&Ð"ð
 !ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ð "Ÿ[œ[ˆMÙ)Øðà5×6RÑ6RÑSØ$7×8TÑ8TÑ$Uñð ñ	ŠMñ )ð Ÿ™¨Ó6ˆä°Ñ?Ð?rx   )r›   rL  rO  rQ  rN  rP  )NNNN)r†   r‡   rˆ   r‰   r)   r¡   rV  rY  r!   r"   r   r³   r´   rµ   r   r   r   r°   r•   r–   r—   s   @rg   rJ  rJ  Ì  s¼   ø† ð
Ð/÷ 
ò.ò/ð  ØØð .2Ø.2Ø04Ø-1ñ,@à×#Ñ# dÑ*ð,@ð Ÿ™ tÑ+ð,@ð ×&Ñ&¨Ñ-ð	,@ð
 —|‘| dÑ*ð,@ð Ð+Ñ,ð,@ð 
ô,@ó ó ó  ö,@rx   rJ  c                   ój   ^ • \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )r1  i  r›   c                 óF  >• [         TU ]  5         Xl        [        R                  " UR
                  UR
                  UR                  5      U l        [        UR                     U l
        [        R                  " UR
                  UR                  UR                  S9U l        g )Nrž   )ra   r¡   r›   r   r»   r0   rU   r2  r   rV   r½   r¤   r<   r=   r¥   r¨   s     €rg   r¡   Ú!ModernBertPredictionHead.__init__  so   ø€ Ü‰ÑÔØŒÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÀv×G]ÑG]Ó^ˆŒ
Ü˜&×6Ñ6Ñ7ˆŒÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ñ`ˆ�	rx   r¯   r¬   c                 ó`   • U R                  U R                  U R                  U5      5      5      $ r®   )r¥   r½   r2  )rc   r¯   s     rg   r°   Ú ModernBertPredictionHead.forward  s#   € Ø�y‰y˜Ÿ™ $§*¡*¨]Ó";Ó<Ó=Ð=rx   )r½   r›   r2  r¥   )r†   r‡   rˆ   r‰   r)   r¡   r³   rµ   r°   r•   r–   r—   s   @rg   r1  r1    s2   ø† ðaÐ/÷ að> U§\¡\ð >°e·l±l÷ >ò >rx   r1  zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   óP  ^ • \ rS rSrSS0rS\4U 4S jjrS rS\R                  4S jr
\\     SS
\R                  S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r3  i  zdecoder.weightz&model.embeddings.tok_embeddings.weightr›   c                 ón  >• [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        [        R                  " UR                  UR                  UR                  S9U l        U R                  R                  U l        U R                  R                  U l        U R                  5         g )Nrº   )ra   r¡   r›   rJ  r  r1  Úheadr   r»   r0   r/   rP   r4  rX   rY   rR  r¨   s     €rg   r¡   ÚModernBertForMaskedLM.__init__&  sƒ   ø€ Ü‰Ñ˜Ô ØŒÜ$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—y’y ×!3Ñ!3°V×5FÑ5FÈV×M`ÑM`ÑaˆŒà!%§¡×!>Ñ!>ˆÔØ(,¯©×(LÑ(LˆÔ%ð 	�‰Õrx   c                 ó   • U R                   $ r®   ©r4  r€   s    rg   Úget_output_embeddingsÚ+ModernBertForMaskedLM.get_output_embeddings3  s   € Ø�|‰|Ðrx   Únew_embeddingsc                 ó   • Xl         g r®   rq  )rc   rt  s     rg   Úset_output_embeddingsÚ+ModernBertForMaskedLM.set_output_embeddings6  s   € Ø%�rx   Nrª   rð   r[  r«   Úlabelsrd   r¬   c                 óÊ  • U R                   " SUUUUS.UD6nUS   nU R                  (       aI  UbF  UR                  S5      nUR                  UR                  S   S5      nXPR                  :g  n	X‰   nXY   nU R                  U R                  U5      5      n
S nUb)  U R                  " X¥4SU R                  R                  0UD6n[        UU
UR                  UR                  S9$ )N©rª   rð   r[  r«   r   rÁ   r/   ©ÚlossÚlogitsr¯   r  r]   )r  rX   rø   r÷   rY   r4  rn  Úloss_functionr›   r/   r   r¯   r  )rc   rª   rð   r[  r«   rx  rd   Úoutputsr^  Úmask_tokensr}  r|  s               rg   r°   ÚModernBertForMaskedLM.forward9  sý   € ð —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà×!×! fÑ&8à—[‘[ “_ˆFØ 1× 6Ñ 6°v·|±|ÀA±ÈÓ KÐð !×$AÑ$AÑAˆKØ 1Ñ >ÐØÑ(ˆFà—‘˜dŸi™iÐ(9Ó:Ó;ˆàˆØÑØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
rx   )r›   r4  rn  r  rY   rX   ©NNNNN)r†   r‡   rˆ   r‰   Ú_tied_weights_keysr)   r¡   rr  r   r»   rv  r    r   r³   r´   rµ   r   r   rÒ   r   r°   r•   r–   r—   s   @rg   r3  r3    sç   ø† ð +Ð,TÐUÐðÐ/÷ òð&°B·I±Iô &ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ'
à×#Ñ# dÑ*ð'
ð Ÿ™ tÑ+ð'
ð —l‘l TÑ)ð	'
ð
 —|‘| dÑ*ð'
ð —‘˜tÑ#ð'
ð Ð+Ñ,ð'
ð 
ˆu�|‰|Ñ	˜~Ñ	-ô'
ó ó ö'
rx   r3  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r5  ie  r›   c                 ón  >• [         TU ]  U5        UR                  U l        Xl        [	        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        U R!                  5         g r®   )ra   r¡   Ú
num_labelsr›   rJ  r  r1  rn  r³   r   r¦   rT   r§   r»   r0   r9  rR  r¨   s     €rg   r¡   Ú,ModernBertForSequenceClassification.__init__k  s€   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õrx   Nrª   rð   r[  r«   rx  rd   r¬   c                 ób  • U R                   " SUUUUS.UD6nUS   nU R                  R                  S:X  a
  USS2S4   nO‰U R                  R                  S:X  ao  Uc;  [        R                  " UR
                  SS UR                  [        R                  S9nX‚R                  S5      -  R                  S	S
9UR                  S	SS9-  nU R                  U5      n	U R                  U	5      n	U R                  U	5      n
SnUGbŽ  U R                  R                  c‘  U R                  S	:X  a  SU R                  l        OoU R                  S	:”  aN  UR                  [        R                   :X  d  UR                  [        R"                  :X  a  SU R                  l        OSU R                  l        U R                  R                  S:X  aI  [%        5       nU R                  S	:X  a&  U" U
R'                  5       UR'                  5       5      nOŒU" X¥5      nOƒU R                  R                  S:X  a=  [)        5       nU" U
R+                  SU R                  5      UR+                  S5      5      nO,U R                  R                  S:X  a  [-        5       nU" X¥5      n[/        UU
UR0                  UR2                  S9$ )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
rz  r   rQ   NrR   r#   )rÌ   rÕ   rÁ   ræ   rÂ   T©rÃ   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationr{  r]   )r  r›   rS   r³   Úonesr÷   rÌ   r`   rÖ   Úsumrn  r§   r9  Úproblem_typer†  rÕ   Úlongr�   r	   Úsqueezer   rø   r   r   r¯   r  )rc   rª   rð   r[  r«   rx  rd   r  r^  Úpooled_outputr}  r|  Úloss_fcts                rg   r°   Ú+ModernBertForSequenceClassification.forwardx  s]  € ð" —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà�;‰;×)Ñ)¨UÓ2Ø 1²!°Q°$Ñ 7ÑØ�[‰[×+Ñ+¨vÓ5ØÑ%Ü!&§¢Ø%×+Ñ+¨B¨QÐ/Ð8I×8PÑ8PÔX]×XbÑXbñ"�ð "3×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÐ ]Ð`n×`rÑ`rØ˜tð asð añ !Ðð Ÿ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆàˆØÒØ�{‰{×'Ñ'Ñ/Ø—?‘? aÓ'Ø/;�D—K‘KÕ,Ø—_‘_ qÓ(¨f¯l©l¼e¿j¹jÓ.HÈFÏLÉLÔ\a×\eÑ\eÓLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ó7Ü"›9�Ø—?‘? aÓ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÓJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÓIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
rx   )r9  r›   r§   rn  r  r†  r‚  )r†   r‡   rˆ   r‰   r)   r¡   r    r   r³   r´   rµ   r   r   rÒ   r   r°   r•   r–   r—   s   @rg   r5  r5  e  sÎ   ø† ðÐ/÷ ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð7Ñ	7ôC
ó ó öC
rx   r5  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r7  iÀ  r›   c                 ób  >• [         TU ]  U5        UR                  U l        [        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        U R                  5         g r®   ©ra   r¡   r†  rJ  r  r1  rn  r³   r   r¦   rT   r§   r»   r0   r9  rR  r¨   s     €rg   r¡   Ú)ModernBertForTokenClassification.__init__Æ  s{   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õrx   Nrª   rð   r[  r«   rx  rd   r¬   c                 ób  • U R                   " SUUUUS.UD6nUS   nU R                  U5      nU R                  U5      nU R                  U5      n	Sn
Ub<  [	        5       nU" U	R                  SU R                  5      UR                  S5      5      n
[        U
U	UR                  UR                  S9$ )zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
rz  r   NrÁ   r{  r]   )
r  rn  r§   r9  r   rø   r†  r   r¯   r  )rc   rª   rð   r[  r«   rx  rd   r  r^  r}  r|  r”  s               rg   r°   Ú(ModernBertForTokenClassification.forwardÒ  sÃ   € ð —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆàˆØÑÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
rx   ©r9  r§   rn  r  r†  r‚  )r†   r‡   rˆ   r‰   r)   r¡   r    r   r³   r´   rµ   r   r   rÒ   r   r°   r•   r–   r—   s   @rg   r7  r7  À  sÅ   ø† ð
Ð/÷ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ$
à×#Ñ# dÑ*ð$
ð Ÿ™ tÑ+ð$
ð —l‘l TÑ)ð	$
ð
 —|‘| dÑ*ð$
ð —‘˜tÑ#ð$
ð Ð+Ñ,ð$
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ô$
ó ó ö$
rx   r7  c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\
\   S\\R                     \-  4S jj5       5       rSrU =r$ )r8  iû  r›   c                 ób  >• [         TU ]  U5        UR                  U l        [        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        U R                  5         g r®   r˜  r¨   s     €rg   r¡   Ú'ModernBertForQuestionAnswering.__init__ý  sy   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰Õrx   Nrª   rð   r[  Ústart_positionsÚend_positionsrd   r¬   c                 ó¸  • U R                   " U4UUS.UD6nUS   nU R                  U5      nU R                  U5      nU R                  U5      n	U	R	                  SSS9u  p«U
R                  S5      R                  5       n
UR                  S5      R                  5       nS nUb  Ub  U R                  " X«XE40 UD6n[        UU
UUR                  UR                  S9$ )N)rð   r[  r   ræ   rÁ   rÂ   )r|  Ústart_logitsÚ
end_logitsr¯   r  )r  rn  r§   r9  Úsplitr’  rÿ   r~  r   r¯   r  )rc   rª   rð   r[  r   r¡  rd   r  r^  r}  r£  r¤  r|  s                rg   r°   Ú&ModernBertForQuestionAnswering.forward  sü   € ð —*’*Øð
à)Ø%ñ
ð ñ	
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆà#)§<¡<°°r <Ð#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÑ&¨=Ñ+DØ×%Ò% lÀÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
rx   rœ  r‚  )r†   r‡   rˆ   r‰   r)   r¡   r    r   r³   rµ   r   r   rÒ   r   r°   r•   r–   r—   s   @rg   r8  r8  û  sÃ   ø† ð	Ð/÷ 	ð Øð *.Ø.2Ø,0Ø/3Ø-1ñ#
à—<‘< $Ñ&ð#
ð Ÿ™ tÑ+ð#
ð —l‘l TÑ)ð	#
ð
 Ÿ™¨Ñ,ð#
ð —|‘| dÑ*ð#
ð Ð+Ñ,ð#
ð 
ˆu�|‰|Ñ	Ð;Ñ	;ô#
ó ó ö#
rx   r8  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r6  i0  r›   c                 ó8  >• [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  S5      U l        U R                  5         g )Nræ   )ra   r¡   r›   rJ  r  r1  rn  r³   r   r¦   rT   r§   r»   r0   r9  rR  r¨   s     €rg   r¡   Ú$ModernBertForMultipleChoice.__init__6  sm   ø€ Ü‰Ñ˜Ô ØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õrx   Nrª   rð   r[  r«   rx  rd   r¬   c                 óÄ  • Ub  UR                   S   OUR                   S   nUb!  UR                  SUR                  S5      5      OSnUb!  UR                  SUR                  S5      5      OSnUb!  UR                  SUR                  S5      5      OSnUb1  UR                  SUR                  S5      UR                  S5      5      OSnU R                  " S
UUUUS.UD6nUS   n	U R                  R
                  S:X  aŽ  [        R                  " U	R                   S   U	R                  S9n
Ub)  UR                  SS	9R                  U	R                  5      nO/[        R                  " S
S[        R                  U	R                  S9nXšU4   n	OMU R                  R
                  S:X  a3  UR                  SSS9nX’R                  S5      -  R                  SS	9U-  n	U R                  U	5      nU R!                  U5      nU R#                  U5      nUR                  SU5      nSnUb  [$        R&                  " 5       nU" Xõ5      n[)        UUUR*                  UR,                  S9$ )a  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
Nræ   rÁ   éþÿÿÿrz  r   rQ   r]  rÂ   r]   )rÕ   rÌ   rR   Tr‰  r{  )r÷   rø   Úsizer  r›   rS   r³   r_  rÌ   Úargmaxr×   Úfullr‘  r�  rÖ   rn  r§   r9  r   r   r   r¯   r  )rc   rª   rð   r[  r«   rx  rd   Únum_choicesr  r^  Ú	indices_0Úcls_maskÚnum_non_pad_tokensr“  r}  Úreshaped_logitsr|  r”  s                     rg   r°   Ú#ModernBertForMultipleChoice.forwardB  sh  € ð  -6Ñ,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÑ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÑMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÑG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ñ(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐð �;‰;×)Ñ)¨UÓ2ÜŸšÐ%6×%<Ñ%<¸QÑ%?ÐHY×H`ÑH`ÑaˆIàÑ)Ø)×0Ñ0°RÐ0Ð8×;Ñ;Ð<M×<TÑ<TÓU‘ô !Ÿ:š: b¨!´5·:±:ÐFW×F^ÑF^Ñ_�à 1¸XÐ2EÑ FÑð �[‰[×+Ñ+¨vÓ5Ø!/×!3Ñ!3¸À4Ð!3Ð!HÐØ!2×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÐ ]Ð`rÑ rÐàŸ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÑÜ×*Ò*Ó,ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ñ	
ð 	
rx   )r9  r›   r§   rn  r  r‚  )r†   r‡   rˆ   r‰   r)   r¡   r    r   r³   r´   rµ   r   r   rÒ   r   r°   r•   r–   r—   s   @rg   r6  r6  0  sÎ   ø† ð
Ð/÷ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð8Ñ	8ôC
ó ó öC
rx   r6  )r)   rJ  r  r3  r5  r7  r8  r6  )ræ   )Lr/  Úcollections.abcr   Útypingr   r³   Úhuggingface_hub.dataclassesr   r   Útorch.nnr   r   r	   Ú r   r  Úactivationsr   Úconfiguration_utilsr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   Úalign.modeling_alignr$   Úgemma3.modeling_gemma3r%   r&   Ú
get_loggerr†   Úloggerr)   r.  r™   r·   rÉ   rà   râ   r
  r  rJ  r1  r3  r5  r7  r8  r6  Ú__all__r]   rx   rg   Ú<module>rË     s5  ðó  Ý $Ý ã Ý .Ý ß AÑ Aå &Ý !Ý 3ß Lß `Ý 9÷÷ õ 7ß FÝ &ß @Ñ @ß IÝ 5Ý :ß Gð 
×	Ò	˜HÓ	%€ñ Ð8Ñ9ØôG)Ð'ó G)ó ó :ðG)ôT˜2Ÿ9™9ô ô,:�B—I‘Iô :ô(KÐ 5ô Kñ Ð-Ó.óBó /ðBñ4 Ð)Ó*ôN)˜"Ÿ)™)ó N)ó +ðN)ôbÐ7ô ð@ ôG^ ó G^ó ðG^ðT ôB@Ð/ó B@ó ðB@ôJ	>˜rŸy™yô 	>ñ ðñô
?
Ð5ó ?
óð
?
ñD ðñô
S
Ð*Có S
óð
S
ñl ðñô
3
Ð'@ó 3
óð
3
ðl ô1
Ð%>ó 1
ó ð1
ñh ðñô
R
Ð";ó R
óð
R
òj	�rx   