ó
    ˆŽ¡jŽ  ã                   óÚ  • S SK r S SKJr  S SKrS SKJr  S SKJrJrJr  SSK	J
r  SSKJr  SSKJrJr  SS	KJrJr  SS
KJr  SSKJrJrJrJrJrJr  SSKJrJr  SSK J!r!J"r"  SSK#J$r$  SSK%J&r&J'r'  SSK(J)r)  SSK*J+r+J,r,J-r-  SSK.J/r/  SSK0J1r1   " S S\Rd                  5      r3 " S S\Rd                  5      r4 " S S\Rd                  5      r5 S@S\Rd                  S\Rl                  S\Rl                  S\Rl                  S\Rl                  S-  S \7S!\74S" jjr8S# r9\" S$5      SAS% j5       r:\" \:5       " S& S'\Rd                  5      5       r; " S( S)\5      r<\' " S* S+\"5      5       r=\' " S, S-\=5      5       r> " S. S/\Rd                  5      r?\'" S0S19 " S2 S3\=5      5       r@\'" S4S19 " S5 S6\=5      5       rA\'" S7S19 " S8 S9\=5      5       rB\' " S: S;\=5      5       rC\'" S<S19 " S= S>\=5      5       rD/ S?QrEg)Bé    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚModernBertConfigc                   ó˜   ^ • \ rS rSrSrS\4U 4S jjr SS\R                  S-  S\R                  S-  S\R                  4S	 jjr
S
rU =r$ )ÚModernBertEmbeddingsé4   zN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
Úconfigc                 ó\  >• [         TU ]  5         Xl        [        R                  " UR
                  UR                  UR                  S9U l        [        R                  " UR                  UR                  UR                  S9U l        [        R                  " UR                  5      U l        g )N)Úpadding_idx©ÚepsÚbias)ÚsuperÚ__init__r'   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚtok_embeddingsÚ	LayerNormÚnorm_epsÚ	norm_biasÚnormÚDropoutÚembedding_dropoutÚdrop©Úselfr'   Ú	__class__s     €ÚÀC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/modernbert/modeling_modernbert.pyr.   ÚModernBertEmbeddings.__init__9   su   ø€ Ü‰ÑÔØŒÜ Ÿlšl¨6×+<Ñ+<¸f×>PÑ>PÐ^d×^qÑ^qÑrˆÔÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ñ`ˆŒ	Ü—J’J˜v×7Ñ7Ó8ˆ�	ó    NÚ	input_idsÚinputs_embedsÚreturnc                 ó®   • Ub"  U R                  U R                  U5      5      nU$ U R                  U R                  U R                  U5      5      5      nU$ ©N)r:   r7   r3   )r<   rA   rB   Úhidden_statess       r>   ÚforwardÚModernBertEmbeddings.forward@   sS   € ð Ñ$Ø ŸI™I d§i¡i°Ó&>Ó?ˆMð Ðð !ŸI™I d§i¡i°×0CÑ0CÀIÓ0NÓ&OÓPˆMØÐr@   )r'   r:   r7   r3   ©NN)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r#   r.   ÚtorchÚ
LongTensorÚTensorrG   Ú__static_attributes__Ú__classcell__©r=   s   @r>   r%   r%   4   sW   ø† ñð9Ð/÷ 9ð _cñØ×)Ñ)¨DÑ0ðØHMÏÉÐW[ÑH[ðà	�‰÷ó r@   r%   c                   ón   ^ • \ rS rSrSrS\4U 4S jjrS\R                  S\R                  4S jr	Sr
U =r$ )	ÚModernBertMLPéJ   a*  Applies the GLU at the end of each ModernBERT layer.

Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
r'   c                 ó¤  >• [         TU ]  5         Xl        [        R                  " UR
                  [        UR                  5      S-  UR                  S9U l	        [        UR                     U l        [        R                  " UR                  5      U l        [        R                  " UR                  UR
                  UR                  S9U l        g )Né   ©r,   )r-   r.   r'   r   ÚLinearr1   ÚintÚintermediate_sizeÚmlp_biasÚWir
   Úhidden_activationÚactr8   Úmlp_dropoutr:   ÚWor;   s     €r>   r.   ÚModernBertMLP.__init__Q   s‘   ø€ Ü‰ÑÔØŒÜ—)’)˜F×.Ñ.´°F×4LÑ4LÓ0MÐPQÑ0QÐX^×XgÑXgÑhˆŒÜ˜&×2Ñ2Ñ3ˆŒÜ—J’J˜v×1Ñ1Ó2ˆŒ	Ü—)’)˜F×4Ñ4°f×6HÑ6HÈvÏÉÑ_ˆ�r@   rF   rC   c                 ó¨   • U R                  U5      R                  SSS9u  p#U R                  U R                  U R	                  U5      U-  5      5      $ )NrY   éÿÿÿÿ©Údim)r_   Úchunkrc   r:   ra   )r<   rF   ÚinputÚgates       r>   rG   ÚModernBertMLP.forwardY   sG   € Ø—g‘g˜mÓ,×2Ñ2°1¸"Ð2Ð=‰ˆØ�w‰w�t—y‘y §¡¨%£°4Ñ!7Ó8Ó9Ð9r@   )r_   rc   ra   r'   r:   )rJ   rK   rL   rM   rN   r#   r.   rO   rQ   rG   rR   rS   rT   s   @r>   rV   rV   J   s7   ø† ñð`Ð/÷ `ð: U§\¡\ð :°e·l±l÷ :ò :r@   rV   c                   óÞ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9 SS\S\S-  S	\	\
R                  \4   4S
 jj5       5       r\
R                  " 5       \S 5       5       rSrU =r$ )ÚModernBertRotaryEmbeddingé^   Údevicez5.18)ÚversionNr'   c           
      óž  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        [        [        UR                  5      5      U l        0 U l	        U R                   HÜ  nU R
                  R                  U   nUc  M!  US   U R                  U'   U R                  nU R                  U   S:w  a  [        U R                  U      nU" U R
                  X#S9u  pg[        X S3[        R                  " USS95        [        X S3[        R                  " UR!                  5       SS95        [        X S3U5        MÞ     g )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r-   r.   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr'   ÚsortedÚsetÚlayer_typesrs   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr   ÚBufferÚclone)	r<   r'   rp   rv   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr=   s	           €r>   r.   Ú"ModernBertRotaryEmbedding.__init___   s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÑAˆKØÑ"Ùà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÓ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÈfÑ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ñ4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÑ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTò +r@   rv   rC   c           	      ó  • U R                   U   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?r   rY   ©Údtype)r�   Úgetattrr1   Únum_attention_headsrO   ÚarangeÚfloatÚto)r'   rp   rv   ÚkwargsÚbaserh   Úattention_factorÚinv_freqs           r>   r‚   Ú9ModernBertRotaryEmbedding.compute_default_rope_parametersu   s€   € ð& ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r@   c                 ó  • [        X S35      n[        X S35      nUS S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        US	S
9   Xg-  R                  SS5      n	[        R                  " X™4SS9n
U
R                  5       U-  nU
R                  5       U-  nS S S 5        WR                  UR                  S9WR                  UR                  S94$ ! , (       d  f       N@= f)Nrw   rz   r   rf   r"   ©r�   rp   ÚmpsÚcpuF)Údevice_typeÚenabledrY   rg   rŽ   )r�   ÚexpandÚshaper”   rO   r“   rp   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosÚsinr�   )r<   ÚxÚposition_idsrv   r˜   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrž   ÚfreqsÚembr§   r¨   s                r>   rG   Ú!ModernBertRotaryEmbedding.forward�   sS  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �Tš1˜d�]Ñ#×*Ñ*¨<×+=Ñ+=¸aÑ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÐoð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   Ã*AE5Å5
F)r'   r€   r|   r}   rs   rE   rI   )rJ   rK   rL   rM   r   r#   r.   Ústaticmethodr¤   ÚtuplerO   rQ   r“   r‚   Úno_gradr   rG   rR   rS   rT   s   @r>   rn   rn   ^   s™   ø† Ù�X vÑ.ñUÐ/÷ Uó /ðUð* Ù�X vÑ.àHLñ5Ø ð5Ø;>À¹:ð5à	ˆu�|‰|˜UÐ"Ñ	#ô5ó /ó ð5ð2 ‡]‚]ƒ_Øñ<ó ó ö<r@   rn   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ó°  • [         R                  " XR                  SS5      5      U-  nUb  X„-   n[        R                  R                  US[         R                  S9R                  UR                  5      n[        R                  R                  X†U R                  S9n[         R                  " Xƒ5      n	U	R                  SS5      R                  5       n	X˜4$ )NrY   r   rf   )rh   r�   )ÚpÚtrainingr"   )rO   Úmatmulr¥   r   Ú
functionalÚsoftmaxÚfloat32r”   r�   rº   r½   Ú
contiguous)
r´   rµ   r¶   r·   r¸   r¹   rº   r•   Úattn_weightsÚattn_outputs
             r>   Úeager_attention_forwardrÅ   ¦   s°   € ô —<’< §}¡}°Q¸Ó':Ó;¸gÑE€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r@   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nrf   rY   rg   )r¡   rO   r¦   )r©   Úx1Úx2s      r>   Úrotate_halfrÉ   ¼   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r@   Úrotary_pos_embc                 ób  • U R                   nUR                  U5      nUR                  U5      nU R                  5       U-  [        U R                  5       5      U-  -   nUR                  5       U-  [        UR                  5       5      U-  -   nUR	                  U5      UR	                  U5      4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)r�   Ú	unsqueezer“   rÉ   r”   )ÚqÚkr§   r¨   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r>   Úapply_rotary_pos_embrÓ   Ã   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐAr@   c                   ó  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr  SS\R                  S\
\R                  \R                  4   S-  S	\R                  S-  S
\\   S\
\R                  \R                  S-  4   4
S jjrSrU =r$ )ÚModernBertAttentionéÞ   an  Performs multi-headed self attention on a batch of unpadded sequences.

If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
which requires padding and unpadding inputs, adding some overhead.

See `forward` method for additional details.
Nr'   Ú	layer_idxc                 óP  >• [         TU ]  5         Xl        X l        UR                  UR
                  -  S:w  a&  [        SUR                   SUR
                   S35      eUR                  U l        UR                  U l        UR                  UR
                  -  U l	        [        R                  " UR                  SU R                  -  UR
                  -  UR                  S9U l        UR                  U   S:X  a  UR                  S-   U l        OS U l        S	U l        [        R                  " UR                  UR                  UR                  S9U l        UR                  S
:”  a&  [        R$                  " UR                  5      U l        g [        R&                  " 5       U l        g )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (Ú)r   rZ   Úsliding_attentionr"   Fç        )r-   r.   r'   r×   r1   r‘   Ú
ValueErrorÚattention_dropoutÚdeterministic_flash_attnr�   r   r[   Úattention_biasÚWqkvr€   Úsliding_windowÚ	is_causalrc   r8   ÚIdentityÚout_drop©r<   r'   r×   r=   s      €r>   r.   ÚModernBertAttention.__init__é   sv  ø€ Ü‰ÑÔØŒØ"Œà×Ñ × :Ñ :Ñ:¸aÓ?ÜØ# F×$6Ñ$6Ð#7Ð7mÐnt÷  oIñ  oIð  nJð  JKð  Lóð ð "(×!9Ñ!9ˆÔØ(.×(GÑ(GˆÔ%Ø×*Ñ*¨f×.HÑ.HÑHˆŒÜ—I’IØ×Ñ  D§M¡MÑ 1°F×4NÑ4NÑ NÐU[×UjÑUjñ
ˆŒ	ð ×Ñ˜iÑ(Ð,?Ó?ð #)×"7Ñ"7¸!Ñ";ˆDÕà"&ˆDÔàˆŒä—)’)˜F×.Ñ.°×0BÑ0BÈ×I^ÑI^Ñ_ˆŒØ@F×@XÑ@XÐ[^Ó@^œŸ
š
 6×#;Ñ#;Ó<ˆ�Ôdf×doÒdoÓdqˆ�r@   rF   Úposition_embeddingsr¸   r•   rC   c                 óÄ  • UR                   S S nU R                  U5      nUR                  " / UQSPSPU R                  P76 nUR	                  SS9u  pxn	UR                  SS5      nUR                  SS5      nU	R                  SS5      n	Uu  p«[        XxX«SS9u  px[        R                  " U R                  R                  [        5      nU" U UUU	U4U R                  (       a  U R                  OSU R                  S	-  U R                  U R                  S
.UD6u  pÞUR                   " / UQSP76 R#                  5       nU R%                  U R'                  U5      5      nXÞ4$ )Nrf   r   éýÿÿÿrg   r"   rY   )rÏ   rÛ   ç      à¿)rº   r¹   rá   Údeterministic)r¡   rà   Úviewr�   Úunbindr¥   rÓ   r   Úget_interfacer'   Ú_attn_implementationrÅ   r½   rÝ   rá   rÞ   ÚreshaperÂ   rä   rc   )r<   rF   rç   r¸   r•   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesr§   r¨   Úattention_interfacerÄ   rÃ   s                  r>   rG   ÚModernBertAttention.forward  so  € ð $×)Ñ)¨#¨2Ð.ˆà�i‰i˜Ó&ˆØ�hŠhÐ:˜Ð: QÐ:¨Ð:¨D¯M©MÒ:ˆØ14·±À°Ð1CÑ.ˆ ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÐjkÑ#lÑ ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯m¯m�D×*Ò*ÀØ—M‘M 4Ñ'Ø×.Ñ.Ø×7Ñ7ñ%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m D§G¡G¨KÓ$8Ó9ˆØÐ(Ð(r@   )
rc   rà   rÝ   r'   rÞ   r�   râ   r×   rä   rá   rE   rI   )rJ   rK   rL   rM   rN   r#   r\   r.   rO   rQ   r²   r   r   rG   rR   rS   rT   s   @r>   rÕ   rÕ   Þ   s¯   ø† ññrÐ/ð r¸CÀ$¹J÷ rð rð@ IMØ.2ñ	')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð')ð Ÿ™ tÑ+ð	')ð
 Ð+Ñ,ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')ó ')r@   rÕ   c                   óÆ   ^ • \ rS rSrSS\S\S-  4U 4S jjjr  SS\R                  S\R                  S-  S\R                  S-  S	\	\
   S
\R                  4
S jjrSrU =r$ )ÚModernBertEncoderLayeri0  Nr'   r×   c                 óÄ  >• [         TU ]  5         Xl        X l        US:X  a  [        R
                  " 5       U l        O9[        R                  " UR                  UR                  UR                  S9U l        [        XS9U l        [        R                  " UR                  UR                  UR                  S9U l        [        U5      U l        UR                   U   U l        g )Nr   r*   )r'   r×   )r-   r.   r'   r×   r   rã   Ú	attn_normr4   r1   r5   r6   rÕ   ÚattnÚmlp_normrV   Úmlpr€   Úattention_typerå   s      €r>   r.   ÚModernBertEncoderLayer.__init__1  s¡   ø€ Ü‰ÑÔØŒØ"ŒØ˜‹>ÜŸ[š[›]ˆD�NäŸ\š\¨&×*<Ñ*<À&Ç/Á/ÐX^×XhÑXhÑiˆDŒNÜ'¨vÑKˆŒ	ÜŸš V×%7Ñ%7¸V¿_¹_ÐSY×ScÑScÑdˆŒÜ  Ó(ˆŒØ$×0Ñ0°Ñ;ˆÕr@   rF   r¸   rç   r•   rC   c                 ó    • U R                   " U R                  U5      4UUS.UD6u  pVX-   nXR                  U R                  U5      5      -   nU$ )N)rç   r¸   )rü   rû   rþ   rý   )r<   rF   r¸   rç   r•   rÄ   Ú_s          r>   rG   ÚModernBertEncoderLayer.forward>  s_   € ð ŸšØ�N‰N˜=Ó)ð
à 3Ø)ñ
ð ñ	
‰ˆð &Ñ3ˆØ%¯©°·±¸}Ó1MÓ(NÑNˆØÐr@   )rÿ   rü   rû   r'   r×   rþ   rý   rE   rI   )rJ   rK   rL   rM   r#   r\   r.   rO   rQ   r   r   rG   rR   rS   rT   s   @r>   rù   rù   0  s‚   ø† ñ<Ð/ð <¸CÀ$¹J÷ <ð <ð  /3Ø37ñ	à—|‘|ðð Ÿ™ tÑ+ðð #Ÿ\™\¨DÑ0ð	ð
 Ð+Ñ,ðð 
�‰÷ó r@   rù   c                   ó¢   ^ • \ rS rSr% \\S'   SrSrSS/rSr	Sr
SrSr\\S.r\R"                  " 5       S\R&                  4U 4S	 jj5       rS
rU =r$ )ÚModernBertPreTrainedModeliP  r'   ÚmodelTr%   rù   )rF   Ú
attentionsr´   c                 óž  >^• [         T	U ]  U5        U R                  R                  mTc  SmS[        R
                  S[        4U4S jjnU R                  R                  U R                  R                  [        R                  " SU R                  R                  -  5      -  U R                  R                  U R                  R                  S-  S.n[        U[        5      (       a  U" UR                  US   5        g [        U[        5      (       a-  U" UR                   US	   5        U" UR"                  US
   5        g [        U[$        5      (       a-  U" UR&                  US	   5        U" UR"                  US
   5        g [        U[(        5      (       a  U" UR*                  US
   5        g [        U[,        5      (       a  U" UR.                  US
   5        g [        U[0        [2        [4        [6        45      (       a  U" UR8                  US   5        g [        U[:        5      (       a¢  UR<                   H‘  nUR>                  nUR@                  U   S:w  a  [B        UR@                  U      nU" UR                  US9u  pg[D        RF                  " [I        X S35      U5        [D        RF                  " [I        X S35      U5        M“     g g )Nr   r´   Ústdc                 óô   >• [         R                  " U R                  SUT* U-  TU-  S9  [        U [        R
                  5      (       a/  U R                  b!  [         R                  " U R                  5        g g g )NrÛ   )Úmeanr	  ÚaÚb)ÚinitÚtrunc_normal_Úweightr¢   r   r[   r,   Úzeros_)r´   r	  Úcutoff_factors     €r>   Úinit_weightÚ<ModernBertPreTrainedModel._init_weights.<locals>.init_weightg  sg   ø€ Ü×ÒØ—‘ØØØ �. 3Ñ&Ø #Ñ%òô ˜&¤"§)¡)×,Ñ,Ø—;‘;Ñ*Ü—K’K §¡Õ,ð +ð -r@   g       @rê   )ÚinÚoutÚ	embeddingÚ	final_outr  r  r  r  rt   ru   rw   ry   )%r-   Ú_init_weightsr'   Úinitializer_cutoff_factorr   ÚModuler“   Úinitializer_rangeÚmathÚsqrtÚnum_hidden_layersr1   r¢   r%   r3   rV   r_   rc   rÕ   rà   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierrn   r€   r‚   rs   r   r  Úcopy_r�   )
r<   r´   r  Ústdsrv   r‡   rˆ   r  r  r=   s
           @€r>   r  Ú'ModernBertPreTrainedModel._init_weights`  s>  ù€ ä‰Ñ˜fÔ%ØŸ™×=Ñ=ˆØÑ ØˆMð	-¤§	¡	ð 	-´÷ 	-ð —+‘+×/Ñ/Ø—;‘;×0Ñ0´4·9²9¸SÀ4Ç;Á;×C`ÑC`Ñ=`Ó3aÑaØŸ™×6Ñ6ØŸ™×0Ñ0°$Ñ6ñ	
ˆô �fÔ2×3Ñ3Ù˜×-Ñ-¨t°KÑ/@ÕAÜ˜¤×.Ñ.Ù˜Ÿ	™	 4¨¡:Ô.Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 3×4Ñ4Ù˜Ÿ™ T¨$¡ZÔ0Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 8×9Ñ9Ù˜Ÿ™ d¨5¡kÕ2Ü˜Ô 5×6Ñ6Ù˜Ÿ™¨¨U©Õ4ÜØä3Ü+Ü0Ü.ð	÷
ñ 
ñ ˜×)Ñ)¨4°Ñ+<Õ=Ü˜Ô 9×:Ñ:Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ó<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ñ#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ò 1ð ;r@   © )rJ   rK   rL   rM   r#   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrù   rÕ   Ú_can_record_outputsrO   r³   r   r  r  rR   rS   rT   s   @r>   r  r  P  sp   ø‡ àÓØÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 0Ø)ñÐð
 ‡]‚]ƒ_ð7^ B§I¡Iö 7^ó ö7^r@   r  c                   óö   ^ • \ rS rSrS\4U 4S jjrS rS r\\	\
    SS\R                  S-  S\R                  S-  S	\R                  S-  S
\R                  S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚModernBertModeli›  r'   c           	      ó¦  >• [         TU ]  U5        Xl        [        U5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l
        [
        R                  " UR                  UR                  UR                  S9U l        [!        US9U l        SU l        U R'                  5         g s  snf )Nr*   )r'   F)r-   r.   r'   r%   Ú
embeddingsr   Ú
ModuleListÚranger  rù   Úlayersr4   r1   r5   r6   Ú
final_normrn   Ú
rotary_embÚgradient_checkpointingÚ	post_initrå   s      €r>   r.   ÚModernBertModel.__init__�  s¡   ø€ Ü‰Ñ˜Ô ØŒÜ.¨vÓ6ˆŒÜ—m’mÜHMÈf×NfÑNfÔHgÓhÒHg¸9Ô# FÖ6ÑHgÑhó
ˆŒô Ÿ,š, v×'9Ñ'9¸v¿¹ÐU[×UeÑUeÑfˆŒÜ3¸6ÑBˆŒØ&+ˆÔ#Ø�‰Õùò is   ÁCc                 ó.   • U R                   R                  $ rE   ©r9  r3   ©r<   s    r>   Úget_input_embeddingsÚ$ModernBertModel.get_input_embeddings©  s   € Ø�‰×-Ñ-Ð-r@   c                 ó$   • XR                   l        g rE   rC  )r<   r·   s     r>   Úset_input_embeddingsÚ$ModernBertModel.set_input_embeddings¬  s   € Ø).�‰Õ&r@   NrA   r¸   rª   rB   r•   rC   c                 ó¬  • US L US L-  (       a  [        S5      eUb  UR                  S   OUR                  S   nUb  UR                  OUR                  nUc#  [        R                  " XgS9R                  S5      nU R                  XS9n[        U=n	[        5      (       d'  U R                  UUS.n
[        S
0 U
D6[        S
0 U
D6S.n	0 n[        U R                  R                  5       H  nU R                  XƒU5      X¼'   M     U R                   H'  nU" U4X�R                      X½R                      S.UD6nM)     U R#                  U5      n[%        US	9$ )Nz:You must specify exactly one of input_ids or inputs_embedsr"   ©rp   r   )rA   rB   )r'   rB   r¸   )Úfull_attentionrÚ   )r¸   rç   )Úlast_hidden_stater,  )rÜ   r¡   rp   rO   r’   rÌ   r9  r¢   Údictr'   r   r   r   r€   r>  r<  rÿ   r=  r   )r<   rA   r¸   rª   rB   r•   Úseq_lenrp   rF   Úattention_mask_mappingÚmask_kwargsrç   rv   Úencoder_layers                 r>   rG   ÚModernBertModel.forward¯  sg  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZà,9Ñ,E�-×%Ñ% aÒ(È9Ï?É?Ð[\ÑK]ˆØ%.Ñ%:�×!Ò!À×@TÑ@TˆàÑÜ Ÿ<š<¨Ñ?×IÑIÈ!ÓLˆLàŸ™°)˜ÐYˆä°NÐBÐ0ÄD×IÑIàŸ+™+Ø!.Ø"0ñˆKô #<Ñ"J¸kÑ"JÜ%MÑ%\ÐP[Ñ%\ñ&Ð"ð
 !ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ð "Ÿ[œ[ˆMÙ)Øðà5×6RÑ6RÑSØ$7×8TÑ8TÑ$Uñð ñ	ŠMñ )ð Ÿ™¨Ó6ˆä°Ñ?Ð?r@   )r'   r9  r=  r?  r<  r>  )NNNN)rJ   rK   rL   rM   r#   r.   rE  rH  r    r!   r   rO   rP   rQ   r   r   r   rG   rR   rS   rT   s   @r>   r7  r7  ›  s¼   ø† ð
Ð/÷ 
ò.ò/ð  ØØð .2Ø.2Ø04Ø-1ñ,@à×#Ñ# dÑ*ð,@ð Ÿ™ tÑ+ð,@ð ×&Ñ&¨Ñ-ð	,@ð
 —|‘| dÑ*ð,@ð Ð+Ñ,ð,@ð 
ô,@ó ó ó  ö,@r@   r7  c                   ój   ^ • \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )r   iá  r'   c                 óF  >• [         TU ]  5         Xl        [        R                  " UR
                  UR
                  UR                  5      U l        [        UR                     U l
        [        R                  " UR
                  UR                  UR                  S9U l        g )Nr*   )r-   r.   r'   r   r[   r1   Úclassifier_biasr!  r
   Úclassifier_activationra   r4   r5   r6   r7   r;   s     €r>   r.   Ú!ModernBertPredictionHead.__init__â  so   ø€ Ü‰ÑÔØŒÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÀv×G]ÑG]Ó^ˆŒ
Ü˜&×6Ñ6Ñ7ˆŒÜ—L’L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ñ`ˆ�	r@   rF   rC   c                 ó`   • U R                  U R                  U R                  U5      5      5      $ rE   )r7   ra   r!  )r<   rF   s     r>   rG   Ú ModernBertPredictionHead.forwardé  s#   € Ø�y‰y˜Ÿ™ $§*¡*¨]Ó";Ó<Ó=Ð=r@   )ra   r'   r!  r7   )rJ   rK   rL   rM   r#   r.   rO   rQ   rG   rR   rS   rT   s   @r>   r   r   á  s2   ø† ðaÐ/÷ að> U§\¡\ð >°e·l±l÷ >ò >r@   r   zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   óP  ^ • \ rS rSrSS0rS\4U 4S jjrS rS\R                  4S jr
\\     SS
\R                  S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r"  ií  zdecoder.weightz&model.embeddings.tok_embeddings.weightr'   c                 ón  >• [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        [        R                  " UR                  UR                  UR                  S9U l        U R                  R                  U l        U R                  R                  U l        U R                  5         g )NrZ   )r-   r.   r'   r7  r  r   Úheadr   r[   r1   r0   Údecoder_biasr#  Úsparse_predictionÚsparse_pred_ignore_indexr@  r;   s     €r>   r.   ÚModernBertForMaskedLM.__init__õ  sƒ   ø€ Ü‰Ñ˜Ô ØŒÜ$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—y’y ×!3Ñ!3°V×5FÑ5FÈV×M`ÑM`ÑaˆŒà!%§¡×!>Ñ!>ˆÔØ(,¯©×(LÑ(LˆÔ%ð 	�‰Õr@   c                 ó   • U R                   $ rE   ©r#  rD  s    r>   Úget_output_embeddingsÚ+ModernBertForMaskedLM.get_output_embeddings  s   € Ø�|‰|Ðr@   Únew_embeddingsc                 ó   • Xl         g rE   rd  )r<   rg  s     r>   Úset_output_embeddingsÚ+ModernBertForMaskedLM.set_output_embeddings  s   € Ø%�r@   NrA   r¸   rª   rB   Úlabelsr•   rC   c                 óÊ  • U R                   " SUUUUS.UD6nUS   nU R                  (       aI  UbF  UR                  S5      nUR                  UR                  S   S5      nXPR                  :g  n	X‰   nXY   nU R                  U R                  U5      5      n
S nUb)  U R                  " X¥4SU R                  R                  0UD6n[        UU
UR                  UR                  S9$ )N©rA   r¸   rª   rB   r   rf   r0   ©ÚlossÚlogitsrF   r  r,  )r  r`  rì   r¡   ra  r#  r^  Úloss_functionr'   r0   r   rF   r  )r<   rA   r¸   rª   rB   rk  r•   ÚoutputsrM  Úmask_tokensrp  ro  s               r>   rG   ÚModernBertForMaskedLM.forward  sý   € ð —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà×!×! fÑ&8à—[‘[ “_ˆFØ 1× 6Ñ 6°v·|±|ÀA±ÈÓ KÐð !×$AÑ$AÑAˆKØ 1Ñ >ÐØÑ(ˆFà—‘˜dŸi™iÐ(9Ó:Ó;ˆàˆØÑØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
r@   )r'   r#  r^  r  ra  r`  ©NNNNN)rJ   rK   rL   rM   Ú_tied_weights_keysr#   r.   re  r   r[   ri  r   r   rO   rP   rQ   r   r   r²   r   rG   rR   rS   rT   s   @r>   r"  r"  í  sç   ø† ð +Ð,TÐUÐðÐ/÷ òð&°B·I±Iô &ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ'
à×#Ñ# dÑ*ð'
ð Ÿ™ tÑ+ð'
ð —l‘l TÑ)ð	'
ð
 —|‘| dÑ*ð'
ð —‘˜tÑ#ð'
ð Ð+Ñ,ð'
ð 
ˆu�|‰|Ñ	˜~Ñ	-ô'
ó ó ö'
r@   r"  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r$  i4  r'   c                 ón  >• [         TU ]  U5        UR                  U l        Xl        [	        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        U R!                  5         g rE   )r-   r.   Ú
num_labelsr'   r7  r  r   r^  rO   r   r8   Úclassifier_dropoutr:   r[   r1   r(  r@  r;   s     €r>   r.   Ú,ModernBertForSequenceClassification.__init__:  s€   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr@   NrA   r¸   rª   rB   rk  r•   rC   c                 ób  • U R                   " SUUUUS.UD6nUS   nU R                  R                  S:X  a
  USS2S4   nO‰U R                  R                  S:X  ao  Uc;  [        R                  " UR
                  SS UR                  [        R                  S9nX‚R                  S5      -  R                  S	S
9UR                  S	SS9-  nU R                  U5      n	U R                  U	5      n	U R                  U	5      n
SnUGbŽ  U R                  R                  c‘  U R                  S	:X  a  SU R                  l        OoU R                  S	:”  aN  UR                  [        R                   :X  d  UR                  [        R"                  :X  a  SU R                  l        OSU R                  l        U R                  R                  S:X  aI  [%        5       nU R                  S	:X  a&  U" U
R'                  5       UR'                  5       5      nOŒU" X¥5      nOƒU R                  R                  S:X  a=  [)        5       nU" U
R+                  SU R                  5      UR+                  S5      5      nO,U R                  R                  S:X  a  [-        5       nU" X¥5      n[/        UU
UR0                  UR2                  S9$ )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
rm  r   ÚclsNr  rY   )rp   r�   rf   r"   rg   T©rh   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrn  r,  )r  r'   Úclassifier_poolingrO   Úonesr¡   rp   ÚboolrÌ   Úsumr^  r:   r(  Úproblem_typery  r�   Úlongr\   r   Úsqueezer   rì   r   r   rF   r  )r<   rA   r¸   rª   rB   rk  r•   rr  rM  Úpooled_outputrp  ro  Úloss_fcts                r>   rG   Ú+ModernBertForSequenceClassification.forwardG  s]  € ð" —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà�;‰;×)Ñ)¨UÓ2Ø 1²!°Q°$Ñ 7ÑØ�[‰[×+Ñ+¨vÓ5ØÑ%Ü!&§¢Ø%×+Ñ+¨B¨QÐ/Ð8I×8PÑ8PÔX]×XbÑXbñ"�ð "3×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÐ ]Ð`n×`rÑ`rØ˜tð asð añ !Ðð Ÿ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆàˆØÒØ�{‰{×'Ñ'Ñ/Ø—?‘? aÓ'Ø/;�D—K‘KÕ,Ø—_‘_ qÓ(¨f¯l©l¼e¿j¹jÓ.HÈFÏLÉLÔ\a×\eÑ\eÓLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ó7Ü"›9�Ø—?‘? aÓ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÓJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÓIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
r@   )r(  r'   r:   r^  r  ry  ru  )rJ   rK   rL   rM   r#   r.   r   r   rO   rP   rQ   r   r   r²   r   rG   rR   rS   rT   s   @r>   r$  r$  4  sÎ   ø† ðÐ/÷ ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð7Ñ	7ôC
ó ó öC
r@   r$  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r&  i�  r'   c                 ób  >• [         TU ]  U5        UR                  U l        [        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        U R                  5         g rE   ©r-   r.   ry  r7  r  r   r^  rO   r   r8   rz  r:   r[   r1   r(  r@  r;   s     €r>   r.   Ú)ModernBertForTokenClassification.__init__•  s{   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr@   NrA   r¸   rª   rB   rk  r•   rC   c                 ób  • U R                   " SUUUUS.UD6nUS   nU R                  U5      nU R                  U5      nU R                  U5      n	Sn
Ub<  [	        5       nU" U	R                  SU R                  5      UR                  S5      5      n
[        U
U	UR                  UR                  S9$ )zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
rm  r   Nrf   rn  r,  )
r  r^  r:   r(  r   rì   ry  r   rF   r  )r<   rA   r¸   rª   rB   rk  r•   rr  rM  rp  ro  r‹  s               r>   rG   Ú(ModernBertForTokenClassification.forward¡  sÃ   € ð —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆàˆØÑÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
r@   ©r(  r:   r^  r  ry  ru  )rJ   rK   rL   rM   r#   r.   r   r   rO   rP   rQ   r   r   r²   r   rG   rR   rS   rT   s   @r>   r&  r&  �  sÅ   ø† ð
Ð/÷ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ$
à×#Ñ# dÑ*ð$
ð Ÿ™ tÑ+ð$
ð —l‘l TÑ)ð	$
ð
 —|‘| dÑ*ð$
ð —‘˜tÑ#ð$
ð Ð+Ñ,ð$
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ô$
ó ó ö$
r@   r&  c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\
\   S\\R                     \-  4S jj5       5       rSrU =r$ )r'  iÊ  r'   c                 ób  >• [         TU ]  U5        UR                  U l        [        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        U R                  5         g rE   r�  r;   s     €r>   r.   Ú'ModernBertForQuestionAnswering.__init__Ì  sy   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰Õr@   NrA   r¸   rª   Ústart_positionsÚend_positionsr•   rC   c                 ó¸  • U R                   " U4UUS.UD6nUS   nU R                  U5      nU R                  U5      nU R                  U5      n	U	R	                  SSS9u  p«U
R                  S5      R                  5       n
UR                  S5      R                  5       nS nUb  Ub  U R                  " X«XE40 UD6n[        UU
UUR                  UR                  S9$ )N)r¸   rª   r   r"   rf   rg   )ro  Ústart_logitsÚ
end_logitsrF   r  )r  r^  r:   r(  Úsplitr‰  rÂ   rq  r   rF   r  )r<   rA   r¸   rª   r—  r˜  r•   rr  rM  rp  rš  r›  ro  s                r>   rG   Ú&ModernBertForQuestionAnswering.forward×  sü   € ð —*’*Øð
à)Ø%ñ
ð ñ	
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆà#)§<¡<°°r <Ð#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÑ&¨=Ñ+DØ×%Ò% lÀÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r@   r“  ru  )rJ   rK   rL   rM   r#   r.   r   r   rO   rQ   r   r   r²   r   rG   rR   rS   rT   s   @r>   r'  r'  Ê  sÃ   ø† ð	Ð/÷ 	ð Øð *.Ø.2Ø,0Ø/3Ø-1ñ#
à—<‘< $Ñ&ð#
ð Ÿ™ tÑ+ð#
ð —l‘l TÑ)ð	#
ð
 Ÿ™¨Ñ,ð#
ð —|‘| dÑ*ð#
ð Ð+Ñ,ð#
ð 
ˆu�|‰|Ñ	Ð;Ñ	;ô#
ó ó ö#
r@   r'  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   ó   ^ • \ rS rSrS\4U 4S jjr\\     SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\\   S\\R                     \-  4S jj5       5       rSrU =r$ )r%  iÿ  r'   c                 ó8  >• [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        [        R                  R                  UR                  5      U l        [        R                  " UR                  S5      U l        U R                  5         g )Nr"   )r-   r.   r'   r7  r  r   r^  rO   r   r8   rz  r:   r[   r1   r(  r@  r;   s     €r>   r.   Ú$ModernBertForMultipleChoice.__init__  sm   ø€ Ü‰Ñ˜Ô ØŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr@   NrA   r¸   rª   rB   rk  r•   rC   c                 óÄ  • Ub  UR                   S   OUR                   S   nUb!  UR                  SUR                  S5      5      OSnUb!  UR                  SUR                  S5      5      OSnUb!  UR                  SUR                  S5      5      OSnUb1  UR                  SUR                  S5      UR                  S5      5      OSnU R                  " S
UUUUS.UD6nUS   n	U R                  R
                  S:X  aŽ  [        R                  " U	R                   S   U	R                  S9n
Ub)  UR                  SS	9R                  U	R                  5      nO/[        R                  " S
S[        R                  U	R                  S9nXšU4   n	OMU R                  R
                  S:X  a3  UR                  SSS9nX’R                  S5      -  R                  SS	9U-  n	U R                  U	5      nU R!                  U5      nU R#                  U5      nUR                  SU5      nSnUb  [$        R&                  " 5       nU" Xõ5      n[)        UUUR*                  UR,                  S9$ )a  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
Nr"   rf   éþÿÿÿrm  r   r}  rK  rg   r,  r›   r  Tr~  rn  )r¡   rì   Úsizer  r'   rƒ  rO   r’   rp   Úargmaxr”   Úfullrˆ  r†  rÌ   r^  r:   r(  r   r   r   rF   r  )r<   rA   r¸   rª   rB   rk  r•   Únum_choicesrr  rM  Ú	indices_0Úcls_maskÚnum_non_pad_tokensrŠ  rp  Úreshaped_logitsro  r‹  s                     r>   rG   Ú#ModernBertForMultipleChoice.forward  sh  € ð  -6Ñ,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÑ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÑMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÑG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ñ(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*’*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐð �;‰;×)Ñ)¨UÓ2ÜŸšÐ%6×%<Ñ%<¸QÑ%?ÐHY×H`ÑH`ÑaˆIàÑ)Ø)×0Ñ0°RÐ0Ð8×;Ñ;Ð<M×<TÑ<TÓU‘ô !Ÿ:š: b¨!´5·:±:ÐFW×F^ÑF^Ñ_�à 1¸XÐ2EÑ FÑð �[‰[×+Ñ+¨vÓ5Ø!/×!3Ñ!3¸À4Ð!3Ð!HÐØ!2×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÐ ]Ð`rÑ rÐàŸ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÑÜ×*Ò*Ó,ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ñ	
ð 	
r@   )r(  r'   r:   r^  r  ru  )rJ   rK   rL   rM   r#   r.   r   r   rO   rP   rQ   r   r   r²   r   rG   rR   rS   rT   s   @r>   r%  r%  ÿ  sÎ   ø† ð
Ð/÷ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð8Ñ	8ôC
ó ó öC
r@   r%  )r7  r  r"  r$  r&  r'  r%  )rÛ   )r"   )Fr  Úcollections.abcr   rO   r   Útorch.nnr   r   r   Ú r	   r  Úactivationsr
   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r    Úutils.output_capturingr!   Úconfiguration_modernbertr#   r  r%   rV   rn   rQ   r“   rÅ   rÉ   rÓ   rÕ   rù   r  r7  r   r"  r$  r&  r'  r%  Ú__all__r,  r@   r>   Ú<module>r½     s[  ðó, Ý $ã Ý ß AÑ Aå &Ý !ß Lß `Ý 9÷÷ ÷ Lß FÝ &ß 7Ý 0ß YÑ YÝ 5Ý 6ô˜2Ÿ9™9ô ô,:�B—I‘Iô :ô(E< §	¡	ô E<ð^ ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð õ%ò,(ñ Ð-Ó.óBó /ðBñ4 Ð)Ó*ôN)˜"Ÿ)™)ó N)ó +ðN)ôbÐ7ô ð@ ôG^ ó G^ó ðG^ðT ôB@Ð/ó B@ó ðB@ôJ	>˜rŸy™yô 	>ñ ðñô
?
Ð5ó ?
óð
?
ñD ðñô
S
Ð*Có S
óð
S
ñl ðñô
3
Ð'@ó 3
óð
3
ðl ô1
Ð%>ó 1
ó ð1
ñh ðñô
R
Ð";ó R
óð
R
òj�r@   