ó
    ˆŽ¡jÆ  ã                   ól   • S SK Jr  S SKJr  SSKJr  SSKJr  \" SS9\ " S S	\5      5       5       rS	/r	g
)é    )ÚLiteral)Ústricté   )ÚPreTrainedConfig)Úauto_docstringzanswerdotai/ModernBERT-base)Ú
checkpointc                   óÀ  ^ • \ rS rSr% SrSrS/rSSS.rSr\	\
S	'   S
r\	\
S'   Sr\	\
S'   Sr\	\
S'   Sr\	\
S'   Sr\\
S'   Sr\	\
S'   Sr\\
S'   Sr\\
S'   Sr\\
S'   Sr\\
S'   Sr\	S-  \
S '   S!r\	\\	   -  S-  \
S"'   S#r\	S-  \
S$'   S#r\	S-  \
S%'   S!r\	S-  \
S&'   Sr\\
S''   S(r\\	-  \
S)'   Sr \\   S-  \
S*'   Sr!\"\#S+   \"4   S-  \
S,'   S-r$\	\
S.'   S(r%\\	-  \
S/'   Sr&\\
S0'   S(r'\\	-  \
S1'   S2r(\\
S3'   S4r)\#S5   \
S6'   S(r*\\	-  \
S7'   Sr+\\
S8'   Sr,\\
S9'   Sr-\\
S:'   Sr.\\
S;'   S<r/\	\
S='   S2r0\\
S>'   U 4S? jr1S@ r2U 4SA jr3\4SB 5       r5\5Rl                  SC 5       r5SDr7U =r8$ )EÚModernBertConfigé   aŸ  
initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
    The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
norm_eps (`float`, *optional*, defaults to 1e-05):
    The epsilon used by the rms normalization layers.
norm_bias (`bool`, *optional*, defaults to `False`):
    Whether to use bias in the normalization layers.
local_attention (`int`, *optional*, defaults to 128):
    The window size for local attention.
mlp_dropout (`float`, *optional*, defaults to 0.0):
    The dropout ratio for the MLP layers.
decoder_bias (`bool`, *optional*, defaults to `True`):
    Whether to use bias in the decoder layers.
classifier_pooling (`str`, *optional*, defaults to `"cls"`):
    The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
    CLS token doesn't attend to all tokens on long sequences.
classifier_bias (`bool`, *optional*, defaults to `False`):
    Whether to use bias in the classifier.
classifier_activation (`str`, *optional*, defaults to `"gelu"`):
    The activation function for the classifier.
deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
    Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
sparse_prediction (`bool`, *optional*, defaults to `False`):
    Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
    The index to ignore for the sparse prediction.

Examples:

```python
>>> from transformers import ModernBertModel, ModernBertConfig

>>> # Initializing a ModernBert style configuration
>>> configuration = ModernBertConfig()

>>> # Initializing a model from the modernbert-base style configuration
>>> model = ModernBertModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```Ú
modernbertÚpast_key_valuesg     ˆAg     ˆÃ@)ÚglobalÚlocaliÀÄ  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚgeluÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg       @Úinitializer_cutoff_factorgñhãˆµøä>Únorm_epsFÚ	norm_biasikÄ  NÚpad_token_idijÄ  Úeos_token_idiiÄ  Úbos_token_idÚcls_token_idÚsep_token_idÚattention_biasg        Úattention_dropoutÚlayer_types)Úfull_attentionÚsliding_attentionÚrope_parametersé€   Úlocal_attentionÚembedding_dropoutÚmlp_biasÚmlp_dropoutTÚdecoder_biasÚcls)r/   ÚmeanÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictioniœÿÿÿÚsparse_pred_ignore_indexÚtie_word_embeddingsc                 óì   >• UR                  SS5      nU R                  c?  [        U R                  5       Vs/ s H  n[	        X2-  5      (       a  SOSPM     snU l        [
        TU ]  " S0 UD6  g s  snf )NÚglobal_attn_every_n_layersr   r'   r&   © )Úgetr%   Úranger   ÚboolÚsuperÚ__post_init__)ÚselfÚkwargsr:   ÚiÚ	__class__s       €ÚÅC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/modernbert/configuration_modernbert.pyr@   ÚModernBertConfig.__post_init__q   sy   ø€ à%+§Z¡ZÐ0LÈaÓ%PÐ"Ø×ÑÑ#ô ˜t×5Ñ5Ô6ó â6�Aô (,¨AÑ,J×'KÑ'KÑ#ÐQaÒaÙ6ñ ˆDÔô
 	‰ÒÑ' Ó'ùò s   ¸ A1c                 ó¾  • UR                  SS 5      nSS0SS0S.nU R                  b  U R                  OUU l        Ub<  U R                  S   R                  U5        U R                  S   R                  U5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  SU R
                  S	   5      5        U R                  R                  S5      c  SS0U R                  S'   U R                  S   R	                  SUR                  S
U R
                  S   5      5        U R                  5         U$ )NÚrope_scalingÚ	rope_typeÚdefault)r'   r&   r&   r'   Ú
rope_thetaÚglobal_rope_thetar   Úlocal_rope_thetar   )Úpopr(   Úupdater<   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rA   rB   rH   Údefault_rope_paramss       rE   Úconvert_rope_params_to_dictÚ,ModernBertConfig.convert_rope_params_to_dict|   sc  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÑ7W˜t×3Ò3Ð]pˆÔØÑ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGØ× Ñ Ð!4Ñ5×<Ñ<¸\ÔJð ×Ñ×#Ñ#Ð$4Ó5Ñ=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9Ø˜&Ÿ*™*Ð%8¸$×:LÑ:LÈXÑ:VÓWô	
ð ×Ñ×#Ñ#Ð$7Ó8Ñ@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™*Ð%7¸×9KÑ9KÈGÑ9TÓUô	
ð
 	×$Ñ$Ô&Øˆó    c                 óH   >• [         TU ]  5       nUR                  SS 5        U$ )NÚreference_compile)r?   Úto_dictrN   )rA   ÚoutputrD   s     €rE   rY   ÚModernBertConfig.to_dictš   s#   ø€ Ü‘‘Ó"ˆØ�
‰
Ð&¨Ô-ØˆrV   c                 ó    • U R                   S-  $ )zKHalf-window size: `local_attention` is the total window, so we divide by 2.é   ©r*   )rA   s    rE   Úsliding_windowÚModernBertConfig.sliding_windowŸ   s   € ð ×#Ñ# qÑ(Ð(rV   c                 ó   • US-  U l         g)z<Set sliding_window by updating local_attention to 2 * value.r]   Nr^   )rA   Úvalues     rE   r_   r`   ¤   s   € ð  % q™yˆÕrV   )r%   r*   r(   )9Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencerQ   r   ÚintÚ__annotations__r   r   r   r   r   Ústrr   r   Úfloatr   r   r   r>   r   r   Úlistr    r!   r"   r#   r$   r%   r(   Údictr   r*   r+   r,   r-   r.   r1   r2   r3   r4   r5   r6   r7   r8   r@   rT   rY   Úpropertyr_   ÚsetterÚ__static_attributes__Ú__classcell__)rD   s   @rE   r
   r
      s  ø‡ ñ(ðT €JØ#4Ð"5ÐØ(°8Ñ<€Mà€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø#Ð�sÓ#Ø#'Ð˜SÓ'Ø#Ð�uÓ#Ø'*Ð˜uÓ*Ø€HˆeÓØ€IˆtÓØ$€L�#˜‘*Ó$Ø+0€L�#˜˜S™	‘/ DÑ(Ó0Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø$(€K��c‘˜TÑ!Ó(ØY]€O�T˜'Ð"GÑHÈ$ÐNÑOÐRVÑVÓ]Ø€O�SÓØ%(Ð�u˜s‘{Ó(Ø€HˆdÓØ"€K�˜‘Ó"Ø€L�$ÓØ16Ð˜ Ñ.Ó6Ø&)Ð˜ ™Ó)Ø!€O�TÓ!Ø!'Ð˜3Ó'Ø%*Ð˜dÓ*Ø#Ð�tÓ#Ø$(Ð˜cÓ(Ø $Ð˜Ó$õ	(òõ<ð
 ñ)ó ð)ð ×Ññ)ó ö)rV   r
   N)
Útypingr   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r
   Ú__all__r;   rV   rE   Ú<module>ry      sK   ðõ, å .å 3Ý #ñ Ð8Ñ9ØôG)Ð'ó G)ó ó :ðG)ðT Ð
�rV   