ó
    ‡Ž¡j4  ã                   ó”   • S r SSKJr  SSKJr  SSKJr  SSKJr  SSK	J
r
  \
R                  " \5      rSS	0r/ S
Qr " S S\5      rS/rg)zTokenization classes for ESMC.é    )Ú	Tokenizer)ÚBPE)ÚTemplateProcessingé   )ÚTokenizersBackend)ÚloggingÚtokenizer_fileztokenizer.json)!ú<cls>ú<pad>ú<eos>ú<unk>ÚLÚAÚGÚVÚSÚEÚRÚTÚIÚDÚPÚKÚQÚNÚFÚYÚMÚHÚWÚCÚXÚBÚUÚZÚOÚ.Ú-Ú|ú<mask>c                   óP   ^ • \ rS rSrSr\rSS/r\r	        SU 4S jjr
SrU =r$ )ÚEsmcTokenizeréC   a1  
Construct an ESMC tokenizer.

This tokenizer is a character-level tokenizer backed by the HuggingFace `tokenizers` library.
It wraps every sequence with ``<cls>`` and ``<eos>`` tokens and supports a ``|`` chain-break
token for multi-chain inputs.

Args:
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token.
    cls_token (`str`, *optional*, defaults to `"<cls>"`):
        The classification token (prepended to every sequence).
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The padding token.
    mask_token (`str`, *optional*, defaults to `"<mask>"`):
        The mask token, used for masked language modelling.
    eos_token (`str`, *optional*, defaults to `"<eos>"`):
        The end-of-sequence token (appended to every sequence).
    bos_token (`str`, *optional*, defaults to `"<cls>"`):
        The beginning-of-sequence token (prepended to every sequence). When unset, uses cls_token.
    chain_break_token (`str`, *optional*, defaults to `"|"`):
        Token inserted between chains in multi-chain protein inputs. Registered as a model-specific
        special token, so it is exposed as `chain_break_token` / `chain_break_token_id`.
    extra_special_tokens (`list[str]`, *optional*):
        Additional special tokens to register with the tokenizer (round-tripped from a saved config).

Examples:

```python
>>> from transformers import EsmcTokenizer

>>> tokenizer = EsmcTokenizer()
>>> tokenizer("ACDEFGHIKLMNPQRSTVWY")["input_ids"]
[0, 5, 23, 13, 9, 18, 6, 21, 12, 15, 4, 20, 17, 14, 16, 10, 8, 11, 7, 22, 19, 2]
```
Ú	input_idsÚattention_maskc	                 ó   >• [        [        5       V
Vs0 s H  u  p«Xº_M	     nn
n[        [        U/ [	        U5      S95      U l        [        TU ]  " SUUc  UOUUUUUUUS.U	D6  [        U R                   SU R                   3U R                  XÀR                     4U R                  XÀR                     4/S9U R
                  l        g s  snn
f )N)ÚmergesÚ	unk_token)r2   Ú	bos_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚ	eos_tokenÚchain_break_tokenÚextra_special_tokensz $A )ÚsingleÚspecial_tokens© )Ú	enumerateÚSEQUENCE_VOCABr   r   ÚstrÚ
_tokenizerÚsuperÚ__init__r   r4   r7   Úpost_processor)Úselfr2   r4   r5   r6   r7   r3   r8   r9   ÚkwargsÚindÚtokÚtoken_to_idÚ	__class__s                €Ú¸C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/esmc/tokenization_esmc.pyrB   ÚEsmcTokenizer.__init__m   sÖ   ø€ ô 1:¼.Ô0IÔJÒ0I¡H C�s’xÑ0IˆÑJô $¤C¨¸BÌ#ÈiË.Ñ$YÓZˆŒä‰Òð 	
Øà#,Ñ#4‘i¸)ØØØ!ØØ/Ø!5ñ	
ð ò	
ô *<Ø—n‘nÐ% T¨$¯.©.Ð)9Ð:à—‘ ¯^©^Ñ!<Ð=Ø—‘ ¯^©^Ñ!<Ð=ðñ*
ˆ�‰Õ&ùó' Ks   ”C
)r@   )r   r
   r   r*   r   Nr)   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   ÚmodelrB   Ú__static_attributes__Ú__classcell__)rI   s   @rJ   r,   r,   C   sF   ø† ñ#ðJ *ÐØ$Ð&6Ð7ÐØ€Eð ØØØØØØØ!÷%
õ %
ó    r,   N)rP   Ú
tokenizersr   Útokenizers.modelsr   Útokenizers.processorsr   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrL   ÚloggerrQ   r>   r,   Ú__all__r<   rW   rJ   Ú<module>r`      sY   ðñ %å  Ý !Ý 4å >Ý ð 
×	Ò	˜HÓ	%€à%Ð'7Ð8Ð ò"€ôJO
Ð%ô O
ðd Ð
�rW   