ó
    ‡Ž¡jŒ  ã                   ód   • S r SSKJrJr  SSKJr  \R                  " \5      r " S S\5      r	S/r
g)zTokenization class for Dia.é   )Ú
AddedTokenÚPreTrainedTokenizer)Úloggingc            	       ó¸   ^ • \ rS rSrSrSS/r    SS\S-  S\S-  S\S-  S	\4U 4S
 jjjr\	S 5       r
S rS\S\\   4S jrS rS rS\\   S\4S jrSrU =r$ )ÚDiaTokenizeré   a€  
Construct a Dia tokenizer. Dia simply uses raw bytes utf-8 encoding except for special tokens `[S1]` and `[S2]`.

This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The token used for padding, for example when batching sequences of different lengths.
    unk_token (`str`, *optional*, defaults to `"<pad>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    max_length (`int`, *optional*, defaults to 1024):
        The maximum length of the sequences when encoding. Sequences longer than this will be truncated.
    offset (`int`, *optional*, defaults to 0):
        The offset of the tokenizer.
Ú	input_idsÚattention_maskÚ	pad_tokenNÚ	unk_tokenÚ
max_lengthÚoffsetc                 ó  >• [        U[        5      (       a  [        U5      OUn[        U[        5      (       a  [        U5      OUnSU l        U[        S5      [        S5      S.U l        X@l        [        TU ]  " S	UUUUSSSS.UD6  g )
Né   z[S1]z[S2])é    é   é   Ú	all_zerosTÚnone)r   r   r   r   Útoken_type_ids_patternÚ%token_type_ids_include_special_tokensÚspecial_tokens_pattern© )Ú
isinstanceÚstrr   Ú_utf_vocab_sizeÚ_added_tokens_decoderr   ÚsuperÚ__init__)Úselfr   r   r   r   ÚkwargsÚ	__class__s         €Ú¶C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/dia/tokenization_dia.pyr   ÚDiaTokenizer.__init__,   s�   ø€ ô .8¸	Ä3×-GÑ-G”J˜yÔ)ÈYˆ	Ü-7¸	Ä3×-GÑ-G”J˜yÔ)ÈYˆ	à#ˆÔØ)2´zÀ&Ó7IÌjÐY_ÓN`Ñ%aˆÔ"ØŒÜ‰Òð 		
ØØØ!ØØ#.Ø26Ø#)ñ		
ð ó		
ó    c                 ó   • U R                   $ ©N)r   )r    s    r#   Ú
vocab_sizeÚDiaTokenizer.vocab_sizeF   s   € à×#Ñ#Ð#r%   c                 óÈ   • [        U R                  U R                  -   5       Vs0 s H  oR                  U5      U_M     nnUR	                  U R
                  5        U$ s  snf r'   )Úranger(   r   Úconvert_ids_to_tokensÚupdateÚadded_tokens_encoder)r    ÚiÚvocabs      r#   Ú	get_vocabÚDiaTokenizer.get_vocabJ   sX   € Ü;@ÀÇÁÐSW×S^ÑS^ÑA^Ô;_Ó`Ò;_°a×+Ñ+¨AÓ.°Ò1Ñ;_ˆÐ`Ø�‰�T×.Ñ.Ô/Øˆùò as   ¥AÚtextÚreturnc                 ód   • UR                  S5       Vs/ s H  n[        U5      PM     nnU$ s  snf )zPTake as input a string and return a list of strings (tokens) for words/sub-wordsúutf-8)ÚencodeÚchr)r    r3   r/   Útokenss       r#   Ú	_tokenizeÚDiaTokenizer._tokenizeO   s/   € à"&§+¡+¨gÔ"6Ó7Ò"6˜Q”#�a–&Ñ"6ˆÐ7Øˆùò 8s   ”-c                 ó\   • [        U5      S:w  a  SnU$ [        U5      U R                  -   nU$ )z0Converts a token (str) in an id using the vocab.r   N)ÚlenÚordr   )r    ÚtokenÚtoken_ids      r#   Ú_convert_token_to_idÚ!DiaTokenizer._convert_token_to_idT   s4   € ô ˆu‹:˜‹?ØˆHð ˆô ˜5“z D§K¡KÑ/ˆHàˆr%   c                 ó4   • [        XR                  -
  5      nU$ )z=Converts an index (integer) in a token (str) using the vocab.)r8   r   )r    Úindexr?   s      r#   Ú_convert_id_to_tokenÚ!DiaTokenizer._convert_id_to_token^   s   € ä�EŸK™KÑ'Ó(ˆØˆr%   r9   c                 ó  • SnU Hr  nX0R                   ;   a*  U R                   U   n[        U5      R                  S5      nO2X0R                  ;   a  UR                  S5      nOUR                  S5      nX%-  nMt     UR	                  SSS9nU$ )z:Converts a sequence of tokens (string) in a single string.r%   r6   Úignore)Úerrors)r   r   r7   Ú_added_tokens_encoderÚdecode)r    r9   Úbstringr?   Úadded_token_objÚ
tok_stringÚstrings          r#   Úconvert_tokens_to_stringÚ%DiaTokenizer.convert_tokens_to_stringc   s‹   € àˆÛˆEØ×2Ñ2Ó2Ø"&×"<Ñ"<¸UÑ"C�Ü  Ó1×8Ñ8¸ÓA‘
Ø×4Ñ4Ó4Ø"Ÿ\™\¨'Ó2‘
à"Ÿ\™\¨'Ó2�
ØÑ!ŠGñ ð —‘ °�Ð9ˆØˆr%   )r   r   r   )ú<pad>rR   i   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úmodel_input_namesr   Úintr   Úpropertyr(   r1   Úlistr:   rA   rE   rP   Ú__static_attributes__Ú__classcell__)r"   s   @r#   r   r      s¶   ø† ñð$ %Ð&6Ð7Ðð !(Ø 'Ø!%Øñ
à˜‘:ð
ð ˜‘:ð
ð ˜$‘Jð	
ð
 ÷
ð 
ð4 ñ$ó ð$òð
˜cð  d¨3¡iô ò
òð
¨t°C©yð ¸S÷ ò r%   r   N)rW   Útokenization_pythonr   r   Úutilsr   Ú
get_loggerrS   Úloggerr   Ú__all__r   r%   r#   Ú<module>rc      s<   ðñ "ç BÝ ð 
×	Ò	˜HÓ	%€ôYÐ&ô Yðx Ð
�r%   