ó
    ‡Ž¡j  ã                   óˆ   • S SK JrJrJrJr  S SKJr  SSKJr  SSK	J
r
  \
R                  " \5      rSS0r " S S	\5      rS	/rg
)é    )Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackend)ÚloggingÚtokenizer_fileztokenizer.jsonc                   óš   ^ • \ rS rSrSr\rSrSS/r\	r
       SS\\\\4   -  S-  S\\\   -  S-  S	\S
\S\S\S\4U 4S jjjrSrU =r$ )ÚGemmaTokenizeré   u…  
Construct a fast Gemma tokenizer (backed by HuggingFace's tokenizers library).

This tokenizer uses a BPE model with byte fallback, no prefix space, and a normalizer that replaces
spaces with "â–�".

Args:
    tokenizer_file (`str`, optional):
        A tokenizers JSON file containing the serialization of a tokenizer.
    unk_token (`str`, optional, defaults to "<unk>"):
        The unknown token.
    bos_token (`str`, optional, defaults to "<bos>"):
        The beginning of sequence token.
    eos_token (`str`, optional, defaults to "<eos>"):
        The end of sequence token.
    pad_token (`str`, optional, defaults to "<pad>"):
        The padding token.
    mask_token (`str`, optional, defaults to "<mask>"):
        The mask token.
    add_bos_token (`bool`, optional, defaults to True):
        Whether or not to add a `bos_token` at the start of sequences.
    add_eos_token (`bool`, optional, defaults to False):
        Whether or not to add an `eos_token` at the end of sequences.
    vocab (`str` or `dict[str, int]`, optional):
        Custom vocabulary dict. If not provided, a minimal vocabulary is created using the special tokens.
ÚleftÚ	input_idsÚattention_maskNÚvocabÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ
mask_tokenc                 ó¤  >• Uc9  [        U5      S[        U5      S[        U5      S[        U5      S[        U5      S0nXl        U=(       d    / U l        [        [	        U R                  U R                  S[        U5      S SS95      U l        [        R                  " SS	S
S9U R
                  l        [        R                  " [        R                  " SS5      [        R                  " 5       [        R                  " 5       /5      U R
                  l        [        R                  " SS5      U R
                  l        ["        T	U ]H  " SUUUUUS.UD6  g )Nr   é   é   r   é   T)r   r   Úfuse_unkr   ÚdropoutÚbyte_fallbackÚ Úmerged_with_previousF)ÚpatternÚbehaviorÚinvertu   â–�)r   r   r   r   r   © )ÚstrÚ_vocabÚ_mergesr   r   Ú
_tokenizerr   ÚSplitÚpre_tokenizerr   ÚSequenceÚReplaceÚByteFallbackÚFuseÚdecoderr   Ú
normalizerÚsuperÚ__init__)
Úselfr   r   r   r   r   r   r   ÚkwargsÚ	__class__s
            €ÚºC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/gemma/tokenization_gemma.pyr3   ÚGemmaTokenizer.__init__;   s(  ø€ ð ‰=ä�I“ Ü�I“ Ü�I“ Ü�I“ Ü�J“ ðˆEð ŒØ—| ˆŒä#ÜØ—k‘kØ—|‘|ØÜ˜i›.ØØ"ñó	
ˆŒô )7×(<Ò(<ØÐ"8Àñ)
ˆ�‰Ô%ô #+×"3Ò"3Ü×Ò˜e SÓ)¬8×+@Ò+@Ó+BÄHÇMÂMÃOÐTó#
ˆ�‰Ôô &1×%8Ò%8¸¸eÓ%Dˆ�‰Ô"Ü‰Òð 	
ØØØØØ!ñ	
ð ó	
ó    )r(   r)   r'   )NNz<unk>z<bos>z<eos>z<pad>z<mask>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚpadding_sideÚmodel_input_namesr   Úmodelr&   ÚdictÚintÚlistr3   Ú__static_attributes__Ú__classcell__)r6   s   @r7   r   r      sª   ø† ñð6 *ÐØ€LØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø Ø Ø Ø Ø"ñ/
à�T˜#˜s˜(‘^Ñ# dÑ*ð/
ð �d˜3‘i‘ $Ñ&ð/
ð ð	/
ð
 ð/
ð ð/
ð ð/
ð ÷/
ö /
r9   r   N)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   Úutilsr
   Ú
get_loggerr:   Úloggerr?   r   Ú__all__r%   r9   r7   Ú<module>rP      sO   ð÷ HÓ GÝ !å >Ý ð 
×	Ò	˜HÓ	%€Ø%Ð'7Ð8Ð ôP
Ð&ô P
ðf Ð
�r9   