ó
    ˆŽ¡j	  ã                   ó�   • S r SSKJrJrJrJr  SSKJr  SSKJ	r	  SSK
Jr  \R                  " \5      rSSS	S
.r " S S\	5      rS/rg)z$Tokenization classes for OpenAI GPT.é    )Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackend)Úloggingz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filec                   óŽ   ^ • \ rS rSrSr\rSS/r\r	   SS\
\\
\4   -  S-  S\
\\
   -  S-  S\
4U 4S	 jjjr\S
 5       rSrU =r$ )ÚOpenAIGPTTokenizeré   a`  
Construct a GPT Tokenizer (backed by HuggingFace's *tokenizers* library). Based on Byte-Pair-Encoding with
the following peculiarities:

- lower case all inputs
- uses BERT's BasicTokenizer for pre-BPE tokenization

This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    vocab_file (`str`, *optional*):
        Path to the vocabulary file.
    merges_file (`str`, *optional*):
        Path to the merges file.
    tokenizer_file (`str`, *optional*):
        Path to a tokenizers JSON file containing the serialization of a tokenizer.
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    vocab (`str` or `dict[str, int]`, *optional*):
        Custom vocabulary dictionary. If not provided, a blank vocabulary is initialized.
    merges (`str` or `list[str]`, *optional*):
        Custom merges list. If not provided, an empty list is used.
Ú	input_idsÚattention_maskNÚvocabÚmergesÚ	unk_tokenc                 óÀ  >• Ub  UO[        U5      S0U l        U=(       d    / U l        [        [	        U R                  U R                  S SSS[        U5      S95      U l        [        R                  " SS9U R
                  l        [        R                  " 5       U R
                  l        [        R                  " SS9U R
                  l        [        TU ]@  " S
S	U0UD6  g )Nr   Ú z</w>F)r   r   ÚdropoutÚcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkr   T)Ú	lowercase)Úsuffixr   © )ÚstrÚ_vocabÚ_mergesr   r   Ú
_tokenizerr   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerr   Ú
BPEDecoderÚdecoderÚsuperÚ__init__)Úselfr   r   r   ÚkwargsÚ	__class__s        €Ú¼C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/openai/tokenization_openai.pyr*   ÚOpenAIGPTTokenizer.__init__;   s¾   ø€ ð  %Ñ0‘e´s¸9³~ÀqÐ6IˆŒØ—| ˆŒä#ÜØ—k‘kØ—|‘|ØØ*,Ø#)ØÜ˜i›.ñó

ˆŒô &1×%?Ò%?È$Ñ%Oˆ�‰Ô"ä(6×(GÒ(GÓ(Iˆ�‰Ô%Ü"*×"5Ò"5¸VÑ"Dˆ�‰Ôä‰Òñ 	
Øð	
àó	
ó    c                 ó   • g)NTr   )r+   s    r.   Údo_lower_caseÚ OpenAIGPTTokenizer.do_lower_case]   s   € àr0   )r!   r"   r    )NNz<unk>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr   ÚdictÚintÚlistr*   Úpropertyr2   Ú__static_attributes__Ú__classcell__)r-   s   @r.   r   r      s†   ø† ñð4 *ÐØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø ñ	 
à�T˜#˜s˜(‘^Ñ# dÑ*ð 
ð �d˜3‘i‘ $Ñ&ð 
ð ÷	 
ð  
ðD ñó ör0   r   N)r8   Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   Úutilsr
   Ú
get_loggerr4   Úloggerr9   r   Ú__all__r   r0   r.   Ú<module>rJ      sS   ðñ +ç GÓ GÝ !å >Ý ð 
×	Ò	˜HÓ	%€à#/ÀÐ`pÑqÐ ôCÐ*ô CðL  Ð
 �r0   