ó
    ˆŽ¡j2
  ã                   ó0   • S r SSKJrJrJr   " S S5      rg)z Tokenization utils for RoFormer.é    )ÚNormalizedStringÚPreTokenizedStringÚnormalizersc                   óH   • \ rS rSrSS jrS\S\S\\   4S jrS\	4S	 jr
S
rg)ÚJiebaPreTokenizeré   ÚreturnNc                 ó�   • Xl         [        R                  " SSSSS9U l         SS KnX l        g ! [         a    [	        S5      ef = f)NFT)Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaser   zkYou need to install rjieba to use RoFormerTokenizer. See https://pypi.org/project/rjieba/ for installation.)Úvocabr   ÚBertNormalizerÚrjiebaÚImportErrorÚjieba)Úselfr   r   s      Ú½C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/roformer/tokenization_utils.pyÚ__init__ÚJiebaPreTokenizer.__init__   sY   € ØŒ
Ü&×5Ò5ØØ!%ØØñ	
ˆÔð	Ûð �
øô ó 	ÜðIóð ð	ús	   ¤/ ¯AÚiÚnormalized_stringc                 óh  • / nU R                   R                  [        U5      SS9 Hˆ  u  pEnX@R                  ;   a  UR	                  X%U 5        M*  U R
                  R                  U5      R                  5       nU H/  nU(       d  M  U[        U5      -   nUR	                  X%U 5        UnM1     MŠ     U$ )NF)Úhmm)	r   ÚtokenizeÚstrr   Úappendr   Únormalize_strÚsplitÚlen)r   r   r   ÚsplitsÚtokenÚstartÚendÚ
token_lists           r   Újieba_splitÚJiebaPreTokenizer.jieba_split%   sª   € Øˆð "&§¡×!4Ñ!4´SÐ9JÓ5KÐQVÐ!4Ó!WÑˆE˜#ØŸ
™
Ó"Ø—‘Ð/°cÐ:Ö;à!×-Ñ-×;Ñ;¸EÓB×HÑHÓJ�
Û'�Eß�uØ#¤c¨%£jÑ0˜ØŸ™Ð&7¸cÐ&BÔCØ #šó	 (ñ "Xð* ˆó    Úpretokc                 ó:   • UR                  U R                  5        g )N)r    r'   )r   r*   s     r   Úpre_tokenizeÚJiebaPreTokenizer.pre_tokenize@   s   € Ø�‰�T×%Ñ%Õ&r)   )r   r   r   )r	   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Úintr   Úlistr'   r   r,   Ú__static_attributes__© r)   r   r   r      s5   † ôð"˜Sð Ð5Eð È$ÐO_ÑJ`ô ð6'Ð#5÷ 'r)   r   N)Ú__doc__Ú
tokenizersr   r   r   r   r5   r)   r   Ú<module>r8      s   ðñ 'ç HÑ H÷.'ò .'r)   