ó
    ‡Ž¡j�' ã                   ór  • S r SSKrSSKrSSKJr  SSKJr  SSKJrJ	r	J
r
JrJrJrJr  SSKJrJrJr  SSKJr  SS	KJrJrJrJr  SS
KJr  \R8                  " \5      r/ SQr\/ SQ-   r S§S jr!S\"S\#4S jr$S¨S\\#   S-  4S jjr% " S S5      r& " S S\&5      r'S\#S\"4S jr( " S S5      r) " S S\)5      r* " S S\)5      r+ " S S \)5      r, " S! S"\)5      r- " S# S$\)5      r. " S% S&\)5      r/ " S' S(\)5      r0 " S) S*\)5      r1 " S+ S,\)5      r2 " S- S.\)5      r3 " S/ S0\)5      r4 " S1 S2\)5      r5 " S3 S4\55      r6 " S5 S6\55      r7 " S7 S8\55      r8 " S9 S:\55      r9 " S; S<\55      r: " S= S>\55      r; " S? S@\55      r< " SA SB\55      r= " SC SD\55      r> " SE SF\55      r? " SG SH\55      r@ " SI SJ\55      rA " SK SL\55      rB " SM SN\55      rC " SO SP\55      rD " SQ SR\55      rE " SS ST\D5      rF " SU SV\)5      rG " SW SX\55      rH " SY SZ\)5      rI " S[ S\\)5      rJ " S] S^\)5      rK " S_ S`\55      rL " Sa Sb\55      rM " Sc Sd\55      rN " Se Sf\)5      rO " Sg Sh\55      rP " Si Sj\55      rQ " Sk Sl\55      rR " Sm Sn\R5      rSSo rT " Sp Sq5      rU0 Sr\6_Ss\2_St\7_Su\*_Sv\H_Sw\K_Sx\8_Sy\I_Sz\/_S{\*_S|\4_S}\9_S~\*_S\*_S€\*_S�\*_S‚\*_0 Sƒ\6_S„\,_S…\/_S†\0_S‡\*_Sˆ\*_S‰\2_SŠ\>_S‹\2_SŒ\2_S�\*_SŽ\O_S�\:_S�\;_S‘\-_S’\*_S“\2_E0 S”\<_S•\._S–\C_S—\1_S˜\@_S™\A_Sš\2_S›\3_Sœ\=_S�\*_Sž\D_SŸ\E_S \F_S¡\G_S¢\>_S£\?_S¤\+_E\L\N\N\M\NS¥.ErVS©S\
4S¦ jjrWg)ªzù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
é    N)Ú
Collection)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPieceé   )Úis_tekken_vocab_filename)Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERROR)Úar_ARÚcs_CZÚde_DEÚen_XXÚes_XXÚet_EEÚfi_FIÚfr_XXÚgu_INÚhi_INÚit_ITÚja_XXÚkk_KZÚko_KRÚlt_LTÚlv_LVÚmy_MMÚne_NPÚnl_XXÚro_ROÚru_RUÚsi_LKÚtr_TRÚvi_VNÚzh_CN)Úaf_ZAÚaz_AZÚbn_INÚfa_IRÚhe_ILÚhr_HRÚid_IDÚka_GEÚkm_KHÚmk_MKÚml_INÚmn_MNÚmr_INÚpl_PLÚps_AFÚpt_XXÚsv_SEÚsw_KEÚta_INÚte_INÚth_THÚtl_XXÚuk_UAÚur_PKÚxh_ZAÚgl_ESÚsl_SIc                 ó8  • [        5       (       a  SSKJn  U$ [        5       (       aV  SS Kn[
        R                  " UR                  R                  5      [
        R                  " S5      :  a  SSK	Jn  U$ SSK	J
n  U$ [        [        R                  " U 5      5      e)Nr   )Úsentencepiece_model_pb2z4.0.0)Úsentencepiece_model_pb2_new)r   ÚsentencepiecerK   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsrL   ÚImportErrorr   Úformat)Úerror_messagerK   Úgoogles      Ú±C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/convert_slow_tokenizer.pyÚimport_protobufrX   _   sr   € Ü!×#Ñ#Ý9à&Ð&Ü×ÑÛä�=Š=˜Ÿ™×4Ñ4Ó5¼¿ºÀgÓ8NÓNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ò6°}ÓEÓFÐFó    Úadd_prefix_spaceÚreturnc                 óH   • U (       a  Sn[        USS5      (       d  SnU$ SnU$ )NÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)rZ   Úoriginal_tokenizerÚprepend_schemes      rW   Ú_get_prepend_schemerd   p   s4   € ÞØ!ˆÜÐ)¨8°T×:Ñ:Ø$ˆNð Ðð !ˆØÐrY   Úskip_tokensc                 óè  ^ • Ub  [        U5      O	[        5       nUS LnU(       a  [        U5      OT n/ nUR                  5        H€  u  pVXR;   a  M  / n[        S[	        U5      5       H:  nUS U XXS  p©X’;   d  X¢;   a  M  U	T ;   d  M  U
T ;   d  M'  UR                  XšU45        M<     [        UU 4S jS9nUR                  U5        M‚     [        US US9nU Vs/ s H  o»S   US   4PM     nnU$ s  snf )Nr   c                 ó$   >• TU S      TU S      4$ ©Nr   r   © )ÚxÚvocabs    €rW   Ú<lambda>Ú!generate_merges.<locals>.<lambda>Š   s   ø€ ¨U°1°Q±4©[¸%ÀÀ!Á¹+Ñ,FrY   ©Úkeyc                 óB   • U S   [        U S   5      [        U S   5      4$ )Né   r   r   )Úlen©Úvals    rW   rl   rm   �   s   € ¨S°©V´S¸¸Q¹³[Ä#ÀcÈ!ÁfÃ+Ñ,NrY   ©ro   Úreverser   )ÚsetÚdictÚitemsÚrangerr   ÚappendÚsortedÚextend)rk   Úvocab_scoresre   rv   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrt   s   `           rW   Úgenerate_mergesr†   z   s  ø€ Ø&1Ñ&=”#�kÔ"Ä3Ã5€KØ $Ð&€GÞ)0”4˜Ô%°e€Là€FØ*×0Ñ0Ö2ÑˆØÓÙØˆÜ˜1œc %›jÖ)ˆEØ$ V e˜}¨e°F¨m�WØÓ%¨Ó)?ÙØ˜%Õ G¨uÕ$4Ø—‘˜g°Ð<Ö=ñ *ô �uÔ"FÑGˆØ�‰�eÖñ 3ô �FÑ NÐX_Ñ`€FÙ*0Ó1ª& 3�1‰v�s˜1‘vÓ©&€FÐ1Ø€Mùò 2s   ÃC/c                   óN   • \ rS rSrSrS\4S jrS\\\\	4   \
\   4   4S jrSrg)	ÚSentencePieceExtractoré’   zd
Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
Úmodelc                 óþ   • [        U S5        [        U S5        [        5       nUR                  5       n[        US5       nUR	                  UR                  5       5        S S S 5        X0l        g ! , (       d  f       N= f)NrM   rP   Úrb)r   rX   Ú
ModelProtoÚopenÚParseFromStringÚreadÚproto)ÚselfrŠ   Ú	model_pb2ÚmÚfs        rW   Ú__init__ÚSentencePieceExtractor.__init__—   sa   € Ü˜$ Ô0Ü˜$ 
Ô+ô $Ó%ˆ	à× Ñ Ó"ˆÜ�%˜Ô !Ø×Ñ˜aŸf™f›hÔ'÷ à�
÷ Õús   ¿ A.Á.
A<r[   c                 óš  • U R                   R                  R                    Uc0  SSKJnJn  U R                   R                  R                  S:X  a  UOUnU R                   R                   Vs/ s H  oUR                  UR                  4PM     nnUR                  S:w  a(  U R                   R                  R                  US'   XbS'   O:SSKJn  [        U5       VV	V
s0 s H
  u  nu  pšX˜_M     nn	nn
U" U5      nXbS'   X²S	'   [        U R                   R                  5       VVs/ s H2  u  pÍUR                  S
;   d  M  XÍR                  UR                  S:H  4PM4     nnn[        US S9 VVVs/ s H  u  pÏn[!        USUS9PM     snnnUS'   [#        U R                   R$                  SS5      US'   U$ s  snf s  sn
n	nf s  snnf s  snnnf )úª
By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
order the merges with respect to the piece scores instead.
Nr   )r   r   r   r   Úunk_idrk   )r†   r   ©é   é   rœ   c                 ó   • U S   $ ©Nr   ri   ©rj   s    rW   rl   Ú0SentencePieceExtractor.extract.<locals>.<lambda>Á   s   € ÈQÈqÊTrY   rn   F©Ú
normalizedÚspecialÚadditional_special_tokensÚprecompiled_charsmapÚ_spm_precompiled_charsmap)r‘   Útrainer_specrš   Útokenizers.modelsr   r   Ú
model_typeÚpiecesÚpieceÚscoreÚ__name__Útokenization_utils_baser†   Ú	enumerateÚtyper|   r   ra   Únormalizer_spec)r’   rª   Úkwargsr   r   r¬   rk   r†   ÚiÚwordr­   r   ÚidÚpÚspm_added_tokensÚtokenr¤   s                    rW   ÚextractÚSentencePieceExtractor.extract£   s¡  € ð
 	�
‰
×Ñ×&Ò&ØÑß6à$(§J¡J×$;Ñ$;×$FÑ$FÈ!Ó$K™ÐQTˆJØ9=¿¹×9JÒ9JÓKÒ9J°—+‘+˜uŸ{™{Ó+Ñ9JˆÐKà×Ñ %Ó'Ø#Ÿz™z×6Ñ6×=Ñ=ˆF�8ÑØ#�7ŠOå@ä5>¸uÔ5EÕFÒ5EÑ!1 ¡M T�T’WÑ5EˆEÒFÙ$ UÓ+ˆFØ#�7‰OØ%�8Ñô ENÈdÏjÉj×N_ÑN_ÔD`ÔuÒD`¹5¸2Ðde×djÑdjÐntÑdtÓ6˜R§¡¨!¯&©&°A©+Ó6ÑD`ÐÑuô '-Ð-=Á>Ò&Rõ/
â&RÑ"�˜7ô �u¨¸Ô@Ù&Ró/
ˆÐ*Ñ+ô /6°d·j±j×6PÑ6PÐRhÐjnÓ.oˆÐ*Ñ+Øˆùò/ Lùô Gùó vùô/
s   Á,!F4ÃF9Ä#G Ä=G Å.G©r‘   N)r®   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ústrr–   Útuplerx   ÚintÚlistrº   Ú__static_attributes__ri   rY   rW   rˆ   rˆ   ’   s5   † ñð
˜cô 
ð!¨u°T¸#¸s¸(±^ÀTÈ%Á[Ð5PÑ/Q÷ !rY   rˆ   c                   ó@   • \ rS rSrSS\\\\4   \\   4   4S jjr	Sr
g)ÚGemmaSentencePieceExtractoréÇ   Nr[   c                 óä   • U R                   n[        UR                  5       5       Vs0 s H  o2R                  U5      U_M     nnSU;  a  UR	                  S5      US'   [        XA5      nXE4$ s  snf )r™   Ú	ú<0x09>)Úsprz   ÚGetPieceSizeÚid_to_pieceÚgetr†   )r’   r~   rÌ   rƒ   rk   r   s         rW   rº   Ú#GemmaSentencePieceExtractor.extractÈ   so   € ð
 �W‰WˆÜ;@ÀÇÁÓARÔ;SÓTÒ;S°%—‘ Ó&¨Ò-Ñ;SˆÐTð �uÓØŸ)™) HÓ-ˆE�$‰KÜ  Ó5ˆØˆ}Ðùò Us   ¨A-ri   ©N)r®   r½   r¾   r¿   rÂ   rx   rÁ   rÃ   rÄ   rº   rÅ   ri   rY   rW   rÇ   rÇ   Ç   s)   † ñ¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,N÷ ð rY   rÇ   r¬   c                 óz   • [        U 5      S:  =(       d'    U S   S:g  =(       d    U S   R                  5       (       + $ )Nrq   éÿÿÿÿÚ,éþÿÿÿ)rr   Úisdigit)r¬   s    rW   Úcheck_number_commar×   Ø   s3   € Üˆu‹:˜‰>×H˜U 2™Y¨#Ñ-×H°U¸2±Y×5FÑ5FÓ5HÔ1HÐHrY   c                   ó(   • \ rS rSrS rS\4S jrSrg)Ú	ConverteréÜ   c                 ó   • Xl         g rÑ   ©rb   )r’   rb   s     rW   r–   ÚConverter.__init__Ý   s   € Ø"4ÕrY   r[   c                 ó   • [        5       erÑ   )ÚNotImplementedError©r’   s    rW   Ú	convertedÚConverter.convertedà   s   € Ü!Ó#Ð#rY   rÜ   N)r®   r½   r¾   r¿   r–   r   rá   rÅ   ri   rY   rW   rÙ   rÙ   Ü   s   † ò5ð$˜9÷ $rY   rÙ   c                   ó"   • \ rS rSrS\4S jrSrg)ÚBertConverteréä   r[   c           	      ób  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	3Xh4Xy4/S
9Ul        [0        R                  " SS9Ul        U$ )N©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©rb   rk   r   r   rÁ   rè   Úhasattrré   Útokenize_chinese_charsrí   Údo_lower_caser	   ÚBertNormalizerÚ
normalizerr
   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr   Údecoder©
r’   rk   Ú	tokenizerrü   rí   rý   ÚclsÚsepr  r  s
             rW   rá   ÚBertConverter.convertedå   ó„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrY   ri   N©r®   r½   r¾   r¿   r   rá   rÅ   ri   rY   rW   rä   rä   ä   ó   † ð#˜9÷ #rY   rä   c                   ó"   • \ rS rSrS\4S jrSrg)ÚSplinterConverteri  r[   c           
      óv  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      n[	        U R                   R&                  5      nSn	U R                   R(                  n
U R                   R*                  nU R                   R,                  nU R                   R/                  S5      nU R                   R0                  S:X  a  U SU S	U	 S	U S
U S3
nOU SU S
U S	U	 S	U S3
n[2        R4                  " U SU S3UXj4X{4XŒ4X�4/S9Ul        [8        R                  " SS9Ul        U$ )Nrç   Fré   Trê   Ú.Úrightrï   Ú rñ   rò   rð   ró   r÷   rø   )rb   rk   r   r   rÁ   rè   rû   ré   rü   rí   rý   r	   rþ   rÿ   r
   r   r  r  r  Úquestion_tokenr  r  Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider   r  r  r   r  )r’   rk   r
  rü   rí   rý   r  r  ÚquestionÚdotr  r  r  Údot_token_idrõ   s                  rW   rá   ÚSplinterConverter.converted  s  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ó:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ò#@Ø�U˜( 3 % rÐ*ØàÐ#ØÐ#ØÐ-ØÐ#ð	ñ	$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrY   ri   Nr  ri   rY   rW   r  r    s   † ð.˜9÷ .rY   r  c                   ó"   • \ rS rSrS\4S jrSrg)ÚFunnelConverteri=  r[   c           	      ób  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	3Xh4Xy4/S
9Ul        [0        R                  " SS9Ul        U$ )Nrç   Fré   Trê   z:2 $A:0 rð   rñ   rò   ró   r÷   rø   rú   r	  s
             rW   rá   ÚFunnelConverter.converted>  r  rY   ri   Nr  ri   rY   rW   r   r   =  r  rY   r   c                   ó"   • \ rS rSrS\4S jrSrg)ÚMPNetConverterid  r[   c                 óh  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	U S
3Xh4Xy4/S9Ul        [0        R                  " SS9Ul        U$ )Nrç   Fré   Trê   rï   rð   z:0 rñ   rò   ró   r÷   rø   rú   r	  s
             rW   rá   ÚMPNetConverter.convertede  sŠ  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrY   ri   Nr  ri   rY   rW   r$  r$  d  r  rY   r$  c                   ó"   • \ rS rSrS\4S jrSrg)ÚOpenAIGPTConverteri‹  r[   c                 óü  • U R                   R                  n[        U R                   R                  R	                  5       5      nU R                   R
                  n[        [        UUS [        U5      SSS95      nUR                  [        U5      5      b  UR                  [        U5      /5        [        R                  " SS9Ul        [        R                  " 5       Ul        ["        R$                  " SS9Ul        U$ )Nú</w>F)rk   r   Údropoutrè   Úend_of_word_suffixÚfuse_unkT)rî   ©Úsuffix)rb   ÚencoderrÄ   Ú	bpe_ranksÚkeysrè   r   r   rÁ   Útoken_to_idÚadd_special_tokensr	   rþ   rÿ   r
   r   r  r   Ú
BPEDecoderr  ©r’   rk   r   rè   r
  s        rW   rá   ÚOpenAIGPTConverter.convertedŒ  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øñó	
ˆ	ð × Ñ ¤ Y£Ó0Ñ<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ò9ÀDÑIˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°vÑ>ˆ	ÔàÐrY   ri   Nr  ri   rY   rW   r(  r(  ‹  s   † ð˜9÷ rY   r(  c                   óT   • \ rS rSrSS\\\4   S-  S\\\\4      S-  S\	4S jjr
Srg)	ÚGPT2Converteri¦  Nrk   r   r[   c                 óZ  • U(       d  U R                   R                  nU(       d  [        U R                   R                  5      n[	        [        UUS SSSS95      n[        U R                   SS5      n[        R                  " US9Ul	        [        R                  " 5       Ul        [        U R                   SS5      (       aQ  U R                   R                  nU R                   R                  n[        R                  " U S3U S3XV4/S	9Ul        U$ [        R                  " SS
9Ul        U$ )NÚ F©rk   r   r+  Úcontinuing_subword_prefixr,  r-  rZ   ©rZ   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1ró   ©Útrim_offsets)rb   r0  rÄ   r1  r   r   ra   r
   Ú	ByteLevelr  r   r  Ú	bos_tokenÚbos_token_idr   r  r  )r’   rk   r   r
  rZ   ÚbosrD  s          rW   rá   ÚGPT2Converter.converted§  s  € ÞØ×+Ñ+×3Ñ3ˆEÞÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øñó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ò":ÐL\Ñ"]ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸U×CÑCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÒ'DØ˜˜g�Ø�u˜LÐ)àÐ'ð ñ(ˆIÔ$ð Ðô (2×';Ò';ÈÑ'OˆIÔ$ØÐrY   ri   ©NN©r®   r½   r¾   r¿   rx   rÁ   rÃ   rÄ   rÂ   r   rá   rÅ   ri   rY   rW   r9  r9  ¦  sE   † ñ"˜t C¨ H™~°Ñ4ð "ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað "Ðmv÷ "ð "rY   r9  c                   ó"   • \ rS rSrS\4S jrSrg)ÚHerbertConverteriÌ  r[   c           
      ó‚  • SnSnU R                   R                  n[        U R                   R                  R	                  5       5      nXS   S   ;   a  USS  n[        [        UUS U R                   R                  US95      n[        R                  " SSS9Ul
        [        R                  " 5       Ul        [        R                  " US9Ul        ["        R$                  " U R                   R&                  U R                   R(                  4U R                   R*                  U R                   R,                  45      Ul        U$ )	Nz	#version:r*  r   r   )r+  rè   r,  F)rî   rí   r.  )rb   r0  rÄ   r1  r2  r   r   rè   r	   rþ   rÿ   r
   r   r  r   r5  r  r   ÚBertProcessingr  r  r  r  r  )r’   Útokenizer_info_strÚtoken_suffixrk   r   r
  s         rW   rá   ÚHerbertConverter.convertedÍ  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ¨¡¨1¡Ó-Ø˜A˜B�ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ñó
ˆ	ô  +×9Ò9ÀEÐY^Ñ_ˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°|ÑDˆ	ÔÜ#-×#<Ò#<Ø×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUØ×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUó$
ˆ	Ô ð
 ÐrY   ri   Nr  ri   rY   rW   rJ  rJ  Ì  ó   † ð˜9÷ rY   rJ  c                   óT   • \ rS rSrSS\\\4   S-  S\\\\4      S-  S\	4S jjr
Srg)	ÚQwen2Converterië  Nrk   r   r[   c                 ó8  • U(       d  U R                   R                  nU(       d-  [        U R                   R                  R	                  5       5      n[        [        UUS S SSSSS95      n[        R                  " 5       Ul	        [        R                  " [        R                  " [        S5      SSS9[        R                  " [        U R                   SS5      SS9/5      Ul        ["        R                  " 5       Ul        [&        R                  " SS	9Ul        U$ )
Nr;  F)rk   r   r+  rè   r=  r,  r-  Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertrZ   ©rZ   Ú	use_regexr@  )rb   r0  rÄ   r1  r2  r   r   r	   ÚNFCrÿ   r
   ÚSequenceÚSplitr   rB  ra   r  r   r  r   r  )r’   rk   r   r
  s       rW   rá   ÚQwen2Converter.convertedì  s  € ÞØ×+Ñ+×3Ñ3ˆEÞÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ñ	ó
ˆ	ô  +ŸšÓ0ˆ	Ôä"0×"9Ò"9ä×$Ò$Üð Nóð (Ø ñô ×(Ò(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ñðó#
ˆ	Ôô  %×.Ò.Ó0ˆ	ÔÜ#-×#7Ò#7ÀUÑ#Kˆ	Ô àÐrY   ri   rG  rH  ri   rY   rW   rR  rR  ë  sE   † ñ(˜t C¨ H™~°Ñ4ð (ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað (Ðmv÷ (ð (rY   rR  c                   ó"   • \ rS rSrS\4S jrSrg)ÚRobertaConverteri  r[   c                 óÌ  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " UR                  S9Ul
        [        R                  " 5       Ul        [        R                  " UR                  UR                   4UR"                  UR$                  4SUR                  5      Ul        U$ )Nr;  Fr<  r>  T)rb   r0  rÄ   r1  r2  r   r   r
   rB  rZ   r  r   r  r   ÚRobertaProcessingr  r  r  r  r  ©r’   Úotrk   r   r
  s        rW   rá   ÚRobertaConverter.converted  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#?Ò#?Ø�\‰\˜2Ÿ?™?Ð+Ø�\‰\˜2Ÿ?™?Ð+ØØ×Ñó	$
ˆ	Ô ð ÐrY   ri   Nr  ri   rY   rW   r`  r`    ó   † ð˜9÷ rY   r`  c                   ó"   • \ rS rSrS\4S jrSrg)ÚRoFormerConverteri4  r[   c           	      óJ  • SSK Jn  U R                  R                  n[	        [        U[        U R                  R                  5      S95      nSnSn[        U R                  S5      (       a@  U R                  R                  R                  nU R                  R                  R                  n[        R                  " SSUUS9Ul        [        R                   R#                  U" U5      5      Ul        [        U R                  R&                  5      n[        U R                  R(                  5      nU R                  R*                  nU R                  R,                  n	[.        R0                  " U SU S	3U SU S
U S3Xh4Xy4/S9Ul        [4        R
                  " SS9Ul        U$ )Nr   )ÚJiebaPreTokenizerrç   Fré   Trê   rï   rð   rñ   rò   ró   r÷   rø   )Ú"models.roformer.tokenization_utilsrj  rb   rk   r   r   rÁ   rè   rû   ré   rí   rý   r	   rþ   rÿ   r
   ÚPreTokenizerÚcustomr  r  r  r  r  r   r  r  r   r  )
r’   rj  rk   r
  rí   rý   r  r  r  r  s
             rW   rá   ÚRoFormerConverter.converted5  sx  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!&Ø'Ø#ñ	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrY   ri   Nr  ri   rY   rW   rh  rh  4  r  rY   rh  c                   ó"   • \ rS rSrS\4S jrSrg)ÚDebertaConverteri[  r[   c                 óÊ  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " UR                  S9Ul
        [        R                  " 5       Ul        [        R                  " SSSU R                   R                  S5      4SU R                   R                  S5      4/S	9Ul        U$ )
Nr;  Fr<  r>  ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]ró   )rb   r0  rÄ   r1  r2  r   r   r
   rB  rZ   r  r   r  r   r  r  r  rc  s        rW   rá   ÚDebertaConverter.converted\  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðñ$
ˆ	Ô ð ÐrY   ri   Nr  ri   rY   rW   rp  rp  [  rP  rY   rp  c                   ó”   ^ • \ rS rSrSr\r0 r\SS j5       r	\
SS j5       rU 4S jrS rS rS rS	 rS
 rS rS rS\4S jrSrU =r$ )ÚSpmConverteriz  Fc                 ó¦  • U R                   R                  nU R                   R                  nU R                  R                  n[        U[        5      (       a!  [        [        UU=(       d    / USUSS95      nOe[        U[        5      (       aO  U(       aH  [        US   [        [        -  5      (       a)  [        [        UU R                   R                  US95      nOg[        R                  " SS5      /nU(       a&  UR                  S[        R                   " U5      5        [        R"                  " U5      Ul        U(       a[  [&        R"                  " [&        R                  " SS5      [&        R(                  " 5       [&        R*                  " 5       /5      Ul        U$ [&        R"                  " [&        R                  " SS5      /5      Ul        U$ )zç
Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file
TN)rk   r   rè   r-  rT  r+  r   )rk   rš   rT  r  õ   â–�)r¨   rT  Ú	unk_piecer²   r¦   Ú
isinstancerx   r   r   rÄ   rÂ   r   rš   r	   ÚReplaceÚinsertÚPrecompiledr\  rÿ   r   ÚByteFallbackÚFuser  )r‘   rk   r   rT  r{  r¦   r
  Ú_normalizerss           rW   Úbuild_tokenizer_from_spm_protoÚ+SpmConverter.build_tokenizer_from_spm_proto  s{  € ð
 ×*Ñ*×8Ñ8ˆØ×&Ñ&×0Ñ0ˆ	Ø$×4Ñ4×IÑIÐô �eœT×"Ñ"Ü!ÜØØ!Ÿ< RØ'Ø!Ø"/Ø ñó	‰Iô ˜œt×$Ñ$®´:¸eÀA¹hÌÔPTÉ×3UÑ3UÜ!ÜØØ ×-Ñ-×4Ñ4Ø"/ñó‰Ið ô $×+Ò+¨C°Ó7Ð8ˆÞØ×Ñ ¤;×#:Ò#:Ð;OÓ#PÔQÜ*×3Ò3°LÓAˆ	Ôö Ü (× 1Ò 1Ü×!Ò! %¨Ó-¬x×/DÒ/DÓ/FÌÏÊËÐXó!ˆIÔð Ðô !)× 1Ò 1´8×3CÒ3CÀEÈ3Ó3OÐ2PÓ QˆIÔàÐrY   c                 ó   • Ub  XS'   U$ )z‡
Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
By default, return kwargs unchanged.
rk   ri   )r  rk   r³   s      rW   Úconvert_from_spmÚSpmConverter.convert_from_spm¯  s   € ð ÑØ#�7‰OØˆrY   c                 óÆ  >• [        U S5        [        TU ]  " U6   [        5       nUR	                  5       n[        U R                  R                  S5       nUR                  UR                  5       5        S S S 5        X0l
        U R                  R                  R                  (       a)  U R                  (       d  [        R                  " S5        g g g ! , (       d  f       Nc= f)NrP   rŒ   a  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   Úsuperr–   rX   r�   rŽ   rb   Ú
vocab_filer�   r�   r‘   r¨   rT  Úhandle_byte_fallbackÚwarningsÚwarn)r’   Úargsr“   r”   r•   Ú	__class__s        €rW   r–   ÚSpmConverter.__init__¹  s§   ø€ Ü˜$ 
Ô+ä‰Ò˜$Ñô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°dÔ;¸qØ×Ñ˜aŸf™f›hÔ'÷ <àŒ
à�:‰:×"Ñ"×0×0¸×9R×9RÜ�MŠMðeõð :SÐ0÷	 <Õ;ús   Á CÃ
C c                 óp   • UR                    Vs/ s H  o"R                  UR                  4PM     sn$ s  snf rÑ   ©r«   r¬   r­   ©r’   r‘   r¬   s      rW   rk   ÚSpmConverter.vocabÎ  s)   € Ø8=¿ºÓEº¨u—‘˜eŸk™kÓ*¹ÑEÐEùÒEs   �!3c                 ó.   • UR                   R                  $ rÑ   )r¨   rš   ©r’   r‘   s     rW   rš   ÚSpmConverter.unk_idÑ  s   € Ø×!Ñ!×(Ñ(Ð(rY   c                 ó„  • UR                   R                  nU R                  U5      nUS:X  a.  [        [	        UU R                  U5      U R                  S95      nO¡US:X  a�  U R                  U R                  R                  5      R                  U5      u  pV[        U5       VVV	s0 s H
  u  nu  p‰X‡_M     n
nnn	[        [        U
UUR                   R                  SU R                  S S95      nO[        S5      e[        UR                  5       VVs/ s HR  u  p¼UR                   S;   d  M  X¼R"                  UR                   S:H  =(       d    UR"                  U R$                  ;   4PMT     nnnUR'                  [)        US	 S
9 VVVs/ s H  u  p¾n[+        USUS9PM     snnn5        U$ s  sn	nnf s  snnf s  snnnf )Nr   ©rš   rT  rq   T©rè   r-  rT  r+  z]You're trying to run a `Unigram` model but you're file was trained with a different algorithmr›   rœ   c                 ó   • U S   $ rŸ   ri   r    s    rW   rl   Ú(SpmConverter.tokenizer.<locals>.<lambda>   ó   € ÐQRÐSTÒQUrY   rn   Fr¢   )r¨   rª   rk   r   r   rš   r‹  ÚSpmExtractorrb   rŠ  rº   r°   r   r{  Ú	Exceptionr«   r±   r¬   rö   Ú
add_tokensr|   r   )r’   r‘   rª   r~   r
  Ú_r   r´   rµ   r­   Ú	bpe_vocabr¶   r·   r¸   r¹   r¤   s                   rW   r
  ÚSpmConverter.tokenizerÔ  s®  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜‹?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ñó‰Ið ˜1‹_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAÜ9BÀ<Ô9PÕQÒ9PÑ%5 Q©¨˜šÑ9PˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ñó	‰Iô Øoóð ô # 5§<¡<Ô0ô
â0‘�Ø�v‰v˜Ñó IˆR—‘˜!Ÿ&™& A™+×G¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Ò*Võâ*VÑ&�B˜wô ˜5¨U¸GÔDÙ*Vóô	
ð ÐùôC Rùó*
ùôs   Â'F.ÄF5Ä/?F5ÆF;c                 ó.  • UR                   R                  n[        R                  " SSS9[        R                  " [        S5      S5      /nU(       d  [        R                  " U5      $ [        R                  " [        R                  " U5      /U-   5      $ )NFT)Úleftr  ú {2,}rz  )r²   r¦   r	   ÚStripr}  r   r\  r  ©r’   r‘   r¦   r‚  s       rW   rÿ   ÚSpmConverter.normalizer  s{   € Ø$×4Ñ4×IÑIÐä×Ò 5°Ñ5Ü×Ò¤ g£°Ó6ð
ˆö $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrY   c                 óT   • [        X R                  5      n[        R                  " XS9$ ©N©Úreplacementrc   )rd   rb   r
   Ú	Metaspace©r’   r­  rZ   rc   s       rW   r  ÚSpmConverter.pre_tokenizer  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×'Ò'°KÑ_Ð_rY   c                 ó   • g rÑ   ri   rà   s    rW   r  ÚSpmConverter.post_processor  ó   € ØrY   c                 óT   • [        X R                  5      n[        R                  " XS9$ r«  )rd   rb   r   r®  r¯  s       rW   r  ÚSpmConverter.decoder  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×!Ò!¨kÑYÐYrY   r[   c                 óˆ  • U R                  U R                  5      nU R                  U R                  5      nUb  X!l        SnSn[        U R                  S5      (       a  U R                  R
                  nU R                  X45      nUb  XQl        U R                  X45      Ul        U R                  5       nU(       a  Xal        U$ )Nrz  TrZ   )	r
  r‘   rÿ   rû   rb   rZ   r  r  r  )r’   r
  rÿ   r­  rZ   r  r  s          rW   rá   ÚSpmConverter.converted  s¯   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÑ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ñ?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÑ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆÞØ'5Ô$àÐrY   r¼   rÑ   )r®   r½   r¾   r¿   r‹  rˆ   rž  rö   Ústaticmethodrƒ  Úclassmethodr†  r–   rk   rš   r
  rÿ   r  r  r  r   rá   rÅ   Ú__classcell__©r�  s   @rW   rx  rx  z  sx   ø† Ø ÐØ)€LØ€Nàó-ó ð-ð^ óó ðõò*Fò)ò0òd	hò`òòZð˜9÷ ò rY   rx  c                   ó&   • \ rS rSrS rS rS rSrg)ÚAlbertConverteri5  c                 óÜ   • UR                    Vs/ s HP  n[        UR                  5      (       a  UR                  UR                  4OUR                  UR                  S-
  4PMR     sn$ s  snf ©Néd   ©r«   r×   r¬   r­   r“  s      rW   rk   ÚAlbertConverter.vocab6  óf   € ð Ÿšó
â%�ô +=¸U¿[¹[×*IÑ*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÑ]nÐOoÒoÙ%ñ
ð 	
ùò 
ó   �AA)c                 ó°  • [         R                  " SS5      [         R                  " SS5      /nU R                  R                  (       dH  UR	                  [         R
                  " 5       5        UR	                  [         R                  " 5       5        U R                  R                  (       a$  UR	                  [         R                  " 5       5        UR                  R                  nU(       a%  UR	                  [         R                  " U5      5        UR	                  [         R                  " [        S5      S5      5        [         R                  " U5      $ ©Nz``Ú"z''r¦  r  ©r	   r}  rb   Úkeep_accentsr{   ÚNFKDÚStripAccentsrý   Ú	Lowercaser²   r¦   r  r   r\  ©r’   r‘   Úlist_normalizersr¦   s       rW   rÿ   ÚAlbertConverter.normalizer<  óï   € ä×Ò  cÓ*Ü×Ò  cÓ*ð
Ðð ×&Ñ&×3×3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐæØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRà×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIÜ×#Ò#Ð$4Ó5Ð5rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ ©Nrr  rs  rt  ru  ró   ©r   r  rb   r  rà   s    rW   r  ÚAlbertConverter.post_processorO  óR   € Ü×,Ò,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðñ
ð 	
rY   ri   N©r®   r½   r¾   r¿   rk   rÿ   r  rÅ   ri   rY   rW   r½  r½  5  ó   † ò
ò6õ&
rY   r½  c                   ó    • \ rS rSrS rS rSrg)ÚBarthezConverteriZ  c                 ó
   • SnU$ ©Nrœ   ri   ©r’   r‘   rš   s      rW   rš   ÚBarthezConverter.unk_id[  ó   € ØˆØˆrY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ ©Nz<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>ró   rÓ  rà   s    rW   r  ÚBarthezConverter.post_processor_  óR   € Ü×,Ò,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rY   ri   N)r®   r½   r¾   r¿   rš   r  rÅ   ri   rY   rW   rÙ  rÙ  Z  s   † òõ
rY   rÙ  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚCamembertConverterij  c                 ó’   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nUS/-  nU$ s  snf )N)©z
<s>NOTUSEDç        ©ú<pad>ré  ©z</s>NOTUSEDré  ©ú<unk>ré  )ú<unk>NOTUSEDiœÿÿÿr   ©ú<mask>ré  r’  ©r’   r‘   rk   r¬   s       rW   rk   ÚCamembertConverter.vocabk  sR   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØ�/Ð"Ñ"ˆØˆùò Ló   –!Ac                 ó   • grÛ  ri   r–  s     rW   rš   ÚCamembertConverter.unk_idx  s   € àrY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rà  rÓ  rà   s    rW   r  Ú!CamembertConverter.post_processor|  rä  rY   Nc                 ó,  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      nSUS4S	US4S
/nUb  UR                  [        U5      SS  5        UR	                  US45        XbS'   U$ )NÚ	pad_tokenrë  rè   rî  Ú
mask_tokenrñ  rè  ré  rì  )rï  ç      YÀr   rk   ©rÁ   rÏ   r}   rÄ   r{   )r  rk   r³   rú  rè   rû  Ú
vocab_lists          rW   r†  Ú#CamembertConverter.convert_from_spm†  s¢   € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð  Ø˜ÐØ Ø˜ÐØ$ð
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrY   ri   rÑ   ©
r®   r½   r¾   r¿   rk   rš   r  r¹  r†  rÅ   ri   rY   rW   ræ  ræ  j  s%   † òòò
ð óó órY   ræ  c                   ó&   • \ rS rSrS rS rS rSrg)ÚDebertaV2Converteriš  c                 ó  • / nU R                   R                  (       a#  UR                  [        R                  " SS95        [        X R                   5      nUR                  [        R                  " XS95        [        R                  " U5      $ )NrU  )rW  r¬  )rb   Úsplit_by_punctr{   r
   ÚPunctuationrd   r®  r\  )r’   r­  rZ   Úlist_pretokenizersrc   s        rW   r  Ú DebertaV2Converter.pre_tokenizer›  sl   € ØÐØ×"Ñ"×1×1Ø×%Ñ%¤n×&@Ò&@È*Ñ&UÔVÜ,Ð-=×?VÑ?VÓWˆØ×!Ñ!¤.×":Ò":À{Ñ"rÔsÜ×&Ò&Ð'9Ó:Ð:rY   c                 óÚ  • / nU R                   R                  (       a$  UR                  [        R                  " 5       5        UR                  [        R
                  " 5       5        UR                  R                  nU(       a%  UR                  [        R                  " U5      5        UR                  [        R                  " [        S5      S5      5        [        R                  " U5      $ )Nr¦  r  )rb   rý   r{   r	   rÌ  r§  r²   r¦   r  r}  r   r\  rÍ  s       rW   rÿ   ÚDebertaV2Converter.normalizer£  s¥   € ØÐØ×"Ñ"×0×0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<Ø×Ñ¤× 1Ò 1Ó 3Ô4à$×4Ñ4×IÑIÐÞØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRØ×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIä×#Ò#Ð$4Ó5Ð5rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rÒ  rÓ  rà   s    rW   r  Ú!DebertaV2Converter.post_processor°  rÕ  rY   ri   N)r®   r½   r¾   r¿   r  rÿ   r  rÅ   ri   rY   rW   r  r  š  s   † ò;ò6õ
rY   r  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚMBartConverteri»  c                 ó    • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU/ SQ-  nUS/-  nU$ s  snf )N©©rá  ré  rê  ©râ  ré  rí  rœ   )©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r    ré  ©r!   ré  ©r"   ré  ©r#   ré  ©r$   ré  ©r%   ré  ©r&   ré  ©r'   ré  ©r(   ré  ©r)   ré  ©r*   ré  ©r+   ré  ©r,   ré  ©r-   ré  ©r.   ré  rð  r’  rò  s       rW   rk   ÚMBartConverter.vocab¼  sc   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØò 
ñ 	
ˆð6 	�/Ð"Ñ"ˆØˆùò; Ló   –!Ac                 ó   • grÛ  ri   r–  s     rW   rš   ÚMBartConverter.unk_idâ  ó   € ØrY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz$A </s> en_XXz$A $B </s> en_XXr   râ  ró   rÓ  rà   s    rW   r  ÚMBartConverter.post_processorå  óR   € Ü×,Ò,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rY   Nc                 óÖ  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  S	S
5      5      nUS4US4US4US4/nUb  UR                  [        U5      SS  5        UR                  S [         5       5        UR                  US45        X‚S'   U$ )NrC  rá  rú  rë  Ú	eos_tokenrâ  rè   rî  rû  rñ  ré  rœ   c              3   ó(   #   • U  H  oS 4v •  M
     g7f©ré  Nri   ©Ú.0Ú	lang_codes     rW   Ú	<genexpr>Ú2MBartConverter.convert_from_spm.<locals>.<genexpr>ÿ  s   é € ÐLºO¨y cÕ*ºOùó   ‚rk   )rÁ   rÏ   r}   rÄ   ÚMBART_LANGUAGESr{   ©	r  rk   r³   rC  rú  r4  rè   rû  rþ  s	            rW   r†  ÚMBartConverter.convert_from_spmï  sç   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑL½OÓLÔLØ×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrY   ri   rÑ   r   ri   rY   rW   r  r  »  s&   † ò$òLò
ð óó órY   r  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚMBart50Converteri  c                 ó    • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU/ SQ-  nUS/-  nU$ s  snf )Nr  rœ   )4r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r!  r"  r#  r$  r%  r&  r'  r(  r)  r*  )r/   ré  )r0   ré  )r1   ré  )r2   ré  )r3   ré  )r4   ré  )r5   ré  )r6   ré  )r7   ré  )r8   ré  )r9   ré  )r:   ré  )r;   ré  )r<   ré  )r=   ré  )r>   ré  )r?   ré  )r@   ré  )rA   ré  )rB   ré  )rC   ré  )rD   ré  )rE   ré  )rF   ré  )rG   ré  )rH   ré  )rI   ré  rð  r’  rò  s       rW   rk   ÚMBart50Converter.vocab  sc   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØò  Rñ  	RˆØ�/Ð"Ñ"ˆØˆùò Lr,  c                 ó   • grÛ  ri   r–  s     rW   rš   ÚMBart50Converter.unk_id  r/  rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nzen_XX $A </s>zen_XX $A $B </s>r   râ  ró   rÓ  rà   s    rW   r  ÚMBart50Converter.post_processor  r2  rY   Nc                 óÖ  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  S	S
5      5      nUS4US4US4US4/nUb  UR                  [        U5      SS  5        UR                  S [         5       5        UR                  US45        X‚S'   U$ )Nr  rá  rú  rë  r4  râ  rè   rî  rû  rñ  ré  rœ   c              3   ó(   #   • U  H  oS 4v •  M
     g7fr6  ri   r7  s     rW   r:  Ú4MBart50Converter.convert_from_spm.<locals>.<genexpr>/  s   é € ÐNÒ<M¨y cÕ*Ò<Mùr<  rk   )rÁ   rÏ   r}   rÄ   ÚMBART50_LANGUAGESr{   )	r  rk   r³   r  rú  r4  rè   rû  rþ  s	            rW   r†  Ú!MBart50Converter.convert_from_spm  sè   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑNÕ<MÓNÔNØ×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrY   ri   rÑ   r   ri   rY   rW   rA  rA    s%   † ò
òò
ð óó órY   rA  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚNllbConverteri5  c                 ó†   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU$ s  snf )Nr  rœ   r’  rò  s       rW   rk   ÚNllbConverter.vocab6  óE   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØˆùò Ló   –!>c                 ó   • grÛ  ri   r–  s     rW   rš   ÚNllbConverter.unk_id@  r/  rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nzeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_Latnrâ  ró   rÓ  rà   s    rW   r  ÚNllbConverter.post_processorC  sR   € Ü×,Ò,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rY   Nc                 ó¾  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      nUS	US
USUS0nUbY  [        U[        5      (       a  UR	                  5       OU VV	s/ s H  u  p‰UPM	     sn	nn
U
 H  nX·;   a  M
  [        U5      X{'   M     XrS'   U$ s  sn	nf )NrC  rá  rú  rë  r4  râ  rè   rî  r   r   rq   rœ   rk   )rÁ   rÏ   r|  rx   r2  rr   )r  rk   r³   rC  rú  r4  rè   Úreordered_vocabÚtokr¡  Útokensr¹   s               rW   r†  ÚNllbConverter.convert_from_spmM  sÝ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	ð �qØ�qØ�qØ�qð	
ˆð ÑÜ%/°´t×%<Ñ%<�U—Z‘Z”\ÑUZÔB[ÒUZÉ6È3Ã3ÑUZÒB[ˆFÛ�ØÓ+ÙÜ),¨_Ó)=�Ó&ñ  ð *ˆw‰Øˆùó C\s   Â$Cri   rÑ   r   ri   rY   rW   rN  rN  5  s%   † òòò
ð óó órY   rN  c                   ó&   • \ rS rSrS rS rS rSrg)ÚSeamlessM4TConverterid  c                 ó†   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU$ s  snf )N)rê  rí  r  r  rœ   r’  rò  s       rW   rk   ÚSeamlessM4TConverter.vocabe  rQ  rR  c                 ó.   • U R                   R                  $ rÑ   )rb   Úunk_token_idr–  s     rW   rš   ÚSeamlessM4TConverter.unk_ido  s   € Ø×&Ñ&×3Ñ3Ð3rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__râ  ró   rÓ  rà   s    rW   r  Ú#SeamlessM4TConverter.post_processorr  sR   € Ü×,Ò,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rY   ri   N©r®   r½   r¾   r¿   rk   rš   r  rÅ   ri   rY   rW   r^  r^  d  s   † òò4õ
rY   r^  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚXLMRobertaConverteri}  c                 ó’   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nUS/-  nU$ s  snf )Nr  rœ   rð  r’  rò  s       rW   rk   ÚXLMRobertaConverter.vocab~  sR   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØ�/Ð"Ñ"ˆØˆùò Lrô  c                 ó
   • SnU$ rÛ  ri   rÜ  s      rW   rš   ÚXLMRobertaConverter.unk_id‰  rÞ  rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rà  rÓ  rà   s    rW   r  Ú"XLMRobertaConverter.post_processor�  rä  rY   Nc                 óž  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  S	S
5      5      nUS4US4US4US4/nUb  UR                  [        U5      SS  5        UR	                  US45        X‚S'   U$ )NrC  rá  rú  rë  r4  râ  rè   rî  rû  rñ  ré  rœ   rk   rý  r>  s	            rW   r†  Ú$XLMRobertaConverter.convert_from_spm—  sÓ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrY   ri   rÑ   r   ri   rY   rW   ri  ri  }  s%   † ò	òò
ð óó órY   ri  c                   ó&   • \ rS rSrS rS rS rSrg)ÚXLNetConverteri¬  c                 óÜ   • UR                    Vs/ s HP  n[        UR                  5      (       a  UR                  UR                  4OUR                  UR                  S-
  4PMR     sn$ s  snf r¿  rÁ  r“  s      rW   rk   ÚXLNetConverter.vocab­  rÃ  rÄ  c                 ó°  • [         R                  " SS5      [         R                  " SS5      /nU R                  R                  (       dH  UR	                  [         R
                  " 5       5        UR	                  [         R                  " 5       5        U R                  R                  (       a$  UR	                  [         R                  " 5       5        UR                  R                  nU(       a%  UR	                  [         R                  " U5      5        UR	                  [         R                  " [        S5      S5      5        [         R                  " U5      $ rÆ  rÈ  rÍ  s       rW   rÿ   ÚXLNetConverter.normalizer³  rÐ  rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>ró   rÓ  rà   s    rW   r  ÚXLNetConverter.post_processorÆ  rÕ  rY   ri   NrÖ  ri   rY   rW   rs  rs  ¬  r×  rY   rs  c                   ó   • \ rS rSrSrg)ÚReformerConverteriÑ  ri   N©r®   r½   r¾   r¿   rÅ   ri   rY   rW   r{  r{  Ñ  ó   † ÚrY   r{  c                   ó    • \ rS rSrS rS rSrg)ÚRemBertConverteriÕ  c                 ó�  • [         R                  " SS5      [         R                  " SS5      [         R                  " [        S5      S5      /nU R                  R                  (       dH  UR                  [         R                  " 5       5        UR                  [         R                  " 5       5        U R                  R                  (       a$  UR                  [         R                  " 5       5        UR                  R                  nU(       a%  UR                  [         R                  " U5      5        [         R                  " U5      $ rÆ  )r	   r}  r   rb   rÉ  r{   rÊ  rË  rý   rÌ  r²   r¦   r  r\  rÍ  s       rW   rÿ   ÚRemBertConverter.normalizer×  sä   € ä×Ò  cÓ*Ü×Ò  cÓ*Ü×Ò¤ g£°Ó4ð
Ðð
 ×&Ñ&×3×3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐæØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRä×#Ò#Ð$4Ó5Ð5rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rÒ  rÓ  rà   s    rW   r  ÚRemBertConverter.post_processorê  rÕ  rY   ri   N)r®   r½   r¾   r¿   rÿ   r  rÅ   ri   rY   rW   r  r  Õ  s   † ò6õ&
rY   r  c                   ó   • \ rS rSrSrg)ÚBertGenerationConverteriõ  ri   Nr|  ri   rY   rW   r…  r…  õ  r}  rY   r…  c                   ó@   • \ rS rSrS r\S	S j5       rS rS rS r	Sr
g)
ÚPegasusConverteriù  c                 ó€  • U R                   R                  S4U R                   R                  S4/nU R                   R                  b  X R                   R                  S4/-  nU R                   R                  bI  U R                   R
                  U R                   R                  :  a  X R                   R                  S4/-  nU[        SU R                   R                  5       Vs/ s H  nSU S3S4PM     sn-  nX!R                  SS   Vs/ s H  oDR                  UR                  4PM     sn-  nU$ s  snf s  snf )Nré  rq   ú<unk_Ú>rü  )rb   rú  r4  Úmask_token_sentrû  Úmask_token_idÚoffsetrz   r«   r¬   r­   )r’   r‘   rk   r´   r¬   s        rW   rk   ÚPegasusConverter.vocabú  s)  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ñ>Ø×.Ñ.×>Ñ>ÀÐDÐEÑEˆEð ×#Ñ#×.Ñ.Ñ:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÓVà×.Ñ.×9Ñ9¸3Ð?Ð@Ñ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÔ2ZÓ[Ò2Z¨Q�U˜1˜#˜Q�< Ó(Ñ2ZÑ[Ñ[ˆØ¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØˆùò \ùÚKs   Ã&D6Ä!D;Nc           	      ó2  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      nUS	4US	4/nUS
:w  a  UR                  US	45        US
:w  a  Xe:w  a  UR                  US	45        UR                  [	        SUR                  SS5      5       Vs/ s H  nSU S3S4PM     sn5        Ub  UR                  [        U5      SS  5        XrS'   U$ s  snf )Nrú  rë  r4  râ  rû  z<mask_1>r‹  z<mask_2>ré  ÚNonerq   r�  ég   r‰  rŠ  rü  rk   )rÁ   rÏ   r{   r}   rz   rÄ   )	r  rk   r³   rú  r4  rû  r‹  rþ  r´   s	            rW   r†  Ú!PegasusConverter.convert_from_spm  s  € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ™ L°*Ó=Ó>ˆ
Ü˜fŸj™jÐ):¸JÓGÓHˆð ˜ÐØ˜Ðð
ˆ
ð ˜ÓØ×Ñ˜z¨3Ð/Ô0Ø˜fÓ$¨Ó)FØ×Ñ˜°Ð4Ô5à×Ñ¼5ÀÀFÇJÁJÈxÐY\ÓD]Ô;^Ó_Ò;^°a˜e A 3 a˜L¨&Ó1Ñ;^Ñ_Ô`ØÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø$ˆw‰Øˆùò	 `s   ÃDc                 ó\   • UR                   R                  U R                  R                  -   $ rÑ   )r¨   rš   rb   r�  r–  s     rW   rš   ÚPegasusConverter.unk_id#  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÑIÐIrY   c                 ó¦   • [        X R                  5      n[        R                  " [        R                  " 5       [        R
                  " XS9/5      $ r«  )rd   rb   r
   r\  ÚWhitespaceSplitr®  r¯  s       rW   r  ÚPegasusConverter.pre_tokenizer&  sE   € Ü,Ð-=×?VÑ?VÓWˆÜ×&Ò&ä×.Ò.Ó0Ü×(Ò(°[Ñ`ðó
ð 	
rY   c                 ó”   • U R                   R                  nXR                   R                  4/n[        R                  " SU/SSU/US9$ )Nú$Aú$Bró   )rb   r4  Úeos_token_idr   r  )r’   Úeosrö   s      rW   r  ÚPegasusConverter.post_processor/  sP   € Ø×%Ñ%×/Ñ/ˆà×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ò,°T¸3°KÀtÈTÐSVÐFWÐhvÑwÐwrY   ri   rÑ   )r®   r½   r¾   r¿   rk   r¹  r†  rš   r  r  rÅ   ri   rY   rW   r‡  r‡  ù  s,   † òð& óó ðò*Jò
õxrY   r‡  c                   ó4   • \ rS rSrS rS r\SS j5       rSrg)ÚT5Converteri7  c                 óü   • U R                   R                  nUR                   Vs/ s H  o3R                  UR                  4PM     nnU[        US-
  SS5       Vs/ s H  nSU S3S4PM     sn-  nU$ s  snf s  snf )Nr   rÓ   ú
<extra_id_rŠ  ré  )rb   Ú
_extra_idsr«   r¬   r­   rz   )r’   r‘   Únum_extra_idsr¬   rk   r´   s         rW   rk   ÚT5Converter.vocab8  s|   € Ø×/Ñ/×:Ñ:ˆØ9>¿ºÓFº°—+‘+˜uŸ{™{Ó+¹ˆÐFØ´E¸-È!Ñ:KÈRÐQSÔ4TÓUÒ4T¨q�Z ˜s !Ð$ cÓ*Ñ4TÑUÑUˆØˆùò GùÚUs   ¥!A4ÁA9c                 ón   • [         R                  " SS// SQSU R                  R                  S5      4/S9$ ©Nr™  râ  )r™  râ  rš  râ  ró   rÓ  rà   s    rW   r  ÚT5Converter.post_processor>  ó=   € Ü×,Ò,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rY   Nc                 óø   • UR                  SS5      n[        US-
  SS5       Vs/ s H	  nSU S3PM     nnUb  [        U5      O/ nUR                  S U 5       5        UR	                  SU5        XbS	'   U$ s  snf )
NÚ	extra_idsrÀ  r   rÓ   r¡  rŠ  c              3   ó(   #   • U  H  oS 4v •  M
     g7fr6  ri   )r8  r¹   s     rW   r:  Ú/T5Converter.convert_from_spm.<locals>.<genexpr>L  s   é € ÐA²L¨5 #�,²Lùr<  r¥   rk   )rÏ   rz   rÄ   r}   Ú
setdefault)r  rk   r³   rª  r´   Úextra_tokensrþ  s          rW   r†  ÚT5Converter.convert_from_spmG  s…   € à—J‘J˜{¨CÓ0ˆ	Ü38¸ÀQ¹ÈÈBÔ3OÓPÒ3O¨a˜* Q C qÓ)Ñ3OˆÐPØ$)Ñ$5”T˜%”[¸2ˆ
Ø×ÑÑA±LÓAÔAà×ÑÐ5°|ÔDØ$ˆw‰Øˆùò Qs   ¥A7ri   rÑ   )	r®   r½   r¾   r¿   rk   r  r¹  r†  rÅ   ri   rY   rW   rŸ  rŸ  7  s    † òò
ð óó órY   rŸ  c                   ó   • \ rS rSrS rSrg)ÚUdopConverteriS  c                 ón   • [         R                  " SS// SQSU R                  R                  S5      4/S9$ r¦  rÓ  rà   s    rW   r  ÚUdopConverter.post_processorT  r¨  rY   ri   N©r®   r½   r¾   r¿   r  rÅ   ri   rY   rW   r±  r±  S  s   † õ
rY   r±  c                   ó   • \ rS rSrS rSrg)ÚVideoPrismConverteri^  c                 ó   • g rÑ   ri   rà   s    rW   r  Ú"VideoPrismConverter.post_processor_  r³  rY   ri   Nr´  ri   rY   rW   r¶  r¶  ^  s   † õrY   r¶  c                   ó"   • \ rS rSrS\4S jrSrg)ÚWhisperConverteric  r[   c                 ó¼  • U R                   R                  n[        U R                   R                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " U R                   R                  S9Ul
        [        R                  " 5       Ul        U R                   R                  nU R                   R                  U5      nU R                   R                  nU R                   R                   nSR#                  U Vs/ s H  oˆ S3PM	     sn5      n	[$        R&                  " U	 SU S3U	 SU S	3Xg4/[)        XT5      QS
9Ul        U$ s  snf )Nr;  Fr<  r>  r  rð   z $A:0 z $A:0 $B:1 rò   ró   )rb   r0  rÄ   r1  r2  r   r   r
   rB  rZ   r  r   r  Úprefix_tokensÚconvert_ids_to_tokensr4  r›  Újoinr   r  Úzipr  )
r’   rk   r   r
  Úprefix_token_idsÚprefixesrœ  r›  r¹   Úprefix_templates
             rW   rá   ÚWhisperConverter.convertedd  sO  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈD×LcÑLc×LtÑLtÑ"uˆ	ÔÜ$×.Ò.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¹hÓ#Gºh°U g¨R£L¹hÑ#GÓHˆÜ#-×#@Ò#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7àÐ#ðä�XÓ0ðñ$
ˆ	Ô ð Ðùò $Hs   ÄEri   Nr  ri   rY   rW   rº  rº  c  s   † ð ˜9÷  rY   rº  c                   ó   • \ rS rSrS rSrg)ÚBigBirdConverteri‡  c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rÒ  rÓ  rà   s    rW   r  ÚBigBirdConverter.post_processorˆ  rÕ  rY   ri   Nr´  ri   rY   rW   rÅ  rÅ  ‡  s   † õ
rY   rÅ  c                   ó"   • \ rS rSrS\4S jrSrg)ÚCLIPConverteri“  r[   c                 ón  • U R                   R                  n[        U R                   R                  R	                  5       5      nU R                   R
                  n[        [        UUS SSS[        U5      S95      n[        R                  " [        R                  " 5       [        R                  " [        S5      S5      [        R                  " 5       /5      Ul        [         R                  " [         R"                  " [        S5      SS	S
9[         R$                  " SS9/5      Ul        [(        R$                  " 5       Ul        [,        R.                  " U R                   R0                  U R                   R2                  4U R                   R4                  U R                   R6                  4SS5      Ul        U$ )Nr;  r*  F©rk   r   r+  r=  r,  r-  rè   z\s+r  z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrV  r>  )rb   r0  rÄ   r1  r2  rè   r   r   rÁ   r	   r\  r[  r}  r   rÌ  rÿ   r
   r]  rB  r  r   r  r   rb  r4  r›  rC  rD  r  r6  s        rW   rá   ÚCLIPConverter.converted”  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ñó

ˆ	ô  +×3Ò3Ü�_Š_Ó¤× 3Ò 3´E¸&³MÀ3Ó GÌ×I^ÒI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ò"9ä×$Ò$ÜÐZÓ[Ø&Øñô
 ×(Ò(¸%Ñ@ðó	#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôô $.×#?Ò#?Ø×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUØ×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUØØó	$
ˆ	Ô ð ÐrY   ri   Nr  ri   rY   rW   rÉ  rÉ  “  s   † ð'˜9÷ 'rY   rÉ  c                   ó"   • \ rS rSrS\4S jrSrg)ÚLayoutLMv2Converteri¾  r[   c           	      ób  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	3Xh4Xy4/S
9Ul        [0        R                  " SS9Ul        U$ )Nrç   FTré   rê   rï   rð   rñ   rò   ró   r÷   rø   rú   r	  s
             rW   rá   ÚLayoutLMv2Converter.converted¿  s„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrY   ri   Nr  ri   rY   rW   rÏ  rÏ  ¾  r  rY   rÏ  c                   ó"   • \ rS rSrS\4S jrSrg)ÚBlenderbotConverteriå  r[   c                 ó¢  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " UR                  S9Ul
        [        R                  " 5       Ul        [        R                  " SUR                   S3UR                  UR                   4/S9Ul        U$ )Nr;  Fr<  r>  z$A:0 rð   )rô   rö   )rb   r0  rÄ   r1  r2  r   r   r
   rB  rZ   r  r   r  r   r  r4  r›  r  rc  s        rW   rá   ÚBlenderbotConverter.convertedæ  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðñ$
ˆ	Ô ð ÐrY   ri   Nr  ri   rY   rW   rÓ  rÓ  å  rf  rY   rÓ  c                   ó&   • \ rS rSrS rS rS rSrg)ÚXGLMConverteri  c                 ó”   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU/ SQ-  nU$ s  snf )Nr  rœ   ))z<madeupword0>ré  )z<madeupword1>ré  )z<madeupword2>ré  )z<madeupword3>ré  )z<madeupword4>ré  )z<madeupword5>ré  )z<madeupword6>ré  r’  rò  s       rW   rk   ÚXGLMConverter.vocab  sV   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØò  zñ  	zˆØˆùò Ls   –!Ac                 ó
   • SnU$ rÛ  ri   rÜ  s      rW   rš   ÚXGLMConverter.unk_id  rÞ  rY   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz</s> $Az</s> $A </s> </s> $Brá  râ  ró   rÓ  rà   s    rW   r  ÚXGLMConverter.post_processor  sR   € Ü×,Ò,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rY   ri   Nrg  ri   rY   rW   r×  r×    s   † ò	òõ
rY   r×  c                   óD   • \ rS rSrSr\rSS1r S rS r	S r
S rS	 rS
rg)ÚGemmaConverteri  Tz<start_of_turn>z<end_of_turn>c                 ó0   • [         R                  " SS5      $ ©Nr  rz  )r	   r}  r–  s     rW   rÿ   ÚGemmaConverter.normalizer-  s   € Ü×"Ò" 3¨Ó.Ð.rY   c                 ó€  • U R                   R                  S4U R                   R                  S4U R                   R                  S4/nX!R                  SS   Vs/ s H  o3R
                  UR                  4PM     sn-  n[        S U 5       5      (       d#  [        S [        U5       5       S 5      nUb  SX$'   U$ s  snf )Nré  rœ   c              3   ó0   #   • U  H  oS    S:H  v •  M     g7f)r   rÊ   Nri   )r8  rj   s     rW   r:  Ú'GemmaConverter.vocab.<locals>.<genexpr>9  s   é € Ð/ª A�Q‘4˜4–<ªùs   ‚c              3   ó@   #   • U  H  u  pUS    S:X  d  M  Uv •  M     g7f)r   rË   Nri   )r8  r´   rj   s      rW   r:  rå  :  s!   é € Ð"VÒ1A©¨ÀQÀqÁTÈXÑEU§1¡1Ò1Aùs   ‚•	)rÊ   ré  )
rb   rú  r4  rC  r«   r¬   r­   ÚanyÚnextr°   )r’   r‘   rk   r¬   Úoverride_indexs        rW   rk   ÚGemmaConverter.vocab0  sº   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆô Ñ/©Ó/×/Ñ/Ü!Ñ"V´¸5Ô1AÓ"VÐX\Ó]ˆNØÑ)Ø(3�Ñ%àˆùò Ls   Á!B;c                 ó0   • [         R                  " SS5      $ )Nr  Úmerged_with_previous)r
   r]  ©r’   r­  rZ   s      rW   r  ÚGemmaConverter.pre_tokenizer@  s   € Ü×#Ò# CÐ)?Ó@Ð@rY   c                 ó
   • SnU$ rÛ  ri   rÜ  s      rW   rš   ÚGemmaConverter.unk_idC  rÞ  rY   c                 óª   • [         R                  " [         R                  " SS5      [         R                  " 5       [         R                  " 5       /5      $ )Nrz  r  )r   r\  r}  r€  r�  rí  s      rW   r  ÚGemmaConverter.decoderG  s?   € Ü× Ò ä× Ò  ¨Ó,Ü×%Ò%Ó'Ü—’“ðó
ð 	
rY   ri   N)r®   r½   r¾   r¿   r‹  rÇ   rž  rö   rÿ   rk   r  rš   r  rÅ   ri   rY   rW   rß  rß    s6   † ØÐØ.€Là'¨Ð9€Nðò/òò Aòõ
rY   rß  c                   ó<   • \ rS rSrSrS rS rS rS rS r	S r
S	rg
)ÚLlamaConverteriQ  Tc                 ó*  • U R                   R                  S5      S4U R                   R                  S5      S4U R                   R                  S5      S4/nX!R                  SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU$ s  snf )Nr   ré  r   rq   rœ   )rb   r½  r«   r¬   r­   rò  s       rW   rk   ÚLlamaConverter.vocabT  s�   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØˆùò Ls   Á(!Bc                 ó
   • SnU$ rŸ   ri   rÜ  s      rW   rš   ÚLlamaConverter.unk_id]  rÞ  rY   c                 óî   • [         R                  " SS5      [         R                  " 5       [         R                  " 5       /nU(       a  U[         R                  " SSS9/-  n[         R
                  " U5      $ ©Nrz  r  r   )Úcontentr¥  ©r   r}  r€  r�  r§  r\  ©r’   r­  rZ   Úsequences       rW   r  ÚLlamaConverter.decodera  ó\   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆö
 ØœŸš°¸!Ñ<Ð=Ñ=ˆHÜ× Ò  Ó*Ð*rY   c                 ó  • [        U R                  SS5      (       ae  / n[        U R                  SS5      (       a  U[        R                  " SS9/-  nU[        R                  " SSS9/-  n[        R
                  " U5      $ g )Nr^   TrZ   rz  )Úprependr  )Úpatternrû  )ra   rb   r	   ÚPrependr}  r\  )r’   r‘   rþ  s      rW   rÿ   ÚLlamaConverter.normalizerk  sx   € Ü�4×*Ñ*¨H°d×;Ñ;ØˆHÜ�t×.Ñ.Ð0BÀD×IÑIØœ[×0Ò0¸Ñ?Ð@Ñ@�Øœ×,Ò,°SÀ%ÑHÐIÑIˆHÜ×'Ò'¨Ó1Ð1ØrY   c                 ó�   • [        U R                  SS5      (       d*  [        X R                  5      n[        R                  " XSS9$ g )Nr^   TF©r­  rc   Úsplit)ra   rb   rd   r
   r®  r¯  s       rW   r  ÚLlamaConverter.pre_tokenizert  s?   € Ü�t×.Ñ.°¸$×?Ñ?Ü0Ð1A×CZÑCZÓ[ˆNÜ!×+Ò+¸ÐjoÑpÐpØrY   c                 ó   • g rÑ   ri   rà   s    rW   r  ÚLlamaConverter.post_processorz  s   € àrY   ri   N)r®   r½   r¾   r¿   r‹  rk   rš   r  rÿ   r  r  rÅ   ri   rY   rW   rô  rô  Q  s&   † ØÐòòò+òòõrY   rô  c                   ó"   • \ rS rSrS\4S jrSrg)ÚMarkupLMConverteri  r[   c                 óz  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSU R                   R                  S95      n[        R                  " UR                  S9Ul        [        R                  " 5       Ul        [        U R                   R                  5      n[        U R                   R                   5      nU R                   R"                  nU R                   R$                  n[&        R(                  " U SU 3U SU SU 3XW4Xh4/S9Ul        U$ )Nr;  FrË  r>  z $A z $B ró   )rb   r0  rÄ   r1  r2  r   r   rè   r
   rB  rZ   r  r   r  rÁ   r  r  r  r  r   r  r  )	r’   rd  rk   r   r
  r  r  r  r  s	            rW   rá   ÚMarkupLMConverter.converted€  s(  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ñó

ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+àÐ#ØÐ#ðñ$
ˆ	Ô ð ÐrY   ri   Nr  ri   rY   rW   r  r    s   † ð"˜9÷ "rY   r  c                   ó0   • \ rS rSrSrS rS rS rS rSr	g)	ÚMoshiConverteri¥  Tc                 ó  • [        U S5        [        R                  X5        [        5       nUR	                  5       n[        US5       nUR                  UR                  5       5        S S S 5        X@l        g ! , (       d  f       N= f©NrP   rŒ   ©	r   rÙ   r–   rX   r�   rŽ   r�   r�   r‘   ©r’   rŠ  r³   r“   r”   r•   s         rW   r–   ÚMoshiConverter.__init__¨  se   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÔ# qØ×Ñ˜aŸf™f›hÔ'÷ $à�
÷ $Õ#úó   Á A7Á7
Bc                 óô   • UR                   R                  n[        R                  " SS5      /nU(       d  [        R                  " U5      $ [        R                  " [        R
                  " U5      /U-   5      $ rá  )r²   r¦   r	   r}  r\  r  r¨  s       rW   rÿ   ÚMoshiConverter.normalizerµ  sg   € Ø$×4Ñ4×IÑIÐä×Ò  UÓ+ð
ˆö $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrY   c                 óî   • [         R                  " SS5      [         R                  " 5       [         R                  " 5       /nU(       a  U[         R                  " SSS9/-  n[         R
                  " U5      $ rú  rü  rý  s       rW   r  ÚMoshiConverter.decoder¿  r   rY   c                 ó0   • Sn[         R                  " XSS9$ )Nr_   Fr  )r
   r®  r¯  s       rW   r  ÚMoshiConverter.pre_tokenizerÉ  s   € Ø ˆÜ×'Ò'°KÐfkÑlÐlrY   r¼   N)
r®   r½   r¾   r¿   r‹  r–   rÿ   r  r  rÅ   ri   rY   rW   r  r  ¥  s   † ØÐòòhò+õmrY   r  c                   óL   • \ rS rSrSrSS jrS rS rS rS r	S	 r
S
 rS rSrg)ÚHeliumConverteriÎ  TNc                 ó  • [        U S5        [        R                  X5        [        5       nUR	                  5       n[        US5       nUR                  UR                  5       5        S S S 5        X@l        g ! , (       d  f       N= fr  r  r  s         rW   r–   ÚHeliumConverter.__init__Ñ  sc   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÔ# qØ×Ñ˜aŸf™f›hÔ'÷ $à�
÷ $Õ#úr  c                 óR  • U R                  U5      n[        [        UU R                  U5      U R                  S95      n[        UR                  5       VVs/ s HR  u  pEUR                  S;   d  M  XER                  UR                  S:H  =(       d    UR                  U R                  ;   4PMT     nnnUR                  [        US S9 VVVs/ s H  u  pGn[        USUSS9PM     snnn5        UR                  [        S	SSS
9/5        UR                  SSS9  U$ s  snnf s  snnnf )Nr™  r›   rœ   c                 ó   • U S   $ rŸ   ri   r    s    rW   rl   Ú+HeliumConverter.tokenizer.<locals>.<lambda>ò  r�  rY   rn   FT)r£   r¤   Úsingle_wordÚ
r¢   rë  )rú  Úpad_id)rk   r   r   rš   r‹  r°   r«   r±   r¬   rö   r   r|   r   Úenable_padding)	r’   r‘   r~   r
  r¶   r·   r¸   r¹   r¤   s	            rW   r
  ÚHeliumConverter.tokenizerÝ  s   € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ñó
ˆ	ô # 5§<¡<Ô0ô
â0‘�Ø�v‰v˜Ñó IˆR—‘˜!Ÿ&™& A™+×G¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Ò*Võâ*VÑ&�B˜wô ˜5¨U¸GÐQUÔVÙ*Vóô	
ð 	×Ñœj¨¸%ÈÑOÐPÔQØ× Ñ ¨7¸1Ð Ñ=ØÐùó
ùôs   ÁDÁ1?DÃD"c                 ó®   • / nUR                    HB  nUR                  S:X  a  USUR                  4/-  nM'  X#R                  UR                  4/-  nMD     U$ )Nz<0x0A>r&  r’  rò  s       rW   rk   ÚHeliumConverter.vocabù  sV   € ØˆØ—\”\ˆEØ�{‰{˜hÓ&Ø˜4 §¡Ð-Ð.Ñ.’àŸ;™;¨¯©Ð4Ð5Ñ5’ñ	 "ð
 ˆrY   c                 ó
   • SnU$ rŸ   ri   rÜ  s      rW   rš   ÚHeliumConverter.unk_id  rÞ  rY   c                 óà   • [         R                  " SS5      [         R                  " 5       [         R                  " 5       /nU[         R                  " SSS9/-  n[         R
                  " U5      $ rú  rü  rý  s       rW   r  ÚHeliumConverter.decoder  sY   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆð
 	”X—^’^¨C°aÑ8Ð9Ñ9ˆÜ× Ò  Ó*Ð*rY   c                 ó„   • [         R                  " [         R                  " S5      [         R                  " SS5      /5      $ rá  )r	   r\  r  r}  r–  s     rW   rÿ   ÚHeliumConverter.normalizer  s2   € Ü×#Ò#¤[×%8Ò%8¸Ó%=¼{×?RÒ?RÐSWÐY^Ó?_Ð$`ÓaÐarY   c                 óZ   • [         R                  " [         R                  " SS5      /5      $ )Nr&  Ú
contiguous)r
   r\  r]  rí  s      rW   r  ÚHeliumConverter.pre_tokenizer  s#   € Ü×&Ò&¬×(<Ò(<¸TÀ<Ó(PÐ'QÓRÐRrY   c                 ó8   • [         R                  " SS// SQS/S9$ )Nrá  r™  )rá  r™  rá  rš  )rá  r   ró   )r   r  rà   s    rW   r  ÚHeliumConverter.post_processor  s/   € Ü×,Ò,àØðòð ðñ
ð 	
rY   r¼   rÑ   )r®   r½   r¾   r¿   r‹  r–   r
  rk   rš   r  rÿ   r  r  rÅ   ri   rY   rW   r  r  Î  s2   † ØÐô
òò8òò+òbòSõ
rY   r  c                   ó(   • \ rS rSrSrSS jrS rSrg)ÚParakeetConverteri'  TNc                 ó  • Xl         [        U S5        [        R                  X5        [	        5       nUR                  5       n[        US5       nUR                  UR                  5       5        S S S 5        X@l	        g ! , (       d  f       N= fr  )
rŠ  r   rÙ   r–   rX   r�   rŽ   r�   r�   r‘   )r’   rŠ  rŽ  r“   r”   r•   s         rW   r–   ÚParakeetConverter.__init__*  sh   € Ø$Œä˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	Ø× Ñ Ó"ˆÜ�*˜dÔ# qØ×Ñ˜aŸf™f›hÔ'÷ $à�
÷ $Õ#ús   Á A=Á=
Bc                 óv  • U R                  U5      n[        U5       VVVs0 s H
  u  nu  pEXC_M     nnnn[        Xb5      n[        [	        UUUR
                  R                  SU R                  S S95      n[        UR                  5       V	V
s/ s HR  u  pšU
R                  S;   d  M  XšR                  U
R                  S:H  =(       d    U
R                  U R                  ;   4PMT     nn	n
UR                  [        US S9 V	VVs/ s H  u  pœn[        USUS9PM     snnn	5        U$ s  snnnf s  sn
n	f s  snnn	f )	NTrš  r›   rœ   c                 ó   • U S   $ rŸ   ri   r    s    rW   rl   Ú-ParakeetConverter.tokenizer.<locals>.<lambda>P  r�  rY   rn   Fr¢   )rk   r°   r†   r   r   r¨   r{  r‹  r«   r±   r¬   rö   r   r|   r   )r’   r‘   r~   r´   rµ   r­   r¢  r   r
  r¶   r·   r¸   r¹   r¤   s                 rW   r
  ÚParakeetConverter.tokenizer7  s0  € Ø—z‘z %Ó(ˆä5>¸|Ô5LÕMÒ5LÑ!1 ¡M T�T’WÑ5Lˆ	ÒMÜ  Ó9ˆÜÜØØØ×,Ñ,×6Ñ6ØØ"×7Ñ7Øñó	
ˆ	ô # 5§<¡<Ô0ô
â0‘�Ø�v‰v˜Ñó IˆR—‘˜!Ÿ&™& A™+×G¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Ò*Võâ*VÑ&�B˜wô ˜5¨U¸GÔDÙ*Vóô	
ð Ðùô5 Nùó
ùôs   ¡D'ÂD.Â(?D.ÄD4)r‘   rŠ  rÑ   )r®   r½   r¾   r¿   r‹  r–   r
  rÅ   ri   rY   rW   r8  r8  '  s   † ØÐôõrY   r8  c                   ó,   ^ • \ rS rSrSU 4S jjrSrU =r$ )ÚCanaryConverteriW  c                 óJ  >• [         TU ]  " U/UQ76   U R                  R                   Vs1 s He  nUR                  S:X  d  M  UR
                  R                  S5      (       d  M7  UR
                  R                  S5      (       d  MY  UR
                  iMg     snU l        g s  snf )Nr�   z<|z|>)	r‰  r–   r‘   r«   r±   r¬   Ú
startswithÚendswithrö   )r’   rŠ  rŽ  r¬   r�  s       €rW   r–   ÚCanaryConverter.__init__X  s†   ø€ Ü‰Ò˜Ð+ dÓ+ð Ÿ™×*Ò*ó
â*�Ø�z‰z˜Q‰ó à#(§;¡;×#9Ñ#9¸$×#?ó àDIÇKÁK×DXÑDXÐY]×D^ó ˆE�KŒKÙ*ñ
ˆÕùò 
s   «B ÁB Á%B ÂB )rö   rÑ   )r®   r½   r¾   r¿   r–   rÅ   rº  r»  s   @rW   r@  r@  W  s   ø† ÷
õ 
rY   r@  c            	      óþ  • [        [        [        S5      [        S5      S-   5      5      [        [        [        S5      [        S5      S-   5      5      -   [        [        [        S5      [        S5      S-   5      5      -   n U SS nS	n[        S
5       H4  nX0;  d  M
  U R                  U5        UR                  S
U-   5        US-  nM6     U Vs/ s H  n[	        U5      PM     nn[        [        X5      5      $ s  snf )a  
Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
characters the bpe code barfs on.

The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
tables between utf-8 bytes and unicode strings.
Ú!Ú~r   ô   Â¡ô   Â¬ô   Â®ô   Ã¿Nr   é   )rÄ   rz   Úordr{   Úchrrx   r¿  )ÚbsÚcsÚnÚbs       rW   Úbytes_to_unicoderS  b  sÞ   € ô 	ŒU”3�s“8œS ›X¨™\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÁMÓ3RÓ.SÑSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÑlyÓ[zÓV{Ñ{ð ð 
‰Aˆ€BØ	€AÜ�4Ž[ˆØ�;Ø�I‰I�aŒLØ�I‰I�d˜Q‘hÔØ�‰FŠAñ	 ñ
 Ó	š"�QŒ#ˆaŽ&™"€BÐ	Ü”�B“ÓÐùò 
s   ÃC:c                   ór   • \ rS rSrSr    SS jr\S\S\\	\
4   4S j5       rS\4S jrS	 rS\4S
 jrSrg)ÚTikTokenConverteriz  z
A general tiktoken converter.
Nc                 óŒ   • Xl         X l        X0l        [        U[        5      (       a  UR                  5       U l        g UU l        g rÑ   )rŠ  r  rZ   r|  rx   r2  Úextra_special_tokens)r’   rŠ  r  rZ   rW  r³   s         rW   r–   ÚTikTokenConverter.__init__  sA   € ð %ŒØŒØ 0Ôä+5Ð6JÌD×+QÑ+QÐ ×%Ñ%Ó'ð 	Õ!ØWkð 	Õ!rY   Útiktoken_urlr[   c                 óR   •  SSK Jn  U" U 5      $ ! [         a    [        S5      ef = f)Nr   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.)Útiktoken.loadr[  rŸ  Ú
ValueError)rY  r[  s     rW   r[  Ú#TikTokenConverter.load_tiktoken_bpeŽ  s8   € ð	Ý7ñ
 ! Ó.Ð.øô	 ó 	ÜØkóð ð	ús   ‚ �&c                 ó  ^^• U R                  U5      m[        5       mU4S jn/ n0 nTR                  5        H“  u  pVXdU" U5      '   [        U5      S:X  a  M   / n[	        S[        U5      5       H8  nUS U XXS  p©U	T;   d  M  U
T;   d  M  Xš-   T;   d  M%  UR                  XšU45        M:     [        UU4S jSS9nUR                  U5        M•     [        US SS9nU Vs/ s H  o²" US   5      U" US   5      4PM     nnXC4$ s  snf )Nc           	      óˆ   >• SR                  U R                  S5       Vs/ s H  nT[        U5         PM     sn5      $ s  snf )Nr;  zlatin-1)r¾  ÚdecoderM  )rR  ÚcharÚbyte_encoders     €rW   Útoken_bytes_to_stringÚPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_stringœ  s8   ø€ Ø—7‘7ÀÇÁÈÔ@SÓTÒ@S¸˜L¬¨T«Ô3Ñ@SÑTÓUÐUùÒTs    ?r   c                 ó$   >• TU S      TU S      4$ rh   ri   )rj   r1  s    €rW   rl   ÚCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>ª  s   ø€ °¸1¸Q¹4±À)ÈAÈaÉDÁ/Ñ0RrY   Fru   c                 ó   • U S   $ )Nrq   ri   rs   s    rW   rl   rg  ¬  s   € °°A²rY   r   )r[  rS  ry   rr   rz   r{   r|   r}   )r’   rY  rd  r   rk   r¹   Úrankr‚   rƒ   r„   r…   rt   r1  rc  s               @@rW   Úextract_vocab_merges_from_modelÚ1TikTokenConverter.extract_vocab_merges_from_model˜  s  ù€ Ø×*Ñ*¨<Ó8ˆ	Ü'Ó)ˆõ	Vð ˆØˆØ$Ÿ?™?Ö,‰KˆEØ26Ñ'¨Ó.Ñ/Ü�5‹z˜Q‹ÙØˆEÜ˜q¤# e£*Ö-�Ø#(¨¨% =°%¸°-˜Ø˜iÕ'¨G°yÕ,@ÀgÑFWÐ\eÕEeØ—L‘L '°DÐ!9Ö:ñ .ô ˜5Ô&RÐ\aÑbˆEØ�M‰M˜%Ö ñ -ô ˜Ñ$6ÀÑFˆÙ\bÓcÒ\bÐUXÐ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÓPÑ\bˆÐcØˆ}Ðùò ds   ÃDc                 ó¾   • U R                  U R                  5      u  p[        [        XSS95      n[	        UR
                  S5      (       a  SUR
                  l        U$ )NF)r-  Úignore_mergesT)rj  rŠ  r   r   rû   rŠ   rm  )r’   r~   r   r
  s       rW   r
  ÚTikTokenConverter.tokenizer°  sM   € Ø#×CÑCÀDÇOÁOÓTÑˆÜœc ,ÀÑGÓHˆ	Ü�9—?‘? O×4Ñ4Ø,0ˆI�O‰OÔ)ØÐrY   c                 óÜ  • U R                  5       n[        R                  " [        R                  " [	        U R
                  5      SSS9[        R                  " U R                  SS9/5      Ul        [        R                  " 5       Ul
        U R                  b5  UR                  U R                   Vs/ s H  n[        USSS9PM     sn5        [        R                  " SS9Ul        U$ s  snf )NrU  FrV  rY  Tr¢   r@  )r
  r
   r\  r]  r   r  rB  rZ   r  r   r  rW  r4  r   r   r  )r’   r
  r¹   s      rW   rá   ÚTikTokenConverter.converted·  sÊ   € Ø—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ñ\Ü×(Ò(¸$×:OÑ:OÐ[`Ñaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôà×$Ñ$Ñ0Ø×(Ñ(ØPT×PiÒPiÓjÒPiÀu”˜E¨e¸TÔBÑPiÑjôô $.×#7Ò#7ÀUÑ#Kˆ	Ô àÐùò ks   Â2C))rZ   rW  r  rŠ  )Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)r®   r½   r¾   r¿   rÀ   r–   r¸  rÁ   rx   ÚbytesrÃ   r[  rj  r
  r   rá   rÅ   ri   rY   rW   rU  rU  z  se   † ñð ð KØØ!ô
ð ð/¨ð /°°U¸C°ZÑ0@ó /ó ð/ð¸Cô ò0ð˜9÷ rY   rU  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚReformerTokenizerÚRemBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚVideoPrismTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizerÚSplinterTokenizer)ÚXGLMTokenizerÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                 ó˜  • U R                   R                  nU[        ;   a&  U(       d  [        U   nU" U 5      R                  5       $ U R                  n[        U[        5      (       ak  [        R                  R                  U5      (       aG  [        U5      (       a7  SSKJn  X l        [        R                  S5        U" U5      R                  5       $  [        R                  S5        [!        U R                  U R"                  S9R                  5       $ ! [$         a*    ['        S[)        [        R+                  5       5       35      ef = f)a\  
Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

Args:
    transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
        Instance of a slow tokenizer to convert in the backend tokenizer for
        [`~tokenization_utils_base.PreTrainedTokenizerFast`].
   from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
        Defaults to False.

Return:
    A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
    [`~tokenization_utils_base.PreTrainedTokenizerFast`]
r   )ÚMistralConverterz#Converting from Mistral tekken.jsonzConverting from Tiktoken)rŠ  rW  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )r�  r®   ÚSLOW_TO_FAST_CONVERTERSrá   rŠ  r|  rÁ   ÚosÚpathÚisfiler   Úintegrations.mistral.tokenizerr«  rb   ÚloggerÚinforU  rW  rŸ  r]  rÄ   r2  )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classrŠ  r«  s         rW   Úconvert_slow_tokenizerr·    s  € ð  1×:Ñ:×CÑCÐØÔ6Ó6¾}Ü1Ð2FÑGˆÙÐ4Ó5×?Ñ?ÓAÐAà&×1Ñ1€JÜ�*œc×"Ñ"¤r§w¡w§~¡~°j×'AÑ'AÔF^Ð_i×FjÑFjÝDà3HÔ0Ü�‰Ð9Ô:Ù 
Ó+×5Ñ5Ó7Ð7ð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø%:×%OÑ%Oñ÷ ‰i‹kðøô ó 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   ÃA D Ä4E	)r;  rÑ   )F)XrÀ   r­  rŒ  Úcollections.abcr   Ú	packagingr   Ú
tokenizersr   r   r   r   r	   r
   r   r©   r   r   r   Úintegrations.mistral.constantsr   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerr®   r±  r=  rK  rX   ÚboolrÁ   rd   r†   rˆ   rÇ   r×   rÙ   rä   r  r   r$  r(  r9  rJ  rR  r`  rh  rp  rx  r½  rÙ  ræ  r  r  rA  rN  r^  ri  rs  r{  r  r…  r‡  rŸ  r±  r¶  rº  rÅ  rÉ  rÏ  rÓ  r×  rß  rô  r  r  r  r8  r@  rS  rU  r¬  r·  ri   rY   rW   Ú<module>rÀ     sÃ  ðñó 
Û Ý &å ß f× fÑ fß 5Ñ 5å Dß `Ó `Ý 5ð 
×	Ò	˜HÓ	%€ò€ð8 $ò 'ñ Ð ô>Gð"¨$ð Àsô ñ°jÀ±oÈÑ6Lõ ÷02ñ 2ôjÐ"8ô ð"I˜cð I dô I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6#�Iô #ôL�yô ô>)�Yô )ôX�yô ô:$˜	ô $ôN�yô ô>x�9ô xôv"
�lô "
ôJ
�|ô 
ô -˜ô -ô`
˜ô 
ôBG�\ô GôT-�|ô -ô`,�Lô ,ô^
˜<ô 
ô2,˜,ô ,ô^"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô;x�|ô ;xô|�,ô ô8
�Lô 
ô˜+ô ô
!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
ôr-˜ô -ô`
Ð'ô 
ò÷0Nñ Nðb9Ø�ð9àÐ%ð9ð Ð(ð9ð �]ð	9ð
 Ð(ð9ð Ð.ð9ð Ð,ð9ð �]ð9ð ˜ð9ð ˜ð9ð Ð(ð9ð Ð,ð9ð ˜=ð9ð ˜-ð9ð " =ð9ð  ! -ð!9ð" ˜ñ#9ð$ �_ð%9ð& �ð'9ð( �]ð)9ð* Ð(ð+9ð, ˜ð-9ð. ˜=ð/9ð0 Ð+ð19ð2 Ð-ð39ð4 Ð+ð59ð6 Ð$ð79ð8 �}ð99ð: Ð*ð;9ð< �nð=9ð> Ð(ð?9ð@ �nðA9ðB ˜=ðC9ðD Ð$òE9ðF �]ðG9ðH Ð,ðI9ðJ Ð(ðK9ðL �nðM9ðN Ð*ðO9ðP Ð(ðQ9ðR Ð(ðS9ðT Ð*ðU9ðV Ð0ðW9ðX ˜MðY9ðZ �;ð[9ð\ �]ð]9ð^ Ð.ð_9ð` Ð(ða9ðb Ð.ðc9ðd �nðe9ðf Ð*ñg9ðh #Ø$Ø(Ø$Ø#òq9Ð ñx(È)ö (rY   