ó
    ÷“,j˜X  ã                   óÆ  • S r SSKrSSKrSSKrSSKrSSKrSSKJrJr  SSK	J
r
JrJrJrJr  SSKrSSKrSSKrSSKrSSKrS\R,                  S'   SqSr\" 5       S	 5       r\" 5       S
 5       rS rS rS rS rS rS r S\!4S jr"S\!RG                  SS\RH                  5      S.S\%4S jjr& " S S\'5      r(\(" 5       r)S\RT                  4S jr+\4S\\!\\!   4   S\,S\RZ                  4S jjr. S+S\\!\\!   4   S\,S\,S \,S!\
S"\/4S# jjr0S\\!\\!   4   S\,S\,S \,S!\
4
S$ jr1S\\!\\!   4   S\,S\,S \,S!\
S\RZ                  4S% jr2S\!4S& jr3 " S' S(5      r4 " S) S*5      r5g),zoCLIP tokenizer

Copied from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI.
é    N)Ú	lru_cacheÚpartial)ÚCallableÚListÚOptionalÚUnionÚDictÚfalseÚTOKENIZERS_PARALLELISMFéM   c                  ó¾   • [         R                  R                  [         R                  R                  [         R                  R	                  [
        5      5      S5      $ )Nzbpe_simple_vocab_16e6.txt.gz)ÚosÚpathÚjoinÚdirnameÚabspathÚ__file__© ó    Ú¡C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\open_clip/tokenizer.pyÚdefault_bper      s2   € ä�7‰7�<‰<œŸ™Ÿ™¬¯©¯©¼Ó(AÓBÐDbÓcÐcr   c            	      óþ  • [        [        [        S5      [        S5      S-   5      5      [        [        [        S5      [        S5      S-   5      5      -   [        [        [        S5      [        S5      S-   5      5      -   n U SS nS	n[        S
5       H4  nX0;  d  M
  U R                  U5        UR                  S
U-   5        US-  nM6     U Vs/ s H  n[	        U5      PM     nn[        [        X5      5      $ s  snf )a  
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a significant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
Ú!Ú~é   ô   Â¡ô   Â¬ô   Â®ô   Ã¿Nr   é   )ÚlistÚrangeÚordÚappendÚchrÚdictÚzip)ÚbsÚcsÚnÚbs       r   Úbytes_to_unicoder,      s×   € ô 
Œe”C˜“Hœc #›h q™jÓ)Ó	*¬4´´c¸$³iÄÀTÃÈ1ÁÓ0MÓ+NÑ	NÌtÔTYÔZ]Ð^bÓZcÔehÐimÓenÐopÑepÓTqÓOrÑ	r€BØ	‰Aˆ€BØ	€AÜ�4Ž[ˆØ�;Ø�I‰I�aŒLØ�I‰I�d˜1‘fÔØ�‰FŠAñ	 ñ
 Ó	š"�QŒ#ˆaŽ&™"€BÐ	Ü”�B“ÓÐùò 
s   ÃC:c                 ód   • [        5       nU S   nU SS  H  nUR                  X#45        UnM     U$ )zwReturn set of symbol pairs in a word.
Word is represented as tuple of symbols (symbols being variable-length strings).
r   r   N)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars       r   Ú	get_pairsr4   6   s?   € ô ‹E€EØ�Q‘€IØ�Q�R“ˆØ�	‰	�9Ð#Ô$ØŠ	ñ ð €Lr   c                 ó¢   • [         R                  " U 5      n [        R                  " [        R                  " U 5      5      n U R	                  5       $ ©N)ÚftfyÚfix_textÚhtmlÚunescapeÚstrip©Útexts    r   Úbasic_cleanr>   B   s3   € Ü�=Š=˜Ó€DÜ�=Š=œŸš tÓ,Ó-€DØ�:‰:‹<Ðr   c                 ód   • SR                  U R                  5       5      n U R                  5       n U $ )NÚ )r   Úsplitr;   r<   s    r   Úwhitespace_cleanrB   H   s&   € Ø�8‰8�D—J‘J“LÓ!€DØ�:‰:‹<€DØ€Kr   c                 ó*   • [        [        U 5      5      $ r6   )Úcanonicalize_textr>   ©Úxs    r   Ú_clean_canonicalizerG   N   s   € äœ[¨›^Ó,Ð,r   c                 óF   • [        [        U 5      5      R                  5       $ r6   )rB   r>   ÚlowerrE   s    r   Ú_clean_lowerrJ   S   s   € äœK¨›NÓ+×1Ñ1Ó3Ð3r   c                 ó*   • [        [        U 5      5      $ r6   )rB   r>   rE   s    r   Ú_clean_whitespacerL   X   s   € äœK¨›NÓ+Ð+r   Útypec                 ó`   • U S:X  a  [         $ U S:X  a  [        $ U S:X  a  [        $  SU  S35       e)NÚcanonicalizerI   Ú
whitespacezInvalid clean function (z).)rG   rJ   rL   ©rM   s    r   Úget_clean_fnrR   ]   s?   € Øˆ~ÓÜ"Ð"Ø	�‹ÜÐØ	�Ó	Ü Ð à9Ð0°°°bÐ9Ó9ˆur   Ú )Úkeep_punctuation_exact_stringÚtrans_punctuationrU   c                ó,  ^• U R                  SS5      n U(       a+  UR                  U4S jU R                  U5       5       5      n OU R                  T5      n U R	                  5       n SR                  U R                  5       5      n U R                  5       $ )aÜ  Returns canonicalized `text` (lowercase and punctuation removed).

From: https://github.com/google-research/big_vision/blob/53f18caf27a9419231bbf08d3388b07671616d3d/big_vision/evaluators/proj/image_text/prompt_engineering.py#L94

Args:
  text: string to be canonicalized.
  keep_punctuation_exact_string: If provided, then this exact string kept.
    For example providing '{}' will keep any occurrences of '{}' (but will
    still remove '{' and '}' that appear separately).
Ú_r@   c              3   óF   >#   • U  H  nUR                  T5      v •  M     g 7fr6   )Ú	translate)Ú.0ÚpartrU   s     €r   Ú	<genexpr>Ú$canonicalize_text.<locals>.<genexpr>z   s%   øé € ð 2
âA�ð �N‰NÐ,×-Ð-ÚAùs   ƒ!)Úreplacer   rA   rY   rI   r;   )r=   rT   rU   s     `r   rD   rD   h   s{   ø€ ð  �<‰<˜˜SÓ!€DÞ$Ø,×1Ñ1ô 2
àŸ
™
Ð#@ÔAó2
ó 
‰ð
 �~‰~Ð/Ó0ˆØ�:‰:‹<€DØ�8‰8�D—J‘J“LÓ!€DØ�:‰:‹<Ðr   c                   ó²   • \ rS rSr\" 5       S\SS4S\S\\\      S\\	   S\S	\4
S
 jjr
S rS rS rSS\\\\   4   S\\	   S\R                   4S jjrSrg)ÚSimpleTokenizeré…   NrI   rS   Úbpe_pathÚadditional_special_tokensÚcontext_lengthÚcleanÚreduction_maskc           
      ó¦  • [        5       U l        U R                  R                  5        VVs0 s H  u  pgXv_M	     snnU l        [        R
                  " U5      R                  5       R                  S5      R                  S5      nUSS nU V	s/ s H  n	[        U	R                  5       5      PM     nn	[        [        5       R                  5       5      n
Xª Vs/ s H  owS-   PM	     sn-   n
U H#  n	U
R                  SR                  U	5      5        M%     SS/nU(       a  X²-  nU
R                  U5        [        [!        U
[#        [%        U
5      5      5      5      U l        U R&                  R                  5        VVs0 s H  u  pgXv_M	     snnU l        [        [!        U[#        [%        U5      5      5      5      U l        U Vs0 s H  oÌU_M     snU l        S	R                  U5      n[.        R0                  " US
-   [.        R2                  5      U l        [%        U R&                  5      U l        U Vs/ s H  oÀR&                  U   PM     snU l        U R8                  S   U l        U R8                  S   U l        X0l        [A        U5      U l!        U(       a  [E        U5      U l#        g S U l#        g s  snnf s  sn	f s  snf s  snnf s  snf s  snf )Núutf-8Ú
r   iÿ¾  ú</w>rS   z<start_of_text>z<end_of_text>Ú|z:|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+r   )$r,   Úbyte_encoderÚitemsÚbyte_decoderÚgzipÚopenÚreadÚdecoderA   Útupler!   Úvaluesr$   r   Úextendr&   r'   r"   ÚlenÚencoderÚdecoderÚ	bpe_ranksÚcacheÚreÚcompileÚ
IGNORECASEÚpatÚ
vocab_sizeÚall_special_idsÚsot_token_idÚeot_token_idrd   rR   Úclean_fnÚget_reduction_mask_fnÚreduction_fn)Úselfrb   rc   rd   re   rf   ÚkÚvÚmergesÚmergeÚvocabÚspecial_tokensÚtÚspecials                 r   Ú__init__ÚSimpleTokenizer.__init__†   sF  € ô -Ó.ˆÔØ.2×.?Ñ.?×.EÑ.EÔ.GÔHÒ.G¡d a˜QšTÑ.GÒHˆÔÜ—’˜8Ó$×)Ñ)Ó+×2Ñ2°7Ó;×AÑAÀ$ÓGˆØ˜˜-Ð(ˆÙ4:Ó;²F¨5”%˜Ÿ™›Ö&±FˆÐ;ÜÔ%Ó'×.Ñ.Ó0Ó1ˆØ¨5Ó1ª5 a˜6œ©5Ñ1Ñ1ˆÛˆEØ�L‰L˜Ÿ™ ›Ö(ñ à+¨_Ð=ˆÞ$ØÑ7ˆNØ�‰�^Ô$ÜœC ¤u¬S°«ZÓ'8Ó9Ó:ˆŒØ)-¯©×);Ñ);Ô)=Ô>Ò)=¡ ˜šÑ)=Ò>ˆŒÜœc &¬%´°F³Ó*<Ó=Ó>ˆŒÙ#1Ó2¢>˜a˜’c¡>Ñ2ˆŒ
Ø—(‘(˜>Ó*ˆÜ—:’:ØÐWÑWÜ�M‰Mó
ˆŒô ˜dŸl™lÓ+ˆŒÙ9GÓHº°A§¡¨Q¤¹ÑHˆÔØ ×0Ñ0°Ñ3ˆÔØ ×0Ñ0°Ñ3ˆÔØ,ÔÜ$ UÓ+ˆŒÞESÔ1°.ÓAˆÕÐY]ˆÕùó7 Iùò <ùâ1ùó ?ùâ2ùò  Is#   ­J3Â#J9ÃJ>Å>KÇK	È5Kc                 ó
  ^ • UT R                   ;   a  T R                   U   $ [        US S 5      US   S-   4-   n[        U5      nU(       d  US-   $  [        UU 4S jS9nUT R                  ;  a  OÏUu  pV/ nSnU[        U5      :  aŽ   UR                  XX5      n	UR                  X(U	 5        U	nX(   U:X  a6  U[        U5      S-
  :  a$  X(S-      U:X  a  UR                  XV-   5        US-  nOUR                  X(   5        US-  nU[        U5      :  a  MŽ  [        U5      nUn[        U5      S:X  a  O[        U5      nMî  SR                  U5      nUT R                   U'   U$ ! [         a    UR                  X(S  5         Ml  f = f)	Néÿÿÿÿrj   c                 óN   >• TR                   R                  U [        S5      5      $ )NÚinf)ry   ÚgetÚfloat)Úpairr†   s    €r   Ú<lambda>Ú%SimpleTokenizer.bpe.<locals>.<lambda>¶   s   ø€ °4·>±>×3EÑ3EÀdÌEÐRWËLÔ3Yr   )Úkeyr   r   é   r@   )rz   rs   r4   Úminry   rv   Úindexru   Ú	Exceptionr$   r   )
r†   Útokenr0   r1   ÚbigramÚfirstÚsecondÚnew_wordÚiÚjs
   `         r   ÚbpeÚSimpleTokenizer.bpe¬   sŒ  ø€ Ø�D—J‘JÓØ—:‘:˜eÑ$Ð$Ü�U˜3˜B�ZÓ  U¨2¡Y°Ñ%7Ð#9Ñ9ˆÜ˜$“ˆæØ˜‘<ÐàÜ˜Ô&YÑZˆFØ˜TŸ^™^Ó+ØØ"‰MˆEØˆHØˆAØ”c˜$“i“-ðØŸ
™
 5Ó,�AØ—O‘O D¨1 IÔ.Ø�Að
 ‘7˜eÓ#¨¬C°«I°a©K«¸DÀ1Á¹IÈÓ<OØ—O‘O E¡LÔ1Ø˜‘F‘Aà—O‘O D¡GÔ,Ø˜‘F�Að ”c˜$“i•-ô ˜X“ˆHØˆDÜ�4‹y˜A‹~Øä! $›�ñ7 ð8 �x‰x˜‹~ˆØ ˆ�
‰
�5ÑØˆøô% !ó Ø—O‘O D¨ HÔ-Úðús   Â&E! Å!FÆFc                 óD  ^ • / nT R                  U5      n[        R                  " T R                  U5       Hf  nSR	                  U 4S jUR                  S5       5       5      nUR                  U 4S jT R                  U5      R                  S5       5       5        Mh     U$ )NrS   c              3   óB   >#   • U  H  nTR                   U   v •  M     g 7fr6   )rl   )rZ   r+   r†   s     €r   r\   Ú)SimpleTokenizer.encode.<locals>.<genexpr>Ù   s   øé € ÐPÒ:O°Q˜D×-Ñ-¨aÖ0Ò:Oùó   ƒrh   c              3   óB   >#   • U  H  nTR                   U   v •  M     g 7fr6   )rw   )rZ   Ú	bpe_tokenr†   s     €r   r\   rª   Ú   s   øé € ÐbÒGa¸)˜dŸl™l¨9Ö5ÒGaùr«   r@   )	rƒ   r{   Úfindallr~   r   Úencoderu   r¦   rA   )r†   r=   Ú
bpe_tokensrŸ   s   `   r   r¯   ÚSimpleTokenizer.encodeÕ   s}   ø€ Øˆ
Ø�}‰}˜TÓ"ˆÜ—Z’Z §¡¨$Ö/ˆEØ—G‘GÔP¸%¿,¹,ÀwÔ:OÓPÓPˆEØ×ÑÔbÀtÇxÁxÐPUÃ×G\ÑG\Ð]`ÔGaÓbÖbñ 0ð Ðr   c                 ó  • SR                  U Vs/ s H  o R                  U   PM     sn5      n[        U Vs/ s H  o@R                  U   PM     sn5      R	                  SSS9R                  SS5      nU$ s  snf s  snf )NrS   rh   r^   )Úerrorsrj   r@   )r   rx   Ú	bytearrayrn   rr   r^   )r†   ÚtokensrŸ   r=   Úcs        r   rr   ÚSimpleTokenizer.decodeÝ   sv   € Ø�w‰w¹Ó@º°Ÿ™ UÔ+¹Ñ@ÓAˆÜ¹Ó=º°1×+Ñ+¨AÔ.¹Ñ=Ó>×EÑEÀgÐV_ÐEÐ`×hÑhÐioÐqtÓuˆØˆùò AùÚ=s
   �A7¸A<ÚtextsÚreturnc                 óŒ  • [        U[        5      (       a  U/nU=(       d    U R                  nU(       d   S5       eU R                  b1  U R                  UUU R                  U R
                  U R                  S9$ U Vs/ s H/  o0R                  /U R                  U5      -   U R
                  /-   PM1     nn[        R                  " [        U5      U[        R                  S9n[        U5       HM  u  pg[        U5      U:”  a  USU nU R
                  US'   [        R                  " U5      XVS[        U5      24'   MO     U$ s  snf )aŒ  Returns the tokenized representation of given input string(s)

Parameters
----------
texts : Union[str, List[str]]
    An input string or a list of input strings to tokenize
context_length : int
    The context length to use; all CLIP models use 77 as the context length

Returns
-------
A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
z!Please set a valid context lengthN)rd   r�   r‚   Ú	encode_fn©Údtyper’   )Ú
isinstanceÚstrrd   r…   r�   r‚   r¯   ÚtorchÚzerosrv   ÚlongÚ	enumerateÚtensor)r†   r¸   rd   r=   Ú
all_tokensÚresultr¤   rµ   s           r   Ú__call__ÚSimpleTokenizer.__call__â   s3  € ô �eœS×!Ñ!Ø�GˆEà'×>¨4×+>Ñ+>ˆÞÐBÐBÓBˆ~à×ÑÑ(à×$Ñ$ØØ-Ø!×.Ñ.Ø!×.Ñ.ØŸ+™+ð %ð ð ñ afÓfÒ`eÐX\×(Ñ(Ð)¨D¯K©K¸Ó,=Ñ=À×ARÑARÐ@SÔSÑ`eˆ
ÐfÜ—’œS ›_¨nÄEÇJÁJÑOˆä" :Ö.‰IˆAÜ�6‹{˜^Ó+Ø  Ð0�Ø!×.Ñ.��r‘
Ü&+§l¢l°6Ó&:ˆF�l”s˜6“{�l�?Ó#ñ	 /ð ˆùò gs   Á>6E)r€   ry   rn   rl   rz   rƒ   rd   rx   rw   r‚   r~   r…   r�   r   r6   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   ÚDEFAULT_CONTEXT_LENGTHr¿   r   r   Úintr�   r¦   r¯   rr   r   rÀ   Ú
LongTensorrÇ   Ú__static_attributes__r   r   r   r`   r`   …   s¥   † ñ (›MØ=AØ,BØ Ø"$ñ$^àð$^ð (0°°S±	Ñ':ð$^ð % S™Mð	$^ð
 ð$^ð  õ$^òL'òRòñ
'˜e C¨¨c© NÑ3ð 'ÀXÈcÁ]ð 'Ð^c×^nÑ^n÷ 'ð 'r   r`   Ú
output_idsc                 óh   • U R                  5       R                  5       n [        R                  U 5      $ r6   )ÚcpuÚnumpyÚ
_tokenizerrr   )rÑ   s    r   rr   rr     s(   € Ø—‘Ó!×'Ñ'Ó)€JÜ×Ñ˜ZÓ(Ð(r   r¸   rd   r¹   c                 ó   • [        XS9$ )N)rd   )rÕ   )r¸   rd   s     r   Útokenizer×     s   € Ü�eÑ;Ð;r   r�   r‚   r»   Úshufflec                 óÐ  • U  Vs/ s H
  od" U5      PM     nn[         R                  " [        U5      U[         R                  S9n[	        U5       H�  u  pš[         R
                  " U
5      n
[        U
5      nX±S-
  :”  aF  US-
  n[         R                  " [        U
5      5      nUS U nU(       d  UR                  5       nX­   n
UnX(U	S4'   X¨U	SUS-   24'   X8X›S-   4'   M�     U$ s  snf )Nr¼   r›   r   r   )rÀ   rÁ   rv   rÂ   rÃ   rÄ   ÚrandpermÚmsort)r¸   rd   r�   r‚   r»   rØ   r=   rÅ   rÆ   r¤   rµ   Ú
num_tokensÚnum_keepÚindicess                 r   Úrandom_mask_tokenizerß     sç   € ñ /4Ó4ªe d�)˜D–/©e€JÐ4Ü�[Š[œ˜Z›¨.ÄÇ
Á
ÑK€Fä˜zÖ*‰	ˆÜ—’˜fÓ%ˆÜ˜“[ˆ
Ø¨Ñ*Ó*Ø%¨Ñ)ˆHÜ—n’n¤S¨£[Ó1ˆGØ˜i˜xÐ(ˆGÞØ!Ÿ-™-›/�Ø‘_ˆFØ!ˆJØ#ˆq�!ˆt‰Ø&,ˆq�!�J ‘NÐ"Ð"Ñ#Ø$0ˆq˜q‘.Ð Ó!ñ +ð €Mùò% 5s   …C#c                 óŒ  • U  Vs/ s H
  oT" U5      PM     nn[         R                  " [        U5      U[         R                  S9n[	        U5       Hk  u  p‰[        U	5      n
X¡S-
  :”  a$  US-
  n[
        R                  " SX«-
  5      nXœXË-    n	U/U	-   U/-   n	[         R                  " U	5      XxS [        U	5      24'   Mm     U$ s  snf )Nr¼   r›   r   )rÀ   rÁ   rv   rÂ   rÃ   ÚrandomÚrandintrÄ   )r¸   rd   r�   r‚   r»   r=   rÅ   rÆ   r¤   rµ   rÜ   rÝ   Ústart_indexs                r   Úsimple_mask_tokenizerä   5  sÁ   € ñ /4Ó4ªe d�)˜D–/©e€JÐ4Ü�[Š[œ˜Z›¨.ÄÇ
Á
ÑK€Fä˜zÖ*‰	ˆÜ˜“[ˆ
Ø¨Ñ*Ó*Ø%¨Ñ)ˆHÜ Ÿ.š.¨¨JÑ,AÓBˆKØ¨Ñ)?Ð@ˆFØ� &Ñ(¨L¨>Ñ9ˆÜ"'§,¢,¨vÓ"6ˆ�,”3�v“;�,ˆÓñ +ð €Mùò 5s   …Cc                 ó¶  • SSK n[        (       d$  UR                  S5        UR                  S5        SqS n/ nU  Hí  nUR                  R	                  U5      n	UR                  U	5      n
U
 VVs/ s H  u  p¼U" U5      PM     nnn[        R                  " [        R                  " U5      5      n[        USUS-
   5      n[        R                  " [        R                  " U	5      USS9nS	nU H  nU[        U5      -   S
-   nM     UR                  5       nUR                  U5        Mï     Un U  Vs/ s H  o‚/U" U5      -   U/-   PM     nn[        R                  " [!        U5      U[        R"                  S9n[%        U5       HE  u  nn[!        U5      U:”  a
  USU nUUS'   [        R&                  " U5      UUS[!        U5      24'   MG     U$ s  snnf s  snf )zeReturns the tokenized representation of given input string(s).
Apply syntax masking before tokenize.
r   NÚpunktÚaveraged_perceptron_taggerTc                 óŽ   • U R                  S5      (       a  gU R                  S5      (       a  gU R                  S5      (       a  gg)NÚNNr   ÚJJr›   ÚVBé   é   )Ú
startswithrE   s    r   Ú	get_orderÚ'syntax_mask_tokenize.<locals>.get_order]  s;   € Ø�<‰<˜×ÑØØ�\‰\˜$×ÑØØ�\‰\˜$×ÑØàr   r›   )ÚaxisrS   r@   r¼   r’   )ÚnltkÚ
_nltk_initÚdownloadr×   Úword_tokenizeÚpos_tagÚnpÚargsortÚarrayÚsortedÚtaker¿   r;   r$   rÀ   rÁ   rv   rÂ   rÃ   rÄ   )r¸   rd   r�   r‚   r»   rò   rï   Ú	new_textsr=   Úlist_tokensÚpos_tagsrW   ÚtagÚ
order_listÚ
sorted_idsÚsampled_idsÚsampled_tokensÚnew_textrŸ   rÅ   rÆ   r¤   rµ   s                          r   Úsyntax_mask_tokenizer  K  s­  € ó çŠ:à�‰�gÔØ�‰Ð2Ô3Øˆ
òð €IÛˆØ—m‘m×1Ñ1°$Ó7ˆØ—<‘< Ó,ˆá3;Ô<²8©¨‘i –n±8ˆ
Ñ<Ü—Z’Z¤§¢¨Ó 4Ó5ˆ
Ü˜ZÐ(;¨¸!Ñ);Ð<Ó=ˆÜŸš¤§¢¨+Ó!6¸È!ÑLˆàˆÛ#ˆEØ¤# e£*Ñ,¨sÑ2ŠHñ $à—>‘>Ó#ˆØ×Ñ˜Ö"ñ ð €EáPUÓVÒPUÈ�.¡9¨T£?Ñ2°l°^ÔCÑPU€JÐVÜ�[Š[œ˜Z›¨.ÄÇ
Á
ÑK€Fä˜zÖ*‰	ˆˆ6äˆv‹;˜Ó'Ø˜O˜^Ð,ˆFØ%ˆF�2‰JÜ"'§,¢,¨vÓ"6ˆˆq�,”3�v“;�,ˆÓñ +ð €Mùó- =ùò Ws   Á/GÄ2Gc                 ó˜   • U S;   d   eU S:X  a  [         $ U S:X  a  [        $ U S:X  a  [        [        SS9$ U S:X  a  [        $  SU  S	35       e)
zNChoose strategy for dropping (masking) tokens to achieve target context length)Úsimplerá   rØ   Úsyntaxr  rá   rØ   T)rØ   r  zUnknown type Ú.)rä   rß   r   r  rQ   s    r   r„   r„   †  sd   € àÐ<Ó<Ð<Ð<ØˆxÓÜ#Ð#Ø	�Ó	Ü#Ð#Ø	�Ó	ÜÔ+°TÑ:Ð:Ø	�Ó	Ü#Ð#à-˜ d V¨1Ð-Ó-ˆur   c                   ó  • \ rS rSrSr\SSSSS4S\S\\   S\S	\	S
\\   S\\   S\\   4S jjr
S rSS\\\\   4   S\\   S\R                  4S jjrS rS\\   S\S\R                  4S jr  SS\\   S\S\S\S\\   4
S jjrSrg)ÚHFTokenizeri•  zHHuggingFace tokenizer wrapper with support for custom tokenization modesrP   FNÚtokenizer_namerd   re   Ústrip_sep_tokenÚlanguageÚ	cache_dirÚtokenizer_modec                 ó   • U=(       d    SU l         X l        [        U5      U l        X@l        SSKJn	  U	R                  " U4SU0UD6U l        [        U R                  SS 5      n
[        U
5      (       a  X l        Ub  U R                  U5        g g )NrS   r   )ÚAutoTokenizerr  Úset_src_lang_special_tokens)r  rd   rR   rƒ   r  Útransformersr  Úfrom_pretrainedÚ	tokenizerÚgetattrÚcallableÚset_lang_fnÚset_language)r†   r  rd   re   r  r  r  r  Úkwargsr  r  s              r   r�   ÚHFTokenizer.__init__˜  s”   € ð -×2°ˆÔØ,ÔÜ$ UÓ+ˆŒØ.Ôõ: 	/Ø&×6Ò6Øñ
àð
ð ñ
ˆŒô ˜dŸn™nÐ.KÈTÓRˆÜ�K× Ñ Ø*ÔØÑØ×Ñ˜hÕ'ð  r   c                 ó:   • U R                   R                  U5        g r6   ©r  Úsave_pretrained©r†   Údests     r   r  ÚHFTokenizer.save_pretrainedÑ  ó   € Ø�‰×&Ñ& tÕ,r   r¸   r¹   c                 óè  • [        U[        5      (       a  U/nU=(       d    U R                  nU(       d   S5       eU Vs/ s H  o0R                  U5      PM     nnU R                  S:X  a  U R                  X5      $ U R                  USUSSS9R                  nU R                  (       aB  [        R                  " X@R                  R                  :H  [        R                  " U5      U5      nU$ s  snf )Nú8Please set a valid context length in class init or call.ÚclipsÚptÚ
max_lengthT©Úreturn_tensorsr(  ÚpaddingÚ
truncation)r¾   r¿   rd   rƒ   r  Ú_clips_tokenizer  Ú	input_idsr  rÀ   ÚwhereÚsep_token_idÚ
zeros_like)r†   r¸   rd   r=   r.  s        r   rÇ   ÚHFTokenizer.__call__Ô  sç   € ô �eœS×!Ñ!Ø�GˆEà'×>¨4×+>Ñ+>ˆÞÐYÐYÓYˆ~á16Ó7²¨—‘˜tÖ$±ˆÐ7ð ×Ñ 'Ó)Ø×'Ñ'¨Ó>Ð>ð Ÿ™ØØ#Ø)Ø$Øð 'ð ÷ ‰ið ð ×#×#Ü!ŸKšKØ§¡×!<Ñ!<Ñ<Ü×$Ò$ YÓ/Øó�	ð Ðùò- 8s   Á C/c                 óv   • [        U S5      (       a  U R                  U5        g [        R                  " S5        g )Nr  z&Cannot set language for the tokenizer.)Úhasattrr  ÚwarningsÚwarn)r†   Úsrc_langs     r   r  ÚHFTokenizer.set_languageõ  s*   € Ü�4˜×'Ñ'Ø×Ñ˜XÕ&ä�MŠMÐBÕCr   c                 ó  • U R                  USSSSS9n/ nUS    HN  nUSUS-
   nU R                   R                  /U-   U R                   R                  /-   nUR                  U5        MP     [        R
                  " [        U5      U[        R                  S9n[        U5       Hd  u  puU R                  UUU R                   R                  U R                   R                  S9n[        R                  " U5      XgS[        U5      24'   Mf     U$ )z:Use standard HF tokenizer but apply custom post-processingFN)Úadd_special_tokensr+  r,  r*  r.  rì   r¼   )r(  Úpad_token_idÚcls_token_id)r  Úbos_token_idÚeos_token_idr$   rÀ   rÁ   rv   rÂ   rÃ   Ú_pad_and_add_class_tokenr;  r<  rÄ   )	r†   r¸   rd   Úencoded_outputsÚencodedrµ   rÆ   r¤   Úpadded_tokenss	            r   r-  ÚHFTokenizer._clips_tokenizeû  s  € ð Ÿ.™.ØØ$ØØØð )ð 
ˆð ˆØ% kÔ2ˆFØÐ/˜^¨aÑ/Ð0ˆFØ—n‘n×1Ñ1Ð2°VÑ;¸t¿~¹~×?ZÑ?ZÐ>[Ñ[ˆFØ�N‰N˜6Ö"ñ 3ô —’œS ›\¨>ÄÇÁÑLˆÜ" 7Ö+‰IˆAØ ×9Ñ9ØØ)Ø!Ÿ^™^×8Ñ8Ø!Ÿ^™^×8Ñ8ð	 :ð ˆMô .3¯\ª\¸-Ó-HˆFÐ)”s˜=Ó)Ð)Ð)Ó*ñ ,ð ˆr   rµ   r(  r;  r<  c                 ó–   • [        U5      US-
  :”  a  USUS-
   n[        U5      US-
  :  a  X/US-
  [        U5      -
  -  -   nX/-   nU$ )z(Add padding with class token at the end r   N)rv   )r†   rµ   r(  r;  r<  s        r   r?  Ú$HFTokenizer._pad_and_add_class_token  sc   € ô ˆv‹;˜ a™Ó'Ø˜O˜Z¨!™^Ð,ˆFô ˆv‹;˜ a™Ó'Ø˜n°
¸Q±ÄÀVÃÑ0LÑMÑMˆFð ˜.Ñ(ˆØˆr   )rƒ   rd   r  r  r  r  r6   )r   ée   )rÉ   rÊ   rË   rÌ   Ú__doc__rÍ   r¿   r   rÎ   Úboolr�   r  r   r   rÀ   ÚTensorrÇ   r  r-  r?  rÐ   r   r   r   r  r  •  s  † ÙRð
 -CØ%Ø$)Ø&*Ø'+Ø,0ñ7(àð7(ð % S™Mð7(ð ð	7(ð
 "ð7(ð ˜s‘mð7(ð   ‘}ð7(ð % S™Mõ7(òr-ñ˜e C¨¨c© NÑ3ð ÀXÈcÁ]ð Ð^c×^jÑ^jõ òBDð T¨#¡Yð Àð ÈÏÉô ðD !"Ø #ñà˜‘Iðð ðð ð	ð
 ðð 
ˆc‰÷ð r   r  c                   óŒ   • \ rS rSrSrSSSS.r SS\S\\   4S	 jjr	S
 r
SS\\\\   4   S\\   S\R                  4S jjrSrg)ÚSigLipTokenizeri-  zûHuggingFace tokenizer wrapper for SigLIP T5 compatible sentencepiece vocabs

NOTE: this is not needed in normal library use, but is used to import new sentencepiece tokenizers
into OpenCLIP. Leaving code here in case future models use new tokenizers.
zLhttp://storage.googleapis.com/t5-data/vocabs/cc_en.32000/sentencepiece.modelzThttp://storage.googleapis.com/t5-data/vocabs/mc4.250000.100extra/sentencepiece.modelz>http://storage.googleapis.com/big_vision/gemma_tokenizer.model)zc4-enÚmc4Úgemmar  rd   c                 ó4  • SU;   a  SSK Jn  [        USSSS9nOSSK Jn  [        USS	9nXR                  ;   a‚  SS KnSS KnU R                  U   nUR                  S
5       n	UR                  US5       n
U	R                  U
R                  5       5        S S S 5        U" U	R                  SS9U l        S S S 5        OU" USS9U l        SU;   a  SOSU R                  l        SU R                  l        X l        g ! , (       d  f       Nj= f! , (       d  f       NP= f)NrM  r   )ÚGemmaTokenizerFastÚrightFT)Úpadding_sideÚadd_bos_tokenÚadd_eos_token)ÚT5TokenizerFast)Ú	extra_idsÚwbÚrb)Úlegacyr   )r  rO  r   rT  ÚVOCAB_FILESÚtempfileÚfsspecÚNamedTemporaryFilerp   Úwriterq   Únamer  r;  r>  rd   )r†   r  rd   rO  Útokenizer_clsrT  rZ  r[  Ú
vocab_fileÚdstÚsrcs              r   r�   ÚSigLipTokenizer.__init__<  sñ   € ð
 �nÓ$Ý7Ü#Ø"°ÈÐ]añc‰Mõ 5Ü# O¸qÑAˆMà×-Ñ-Ó-ãÛØ×)Ñ)¨.Ñ9ˆJØ×,Ñ,¨TÔ2°cØ—[‘[ ¨TÔ2°cØ—I‘I˜cŸh™h›jÔ)÷ 3á!.¨s¯x©xÀÑ!F�”÷ 3Ð2ñ
 +¨>À%ÑHˆDŒNà+2°nÓ+D¡aÈ!ˆ�‰Ô#Ø&'ˆ�‰Ô#Ø,Õ÷ 3Õ2ú÷ 3Õ2ús$   Á!D	Á4 C8ÂD	Ã8
D	ÄD	Ä	
Dc                 ó:   • U R                   R                  U5        g r6   r  r   s     r   r  ÚSigLipTokenizer.save_pretrainedY  r#  r   Nr¸   r¹   c                 ó  • [        U[        5      (       a  U/nU=(       d    U R                  nU(       d   S5       eU Vs/ s H  n[        [	        U5      5      PM     nnU R                  USUSSS9nUR                  $ s  snf )Nr%  r'  r(  Tr)  )r¾   r¿   rd   rD   r>   r  r.  )r†   r¸   rd   r=   Úoutputs        r   rÇ   ÚSigLipTokenizer.__call__\  s‹   € ô �eœS×!Ñ!Ø�GˆEà'×>¨4×+>Ñ+>ˆÞÐYÐYÓYˆ~áBGÓHÂ%¸$Ô"¤;¨tÓ#4Ö5Á%ˆÐHØ—‘ØØØ%Ø Øð  ð 
ˆð ×ÑÐùò Is   Á A?)rd   r  )é@   r6   )rÉ   rÊ   rË   rÌ   rG  rY  r¿   r   rÎ   r�   r  r   r   rÀ   rI  rÇ   rÐ   r   r   r   rK  rK  -  sv   † ñð `àeàQñ€Kð -/ñ-àð-ð % S™Mõ-ò:-ñ ˜e C¨¨c© NÑ3ð  ÀXÈcÁ]ð  Ð^c×^jÑ^j÷  ð  r   rK  )F)6rG  ro   r9   r   rá   ÚstringÚ	functoolsr   r   Útypingr   r   r   r   r	   r5  r7   rÔ   r÷   Úregexr{   rÀ   Úenvironró   rÍ   r   r,   r4   r>   rB   rG   rJ   rL   r¿   rR   Ú	maketransÚpunctuationr&   rD   Úobjectr`   rÕ   rI  rr   rÎ   rÏ   r×   rH  rß   rä   r  r„   r  rK  r   r   r   Ú<module>rr     s4  ðñó Û Û 	Û Û ß (ß 8Õ 8Û ã Û Û Û ð (/€‡
�
Ð#Ñ $Ø€
àÐ ñ ƒñdó ðdñ ƒñó ðò,	òòò-ò
4ò
,ð
:�sô :ð #'Ø!Ÿm™m¨B°°F×4FÑ4FÓGò	ð õ	ô:D�fô DñN Ó€
ð)�u—|‘|ô )ð
 BXñ <�E˜#˜t C™y˜.Ñ)ð <¸3ð <Ð\a×\lÑ\lõ <ð ñØ�S˜$˜s™)�^Ñ$ðàðð ðð ð	ð
 ðð õð:Ø�S˜$˜s™)�^Ñ$ðàðð ðð ð	ð
 ôð,8Ø�S˜$˜s™)�^Ñ$ð8àð8ð ð8ð ð	8ð
 ð8ð ×Ñô8ðv. ô .÷Uñ U÷p@ ò @ r   