ó
    Î’ŒjÚ  ã                   óœ  • S SK r S SKrS SKrS SKJrJr  S SKJr  S SKJ	r	  S SK
Jr  S SKJrJrJrJrJr  SSKJrJr  S	 r\R,                  R/                  S
5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       rS r\R,                  R9                  S/ SQ5      S 5       rS rg)é    N)ÚENT_IOBÚENT_TYPE)ÚEnglish)Ú	Tokenizer)ÚDoc)Úcompile_infix_regexÚcompile_prefix_regexÚcompile_suffix_regexÚget_lang_classÚ
load_modelé   )Úassert_packed_msg_equalÚmake_tempdirc                 ó\   • [        S5      " 5       R                  nUR                  U 5        U$ )NÚen)r   Ú	tokenizerÚ
from_bytes)ÚbÚtoks     Ú¼C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\spacy/tests/serialize/test_serialize_tokenizer.pyÚload_tokenizerr      s&   € Ü
˜Ô
Ó
 ×
*Ñ
*€CØ‡N�N�1ÔØ€Jó    i  c                 óP  • [        U SS/S9n[        R                  " [        5         [        R
                  " US   5        SSS5        [        R                  " [        5         [        R
                  " USS 5        SSS5        g! , (       d  f       NK= f! , (       d  f       g= f)zATest that a custom error is raised if a token or span is pickled.ÚHelloÚworld)Úwordsr   Nr   )r   ÚpytestÚraisesÚNotImplementedErrorÚpickleÚdumps)Úen_vocabÚdocs     r   Útest_issue2833r$      sn   € ô ˆh˜w¨Ð0Ñ
1€CÜ	�ŠÔ*Õ	+Ü�Š�S˜‘VÔ÷ 
,ä	�ŠÔ*Õ	+Ü�Š�S˜˜1�XÔ÷ 
,Ð	+÷ 
,Õ	+úç	+Õ	+ús   §BÁ#BÂ
BÂ
B%iÄ  c                 óž  • / SQn/ SQn/ SQn/ SQn[        XX#US9nUR                  S5      (       d   eSnUS   R                  US   R                  US   R                  US   R
                  4U:X  d   e[        [        /nUR                  U5      nUR                  Xx5        US   R                  US   R                  US   R                  US   R
                  4U:X  d   eUR                  5       n	[        U 5      R                  U	5      n
U
S   R                  U
S   R                  U
S   R                  U
S   R
                  4U:X  d   eg	)
zeTest that the is_tagged attribute doesn't get overwritten when we from_array
without tag information.)ÚThisÚisÚ10Ú%Ú.)ÚDTÚVBZÚCDÚNNr*   )ÚDETÚVERBÚNUMÚNOUNÚPUNCT)ÚOr4   z	B-PERCENTz	I-PERCENTr4   )r   ÚtagsÚposÚentsÚTAG)r(   r1   r-   ÚPERCENTr   N)r   Úhas_annotationÚtextÚpos_Útag_Ú	ent_type_r   r   Úto_arrayÚ
from_arrayÚto_bytesr   )r"   r   r5   r6   r7   r#   ÚexpectedÚheaderÚ	ent_arrayÚ	doc_bytesÚdoc2s              r   Útest_issue3012rG   %   s0  € ò +€EÚ)€DÚ
1€CÚ4€DÜ
ˆh¨$¸dÑ
C€CØ×Ñ˜e×$Ñ$Ð$Ð$Ø-€HØ�‰F�K‰K˜˜Q™Ÿ™ c¨!¡f§k¡k°3°q±6×3CÑ3CÐDÈÓPÐPÐPÜ”xÐ €FØ—‘˜VÓ$€IØ‡N�N�6Ô%Ø�‰F�K‰K˜˜Q™Ÿ™ c¨!¡f§k¡k°3°q±6×3CÑ3CÐDÈÓPÐPÐPà—‘“€IÜˆx‹=×#Ñ# IÓ.€DØ�‰G�L‰L˜$˜q™'Ÿ,™,¨¨Q©¯©°d¸1±g×6GÑ6GÐHÈHÓTÐTÑTr   i^  c                  óæ  • S n Sn[        5       nU" U5      nU Vs/ s H  oDR                  PM     nnU " U5        U" U5      nU Vs/ s H  oDR                  PM     nn[        5        nUR                  U5        [	        U5      n	S S S 5        W	" U5      n
U
 Vs/ s H  oDR                  PM     nnX{:X  d   eU	R
                  R                  SL d   eg s  snf s  snf ! , (       d  f       N^= fs  snf )Nc           
      ó"  • [        U R                  R                  5      n[        U R                  R                  5      n[        U R                  R                  5      n[        U R                  R                  5      R                  5        VVs0 s H!  u  pE[        U5      S:X  a  US   S:X  a  M  XE_M#     nnn[        U R                  UUR                  UR                  UR                  U R                  R                   SS9nXpl        g s  snnf )Nr   é   r*   F)Úprefix_searchÚsuffix_searchÚinfix_finditerÚtoken_matchÚfaster_heuristics)r	   ÚDefaultsÚprefixesr
   Úsuffixesr   ÚinfixesÚdictÚtokenizer_exceptionsÚitemsÚlenr   ÚvocabÚsearchÚfinditerr   rN   )ÚnlpÚ	prefix_reÚ	suffix_reÚinfix_reÚkÚvÚ
exceptionsÚnew_tokenizers           r   Úcustomize_tokenizerÚ+test_issue4190.<locals>.customize_tokenizer=   sà   € Ü(¨¯©×)>Ñ)>Ó?ˆ	Ü(¨¯©×)>Ñ)>Ó?ˆ	Ü& s§|¡|×';Ñ';Ó<ˆô ˜SŸ\™\×>Ñ>Ó?×EÑEÔGô
âG‘�Ü˜“F˜a“K A a¡D¨C¡Kó ˆAŠDÙGð 	ñ 
ô
 "Ø�I‰IØØ#×*Ñ*Ø#×*Ñ*Ø#×,Ñ,ØŸ™×1Ñ1Ø#ñ
ˆð &�ùó
s   ÂDÂ0DzTest c.F)r   r;   r   Úto_diskr   r   rO   )rc   Útest_stringÚnlp_1Údoc_1aÚtokenÚ	result_1aÚdoc_1bÚ	result_1bÚ	model_dirÚnlp_2Údoc_2Úresult_2s               r   Útest_issue4190rq   ;   sÙ   € ò&ð* €Kä‹I€EÙ�;Ó€FÙ)/Ó0ª —”©€IÐ0á˜ÔÙ�;Ó€FÙ)/Ó0ª —”©€IÐ0ä	Œ˜9Ø�‰�iÔ Ü˜9Ó%ˆ÷ 
ñ �+Ó€EÙ(-Ó.ª˜u—
”
©€HÐ.ØÓ Ð Ð Ø�?‰?×,Ñ,°Ò5Ð5Ñ5ùò 1ùò 1ç	�üò
 /s   œCÁCÁ(CÂC.Ã
C+c                 ó   • [        XR                  S9nUR                  5       n[        U 5      R                  U5        [	        S5      " 5       R
                  n[        R                  " S5      R                  Ul	        UR                  0 :w  d   eUR                  c   eUR                  c   eUR                  c   eUR                  c   eUR                  U5        UR                  0 :X  d   eUR                  b   eUR                  b   eUR                  b   eUR                  b   e[        U SSS0SS0/0S	9n0 Ul
        UR                  5       n[        U 5      R                  U5      nUR                  0 :X  d   eg)
zŽTest that custom tokenizer with not all functions defined or empty
properties can be serialized and deserialized correctly (see #2494,
#4991).)rL   r   ÚtestNzABC.ÚORTHÚABCr*   )Úrules)r   rL   rA   r   r   r   ÚreÚcompileÚmatchrN   rv   Ú	url_matchrK   rM   )r"   Úen_tokenizerr   Útokenizer_bytesÚtokenizer_reloadeds        r   Útest_serialize_custom_tokenizerr~   f   s€  € ô ˜(×2LÑ2LÑM€IØ×(Ñ(Ó*€OÜˆhÓ×"Ñ" ?Ô3ô ˜tÔ$Ó&×0Ñ0€IÜŸJšJ vÓ.×4Ñ4€IÔØ�?‰?˜bÓ Ð Ð Ø× Ñ Ñ,Ð,Ð,Ø×ÑÑ*Ð*Ð*Ø×"Ñ"Ñ.Ð.Ð.Ø×#Ñ#Ñ/Ð/Ð/Ø×Ñ˜Ô)Ø�?‰?˜bÓ Ð Ð Ø× Ñ Ñ(Ð(Ð(Ø×ÑÑ&Ð&Ð&Ø×"Ñ"Ñ*Ð*Ð*Ø×#Ñ#Ñ+Ð+Ð+ä˜(¨6°V¸U°OÀfÈcÀ]Ð3SÐ*TÑU€IØ€I„OØ×(Ñ(Ó*€OÜ" 8Ó,×7Ñ7¸ÓHÐØ×#Ñ# rÓ)Ð)Ñ)r   r;   )u   IðŸ’œyouu	   theyâ€™reu   â€œhelloâ€�c                 ó~  • U n[        UR                  5       5      n[        UR                  5       UR                  5       5        UR                  5       UR                  5       :X  d   eU" U5      nU" U5      nU Vs/ s H  ofR                  PM     snU Vs/ s H  ofR                  PM     sn:X  d   eg s  snf s  snf )N)r   rA   r   r;   )r{   r;   r   rb   Údoc1rF   ri   s          r   Ú(test_serialize_tokenizer_roundtrip_bytesr�   „   sŸ   € à€IÜ" 9×#5Ñ#5Ó#7Ó8€MÜ˜M×2Ñ2Ó4°i×6HÑ6HÓ6JÔKØ×!Ñ!Ó# y×'9Ñ'9Ó';Ó;Ð;Ð;Ù�T‹?€DÙ˜Ó€DÙ$(Ó)¢D˜5�JŒJ¡DÑ)ÁdÓ-KÂd¸U¯j¬jÁdÑ-KÓKÐKÑKùÒ)ùÒ-Ks   Á<B5ÂB:c                 óè   • U n[        5        nUS-  nUR                  U5        U R                  U5      nUR                  5       UR                  5       :X  d   e S S S 5        g ! , (       d  f       g = f)Nr   )r   re   Ú	from_diskrA   )r{   r   ÚdÚ	file_pathÚtokenizer_ds        r   Ú'test_serialize_tokenizer_roundtrip_diskr‡   �   s_   € Ø€IÜ	Œ˜1Ø˜‘Oˆ	Ø×Ñ˜)Ô$Ø"×,Ñ,¨YÓ7ˆØ×!Ñ!Ó# {×';Ñ';Ó'=Ó=Ð=Ñ=÷	 
�Žús   �AA#Á#
A1)r    rw   r   Úspacy.attrsr   r   Úspacy.lang.enr   Úspacy.tokenizerr   Úspacy.tokensr   Ú
spacy.utilr   r	   r
   r   r   Úutilr   r   r   ÚmarkÚissuer$   rG   rq   r~   Úparametrizer�   r‡   © r   r   Ú<module>r’      sÍ   ðÛ Û 	ã ç )Ý !Ý %Ý ÷õ ÷ 9òð ‡�×Ñ�4Óñó ðð ‡�×Ñ�4ÓñUó ðUð* ‡�×Ñ�4Óñ'6ó ð'6òT*ð< ‡�×Ñ˜Ò!IÓJñLó KðLó>r   