ó
    Î’Œj«  ã                   ó  • S SK r S SKrS SKrS SKrS SKJrJr  S SKJr  S SK	J
r
  S SKJrJr  S SKJr  S SKJr  S SKJr  S	S
KJr  \R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r\R,                  R/                  S5      S 5       r S r!S r"S r#S  r$S! r%S" r&g)#é    N)ÚDEPÚHEAD)ÚEnglish)ÚLanguage)ÚMatcherÚPhraseMatcher)ÚDoc)ÚVectors)ÚVocabé   )Úmake_tempdiri¿  c                  ó  • [        [        5       5      n [        R                  " SSS9n[	        U/ SQS9nU R                  S5      nUR                  S5        UR                  R                  SS	5      S	:X  d   eX#R                  l
        [        5        nUR                  U5        U R                  S5      R                  U5      nUR                  R                  SS	5      S	:X  d   e S
S
S
5        g
! , (       d  f       g
= f)zbTest that models with no pretrained vectors can be deserialized
correctly after vectors are added.)é   i,  Úf©Údtype)ÚIÚamÚMatt)ÚdataÚkeysÚtaggerÚPRPÚpretrained_dimsr   N)r   r   ÚnumpyÚonesr
   Úcreate_pipeÚ	add_labelÚcfgÚgetÚvocabÚvectorsr   Úto_diskÚ	from_disk)Únlpr   r"   r   Úpaths        Ú¶C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\spacy/tests/serialize/test_serialize_doc.pyÚtest_issue1727r(      sÅ   € ô ”5“7Ó
€CÜ�:Š:�h cÑ*€DÜ˜4Ò&9Ñ:€GØ�_‰_˜XÓ&€FØ
×Ñ�UÔØ�:‰:�>‰>Ð+¨QÓ/°1Ó4Ð4Ð4Ø"‡L�LÔÜ	Œ˜4Ø�‰�tÔØ—‘ Ó*×4Ñ4°TÓ:ˆØ�z‰z�~‰~Ð/°Ó3°qÓ8Ð8Ñ8÷ 
�Žús   ÂAC1Ã1
C?i  c            
      ób  • [         R                  " SS/SS/SS/SS/SS	/S
S/SS//SS9n [        [        5       SR	                  5       S9nUR
                  R                  R                  S5        UR                  [        [        /U 5      n[        [        UR                  5      5      S:X  d   eg)zWTest sentence boundaries are deserialized correctly, even for
non-projective sentences.é   i�  é   i´  r   iª  i’  r   l   LPû^“& l   ÿÿÿÿ i¸  l   þÿÿÿ iº  Úuint64r   zJust what I was looking for .©ÚwordsÚROOTN)r   Úasarrayr	   r   Úsplitr!   ÚstringsÚaddÚ
from_arrayr   r   ÚlenÚlistÚsents)Ú
heads_depsÚdocs     r'   Útest_issue1799r:   #   s²   € ô —’à�ˆHØ�ˆHØ�ˆHØ�ˆHØÐ#Ð$Ø! 3Ð'Ø! 3Ð'ð	
ð ñ€Jô Œe‹gÐ<×BÑBÓDÑ
E€CØ‡I�I×Ñ×Ñ˜&Ô!Ø
�.‰.œ$¤˜ jÓ
1€CÜŒt�C—I‘I‹Ó 1Ó$Ð$Ñ$ó    i*  c            
      ó†  • / SQn [        [        5       U S9nSUS   l        [        UR                  5      R	                  UR                  5       5      nUS   R                  (       d   eUR                  S5      (       a   eUR                  S5      (       a   e[        [        5       U S/[        U 5      -  / SQS/[        U 5      -  S	9n[        UR                  5      R	                  UR                  5       5      nUS   R                  (       d   eUR                  S5      (       d   eUR                  S5      (       d   eg
)zRTest that sentence boundaries & parse/tag flags are not lost
during serialization.)	ÚThisÚisÚaÚfirstÚsentenceÚ.ÚAndÚanotherÚoner-   Té   r   ÚTAG)	r   r   r   r   r   r   rF   rF   rF   Údep)r.   ÚtagsÚheadsÚdepsN)	r	   r   Úis_sent_startr!   Ú
from_bytesÚto_bytesÚ
sent_startÚhas_annotationr5   )r.   r9   Únew_docs      r'   Útest_issue1834rR   9   s  € ò S€EÜ
Œe‹g˜UÑ
#€CØ€Cˆ�FÔÜ�#—)‘)‹n×'Ñ'¨¯©«Ó7€GØ�1‰:× × Ð Ð Ø×%Ñ% e×,Ñ,Ð,Ð,Ø×%Ñ% e×,Ñ,Ð,Ð,Ü
Ü‹ØØˆW”s˜5“zÑ!Ú)ØˆW”s˜5“zÑ!ñ€Cô �#—)‘)‹n×'Ñ'¨¯©«Ó7€GØ�1‰:× × Ð Ð Ø×!Ñ! %×(Ñ(Ð(Ð(Ø×!Ñ! %×(Ñ(Ð(Ñ(r;   i[  c                  ó2  • [        [        5       5      n U R                  SSS0//5        [        U R                  S/S9n[        U " U5      5      S:X  d   e[        R                  " U 5      n[        UR                  S/S9n[        U" U5      5      S:X  d   eg )NÚpat1ÚorthÚhellor-   r*   )r   r   r3   r	   r!   r5   ÚcopyÚdeepcopy)Úmatcherr9   Únew_matcherrQ   s       r'   Útest_issue1883r[   Q   sŠ   € ä”e“gÓ€GØ‡K�K�˜6 7Ð+Ð,Ð-Ô.Ü
ˆg�m‰m G 9Ñ
-€CÜ‰w�s‹|Ó Ó!Ð!Ð!Ü—-’- Ó(€KÜ�+×#Ñ#¨G¨9Ñ5€GÜ‰{˜7Ó#Ó$¨Ó)Ð)Ñ)r;   i
  c                  ó(  • [        5       n U R                  S5      nUR                  S5        U R                  5         U " S5      nUR	                  S5      (       d   eU R                  SS/5      n[        U5      nUR	                  S5      (       d   eg)zQTest the tagger sets has_annotation("TAG") correctly when used via Language.pipe.r   ÚAzhello worldrG   rV   ÚworldN)r   Úadd_piper   Ú
initializerP   ÚpipeÚnext)r%   r   r9   ÚdocsÚ	piped_docs        r'   Útest_issue2564re   \   s…   € ô ‹*€CØ�\‰\˜(Ó#€FØ
×Ñ�SÔØ‡N�NÔÙ
ˆmÓ
€CØ×Ñ˜e×$Ñ$Ð$Ð$Ø�8‰8�W˜gÐ&Ó'€DÜ�T“
€IØ×#Ñ# E×*Ñ*Ð*Ñ*r;   i°  c                  óN  • [        5       n [        U R                  5      nUR                  SU " S5      U " S5      U " S5      /5        UR                  SU " S5      /5        [        R
                  " U5      n[        R                  " U5      n[        U5      [        U5      :X  d   eg)z5Test that the PhraseMatcher can be pickled correctly.ÚTEST1r?   ÚbÚcÚTEST2ÚdN)r   r   r!   r3   ÚpickleÚdumpsÚloadsr5   )r%   rY   r   rZ   s       r'   Útest_issue3248_2ro   j   s�   € ô ‹)€CÜ˜CŸI™IÓ&€GØ‡K�K�™#˜c›(¡C¨£H©c°#«hÐ7Ô8Ø‡K�K�™#˜c›(˜Ô$Ü�<Š<˜Ó €DÜ—,’,˜tÓ$€KÜˆ{Óœs 7›|Ó+Ð+Ñ+r;   iÙ  c                  ó²   • [        5       n U R                  S5        U R                  5       n[        5       nUR                  S5        UR                  U5        g)zaTest that Language.to_bytes handles serializing a pipeline component
with an uninitialized model.ÚtextcatN)r   r_   rN   rM   )r%   Ú
bytes_dataÚnew_nlps      r'   Útest_issue3289rt   v   sE   € ô ‹)€CØ‡L�L�ÔØ—‘“€JÜ‹i€GØ×Ñ�YÔØ×Ñ�zÕ"r;   iŒ  c                  óú  • [        5       n U R                  S5        U " S5      nUS   R                  (       d   eUR                  S5      (       d   e[	        [        UR                  5      5      S:X  d   eUR                  5       n[        U R                  5      R                  U5      nUS   R                  (       d   eUR                  S5      (       d   e[	        [        UR                  5      5      S:X  d   eg)zxTest that sentence boundaries are set correctly so Doc.has_annotation("SENT_START") can
be restored after serialization.ÚsentencizerzHello worldr   Ú
SENT_STARTr*   N)r   r_   rL   rP   r5   r6   r7   rN   r	   r!   rM   )r%   r9   Ú	doc_bytesrQ   s       r'   Útest_issue3468ry   ‚   sÏ   € ô ‹)€CØ‡L�L�ÔÙ
ˆmÓ
€CØˆq‰6××ÐÐØ×Ñ˜l×+Ñ+Ð+Ð+ÜŒt�C—I‘I‹Ó 1Ó$Ð$Ð$Ø—‘“€IÜ�#—)‘)‹n×'Ñ'¨	Ó2€GØ�1‰:×#×#Ð#Ð#Ø×!Ñ! ,×/Ñ/Ð/Ð/ÜŒt�G—M‘MÓ"Ó# qÓ(Ð(Ñ(r;   iw  c                  ób  • [        5       n U " S5      nUS   R                  S:X  d   eSUS   l        US   R                  S:X  d   e[        5        nUS-  nUR                  U5        U " S5      nUR	                  U5        US   R                  S:X  d   e SSS5        g! , (       d  f       g= f)z=Ensure that a modified pos attribute is serialized correctly.zSdisplaCy uses JavaScript, SVG and CSS to show you how computers understand languager   Ú ÚNOUNÚmy_docN)r   Úpos_r   r#   r$   )r%   r9   Útmp_dirÚ	file_pathÚdoc2s        r'   Útest_issue3959r‚   “   s£   € ô ‹)€CÙ
Ø]ó€Cð ˆq‰6�;‰;˜"ÓÐÐØ€Cˆ�F„KØˆq‰6�;‰;˜&Ó Ð Ð ä	Œ˜7Ø˜hÑ&ˆ	Ø�‰�IÔÙ�2‹wˆØ�‰�yÔ!Ø�A‰w�|‰|˜vÓ%Ð%Ñ%÷ 
�Žús   ÁAB Â 
B.c                 ó  • [        U 5      nUR                  5       n[        U 5      nUR                  U5        [        U5      [        U5      :X  d   e[	        X5       H!  u  pEUR
                  UR
                  :X  a  M!   e   g )N)r	   rN   rM   r5   ÚzipÚtext)Úen_vocabr9   r   r�   Útoken1Útoken2s         r'   Útest_serialize_empty_docr‰   ¦   sg   € Ü
ˆh‹-€CØ�<‰<‹>€DÜˆx‹=€DØ‡O�O�DÔÜˆs‹8”s˜4“yÓ Ð Ð Ü˜cž.‰ˆØ�{‰{˜fŸk™kÕ)Ð)Ð)ò )r;   c                 ó®   • [        U SS/S9nSS0Ul        UR                  5       n[        U 5      R                  U5      nUR                  5       U:X  d   eg )NrV   r^   r-   r]   g      à?)r	   ÚcatsrN   rM   )r†   r9   Údoc_brQ   s       r'   Ú"test_serialize_doc_roundtrip_bytesr�   °   sT   € Ü
ˆh˜w¨Ð0Ñ
1€CØ�Sˆz€C„HØ�L‰L‹N€EÜ�(‹m×&Ñ& uÓ-€GØ×ÑÓ Ó&Ð&Ñ&r;   c                 ó  • [        U SS/S9n[        5        nUS-  nUR                  U5        [        U 5      R                  U5      nUR	                  5       UR	                  5       :X  d   e S S S 5        g ! , (       d  f       g = f©NrV   r^   r-   r9   )r	   r   r#   r$   rN   ©r†   r9   rk   r€   Údoc_ds        r'   Ú!test_serialize_doc_roundtrip_diskr’   ¸   sh   € Ü
ˆh˜w¨Ð0Ñ
1€CÜ	Œ˜1Ø˜‘Iˆ	Ø�‰�IÔÜ�H“×'Ñ'¨	Ó2ˆØ�|‰|‹~ §¡Ó!1Ó1Ð1Ñ1÷	 
�Žús   —AA6Á6
Bc                 ó$  • [        U SS/S9n[        5        nUS-  n[        U5      nUR                  U5        [        U 5      R	                  U5      nUR                  5       UR                  5       :X  d   e S S S 5        g ! , (       d  f       g = fr�   )r	   r   Ústrr#   r$   rN   r�   s        r'   Ú*test_serialize_doc_roundtrip_disk_str_pathr•   Á   sq   € Ü
ˆh˜w¨Ð0Ñ
1€CÜ	Œ˜1Ø˜‘Iˆ	Ü˜	“Nˆ	Ø�‰�IÔÜ�H“×'Ñ'¨	Ó2ˆØ�|‰|‹~ §¡Ó!1Ó1Ð1Ñ1÷ 
�Žús   —A BÂ
Bc                 ó   • [        U SS/S9nSUR                  S'   [        U 5      R                  UR                  5       5      nUR                  S   S:X  d   e[        U 5      R                  UR                  5       S/S9nUR                  (       a   e[        U 5      R                  UR                  S/S95      nUR                  (       a   eg )NrV   r^   r-   ÚbarÚfooÚ	user_data)Úexclude)r	   r™   rM   rN   )r†   r9   rQ   s      r'   Útest_serialize_doc_excluder›   Ë   s·   € Ü
ˆh˜w¨Ð0Ñ
1€CØ €C‡M�M�%ÑÜ�(‹m×&Ñ& s§|¡|£~Ó6€GØ×Ñ˜UÑ# uÓ,Ð,Ð,Ü�(‹m×&Ñ& s§|¡|£~À¸}Ð&ÐM€GØ× × Ð Ð Ü�(‹m×&Ñ& s§|¡|¸[¸M |Ð'JÓK€GØ× × Ð Ð Ð r;   c                 óÈ  • [        U / SQS9nUSS nSUl        SUl        SUl        U/UR                  S'   [        U 5      R                  UR                  5       5      n[        UR                  S   5      S	:X  d   eUR                  S   S   R                  S:X  d   eUR                  S   S   R                  S:X  d   eUR                  S   S   R                  S:X  d   eg )
N)rV   r^   Ú!r-   r   r   Ú$test_serialize_doc_span_groups_labelÚ!test_serialize_doc_span_groups_idÚ$test_serialize_doc_span_groups_kb_idÚcontentr*   )r	   Úlabel_Úid_Úkb_id_ÚspansrM   rN   r5   )r†   r9   ÚspanrQ   s       r'   Útest_serialize_doc_span_groupsr§   Ö   sÞ   € Ü
ˆhÒ5Ñ
6€CØˆq�ˆ8€DØ8€D„KØ2€D„HØ8€D„KØ ˜6€C‡I�IˆiÑÜ�(‹m×&Ñ& s§|¡|£~Ó6€GÜˆw�}‰}˜YÑ'Ó(¨AÓ-Ð-Ð-Ø�=‰=˜Ñ# AÑ&×-Ñ-Ð1WÓWÐWÐWØ�=‰=˜Ñ# AÑ&×*Ñ*Ð.QÓQÐQÐQØ�=‰=˜Ñ# AÑ&×-Ñ-Ð1WÓWÐWÑWr;   )'rW   rl   r   ÚpytestÚspacy.attrsr   r   Úspacy.lang.enr   Úspacy.languager   Úspacy.matcherr   r   Úspacy.tokensr	   Úspacy.vectorsr
   Úspacy.vocabr   Úutilr   ÚmarkÚissuer(   r:   rR   r[   re   ro   rt   ry   r‚   r‰   r�   r’   r•   r›   r§   © r;   r'   Ú<module>r´      sƒ  ðÛ Û ã Û ç !Ý !Ý #ß 0Ý Ý !Ý å ð ‡�×Ñ�4Óñ9ó ð9ð  ‡�×Ñ�4Óñ%ó ð%ð* ‡�×Ñ�4Óñ)ó ð)ð. ‡�×Ñ�4Óñ*ó ð*ð ‡�×Ñ�4Óñ
+ó ð
+ð ‡�×Ñ�4Óñ,ó ð,ð ‡�×Ñ�4Óñ#ó ð#ð ‡�×Ñ�4Óñ)ó ð)ð  ‡�×Ñ�4Óñ&ó ð&ò$*ò'ò2ò2ò!óXr;   