ó
    •JÕi×%  ã                   ó¤   • S r SSKrSSKrSSKrSSKrSSKrS/r\R                  " SS5      r	S r
S r " S S5      r " S	 S
5      r " S S5      rg)a  robotparser.py

Copyright (C) 2000  Bastian Kleineidam

You can choose between two licenses when using this package:
1) GNU GPLv2
2) PSF license for Python 2.2

The robots.txt Exclusion Protocol is implemented as specified in
http://www.robotstxt.org/norobots-rfc.txt
é    NÚRobotFileParserÚRequestRatezrequests secondsc                 óz   • [         R                  R                  U SS9n[         R                  R                  USS9$ )NÚsurrogateescape)Úerrors)ÚurllibÚparseÚunquoteÚquote)ÚpathÚunquoteds     ÚLib\urllib\robotparser.pyÚ	normalizer      s7   € Ü�|‰|×#Ñ# DÐ1BÐ#ÐC€HÜ�<‰<×Ñ˜hÐ/@ÐÐAÐAó    c                 ó”   • U R                  S5      u  pn[        U 5      n U(       a!  [        R                  " SS U5      nU SU-   -  n U $ )NÚ?z[^=&]+c                 ó   • [        U S   5      $ )Nr   )r   )Úms    r   Ú<lambda>Ú normalize_path.<locals>.<lambda>    s   € ¬I°a¸±d¬Or   )Ú	partitionr   ÚreÚsub)r   ÚsepÚquerys      r   Únormalize_pathr      sG   € Ø—~‘~ cÓ*Ñ€DˆuÜ�T‹?€DÞ
Ü—’�yÑ";¸UÓCˆØ��e‘ÑˆØ€Kr   c                   ód   • \ rS rSrSrSS jrS rS rS rS r	S r
S	 rS
 rS rS rS rS rSrg)r   é%   zjThis class provides a set of methods to read, parse and answer
questions about a single robots.txt file.

c                 óz   • / U l         / U l        S U l        SU l        SU l        U R                  U5        SU l        g )NFr   )ÚentriesÚsitemapsÚdefault_entryÚdisallow_allÚ	allow_allÚset_urlÚlast_checked©ÚselfÚurls     r   Ú__init__ÚRobotFileParser.__init__+   s;   € ØˆŒØˆŒØ!ˆÔØ!ˆÔØˆŒØ�‰�SÔØˆÕr   c                 ó   • U R                   $ )zŸReturns the time the robots.txt file was last fetched.

This is useful for long-running web spiders that need to
check for new robots.txt files periodically.

)r&   ©r(   s    r   ÚmtimeÚRobotFileParser.mtime4   s   € ð × Ñ Ð r   c                 ó6   • SSK nUR                  5       U l        g)zISets the time the robots.txt file was last fetched to the
current time.

r   N)Útimer&   )r(   r1   s     r   ÚmodifiedÚRobotFileParser.modified=   s   € ó
 	Ø ŸI™I›KˆÕr   c                 ón   • Xl         [        R                  R                  U5      SS u  U l        U l        g)z,Sets the URL referring to a robots.txt file.é   é   N)r)   r   r	   ÚurlsplitÚhostr   r'   s     r   r%   ÚRobotFileParser.set_urlE   s+   € àŒÜ%Ÿ|™|×4Ñ4°SÓ9¸!¸AÐ>ÑˆŒ	�4•9r   c                 óÊ  •  [         R                  R                  U R                  5      nUR	                  5       nU R                  UR                  SS5      R                  5       5        g! [         R                  R                   aY  nUR                  S;   a  SU l        O'UR                  S:¼  a  UR                  S:  a  SU l        UR                  5          SnAgSnAff = f)z4Reads the robots.txt URL and feeds it to the parser.zutf-8r   )i‘  i“  Ti�  iô  N)r   ÚrequestÚurlopenr)   Úreadr	   ÚdecodeÚ
splitlinesÚerrorÚ	HTTPErrorÚcoder#   r$   Úclose)r(   ÚfÚrawÚerrs       r   r=   ÚRobotFileParser.readJ   s¥   € ð
	LÜ—‘×&Ñ& t§x¡xÓ0ˆAð —&‘&“(ˆCØ�J‰J�s—z‘z 'Ð+<Ó=×HÑHÓJÕKøô �|‰|×%Ñ%ó 	Ø�x‰x˜:Ó%Ø$(�Õ!Ø—‘˜S“ S§X¡X°£^Ø!%�”Ø�I‰I�K‰Kûð	ús   ‚)A+ Á+C"Â	ACÃC"c                 ó„   • SUR                   ;   a  U R                  c  Xl        g g U R                  R                  U5        g ©NÚ*)Ú
useragentsr"   r    Úappend)r(   Úentrys     r   Ú
_add_entryÚRobotFileParser._add_entryX   s;   € Ø�%×"Ñ"Ó"à×!Ñ!Ñ)à%*Õ"ð *ð �L‰L×Ñ Õ&r   c                 ó  • Sn[        5       nU R                  5         U GHÇ  nU(       d6  US:X  a  [        5       nSnO#US:X  a  U R                  U5        [        5       nSnUR                  S5      nUS:¼  a  USU nUR	                  5       nU(       d  Mv  UR                  SS5      n[        U5      S:X  d  M™  US   R	                  5       R                  5       US'   US   R	                  5       US'   US   S:X  aD  US:X  a  U R                  U5        [        5       nUR                  R                  US   5        SnGM   US   S:X  a6  US:w  a-  UR                  R                  [        US   S	5      5        SnGM\  GM_  US   S
:X  a6  US:w  a-  UR                  R                  [        US   S5      5        SnGM›  GMž  US   S:X  aG  US:w  a>  US   R	                  5       R                  5       (       a  [        US   5      Ul        SnGMë  GMî  US   S:X  a¦  US:w  a�  US   R                  S5      n[        U5      S:X  au  US   R	                  5       R                  5       (       aO  US   R	                  5       R                  5       (       a)  [        [        US   5      [        US   5      5      Ul        SnGMš  GM�  US   S:X  d  GM©  U R"                  R                  US   5        GMÊ     US:X  a  U R                  U5        gg)z|Parse the input lines from a robots.txt file.

We allow that a user-agent: line is not preceded by
one or more blank lines.
r   r5   é   Ú#NÚ:z
user-agentÚdisallowFÚallowTzcrawl-delayzrequest-rateÚ/Úsitemap)ÚEntryr2   rN   ÚfindÚstripÚsplitÚlenÚlowerrK   rL   Ú	rulelinesÚRuleLineÚisdigitÚintÚdelayr   Úreq_rater!   )r(   ÚlinesÚstaterM   ÚlineÚiÚnumberss          r   r	   ÚRobotFileParser.parsea   s©  € ð ˆÜ“ˆà�‰ŒÜˆDÞØ˜A“:Ü!›G�EØ‘EØ˜a“ZØ—O‘O EÔ*Ü!›G�EØ�Eà—	‘	˜#“ˆAØ�A‹vØ˜B˜Q�x�Ø—:‘:“<ˆDÞÙØ—:‘:˜c 1Ó%ˆDÜ�4‹y˜A�~Ø˜q™'Ÿ-™-›/×/Ñ/Ó1��Q‘Ø˜q™'Ÿ-™-›/��Q‘Ø˜‘7˜lÓ*Ø “zØŸ™¨Ô.Ü %£˜Ø×$Ñ$×+Ñ+¨D°©GÔ4Ø“EØ˜!‘W 
Ó*Ø “zØŸ™×.Ñ.¬x¸¸Q¹ÀÓ/GÔHØ !›ò "ð ˜!‘W Ó'Ø “zØŸ™×.Ñ.¬x¸¸Q¹ÀÓ/FÔGØ !›ò "ð ˜!‘W Ó-Ø “zð   ™7Ÿ=™=›?×2Ñ2×4Ñ4Ü*-¨d°1©g«,˜EœKØ !›ò "ð ˜!‘W Ó.Ø “zØ"& q¡'§-¡-°Ó"4˜ä ›L¨AÓ-°'¸!±*×2BÑ2BÓ2D×2LÑ2L×2NÑ2NØ '¨¡
× 0Ñ 0Ó 2× :Ñ :× <Ñ <Ü-8¼¸WÀQ¹Z»Ì#ÈgÐVWÉjË/Ó-Z˜EœNØ !›ò "ð ˜!‘W 	Ö)ð
 —M‘M×(Ñ(¨¨a©×1ño ðp �A‹:Ø�O‰O˜EÕ"ð r   c                 óô  • U R                   (       a  gU R                  (       a  gU R                  (       d  g[        R                  R                  U5      n[        R                  R                  SS/USS Q75      n[        U5      nU(       d  SnU R                   H,  nUR                  U5      (       d  M  UR                  U5      s  $    U R                  (       a  U R                  R                  U5      $ g)z=using the parsed robots.txt decide if useragent can fetch urlFTÚ rQ   NrV   )r#   r$   r&   r   r	   r7   Ú
urlunsplitr   r    Ú
applies_toÚ	allowancer"   )r(   Ú	useragentr)   Ú
parsed_urlrM   s        r   Ú	can_fetchÚRobotFileParser.can_fetchª   sÅ   € à××ØØ�>�>Øð
 × × Øô —\‘\×*Ñ*¨3Ó/ˆ
Ü�l‰l×%Ñ% r¨2Ð&?°
¸1¸2°Ñ&?Ó@ˆÜ˜SÓ!ˆÞØˆCØ—\”\ˆEØ×Ñ 	×*Ó*Ø—‘ sÓ+Ò+ñ "ð ××Ø×%Ñ%×/Ñ/°Ó4Ð4àr   c                 óì   • U R                  5       (       d  g U R                   H'  nUR                  U5      (       d  M  UR                  s  $    U R                  (       a  U R                  R                  $ g ©N)r.   r    rm   rb   r"   ©r(   ro   rM   s      r   Úcrawl_delayÚRobotFileParser.crawl_delayÆ   sY   € Ø�z‰z�|‰|ØØ—\”\ˆEØ×Ñ 	×*Ó*Ø—{‘{Ò"ñ "ð ××Ø×%Ñ%×+Ñ+Ð+Ør   c                 óì   • U R                  5       (       d  g U R                   H'  nUR                  U5      (       d  M  UR                  s  $    U R                  (       a  U R                  R                  $ g rt   )r.   r    rm   rc   r"   ru   s      r   Úrequest_rateÚRobotFileParser.request_rateÐ   sY   € Ø�z‰z�|‰|ØØ—\”\ˆEØ×Ñ 	×*Ó*Ø—~‘~Ò%ñ "ð ××Ø×%Ñ%×.Ñ.Ð.Ør   c                 ó>   • U R                   (       d  g U R                   $ rt   )r!   r-   s    r   Ú	site_mapsÚRobotFileParser.site_mapsÚ   s   € Ø�}�}ØØ�}‰}Ðr   c                 ó�   • U R                   nU R                  b  XR                  /-   nSR                  [        [        U5      5      $ )Nz

)r    r"   ÚjoinÚmapÚstr)r(   r    s     r   Ú__str__ÚRobotFileParser.__str__ß   s>   € Ø—,‘,ˆØ×ÑÑ)Ø×!3Ñ!3Ð 4Ñ4ˆGØ�{‰{œ3œs GÓ,Ó-Ð-r   )	r$   r"   r#   r    r8   r&   r   r!   r)   N)rk   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r*   r.   r2   r%   r=   rN   r	   rq   rv   ry   r|   r‚   Ú__static_attributes__© r   r   r   r   %   sF   † ñô
ò!ò(ò?ò
Lò'òG#òRò8òòõ
.r   c                   ó*   • \ rS rSrSrS rS rS rSrg)r_   éå   zhA rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path.c                 óN   • US:X  a	  U(       d  Sn[        U5      U l        X l        g )Nrk   T)r   r   rn   )r(   r   rn   s      r   r*   ÚRuleLine.__init__è   s!   € Ø�2‹:žiàˆIÜ" 4Ó(ˆŒ	Ø"�r   c                 ód   • U R                   S:H  =(       d    UR                  U R                   5      $ rI   )r   Ú
startswith)r(   Úfilenames     r   rm   ÚRuleLine.applies_toï   s%   € Ø�y‰y˜CÑ×A 8×#6Ñ#6°t·y±yÓ#AÐAr   c                 óL   • U R                   (       a  SOSS-   U R                  -   $ )NÚAllowÚDisallowz: ©rn   r   r-   s    r   r‚   ÚRuleLine.__str__ò   s   € ØŸ>Ÿ>‘¨z¸TÑAÀDÇIÁIÑMÐMr   r–   N)	r„   r…   r†   r‡   rˆ   r*   rm   r‚   r‰   rŠ   r   r   r_   r_   å   s   † ñ1ò#òBõNr   r_   c                   ó0   • \ rS rSrSrS rS rS rS rSr	g)	rX   éö   z?An entry has one or more user-agents and zero or more rulelinesc                 ó<   • / U l         / U l        S U l        S U l        g rt   )rK   r^   rb   rc   r-   s    r   r*   ÚEntry.__init__ø   s   € ØˆŒØˆŒØˆŒ
Øˆ�r   c                 ó¦  • / nU R                    H  nUR                  SU 35        M     U R                  b  UR                  SU R                   35        U R                  b7  U R                  nUR                  SUR                   SUR
                   35        UR                  [        [        U R                  5      5        SR                  U5      $ )NzUser-agent: zCrawl-delay: zRequest-rate: rV   Ú
)rK   rL   rb   rc   ÚrequestsÚsecondsÚextendr€   r�   r^   r   )r(   ÚretÚagentÚrates       r   r‚   ÚEntry.__str__þ   s¢   € ØˆØ—_”_ˆEØ�J‰J˜ e WÐ-Ö.ñ %à�:‰:Ñ!Ø�J‰J˜ t§z¡z lÐ3Ô4Ø�=‰=Ñ$Ø—=‘=ˆDØ�J‰J˜¨¯© °a¸¿¹°~ÐFÔGØ�
‰
”3”s˜DŸN™NÓ+Ô,Ø�y‰y˜‹~Ðr   c                 ó¬   • UR                  S5      S   R                  5       nU R                   H"  nUS:X  a    gUR                  5       nX!;   d  M"    g   g)z2check if this entry applies to the specified agentrV   r   rJ   TF)r[   r]   rK   )r(   ro   r¢   s      r   rm   ÚEntry.applies_to
  sQ   € ð —O‘O CÓ(¨Ñ+×1Ñ1Ó3ˆ	Ø—_”_ˆEØ˜‹|áØ—K‘K“MˆEØÕ!Ùñ %ð r   c                 ór   • U R                    H'  nUR                  U5      (       d  M  UR                  s  $    g)zJPreconditions:
- our agent applies to this entry
- filename is URL encodedT)r^   rm   rn   )r(   r‘   rf   s      r   rn   ÚEntry.allowance  s0   € ð —N”NˆDØ�‰˜x×(Ó(Ø—~‘~Ò%ñ #ð r   )rb   rc   r^   rK   N)
r„   r…   r†   r‡   rˆ   r*   r‚   rm   rn   r‰   rŠ   r   r   rX   rX   ö   s   † ÙIòò
òõr   rX   )rˆ   Úcollectionsr   Úurllib.errorr   Úurllib.parseÚurllib.requestÚ__all__Ú
namedtupler   r   r   r   r_   rX   rŠ   r   r   Ú<module>r¯      sd   ðñ
ó Û 	Û Û Û àÐ
€à×$Ò$ ]Ð4FÓG€òBò÷~.ñ ~.÷@Nñ N÷"(ò (r   