File "robotparser.cpython-38.pyc"
Full Path: /home/attunedd/public_html/byp/izo/con7ext_sym404/rintoar.txt/lib64/python3.8/urllib/__pycache__/robotparser.cpython-38.pyc
File size: 7.16 KB
MIME-type: text/x-bytecode.python
Charset: 8 bit
U
e5d$ @ s\ d Z ddlZddlZddlZdgZeddZG dd dZG dd dZ G d d
d
Z
dS ) a% robotparser.py
Copyright (C) 2000 Bastian Kleineidam
You can choose between two licenses when using this package:
1) GNU GPLv2
2) PSF license for Python 2.2
The robots.txt Exclusion Protocol is implemented as specified in
http://www.robotstxt.org/norobots-rfc.txt
NRobotFileParser RequestRatezrequests secondsc @ sr e Zd ZdZd ddZdd Zdd Zd d
Zd d Zd
d Z dd Z
dd Z dd Z dd Z
dd Zdd ZdS ) r zs This class provides a set of methods to read, parse and answer
questions about a single robots.txt file.
c C s2 g | _ g | _d | _d| _d| _| | d| _d S )NFr )entriessitemaps
default_entry disallow_all allow_allset_url last_checkedselfurl r */usr/lib64/python3.8/urllib/robotparser.py__init__ s
zRobotFileParser.__init__c C s | j S )zReturns the time the robots.txt file was last fetched.
This is useful for long-running web spiders that need to
check for new robots.txt files periodically.
)r r
r r r mtime% s zRobotFileParser.mtimec C s ddl }| | _dS )zYSets the time the robots.txt file was last fetched to the
current time.
r N)timer )r
r r r r modified. s zRobotFileParser.modifiedc C s& || _ tj|dd \| _| _dS )z,Sets the URL referring to a robots.txt file. N)r urllibparseurlparseZhostpathr r r r r
6 s zRobotFileParser.set_urlc
C s zt j| j}W nR t jjk
rd } z0|jdkr:d| _n|jdkrT|jdk rTd| _W 5 d}~X Y n X | }|
| d dS )z4Reads the robots.txt URL and feeds it to the parser.)i i Ti i Nzutf-8)
r ZrequestZurlopenr errorZ HTTPErrorcoder r readr decode
splitlines)r
ferrrawr r r r ; s
zRobotFileParser.readc C s, d|j kr | jd kr(|| _n | j| d S N*)
useragentsr r append)r
entryr r r
_add_entryH s
zRobotFileParser._add_entryc C sP d}t }| |D ] }|sP|dkr4t }d}n |dkrP| | t }d}|d}|dkrn|d| }| }|s|q|dd}t|dkr|d |d< tj
|d |d< |d dkr|dkr| | t }|j |d d}q|d dkr.|dkr6|j
t|d d d}q|d d
krb|dkr6|j
t|d d d}q|d d kr|dkr6|d rt|d |_d}q|d d
kr|dkr6|d d}t|dkr|d r|d rtt|d t|d |_d}q|d dkr| j |d q|dkrL| | dS )zParse the input lines from a robots.txt file.
We allow that a user-agent: line is not preceded by
one or more blank lines.
r r #N:z
user-agentZdisallowFZallowTz crawl-delayz request-rate/Zsitemap)Entryr r) findstripsplitlenlowerr r unquoter& r' rulelinesRuleLineisdigitintdelayr req_rater )r
linesstater( lineiZnumbersr r r r Q sj
zRobotFileParser.parsec C s | j r
dS | jrdS | js dS tjtj|}tjdd|j|j |j
|j f}tj |}|sfd}| j
D ] }||rl|| S ql| jr| j|S dS )z=using the parsed robots.txt decide if useragent can fetch urlFTr r- )r r r r r r r4
urlunparser ZparamsZqueryZfragmentquoter
applies_to allowancer )r
useragentr Z
parsed_urlr( r r r can_fetch s*
zRobotFileParser.can_fetchc C s> | s d S | jD ]}||r|j S q| jr:| jjS d S N)r r rA r9 r r
rC r( r r r crawl_delay s
zRobotFileParser.crawl_delayc C s> | s d S | jD ]}||r|j S q| jr:| jjS d S rE )r r rA r: r rF r r r request_rate s
z RobotFileParser.request_ratec C s | j s
d S | j S rE )r r r r r site_maps s zRobotFileParser.site_mapsc C s, | j }| jd k r || jg }dtt|S )Nz
)r r joinmapstr)r
r r r r __str__ s
zRobotFileParser.__str__N)r )__name__
__module__ __qualname____doc__r r r r
r r) r rD rG rH rI rM r r r r r s
I
c @ s( e Zd ZdZdd Zdd Zdd ZdS ) r6 zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path.c C s< |dkr|sd}t jt j|}t j|| _|| _d S )Nr T)r r r? r r@ r rB )r
r rB r r r r s
zRuleLine.__init__c C s | j dkp|| j S r$ )r
startswith)r
filenamer r r rA s zRuleLine.applies_toc C s | j r
dndd | j S )NZAllowZDisallowz: )rB r r r r r rM s zRuleLine.__str__N)rN rO rP rQ r rA rM r r r r r6 s r6 c @ s0 e Zd ZdZdd Zdd Zdd Zdd Zd
S ) r. z?An entry has one or more user-agents and zero or more rulelinesc C s g | _ g | _d | _d | _d S rE )r& r5 r9 r: r r r r r s zEntry.__init__c C s g }| j D ]}|d| q
| jd k r<|d| j | jd k rf| j}|d|j d|j |tt| j d
|S )Nz User-agent: z
Crawl-delay: zRequest-rate: r-
) r& r' r9 r: ZrequestsZsecondsextendrK rL r5 rJ )r
ZretagentZrater r r rM s
z
Entry.__str__c C sF | dd }| jD ](}|dkr* dS | }||kr dS qdS )z2check if this entry applies to the specified agentr- r r% TF)r1 r3 r& )r
rC rV r r r rA s
zEntry.applies_toc C s$ | j D ]}||r|j S qdS )zZPreconditions:
- our agent applies to this entry
- filename is URL decodedT)r5 rA rB )r
rS r= r r r rB
s
zEntry.allowanceN)rN rO rP rQ r rM rA rB r r r r r. s
r. ) rQ collectionsZ urllib.parser Zurllib.request__all__
namedtupler r r6 r. r r r r <module> s B