File "robotparser.cpython-37.opt-2.pyc"
Full Path: /home/attunedd/public_html/byp/izo/con7ext_sym404/rintoar.txt/opt/alt/python37/lib64/python3.7/urllib/__pycache__/robotparser.cpython-37.opt-2.pyc
File size: 5.58 KB
MIME-type: text/x-bytecode.python
Charset: 8 bit
B
YI~d" @ sX d dl Z d dlZd dlZdgZe ddZG dd dZG dd dZG dd d Z dS )
NRobotFileParser RequestRatezrequests secondsc @ sf e Zd ZdddZdd Zdd Zdd Zd
d Zd d
Zdd Z dd Z
dd Z dd Z dd Z
dS )r c C s, g | _ d | _d| _d| _| | d| _d S )NFr )entries
default_entry disallow_all allow_allset_url last_checked)selfurl r
7/opt/alt/python37/lib64/python3.7/urllib/robotparser.py__init__ s
zRobotFileParser.__init__c C s | j S )N)r
)r r
r
r mtime$ s zRobotFileParser.mtimec C s dd l }| | _d S )Nr )timer
)r r r
r
r modified- s zRobotFileParser.modifiedc C s& || _ tj|dd \| _| _d S )N )r urllibparseurlparseZhostpath)r r r
r
r r 5 s zRobotFileParser.set_urlc
C s yt j| j}W nR t jjk
rd } z0|jdkr:d| _n|jdkrT|jdk rTd| _W d d }~X Y n X | }|
| d d S )N)i i Ti i zutf-8)
r ZrequestZurlopenr errorZ HTTPErrorcoder r readr decode
splitlines)r ferrrawr
r
r r : s
zRobotFileParser.readc C s, d|j kr | jd kr(|| _n | j| d S )N*)
useragentsr r append)r entryr
r
r
_add_entryG s
zRobotFileParser._add_entryc C s6 d}t }| x|D ]}|sT|dkr8t }d}n |dkrT| | t }d}|d}|dkrr|d | }| }|sq|dd}t|dkr|d |d< tj
|d |d< |d dkr |dkr| | t }|j |d d}q|d dkr4|dkr|j
t|d d d}q|d d krh|dkr|j
t|d d
d}q|d d kr|dkr|d rt|d |_d}q|d d kr|dkr|d d
}t|dkr|d r|d rtt|d t|d |_d}qW |dkr2| | d S )Nr r #:z
user-agentZdisallowFZallowTz crawl-delayz request-rate/)Entryr r% findstripsplitlenlowerr r unquoter" r# rulelinesRuleLineisdigitintdelayr req_rate)r linesstater$ lineiZnumbersr
r
r r P sd
zRobotFileParser.parsec C s | j r
dS | jrdS | js dS tjtj|}tjdd|j|j |j
|j f}tj |}|sfd}x"| j
D ]}||rn||S qnW | jr| j|S dS )NFTr r) )r r r
r r r r0
urlunparser paramsZqueryZfragmentquoter
applies_to allowancer )r useragentr Z
parsed_urlr$ r
r
r can_fetch s$
zRobotFileParser.can_fetchc C s> | s d S x | jD ]}||r|jS qW | jr:| jjS d S )N)r r r> r5 r )r r@ r$ r
r
r crawl_delay s
zRobotFileParser.crawl_delayc C s> | s d S x | jD ]}||r|jS qW | jr:| jjS d S )N)r r r> r6 r )r r@ r$ r
r
r request_rate s
z RobotFileParser.request_ratec C s0 | j }| jd k r || jg }dtt|d S )N
)r r joinmapstr)r r r
r
r __str__ s
zRobotFileParser.__str__N)r )__name__
__module__ __qualname__r r r r r r% r rA rB rC rH r
r
r
r r s
C
c @ s$ e Zd Zdd Zdd Zdd ZdS )r2 c C s< |dkr|sd}t jt j|}t j|| _|| _d S )Nr T)r r r; r r= r r? )r r r? r
r
r r s
zRuleLine.__init__c C s | j dkp|| j S )Nr! )r
startswith)r filenamer
r
r r> s zRuleLine.applies_toc C s | j r
dndd | j S )NZAllowZDisallowz: )r? r )r r
r
r rH s zRuleLine.__str__N)rI rJ rK r r> rH r
r
r
r r2 s r2 c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
r* c C s g | _ g | _d | _d | _d S )N)r" r1 r5 r6 )r r
r
r r s zEntry.__init__c C s g }x | j D ]}|d| q W | jd k r@|d| j | jd k rj| j}|d|j d|j |tt| j |d d
|S )Nz User-agent: z
Crawl-delay: zRequest-rate: r) r rD ) r" r# r5 r6 ZrequestsZsecondsextendrF rG r1 rE )r retagentZrater
r
r rH s
z
Entry.__str__c C sF | dd }x.| jD ]$}|dkr*dS | }||krdS qW dS )Nr) r r! TF)r- r/ r" )r r@ rP r
r
r r> s zEntry.applies_toc C s$ x | j D ]}||r|jS qW dS )NT)r1 r> r? )r rM r9 r
r
r r? s
zEntry.allowanceN)rI rJ rK r rH r> r? r
r
r
r r* s
r* )
collectionsZ urllib.parser Zurllib.request__all__
namedtupler r r2 r* r
r
r
r <module>
s 6