o
    Kg                     @   s   d dl mZmZmZ d dlmZ 	 d dlmZ d dlm	Z
mZ e
e_	ee_dgZG dd deZG dd deZG d	d
 d
eZdS )    )absolute_importdivisionunicode_literalsstr)urllib)parserequestRobotFileParserc                   @   sZ   e Zd ZdZdddZdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd ZdS )r
   zs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

     c                 C   s,   g | _ d | _d| _d| _| | d| _d S )NFr   )entriesdefault_entrydisallow_all	allow_allset_urllast_checkedselfurl r   h/var/www/html/status_management/venv/lib/python3.10/site-packages/future/backports/urllib/robotparser.py__init__   s   

zRobotFileParser.__init__c                 C   s   | j S )zReturns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )r   r   r   r   r   mtime&   s   zRobotFileParser.mtimec                 C   s   ddl }|  | _dS )zYSets the time the robots.txt file was last fetched to the
        current time.

        r   N)timer   )r   r   r   r   r   modified/   s   zRobotFileParser.modifiedc                 C   s&   || _ tj|dd \| _| _dS )z,Sets the URL referring to a robots.txt file.      N)r   r   r   urlparsehostpathr   r   r   r   r   7   s    zRobotFileParser.set_urlc              
   C   s   z	t j| j}W n4 t jjy= } z&|jdv rd| _n|jdkr+d| _W Y d}~dS W Y d}~dS W Y d}~dS d}~ww |	 }| 
|d  dS )z4Reads the robots.txt URL and feeds it to the parser.)i  i  Ti  Nzutf-8)r   r	   urlopenr   error	HTTPErrorcoder   r   readr   decode
splitlines)r   ferrrawr   r   r   r%   <   s   

zRobotFileParser.readc                 C   s2   d|j v r| jd u r|| _d S d S | j| d S N*)
useragentsr   r   append)r   entryr   r   r   
_add_entryI   s
   


zRobotFileParser._add_entryc                 C   sj  d}t  }|D ]}|s#|dkrt  }d}n|dkr#| | t  }d}|d}|dkr2|d| }| }|s9q|dd}t|dkr|d   |d< tj	|d  |d< |d dkrx|dkrm| | t  }|j
|d  d}q|d dkr|dkr|jt|d d	 d}q|d d
kr|dkr|jt|d d d}q|dkr| | dS dS )zParse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r   r      #N:z
user-agentdisallowFallowT)Entryr0   findstripsplitlenlowerr   r   unquoter-   r.   	rulelinesRuleLine)r   linesstater/   lineir   r   r   r   R   sP   



zRobotFileParser.parsec                 C   s   | j rdS | jr
dS tjtj|}tjdd|j|j|j	|j
f}tj|}|s.d}| jD ]}||r?||  S q1| jrI| j|S dS )z=using the parsed robots.txt decide if useragent can fetch urlFTr   /)r   r   r   r   r   r<   
urlunparser    paramsqueryfragmentquoter   
applies_to	allowancer   )r   	useragentr   
parsed_urlr/   r   r   r   	can_fetch   s$   

zRobotFileParser.can_fetchc                 C   s   d dd | jD S )Nr   c                 S   s   g | ]}t |d  qS )
r   ).0r/   r   r   r   
<listcomp>   s    z+RobotFileParser.__str__.<locals>.<listcomp>)joinr   r   r   r   r   __str__      zRobotFileParser.__str__N)r   )__name__
__module____qualname____doc__r   r   r   r   r%   r0   r   rM   rR   r   r   r   r   r
      s    
		3c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	r>   zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.c                 C   s(   |dkr|sd}t j|| _|| _d S )Nr   T)r   r   rH   r    rJ   )r   r    rJ   r   r   r   r      s   
zRuleLine.__init__c                 C   s   | j dkp
|| j S r+   )r    
startswith)r   filenamer   r   r   rI      rS   zRuleLine.applies_toc                 C   s   | j rdpdd | j S )NAllowDisallowz: )rJ   r    r   r   r   r   rR      s   zRuleLine.__str__N)rT   rU   rV   rW   r   rI   rR   r   r   r   r   r>      s
    r>   c                   @   s0   e Zd ZdZdd Zdd Zdd Zdd	 Zd
S )r6   z?An entry has one or more user-agents and zero or more rulelinesc                 C   s   g | _ g | _d S )N)r-   r=   r   r   r   r   r      s   
zEntry.__init__c                 C   sH   g }| j D ]
}|d|dg q| jD ]}|t|dg qd|S )NzUser-agent: rN   r   )r-   extendr=   r   rQ   )r   retagentrA   r   r   r   rR      s   


zEntry.__str__c                 C   sF   | dd  }| jD ]}|dkr dS | }||v r  dS qdS )z2check if this entry applies to the specified agentrC   r   r,   TF)r9   r;   r-   )r   rK   r^   r   r   r   rI      s   
zEntry.applies_toc                 C   s$   | j D ]}||r|j  S qdS )zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)r=   rI   rJ   )r   rY   rA   r   r   r   rJ      s
   


zEntry.allowanceN)rT   rU   rV   rW   r   rR   rI   rJ   r   r   r   r   r6      s    r6   N)
__future__r   r   r   future.builtinsr   future.backportsr   future.backports.urllibr   _parser	   _request__all__objectr
   r>   r6   r   r   r   r   <module>   s     	