Google
Edit File: url_helper.cpython-39.pyc
a '�DgЊ � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm Z m Z m Z d dl m Z d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZm Z m Z d d l m Z m Z m!Z!m"Z" d dl#Z#d d l#m$Z$ d d l%m&Z&m'Z'm(Z( e�)e*�Z+d Z,d d� Z-dd� Z.e/d�dd�Z0dBe1e2e3dd�dd�Z4e1dd�dd �Z5e d d�d d �Z6G d d!� d!�Z7G d"d� de7�Z8G d#d� de7�Z9G d$d%� d%�Z:G d&d'� d'e;�Z<d(d)� Z=dCe>e:d.�d/d0�Z?dDed1ef e1e/ej@ee2 ed2�d3d4�ZAdEed1ef ee1 e2e/e d7�d8d9�ZBe2d:�de+jCdddddd-d+d5f e2ee2 eee ee2 ee eeee2ge2f e1e>e2d;� d<d=�ZDG d>d?� d?�ZEdFd@dA�ZFdS )G� N)�ThreadPoolExecutor� TimeoutError� as_completed)� parsedate)�partial)� NOT_FOUND)�count)�create_default_context)�Any�Callable�Iterator�List�Optional�Tuple�Union)�quote�urlparse�urlsplit� urlunparse)� exceptions)� performance�util�version�REDACTEDc C s< t t| dd��}|d s4|d r4|d |d< d|d< t|�S )N�http)�scheme� � � )�listr r )�urlZ parsed_url� r! �8/usr/lib/python3.9/site-packages/cloudinit/url_helper.py� _cleanurl& s r# c G s$ dd� }| }|D ]}|||�}q|S )Nc S sN t t| ��}|d }|r*|�d�s*|d7 }|tt|�dd�7 }||d< t|�S )Nr �/z/:)�safe)r r �endswithr �strr )r �add_onZ url_parsed�pathr! r! r" �combine_single1 s z#combine_url.<locals>.combine_singler! )�baseZadd_onsr* r r( r! r! r" � combine_url0 s r, ��returnc C sV t jdt jdt jdt jdtdi}|�t| ��}|sRt| t �rD| j }nt � d� d}|S )z4helper for read_ftps to map return codes to a number�, � �� iX i� z9Unexpected exception type while connecting to ftp server.i����) �ftplibZ error_replyZ error_temp� error_permZ error_proto�EOFError�get�type� isinstance�OSError�errno�LOG�warning)�excZftp_error_codes�coder! r! r" �"ftp_get_return_code_from_exception@ s � �r> � @� FtpResponse)r �timeout�kwargsr. c K sr t | �}|js tdtd| d��t�� ��4}|jp2d}|jp<d}d|jk�r&z6t j t � d�}t � d| |� |j|j||pxd d � W nN t jy� } z4t|�} td | � d | � d �| d| d�|�W Y d}~n d}~0 0 t � d|� z*|j||jp�dd� t � d� |�� W nD t j�yJ } z(t �d� tddd| d�|�W Y d}~n d}~0 0 z�zXt � d|j� |jd|j� �|jd� t|�� | �W W t � d� |�� W d � S t j�y� } z4t|�} td| � d | � d �| d| d�|�W Y d}~n d}~0 0 W t � d� |�� nt � d� |�� 0 �n(�zz�t � � } t � d| |� | j|j||�pTd d � t � d|� | j||j�pvdd� t � d|j� | jd|j� �|jd� t|�� | �W W t � d� | �� W d � S t j�y" } z4t|�} td| � d | � d �| d| d�|�W Y d}~n d}~0 0 W t � d� | �� nt � d� | �� 0 W d � n1 �sd0 Y dS )az connect to URL using ftp over TLS and read a file when using strict mode (ftps://), raise exception in event of failure when not using strict mode (ftp://), fall back to using unencrypted ftp url: string containing the desination to read a file from. The url is parsed with urllib.urlsplit to identify username, password, host, path, and port in the following format: ftps://[username:password@]host[:port]/[path] host is the only required component timeout: maximum time for the connection to take kwargs: unused, for compatibility with read_url returns: UrlResponse zInvalid url providedN��causer= �headersr � Z anonymous�ftps)�contextz3Attempting to connect to %s via port [%s] over tls.r? )�host�portrA z1Reading file from server over tls failed for url z [�]z"Attempting to login with user [%s]r )�user�passwdz Creating a secure connectionz�Attempted to connect to an insecure ftp server but used a scheme of ftps://, which is not allowed. Use ftp:// to allow connecting to insecure ftp servers.r1 zReading file: %szRETR )�callbackzClosing connectionz,Reading file from ftp server failed for url z(Attempting to connect to %s via port %s.) r �hostname�UrlErrorr �io�BytesIOrJ �usernamer r2 ZFTP_TLSr r: �debugZconnectZ all_errorsr> Zlogin�passwordZprot_pr3 r; r) Z retrbinary�writer@ �getvalue�closeZFTP) r rA rB Z url_parts�bufferrJ rL Zftp_tls�er= �ftpr! r! r" � read_ftpsW s� � ��� ���� � �� � � ����� � �� � ����� � r\ � FileResponse)r) r. c K s� |� d�rt�d| � zt�| �}t|| �W S tyb } z t|td| d�|�W Y d}~n@d}~0 t y� } z t||j d| d�|�W Y d}~n d}~0 0 dS )zmread a binary file and return a FileResponse matches function signature with read_ftps and read_url �dataz'Unable to post data to file resource %sNrC ) r5 r: r; r Zload_binary_filer] �FileNotFoundErrorrP r �IOErrorr9 )r) rB �contentsrZ r! r! r" � _read_file� s &rb )r] � UrlResponser@ c K s� | � � } z t| �}W n2 tyF } zt|| d�|�W Y d}~n d}~0 0 |j}|dksf| rxd| d krxt|jfi |��S |dv r�t| fi |��S |dv r�t| fi |��S t � d|� t| fi |��S dS ) a0 Wrapper function around readurl to allow passing a file path as url. When url is not a local file path, passthrough any kwargs to readurl. In the case of parameter passthrough to readurl, default values for some parameters. See: call-signature of readurl in this module for param docs. )rD r N�filer$ r )r[ rG )r �httpsz Attempting unknown protocol %s) �lstripr � ValueErrorrP r rb r) r\ �readurlr: r; )r rB �parsedrZ r r! r! r" �read_file_or_url� s $ rj c @ s& e Zd Zd dd�Zdd� Zdd� ZdS ) �StringResponse�� c C s || _ i | _|| _|| _d S �N)r= rE ra r ��selfra r r= r! r! r" �__init__ s zStringResponse.__init__c O s | j dkS )Nrl �r= �ro �argsrB r! r! r" �ok s zStringResponse.okc C s | j �d�S )Nzutf-8)ra �decode�ro r! r! r" �__str__ s zStringResponse.__str__N)rl )�__name__� __module__� __qualname__rp rt rw r! r! r! r" rk s rk c s& e Zd Zdeed�� fdd� Z� ZS )r] rl �ra r c s t � j|||d� d S )Nrq ��superrp rn �� __class__r! r" rp s zFileResponse.__init__)rl �rx ry rz �bytesr' rp � __classcell__r! r! r~ r" r] s c s$ e Zd Zeed�� fdd� Z� ZS )r@ r{ c s t � �||� d S rm r| )ro ra r r~ r! r" rp s zFtpResponse.__init__r� r! r! r~ r" r@ s c @ s� e Zd Zejd�dd�Zeed�dd��Zee d�dd��Z de d�d d �Z ed d � �Z eed�dd��Zdd� Zdee e ee d�dd�ZdS )rc )�responsec C s || _ d S rm )� _response)ro r� r! r! r" rp s zUrlResponse.__init__r- c C s | j jd u rdS | j jS )N� )r� �contentrv r! r! r" ra # s zUrlResponse.contentsc C s | j jS rm )r� r rv r! r! r" r ) s zUrlResponse.urlFc C s2 d}|r d}d| j kr"|k r*n ndS dS d S )Nr/ r0 rl TFrq )ro Z redirects_ok�upperr! r! r" rt - s zUrlResponse.okc C s | j jS rm )r� rE rv r! r! r" rE 6 s zUrlResponse.headersc C s | j jS rm )r� � status_coderv r! r! r" r= : s zUrlResponse.codec C s | j jS rm )r� �textrv r! r! r" rw > s zUrlResponse.__str__r )� chunk_size�decode_unicoder. c c s | j �||�E dH dS )at Iterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. :param chunk_size: Number of bytes it should read into memory. :param decode_unicode: If True, content will be decoded using the best available encoding based on the response. N)r� � iter_content)ro r� r� r! r! r" r� A s zUrlResponse.iter_contentN)F)r F)rx ry rz �requestsZResponserp �propertyr� ra r' r �boolrt rE �intr= rw r r r� r! r! r! r" rc s ��rc c @ s e Zd Zddd�ZdS )rP Nc C s< t �| t|�� || _|| _|| _| jd u r2i | _|| _d S rm )r` rp r' rD r= rE r )ro rD r= rE r r! r! r" rp Q s zUrlError.__init__)NNN)rx ry rz rp r! r! r! r" rP P s rP c C s� i }t | �j}|dkr~|r~d|v r8|d r8|d |d< nd|d< d|v rfd|v rf|d |d g|d< nd|v r~t|d �|d< |S )Nre Zca_certsZverifyTZ cert_fileZkey_fileZcert)r r r' )r � ssl_detailsZssl_argsr r! r! r" � _get_ssl_args[ s � r� r TFr )�streamr. c s t | �} | |d�}|�t| |�� | |d< |s:|r6dnd}||d< |durpt|t�r^||d< ntt|�d�|d< |du r|g }d }|r�tt|�d d �}d t� � }|dur�|� � }ni }|r�||d < |du r�d }| du r�t � � } t � D �] }|r�|| �}d |v�r ||d < ||d< i }|�� D ]p\}� |d k�r4�q |dk�r�|�r�� fdd�|D �}|�r�t �� �||< |D ]}t|| |< �qnn� ||< �q zl|�r�t�d|| �r�dn|| |� | jf i |��}| �r�|�� t�d| |jt|j�|d � t|�W S tj�y2 } zt|| d�|�W Y d}~q�d}~0 tj �y } z�t|tj ��r�t |d��r�t |j d��r�t||j j|j j | d�}n t|| d�}| �r�| |� � |��s�||�| �p�|d |k } | �s�||�|dk�r�|�r�t�d|� t!�"|� W Y d}~q�d}~0 0 q�t#d��dS )a� Wrapper around requests.Session to read the url and retry if necessary :param url: Mandatory url to request. :param data: Optional form data to post the URL. Will set request_method to 'POST' if present. :param timeout: Timeout in seconds to wait for a response. May be a tuple if specifying (connection timeout, read timeout). :param retries: Number of times to retry on exception if exception_cb is None or exception_cb returns True for the exception caught. Default is to fail with 0 retries on exception. :param sec_between: Default 1: amount of seconds passed to time.sleep between retries. None or -1 means don't sleep. :param headers: Optional dict of headers to send during request :param headers_cb: Optional callable returning a dict of values to send as headers during request :param headers_redact: Optional list of header names to redact from the log :param ssl_details: Optional dict providing key_file, ca_certs, and cert_file keys for use on in ssl connections. :param check_status: Optional boolean set True to raise when HTTPError occurs. Default: True. :param allow_redirects: Optional boolean passed straight to Session.request as 'allow_redirects'. Default: True. :param exception_cb: Optional callable which accepts the params msg and exception and returns a boolean True if retries are permitted. :param session: Optional exiting requests.Session instance to reuse. :param infinite: Bool, set True to retry indefinitely. Default: False. :param log_req_resp: Set False to turn off verbose debug messages. :param request_method: String passed as 'method' to Session.request. Typically GET, or POST. Default: POST if data is provided, GET otherwise. :param stream: if False, the response content will be immediately downloaded. )r r� �allow_redirectsZPOSTZGET�methodNrA r r z Cloud-Init/%sr^ ���z User-AgentrE c s g | ]}� � |�r|�qS r! )r5 )�.0�k��vr! r" � <listcomp>� r� zreadurl.<locals>.<listcomp>z'[%s/%s] open '%s' with %s configuration�infinitez(Read from %s (%s, %sb) after %s attempts�r r� r� �r= rE r �1Please wait %s seconds while we wait to try againz"This path should be unreachable...)$r# �updater� r7 �tuple�max�floatr� r Zversion_string�copyr� ZSessionr �items�deepcopyr r: rT ZrequestZraise_for_statusr� �lenr� rc r ZSSLErrorrP ZRequestExceptionZ HTTPError�hasattrr� rE �time�sleep� RuntimeError) r r^ rA ZretriesZ sec_betweenrE � headers_cb�headers_redactr� � check_statusr� � exception_cbZsessionr� Z log_req_resp�request_methodr� Zreq_argsZ manual_triesZ user_agent�iZfiltered_req_argsr� Zmatched_headers�key�rrZ Z url_errorZ will_retryr! r� r" rh m s� 4� �� " ����� �"rh .)�func�addrrA �event�delayr. c C s |r|j |d�rdS | ||�S )z Execute func with optional delay�rA N��wait�r� r� rA r� r� r! r! r" �_run_func_with_delay s r� �333333�?� )r� � addresses� stagger_delayrA r. c s8 d}d}d}g }t �� �tt|�d�� z�z�� ����fdd�t|�D �}t|�d�D ]X} || }| �� } | r|| }|�|� qT| �� }|rT�� � ||f W W � j dd� S qT|r�t � d||� |�nt � d |� td ��W n4 t�y t � d d �|�d �tt|��� Y n0 W � j dd� n� j dd� 0 ||fS ) a execute multiple callbacks in parallel Run blocking func against two different addresses staggered with a delay. The first call to return successfully is returned from this function and remaining unfinished calls are cancelled if they have not yet started N)� max_workersc s, i | ]$\}}� j t�|��|� d �|�qS )r� )Zsubmitr� )r� r� r� ��executorr� Zis_doner� rA r! r" � <dictcomp>I s ��z dual_stack.<locals>.<dictcomp>r� Fr� z<Exception(s) %s during request to %s, raising last exceptionzEmpty result for address %szNo result returnedzJTimed out waiting for addresses: %s, exception(s) raised while waiting: %s� )� threading�Eventr r� � enumerater � exception�append�result�setZshutdownr: r; �errorrg r �join�mapr' ) r� r� r� rA Z return_resultZreturned_addressZlast_exceptionr ZfuturesZfutureZreturn_exceptionr! r� r" � dual_stack/ sN � �� � r� �inf) �max_waitrA � status_cbr� � sleep_timer� � sleep_time_cbr� �connect_synchronously� async_delayc sL t td�� fdd� } dtttd�dd�� dd � ���fd d �����fd d ������� � fdd�} � ��� fdd�}t�� }� r�|r�td��| r�| n|}|p�| }d�d}||��}|||||�}|r�|\}}||jfS � �||�r�q<�d �t�d|� t�|� t�� }|r�|| |� kr��|| }|dkr��q<q�t� d� � dS )a1 urls: a list of urls to try max_wait: roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. a number <= 0 will always result in only one try timeout: the timeout provided to urlopen status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers for request. headers_redact: a list of header names to redact from the log sleep_time: Amount of time to sleep between retries. If this and sleep_time_cb are None, the default sleep time defaults to 1 second and increases by 1 seconds every 5 tries. Cannot be specified along with `sleep_time_cb`. exception_cb: call method with 2 arguments 'msg' (per status_cb) and 'exception', the exception that occurred. sleep_time_cb: call method with 2 arguments (response, loop_n) that generates the next sleep time. Cannot be specified along with 'sleep_time`. request_method: indicate the type of HTTP request, GET, PUT, or POST connect_synchronously: if false, enables executing requests in parallel async_delay: delay before parallel metadata requests, see RFC 6555 returns: tuple of (url, response contents), on failure, (False, None) the idea of this routine is to wait for the EC2 metadata service to come up. On both Eucalyptus and EC2 we have seen the case where the instance hit the MD before the MD service was up. EC2 seems to have permanently fixed this, though. In openstack, the metadata service might be painfully slow, and unable to avoid hitting a timeout of even up to 10 seconds or more (LP: #894279) for a simple GET. Offset those needs with the need to not hang forever (and block boot) on a system where cloud-init is configured to look for EC2 Metadata service but is not going to find one. It is possible that the instance data host (169.254.169.254) may be firewalled off Entirely for a system, meaning that the connection will block forever unless a timeout is set. The default value for max_wait will retry indefinitely. )� loop_numberr. c s � d ur � S |d d S )N� r r! )�_r� )r� r! r" �default_sleep_time� s z(wait_for_url.<locals>.default_sleep_timer �r� � start_timer� c S s0 | t d�dfv rdS | dkp.t�� | | | kS )z4Check if time is up based on start time and max waitr� NFr )r� r� � monotonicr� r! r! r" �timeup� s �z wait_for_url.<locals>.timeupc S sf | j s*d| j }tt|�| j| j|d�}n4| �� sVd| j }tt|�| j| j|d�}nd}d}||fS )z?Map requests response code/contents to internal "UrlError" typezempty response [%s]r� zbad status code [%s]r N)ra r= rP rg rE rt )r� r �reason�url_excr! r! r" �handle_url_response� s& � �z)wait_for_url.<locals>.handle_url_responsec s� d}d}z,| |�\}}� ||�\}}|s2||fW S W n^ t yd } zd| }| }W Y d} ~ n8d} ~ 0 ty� } zd| }| }W Y d} ~ n d} ~ 0 0 tt�� | �} �r�d� nd} d|p�t|dd�| | |f } || � |r�|| |d � dS ) z:Execute request, handle response, optionally log exceptionr Nzrequest error [%s]zunexpected error [%s]z%ssZ unlimitedz Calling '%s' failed [%s/%s]: %sr )�msgr� )rP � Exceptionr� r� r� �getattr) Z url_reader_cb�urlsr� �exc_cb�log_cbr� r r� r� rZ Z time_takenZ max_wait_strZ status_msg)r� r� r! r" �read_url_handle_exceptions� s0 �z0wait_for_url.<locals>.read_url_handle_exceptionsc s$ t | � d u ri n� | ��|d�d�S )NF)rE r� rA r� r� )rh )r rA )r� r� r� r! r" � read_url_cb� s �z!wait_for_url.<locals>.read_url_cbc s� �� fdd�}�D ]l}t �� }�dkrb��| �r6 dS �durb� rb|� | � krbt| � | �� �||| ||�}|r| S qdS )z|iterate over list of urls, request each one and handle responses and thrown exceptions individually per url c s | � | ��fS rm r! r� )r� rA r! r" �url_reader_serial s z@wait_for_url.<locals>.read_url_serial.<locals>.url_reader_serialr N)r� r� r� )r� rA r� r� r� r Znow�out)�loop_nr� r� r� r� r� r� r" �read_url_serial s$ ��� �z%wait_for_url.<locals>.read_url_serialc s, t t�� |d�}�|�| ||�}|r(|S dS )z�pass list of urls to dual_stack which sends requests in parallel handle response and exceptions of the first endpoint to respond )r� rA N)r r� )r� rA r� r� Zurl_reader_parallelr� )r� r� r� r� r! r" �read_url_parallel" s � �z'wait_for_url.<locals>.read_url_parallelz3sleep_time and sleep_time_cb are mutually exclusiveNr r� z$Timed out, no response from urls: %s)FN)r ) r� r� r� r� rg ra r: rT r� r� )r� r� rA r� r� r� r� r� r� r� r� r� r� r� r� r� Z do_read_urlZcalculate_sleep_timer� Zcurrent_sleep_timer �addressZ current_timer! ) r� r� r� r� r� r� r� r� r� r� r� r� r" � wait_for_url� sF 9 � � r� c @ s^ e Zd Zddd�Zdd� Zdd� Zd d � Zd d � Zd d� Zdd� Z dd� Z dd� Z dd� Z dS )�OauthUrlHelperN�/run/oauth_skew.jsonc C st || _ |p d| _|| _|| _|| _d| _d| _| j| j| j f}t|�sNd| _nt|�s^t d��| � � }|pli | _ d S )Nr Tr� FzBall or none of token_key, token_secret, or consumer_key can be set) � consumer_key�consumer_secret� token_key� token_secret�skew_data_file� _do_oauth�skew_change_limit�any�allrg �read_skew_file� skew_data)ro r� r� r� r� r� �required�oldr! r! r" rp b s �zOauthUrlHelper.__init__c C s� | j r�tj�| j �r�t�d| j � ���V t| j dd��(}t�|�W d � W d � S 1 sd0 Y W d � n1 s�0 Y d S )NzReading r� ��mode) r� �osr) �isfiler �Timed�open�json�load)ro �fpr! r! r" r� } s �Tz OauthUrlHelper.read_skew_filec C s� | j s d S | �� }|d u r i }|||< t�d| j � ���N t| j dd�� }|�t�|�� W d � n1 sn0 Y W d � n1 s�0 Y d S )NzWriting �wr )r� r� r r r rV r �dumps)ro rI �valueZcurr r! r! r" �update_skew_file� s �z OauthUrlHelper.update_skew_filec C s� t |t�r |jdks"|jdks"d S d|jvr>t�d|j� d S |jd }zt�t|��}W n4 t y� } z t�d||� W Y d }~d S d }~0 0 t |t�� �}t |j �j }| j�|d�}t|| �| jkr�| �||� t�d||� || j|< d S )Ni� i� �datez$Missing header 'date' in %s responsez#Failed to convert datetime '%s': %sr z$Setting oauth clockskew for %s to %d)r7 rP r= rE r: r; r� �mktimer r� r� r r �netlocr� r5 �absr� r ) ro r� r� r Z remote_timerZ ZskewrI Zold_skewr! r! r" r� � s0 ��� zOauthUrlHelper.exception_cbc C sZ | j s i S d }t|�j}| jr>|| jv r>tt�� �| j| }t|| j| j| j | j |d�S )N)r r� r� r� r� � timestamp) r� r r r� r� r� � oauth_headersr� r� r� r� )ro r r rI r! r! r" r� � s �zOauthUrlHelper.headers_cbc C s: t | j|�d��|d< t | j|�d��|d< ||i |��S )Nr� r� )r � _headers_cbr5 � _exception_cb)ro Z wrapped_funcrs rB r! r! r" �_wrapped� s � �zOauthUrlHelper._wrappedc O s | � t||�S rm )r r� rr r! r! r" r� � s zOauthUrlHelper.wait_for_urlc O s | � t||�S rm )r rh rr r! r! r" rh � s zOauthUrlHelper.readurlc C s6 d }z |r|||�}W | � ||� n| � ||� 0 |S rm )r� )ro Zextra_exception_cbr� r� �retr! r! r" r � s z OauthUrlHelper._exception_cbc C s$ i }|r||�}|� | �|�� |S rm )r� r� )ro Zextra_headers_cbr rE r! r! r" r � s zOauthUrlHelper._headers_cb)NNNNr� ) rx ry rz rp r� r r� r� r r� rh r r r! r! r! r" r� a s � r� c C s~ zdd l m} W n. ty> } ztd�|�W Y d }~n d }~0 0 |rNt|�}nd }|j|||||j|d�}|�| �\} } } | S )Nr z oauth support is not available)Z client_secretZresource_owner_keyZresource_owner_secretZsignature_methodr )Zoauthlib.oauth1�oauth1� ImportError�NotImplementedErrorr' ZClientZSIGNATURE_PLAINTEXT�sign) r r� r� r� r� r r rZ ZclientZ_uriZsigned_headersZ_bodyr! r! r" r � s" �r )r? )NNr r NNNNTTNNFTr F)N)r� r� )N)Gr� r2 rQ r Zloggingr r� r� �concurrent.futuresr r r Z email.utilsr � functoolsr Z http.clientr � itertoolsr Zsslr �typingr r r r r r r � urllib.parser r r r r� r Z cloudinitr r r Z getLoggerrx r: r r# r, r� r> r' r� �dictr\ rb rj rk r] r@ rc r` rP r� r� rh r� r� r� rT r� r� r r! r! r! r" �<module> s� $ � 1 �� 6 � � � � U�� ` �