U
    ¨º|eä>  ã                   @   sø   d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZmZ ddlmZ ddlmZmZmZ ddlmZ e
 d	d
¡ZG dd„ deƒZdd„ ZG dd„ ded�ZG dd„ deƒZG dd„ deeƒZ dS )z.Storage providers backends for Memory caching.é    )ÚPicklingErrorN)ÚABCMetaÚabstractmethodé   )Úconcurrency_safe_rename)ÚmkdirpÚmemstr_to_bytesÚ
rm_subdirs)Únumpy_pickleÚCacheItemInfozpath size last_accessc                   @   s   e Zd ZdZdS )ÚCacheWarningz:Warning to capture dump failures except for PicklingError.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© r   r   úS/var/www/website-v5/atlas_env/lib/python3.8/site-packages/joblib/_store_backends.pyr      s   r   c                 C   s,   t t ¡ ƒ}d ||t ¡ ¡}|| |ƒ |S )z>Writes an object into a unique file in a concurrency-safe way.z{}.thread-{}-pid-{})ÚidÚ	threadingÚcurrent_threadÚformatÚosÚgetpid)Zobject_to_writeÚfilenameÚ
write_funcÚ	thread_idÚtemporary_filenamer   r   r   Úconcurrency_safe_write   s      ÿ
r   c                   @   sp   e Zd ZdZdZedd„ ƒZedd„ ƒZedd„ ƒZed	d
„ ƒZ	edd„ ƒZ
edd„ ƒZedeƒ fdd„ƒZdS )ÚStoreBackendBasezaHelper Abstract Base Class which defines all methods that
       a StorageBackend must implement.Nc                 C   s   dS )aß  Opens an item on the store and return a file-like object.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        f: a file-like object
            The file-like object where an item is stored and retrieved
        mode: string, optional
            the mode in which the file-like object is opened allowed valued are
            'rb', 'wb'

        Returns
        -------
        a file-like object
        Nr   )ÚselfÚfÚmoder   r   r   Ú
_open_item-   s    zStoreBackendBase._open_itemc                 C   s   dS )a¥  Checks if an item location exists in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        location: string
            The location of an item. On a filesystem, this corresponds to the
            absolute path, including the filename, of a file.

        Returns
        -------
        True if the item exists, False otherwise
        Nr   ©r   Úlocationr   r   r   Ú_item_exists@   s    zStoreBackendBase._item_existsc                 C   s   dS )a-  Moves an item from src to dst in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        src: string
            The source location of an item
        dst: string
            The destination location of an item
        Nr   )r   ÚsrcÚdstr   r   r   Ú
_move_itemQ   s    zStoreBackendBase._move_itemc                 C   s   dS )zÎCreates a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory.
        Nr   r#   r   r   r   Úcreate_location_   s    z StoreBackendBase.create_locationc                 C   s   dS )zèClears a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory or a filename absolute path
        Nr   r#   r   r   r   Úclear_locationj   s    zStoreBackendBase.clear_locationc                 C   s   dS )z»Returns the whole list of items available in the store.

        Returns
        -------
        The list of items identified by their ids (e.g filename in a
        filesystem).
        Nr   ©r   r   r   r   Ú	get_itemsu   s    zStoreBackendBase.get_itemsr   c                 C   s   dS )a—  Configures the store.

        Parameters
        ----------
        location: string
            The base location used by the store. On a filesystem, this
            corresponds to a directory.
        verbose: int
            The level of verbosity of the store
        backend_options: dict
            Contains a dictionary of named parameters used to configure the
            store backend.
        Nr   )r   r$   ÚverboseÚbackend_optionsr   r   r   Ú	configure   s    zStoreBackendBase.configure)r   r   r   r   r$   r   r"   r%   r(   r)   r*   r,   Údictr/   r   r   r   r   r   '   s    







	r   )Ú	metaclassc                   @   s¢   e Zd ZdZd&dd„Zd'dd„Zdd	„ Zd
d„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zd(dd„Zdd„ Zdd„ Zdd„ Zd)dd„Zd*d d!„Zd"d#„ Zd$d%„ ZdS )+ÚStoreBackendMixina‰  Class providing all logic for managing the store in a generic way.

    The StoreBackend subclass has to implement 3 methods: create_location,
    clear_location and configure. The StoreBackend also has to provide
    a private _open_item, _item_exists and _move_item methods. The _open_item
    method has to have the same signature as the builtin open and return a
    file-like object.
    r   Nc           	   	   C   sº   t jj| jf|žŽ }|dkrD|dk r4td |¡ƒ ntd ||¡ƒ t| dƒsRdn| j}t j |d¡}|  |¡s|t	d| ƒ‚|dkr¨|  
|d	¡�}t |¡}W 5 Q R X ntj||d
�}|S )zKLoad an item from the store given its path as a list of
           strings.r   é
   z{0}...z{0} from {1}Ú	mmap_modeNú
output.pklzANon-existing item (may have been cleared).
File %s does not existÚrb)r4   )r   ÚpathÚjoinr$   Úprintr   Úhasattrr4   r%   ÚKeyErrorr"   r
   Úload)	r   r7   r-   ÚmsgÚ	full_pathr4   r   r    Úitemr   r   r   Ú	load_itemš   s$    ÿ
ÿzStoreBackendMixin.load_itemc              
      s¤   zht jjˆ jf|žŽ }ˆ  |¡s*ˆ  |¡ t j |d¡}|dkrLtd| ƒ ‡ fdd„}ˆ  |||¡ W n6 tk
rž } zt	 
d|› d�t¡ W 5 d}~X Y nX dS )	zLDump an item in the store at the path given as a list of
           strings.r5   r3   zPersisting in %sc                    sj   ˆ   |d¡�T}ztj| |ˆ jd� W n6 tk
rZ } zt d|› d�t¡ W 5 d }~X Y nX W 5 Q R X d S )NÚwb)ÚcompresszjUnable to cache to disk: failed to pickle output. In version 1.5 this will raise an exception. Exception: Ú.)r"   r
   ÚdumprB   r   ÚwarningsÚwarnÚFutureWarning)Úto_writeÚdest_filenamer    Úer+   r   r   r   À   s    
üz/StoreBackendMixin.dump_item.<locals>.write_funcz`Unable to cache to disk. Possibly a race condition in the creation of the directory. Exception: rC   N)r   r7   r8   r$   r%   r)   r9   Ú_concurrency_safe_writeÚ	ExceptionrE   rF   r   )r   r7   r?   r-   Ú	item_pathr   r   rJ   r   r+   r   Ú	dump_itemµ   s    


ýzStoreBackendMixin.dump_itemc                 C   s,   t jj| jf|žŽ }|  |¡r(|  |¡ dS )z7Clear the item at the path, given as a list of strings.N©r   r7   r8   r$   r%   r*   )r   r7   rM   r   r   r   Ú
clear_itemÕ   s    
zStoreBackendMixin.clear_itemc                 C   s,   t jj| jf|žŽ }t j |d¡}|  |¡S )zLCheck if there is an item at the path, given as a list of
           stringsr5   )r   r7   r8   r$   r%   )r   r7   rM   r   r   r   r   Úcontains_itemÛ   s    zStoreBackendMixin.contains_itemc                 C   s   dt jj| jf|žŽ iS )zReturn information about item.r$   ©r   r7   r8   r$   ©r   r7   r   r   r   Úget_item_infoã   s    ÿzStoreBackendMixin.get_item_infoc              
   C   sr   z\t jj| jf|žŽ }t j |d¡}|  |d¡�$}t | ¡  d¡¡W  5 Q R £ W S Q R X W n   i  Y S X dS )z"Return actual metadata of an item.úmetadata.jsonr6   úutf-8N)	r   r7   r8   r$   r"   ÚjsonÚloadsÚreadÚdecode)r   r7   rM   r   r    r   r   r   Úget_metadataè   s    ,zStoreBackendMixin.get_metadatac                    s\   zJt jjˆ jf|žŽ }ˆ  |¡ t j |d¡}‡ fdd„}ˆ  |||¡ W n   Y nX dS )z Store metadata of a computation.rU   c              	      s2   ˆ   |d¡�}| t | ¡ d¡¡ W 5 Q R X d S )NrA   rV   )r"   ÚwriterW   ÚdumpsÚencode)rH   rI   r    r+   r   r   r   ù   s    z4StoreBackendMixin.store_metadata.<locals>.write_funcN)r   r7   r8   r$   r)   rK   )r   r7   ÚmetadatarM   r   r   r   r+   r   Ústore_metadataò   s    
z StoreBackendMixin.store_metadatac                 C   s   t jj| jf|žŽ }|  |¡S )z,Check cached function is available in store.)r   r7   r8   r$   Zobject_exists©r   r7   Ú	func_pathr   r   r   Úcontains_path  s    zStoreBackendMixin.contains_pathc                 C   s,   t jj| jf|žŽ }|  |¡r(|  |¡ dS )z0Clear all items with a common path in the store.NrO   ra   r   r   r   Ú
clear_path  s    
zStoreBackendMixin.clear_pathc              	   C   sj   t jj| jf|žŽ }|  |¡s(|  |¡ |dk	rft j |d¡}|  |d¡�}| | d¡¡ W 5 Q R X dS )ú&Store the code of the cached function.Núfunc_code.pyrA   rV   )	r   r7   r8   r$   r%   r)   r"   r\   r^   )r   r7   Ú	func_coderb   r   r    r   r   r   Ústore_cached_func_code  s    

z(StoreBackendMixin.store_cached_func_codec              
   C   sf   |dg7 }t jj| jf|žŽ }z4|  |d¡�}| ¡  d¡W  5 Q R £ W S Q R X W n   ‚ Y nX dS )re   rf   r6   rV   N)r   r7   r8   r$   r"   rY   rZ   )r   r7   r   r    r   r   r   Úget_cached_func_code  s    
&z&StoreBackendMixin.get_cached_func_codec                 C   s   dt jj| jf|žŽ iS )z?Return information related to the cached function if it exists.r$   rR   rS   r   r   r   Úget_cached_func_info!  s    z&StoreBackendMixin.get_cached_func_infoc                 C   s   |   | j¡ dS )zClear the whole store content.N)r*   r$   r+   r   r   r   Úclear%  s    zStoreBackendMixin.clearc              	   C   sZ   |   |||¡}|D ]B}| jdkr.td |¡ƒ z|  |j¡ W q tk
rR   Y qX qdS )zX
        Remove the store's oldest files to enforce item, byte, and age limits.
        r3   zDeleting item {0}N)Ú_get_items_to_deleter-   r9   r   r*   r7   ÚOSError)r   Úbytes_limitÚitems_limitÚ	age_limitÚitems_to_deleter?   r   r   r   Úenforce_store_limits)  s      ÿ
z&StoreBackendMixin.enforce_store_limitsc                 C   s  t |tƒrt|ƒ}|  ¡ }tdd„ |D ƒƒ}|dk	r>|| }nd}|dk	rXt|ƒ| }nd}|dk	r†tdd„ |D ƒƒ}tj ¡ | }	nd}	|dkr®|dkr®|	dksª||	kr®g S |j	t
 d¡d� g }
d}d}|D ]H}||krü||krü|	dksö|	|jk rü �q|
 |¡ ||j7 }|d7 }qÐ|
S )	zW
        Get items to delete to keep the store under size, file, & age limits.
        c                 s   s   | ]}|j V  qd S ©N)Úsize©Ú.0r?   r   r   r   Ú	<genexpr>I  s     z9StoreBackendMixin._get_items_to_delete.<locals>.<genexpr>Nr   c                 s   s   | ]}|j V  qd S rs   )Úlast_accessru   r   r   r   rw   V  s     rx   )Úkeyr   )Ú
isinstanceÚstrr   r,   ÚsumÚlenÚminÚdatetimeÚnowÚsortÚoperatorÚ
attrgetterrx   Úappendrt   )r   rn   ro   rp   Úitemsrt   Zto_delete_sizeZto_delete_itemsZ
older_itemÚdeadlinerq   Zsize_so_farZitems_so_farr?   r   r   r   rl   ?  sR    

ÿÿþþÿþýý


z&StoreBackendMixin._get_items_to_deletec                 C   s   t |||ƒ}|  ||¡ dS )z7Writes an object into a file in a concurrency-safe way.N)r   r(   )r   rH   r   r   r   r   r   r   rK   w  s
     ÿz)StoreBackendMixin._concurrency_safe_writec                 C   s   dj | jj| jd�S )z/Printable representation of the store location.z#{class_name}(location="{location}"))Ú
class_namer$   )r   Ú	__class__r   r$   r+   r   r   r   Ú__repr__}  s     ÿzStoreBackendMixin.__repr__)r   N)r   )N)NN)NN)r   r   r   r   r@   rN   rP   rQ   rT   r[   r`   rc   rd   rh   ri   rj   rk   rr   rl   rK   r‰   r   r   r   r   r2   �   s,   	

 


   ÿ
   ÿ
8r2   c                   @   sN   e Zd ZdZeeƒZeejj	ƒZ
eeƒZdd„ Zdd„ Zdd„ Zdd
d„Zd	S )ÚFileSystemStoreBackendz7A StoreBackend used with local or network file systems.c                 C   s&   || j krt|ƒ ntj|dd� dS )zDelete location on store.T)Úignore_errorsN)r$   r	   ÚshutilÚrmtreer#   r   r   r   r*   Š  s    

z%FileSystemStoreBackend.clear_locationc                 C   s   t |ƒ dS )zCreate object location on storeN)r   r#   r   r   r   r)   ‘  s    z&FileSystemStoreBackend.create_locationc           	         sø   g }t  | j¡D ]â\‰ }}t dt j ˆ ¡¡}|rt j ˆ d¡}zt j |¡}W n@ t	k
r�   zt j ˆ ¡}W n t	k
rŠ   Y Y qY nX Y nX t
j
 |¡}z(‡ fdd„|D ƒ}tdd„ |D ƒƒ}W n t	k
rÞ   Y qY nX | tˆ ||ƒ¡ q|S )z7Returns the whole list of items available in the store.z[a-f0-9]{32}r5   c                    s   g | ]}t j ˆ |¡‘qS r   )r   r7   r8   ©rv   Úfn©Údirpathr   r   Ú
<listcomp>ª  s   ÿz4FileSystemStoreBackend.get_items.<locals>.<listcomp>c                 s   s   | ]}t j |¡V  qd S rs   )r   r7   ÚgetsizerŽ   r   r   r   rw   ¬  s   ÿz3FileSystemStoreBackend.get_items.<locals>.<genexpr>)r   Úwalkr$   ÚreÚmatchr7   Úbasenamer8   Úgetatimerm   r   Úfromtimestampr|   r„   r   )	r   r…   Ú_Ú	filenamesZis_cache_hash_dirZoutput_filenamerx   Zfull_filenamesZdirsizer   r�   r   r,   •  s8    
ÿ
ÿÿ

ÿz FileSystemStoreBackend.get_itemsr   Nc                 C   sn   |dkri }|| _ tj | j ¡s*t| j ƒ | dd¡| _| d¡}| jr^|dk	r^tjddd� || _	|| _
dS )zsConfigure the store backend.

        For this backend, valid store options are 'compress' and 'mmap_mode'
        NrB   Fr4   zSCompressed items cannot be memmapped in a filesystem store. Option will be ignored.é   )Ú
stacklevel)r$   r   r7   Úexistsr   ÚgetrB   rE   rF   r4   r-   )r   r$   r-   r.   r4   r   r   r   r/   ¹  s    

þz FileSystemStoreBackend.configure)r   N)r   r   r   r   ÚstaticmethodÚopenr"   r   r7   rž   r%   r   r(   r*   r)   r,   r/   r   r   r   r   rŠ   ƒ  s   $rŠ   )!r   Úpickler   r•   r   Úos.pathr   rW   rŒ   rE   Úcollectionsr‚   r   Úabcr   r   Ú	backportsr   Údiskr   r   r	   Ú r
   Ú
namedtupler   ÚWarningr   r   r   Úobjectr2   rŠ   r   r   r   r   Ú<module>   s0   ÿ
i t