U
    ºmœdõ¥  ã                
   @   s>  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlZddlmZ zddlZW n ek
r¨   dZY nX ddlZddlmZmZmZmZ ddlmZmZ ddlmZmZmZ ddl m!Z!m"Z" dd	l#m$Z$ dd
l%m&Z& ddl%m'Z' ddl%m(Z( ddl%m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ g Z0dZ1e0 2e1¡ e3Z4e0 2e4¡ e5dƒZ6e0 2e6¡ e7dƒZ8e0 2e8¡ e9dƒZ:e0 2e:¡ e;dƒZ<e0 2e<¡ e=dƒZ>e0 2e>¡ dZ?e0 2e?¡ g Z@e0 2e@¡ i ZAe0 2eA¡ eBZCe0 2eC¡ dd„ ZDG dd„ dƒZEG dd„ deFƒZGe0 2eD¡ e0 2eE¡ e0 2eG¡ eEƒ ZHe0 2eH¡ eGƒ ZIe0 2eI¡ edddgƒede0ƒdd„ ƒƒZJdd„ ZKedddeLƒ gƒd d!„ ƒZMeedd"d#dd$d%gƒd&d'„ ƒƒZNed(d)„ ƒZOed*d+„ ƒZPed,d-„ ƒZQed.d/„ ƒZRed0d1„ ƒZSeed2dddgƒd3d4„ ƒƒZTeeedd#d"gƒd5d6„ ƒƒƒZUed7d8„ ƒZVd¶d9d:„ZWed;d<„ ƒZXed=d>„ ƒZYed?d@„ ƒZZedAdBdCgƒdDdE„ ƒZ[edFdGdHdI \dJ¡fdKdL \dJ¡fgƒdMdN„ ƒZ]edOd%dPgƒdQdR„ ƒZ^eeddd$dSgƒedTe+ƒdUdV„ ƒƒƒZ_dWdX„ Z`dYdZ„ Zaed[d\eafd]e`fgƒd^d_„ ƒZbed`dadbdcdddedfdggƒdhdi„ ƒZcedjdk„ ƒZdedldm„ ƒZeedndo„ ƒZfedpdq„ ƒZgedrds„ ƒZhedtdudvdw \e iddx¡dx ¡ jdy¡ gdzd{gd|�ed}dd$d~gƒdd€„ ƒƒZked�ddd‚dƒdi gƒd„d…„ ƒZled†dƒd‡dˆd‰ddŠgƒd‹dŒ„ ƒZmed�ddi gƒdŽd�„ ƒZnedk	�rVG d�d‘„ d‘ejoƒZpd’d“„ ZqG d”d•„ d•ƒZred–d—„ ƒZsd˜d™„ Ztedšd›„ ƒZuedœd�„ ƒZvedždŸ„ ƒZwed d¡„ ƒZxd¢d£„ Zyed¤ddi gƒd¥d¦„ ƒZzd§d¨„ Z{G d©dª„ dªe-ƒZ|G d«d¬„ d¬e-ƒZ}d­d®„ Z~ed¯d°„ ƒZed±d²„ ƒZ€ej�gZ‚ejƒej�k�r$e‚ 2ejƒ¡ eed³e‚ƒd´dµ„ ƒƒZ„dS )·z@Test the numpy pickler as a replacement of the standard pickler.é    N)Úclosing)ÚPath)ÚnpÚ
with_numpyÚwith_lz4Úwithout_lz4)Úwith_memory_profilerÚmemory_used)ÚparametrizeÚraisesÚwarns)Únumpy_pickleÚregister_compressor)Údata)Ú_IO_BUFFER_SIZE)Ú_detect_compressor)Ú#_is_numpy_array_byte_order_mismatch)Ú_ensure_native_byte_order)Ú_COMPRESSORSÚ_LZ4_PREFIXÚCompressorWrapperÚLZ4_NOT_INSTALLED_ERRORÚBinaryZlibFileé   © c                 c   s
   | V  d S ©Nr   )Úxr   r   úV/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/joblib/test/test_numpy_pickle.pyÚ	_functionK   s    r   c                   @   s   e Zd Zdd„ ZdS )Ú_classc                 C   s   d S r   r   ©Úselfr   r   r   Ú_methodP   s    z_class._methodN©Ú__name__Ú
__module__Ú__qualname__r"   r   r   r   r   r   O   s   r   c                   @   s   e Zd Zdd„ ZdS )Ú	_newclassc                 C   s   d S r   r   r    r   r   r   r"   U   s    z_newclass._methodNr#   r   r   r   r   r'   T   s   r'   ÚcompressÚmemberc                 C   sD   |   d¡j}tj|||d� t |¡}|t |¡kr@||ks@t‚d S )Nútest.pkl©r(   )ÚjoinÚstrpathr   ÚdumpÚloadÚcopyÚdeepcopyÚAssertionError)Útmpdirr(   r)   ÚfilenameZ_memberr   r   r   Útest_standard_typese   s
    
r5   c                	   C   s&   t tƒ� t dtƒ ¡ W 5 Q R X d S )NÚfoo)r   Ú
ValueErrorr   r.   Údictr   r   r   r   Útest_value_errorr   s    
r9   Úwrong_compresséÿÿÿÿé
   c              	   C   s<   d  | ¡}ttƒ�}tjdd| d� W 5 Q R X | |¡ d S )Nz%Non valid compress level given: "{0}"Údummyr6   r+   )Úformatr   r7   r   r.   Úmatch)r:   Zexception_msgÚexcinfor   r   r   Útest_compress_level_errorx   s    ÿ
rA   FTé   Úzlibc                 C   sÐ  |   d¡j}tj d¡}| d¡}t|f|jf||f|||gfƒD ]Š\}}tj	|||d�}t
|ƒdksjt‚|d |kszt‚tj |d ¡sŽt‚t |¡}|D ]}	t|	tjƒsœt‚qœtj t |¡t |¡¡ qBtj|d ddtjd	�}tj	|||d�}t
|ƒdk�st‚t |¡}t|ƒtjk	�rBt|d
ƒ�rBt|t|ƒƒ�sBt‚tj ||¡ tƒ }tj	|||d�}t
|ƒdk�sxt‚t |¡}
t|
t|ƒƒ�s–t‚tj |
j|j¡ tj |
j|j¡ tj |
j|j¡ d S )Nr*   r   ©r<   é   r+   r   Úmmapúw+é   )ÚmodeÚshapeÚdtypeZ__array_prepare__)r,   r-   r   ÚrandomÚRandomStateÚrandom_sampleÚ	enumerateÚTr   r.   Úlenr2   ÚosÚpathÚexistsr/   Ú
isinstanceÚndarrayÚtestingÚassert_array_equalÚarrayÚmemmapÚfloat64ÚtypeÚhasattrÚComplexTestObjectÚarray_floatÚ	array_intÚ	array_obj)r3   r(   r4   ÚrndÚaÚindexÚobjÚ	filenamesZobj_ÚitemÚ
obj_loadedr   r   r   Útest_numpy_persistence‚   s:    
(

ÿ
ri   c                 C   sL   t jtd t jd�}|  d¡j}tj||dd� t |¡}t j	 
||¡ d S )Néd   ©rK   r*   Tr+   )r   Úonesr   Úuint8r,   r-   r   r.   r/   rW   rX   )r3   Ú	big_arrayr4   Zarr_reloadedr   r   r   Ú2test_numpy_persistence_bufferred_array_compression¶   s
    
ro   c                 C   sä  t j d¡}| d¡}|  d¡j}t ||¡ tj|dd�}t	|t j
ƒsLt‚|  d¡j}tƒ }t ||¡ tj|dd�}t	|t|ƒƒsŠt‚t	|jt j
ƒsœt‚|jjjrªt‚t	|jt j
ƒs¼t‚|jjjrÊt‚t	|jt j
ƒrÜt‚t j |j|j¡ t j |j|j¡ t j |j|j¡ tj|dd�}|jjj�s0t‚d|jdd…< |jjj�sNt‚d|jdd…< tj|dd�}t j |j|j¡ t j |j|j¡ tj|d	d� |jjj�s¬t‚|jjdk�s¾t‚|jjj�sÎt‚|jjdk�sàt‚d S )
Nr   r<   z	test1.pklÚr©Ú	mmap_modez	test2.pklúr+g      $@rG   )r   rL   rM   rN   r,   r-   r   r.   r/   rU   rZ   r2   r^   r\   r_   ÚflagsZ	writeabler`   ra   rW   rX   rI   )r3   rb   rc   r4   Úbre   rh   Úobj_reloadedr   r   r   Útest_memmap_persistenceÀ   sT    

ÿ
ÿ
ÿ
ÿ
ÿrw   c                 C   s~   t j d¡}| d¡}t jddgtd�}||f}|  d¡j}t 	||¡ tj
|dd�\}}t|t jƒsjt‚t|t jƒrzt‚d S )	Nr   r<   r   ru   rk   r*   rp   rq   )r   rL   rM   rN   rY   Úobjectr,   r-   r   r.   r/   rU   rZ   r2   )r3   rb   rc   ru   Ú	constructr4   Za_cloneZb_cloner   r   r   Ú$test_memmap_persistence_mixed_dtypesò   s    
rz   c                 C   s`   t j d¡}| d¡}t j |d¡}|  d¡j}t 	||¡ tj
|dd�}t|t jjƒs\t‚d S )Nr   r<   g      à?r*   rp   rq   )r   rL   rM   rN   ÚmaZmasked_greaterr,   r-   r   r.   r/   rU   Zmasked_arrayr2   )r3   rb   rc   r4   ru   r   r   r   Útest_masked_array_persistence  s    
r|   c              	   C   sª   t j d¡}| d¡}|  d¡j}tj||dd� tt	ƒ�}tj
|dd� W 5 Q R X d d	d
„ |D ƒ¡}dd
„ |D ƒ}t|ƒdksŠt|ƒ‚t|d ƒd|› d�ks¦t‚d S )Nr   r<   r*   r   r+   rs   rq   Ú
c                 S   s   g | ]}t |ƒ‘qS r   )Ústr©Ú.0Úwr   r   r   Ú
<listcomp>  s     z3test_compress_mmap_mode_warning.<locals>.<listcomp>c                 S   s   g | ]
}|j ‘qS r   )Úmessager   r   r   r   r‚     s     z6mmap_mode "r+" is not compatible with compressed file z. "r+" flag will be ignored.)r   rL   rM   rN   r,   r-   r   r.   r   ÚUserWarningr/   rQ   r2   r~   )r3   rb   rc   Zthis_filenameÚwarninfoZ	debug_msgr   r   r   Útest_compress_mmap_mode_warning  s    



ÿÿr†   Ú
cache_sizec              	   C   s¨   |   d¡j}tj d¡}| d¡}t d¡ tjdd��}t	j
|||d� W 5 Q R X |d k	r`dnd}t|ƒ|kstt‚|D ]*}|jtksŠt‚t|jƒd	 |¡ksxt‚qxd S )
Nr*   r   rD   ÚalwaysT©Úrecord)r‡   r   zzPlease do not set 'cache_size' in joblib.dump, this parameter has no effect and will be removed. You used 'cache_size={0}')r,   r-   r   rL   rM   rN   ÚwarningsÚsimplefilterÚcatch_warningsr   r.   rQ   r2   ÚcategoryÚDeprecationWarningr~   rƒ   r>   )r3   r‡   r4   rb   rc   r…   Úexpected_nb_warningsr�   r   r   r   Útest_cache_size_warning%  s    

þÿr‘   c                 C   sª   |   d¡j}t d¡}tjdtdƒ tjd�}||fD ]n}|jd }|ttj 	dd¡ƒ }t
tj|||d�}td	 }	||	ks€t‚t
tj|ƒ}d
t }
|||
 k s6t‚q6d S )Nr*   ©r<   r<   rj   g    €„.A©rJ   rK   r   éè  r+   g�íµ ÷Æ0@é    )r,   r-   r   rl   Úintrm   Únbytesr~   rL   Úrandintr	   r   r.   r   r2   r/   )r3   r(   r4   Zsmall_arrayrn   re   ÚsizeZobj_filenameZmem_usedZwrite_buf_sizeZread_buf_sizer   r   r   Útest_memory_usage:  s     

  ÿrš   c              
   C   s  t jdt  d¡d�t jdt  d¡d�t jdt  d¡d�t jdt  d¡d�t jdddd	d
œgdd�t jdt jd� ¡ dg}|  d¡j}tj	||dd�}t
|ƒdks¤t‚t |¡}t||ƒD ]H\}}t|t jƒrôt|ƒ}|j|jksät‚t j ||¡ q¸||ks¸t‚q¸d S )Né   ú<i8rk   ú>i8ú<f8ú>f8r   ÚabcrE   ©rc   ru   ÚOé   õ   C'est l'Ã©tÃ© !ztemp.pkl.gzr+   )r   ÚarangerK   rY   rm   Útobytesr,   r-   r   r.   rQ   r2   r/   ÚziprU   rV   r   rW   Úassert_equal)r3   Úexpected_listÚfnameZdumped_filenamesÚresult_listÚresultÚexpectedr   r   r   Ú$test_compressed_pickle_dump_and_loadU  s$    ú
r®   c              
   C   s®  t  d| ¡}t| d¡ƒ}dddœ}| dd¡}| |d¡}||k�rP�zptjdd��.}t d	¡ tjd
ddd� t	j
| |d�}	W 5 Q R X tj | ¡}
d|
ks¦d|
krªdnd}t  d|
¡rÆ|dk	rÆdnd}|| }t|ƒ|ksât‚dd„ |D ƒ}dd„ |D ƒ}|D ] }t|jƒd | ¡k�st‚�q|D ]0}t  | ¡}t  d|› d�t|jƒ¡�s(t‚�q(t|	|ƒD ]P\}}t|tjƒ�r¤t|ƒ}|j|jk�s”t‚tj ||¡ n||k�sdt‚�qdW n” tk
�rL } zt|dk�rþt|tƒ�sæt‚d}|t|ƒk�s<t‚n>|  d¡�r:t j!d �r:t|tƒ�s&t‚t"t|ƒk�s<t‚n‚ W 5 d}~X Y nX nZzt	 
| ¡ tdƒ‚W nB tk
�r¨ } z"d |¡}|t|j!ƒk�s˜t‚W 5 d}~X Y nX dS )z�Helper function to test joblib pickle content.

    Note: currently only pickles containing an iterable are supported
    by this function.
    z.+py(\d)(\d).+r   rE   rB   )rE   rB   rH   Tr‰   rˆ   ÚignoreÚnumpyz"The compiler package is deprecated)Úmodulerƒ   rq   z_0.9z_0.8.4r   z_0.1.+.pkl$Nc                 S   s   g | ]}t |jtƒr|‘qS r   )Ú
issubclassrŽ   r�   r   r   r   r   r‚   �  s     ÿ z!_check_pickle.<locals>.<listcomp>c                 S   s   g | ]}t |jtƒr|‘qS r   )r²   rŽ   r„   r   r   r   r   r‚   �  s     ÿ zkThe file '{0}' has been generated with a joblib version less than 0.10. Please regenerate this pickle file.zmemmapped.+z.+segmentation faultzPYou may be trying to read with python 3 a joblib pickle generated with python 2.ú.lz4z>Numpy pickle loading should have raised a ValueError exceptionz unsupported pickle protocol: {0})#Úrer?   r–   ÚgroupÚgetr‹   r�   rŒ   Úfilterwarningsr   r/   rR   rS   ÚbasenameÚsearchrQ   r2   r~   rƒ   r>   Úescaper§   rU   r   rV   r   rK   rW   r¨   Ú	Exceptionr7   Úendswithr   Úargsr   )r4   r©   rr   Zversion_matchZpy_version_used_for_writingZ%py_version_to_default_pickle_protocolZpickle_reading_protocolZpickle_writing_protocolr…   r«   Úfilename_baseZ expected_nb_deprecation_warningsZexpected_nb_user_warningsr�   Zdeprecation_warningsZuser_warningsr�   Zescaped_filenamer¬   r­   Úexcrƒ   Úer   r   r   Ú_check_picklem  s–    
 ÿ

 þÿÿÿ
ÿþþÿÿÿþÿ

þ

ÿrÁ   c               	      sÈ   t jdt  d¡d�t jdt  d¡d�t jdddddœgd	d�t jd
t jd� ¡ t jdddgt  d¡d�dg} tj 	tj 
tj¡¡‰d‰ td k	r–ˆ d7 ‰ ‡ ‡fdd„t ˆ¡D ƒ}|D ]}t|| ƒ q´d S )Nr›   rœ   rk   rž   r   r    rE   r¡   r¢   r£   r   r¤   )ú.pklú.gzz.gzipú.bz2Úlz4)ú.xzú.lzmac                    s0   g | ](‰ t ‡ fd d„ˆD ƒƒrtj ˆˆ ¡‘qS )c                 3   s   | ]}ˆ   |¡V  qd S r   )r¼   )r€   Úext©Úfnr   r   Ú	<genexpr>Û  s     zGtest_joblib_pickle_across_python_versions.<locals>.<listcomp>.<genexpr>)ÚanyrR   rS   r,   )r€   ©Zpickle_extensionsÚtest_data_dirrÉ   r   r‚   Ù  s   þz=test_joblib_pickle_across_python_versions.<locals>.<listcomp>)r   r¥   rK   rY   rm   r¦   ÚmatrixrR   rS   ÚdirnameÚabspathr   Ú__file__ÚlzmaÚlistdirrÁ   ©r©   Zpickle_filenamesrª   r   rÍ   r   Ú)test_joblib_pickle_across_python_versionsÀ  s     øÿrÖ   c               	      s¶   t jdt  d¡d�t jdt  d¡d�t jdddddœgd	d�t jd
t jd� ¡ t jdddgt  d¡d�dg} tj 	tj 
tj¡¡‰ ‡ fdd„t ˆ ¡D ƒ}|D ]}t|| dd� qžd S )Nr›   rœ   rk   rž   r   r    rE   r¡   r¢   r£   r   r¤   c                    s$   g | ]}|  d ¡rtj ˆ |¡‘qS )rÂ   )r¼   rR   rS   r,   )r€   rÊ   ©rÎ   r   r   r‚   ï  s    
ÿzGtest_joblib_pickle_across_python_versions_with_mmap.<locals>.<listcomp>rp   rq   )r   r¥   rK   rY   rm   r¦   rÏ   rR   rS   rÐ   rÑ   r   rÒ   rÔ   rÁ   rÕ   r   r×   r   Ú3test_joblib_pickle_across_python_versions_with_mmapá  s    ø

þrØ   c                  C   sf  t jddgddgd�t jdt  d¡d�t jdt  d¡d�g} | D ]j}tjd	kr\t|ƒrht‚nt|ƒsht‚t|ƒ}|jj	rš|jj	 
¡ D ]}|d
 jdk q„q@|jjdks@t‚q@t jddgddgd�t jdt  d¡d�t jdt  d¡d�g}|D ]t}tjdk�rt|ƒ�rt‚nt|ƒ�st‚t|ƒ}|jj	�rP|jj	 
¡ D ]}|d
 jdk �q8qì|jjdksìt‚qìd S )N)r   g       @)rB   g      @)Ú r�   )rÙ   rŸ   rk   rB   r�   rŸ   Úbigr   ú=)rÙ   rœ   )rÙ   rž   rœ   rž   Úlittle)r   rY   r¥   rK   ÚsysÚ	byteorderr   r2   r   ÚfieldsÚvalues)Z	be_arraysrY   Z	convertedÚfZ	le_arraysr   r   r   Ú.test_numpy_array_byte_order_mismatch_detectionö  s<    
ÿý

ÿý
râ   Úcompress_tuple)rC   rB   ©ÚgziprB   c              	   C   sJ   |   d¡j}tjd||d� t|dƒ�}t|ƒ|d ks<t‚W 5 Q R X d S )Nr*   r=   r+   Úrbr   ©r,   r-   r   r.   Úopenr   r2   )r3   rã   r4   rá   r   r   r   Útest_compress_tuple_argument  s    ÿré   zcompress_tuple,message))rC   rB   Úextraz9Compress argument tuple should contain exactly 2 elements)ÚwrongrB   z(Non valid compression method given: "{}"rë   )rC   rë   z$Non valid compress level given: "{}"c              	   C   s>   |   d¡j}ttƒ�}tjd||d� W 5 Q R X | |¡ d S )Nr*   r=   r+   )r,   r-   r   r7   r   r.   r?   )r3   rã   rƒ   r4   r@   r   r   r   Ú&test_compress_tuple_argument_exception*  s    
rì   Úcompress_stringrå   c              	   C   sF   |   d¡j}tjd||d� t|dƒ�}t|ƒ|ks8t‚W 5 Q R X d S )Nr*   r=   r+   ræ   rç   )r3   rí   r4   rá   r   r   r   Útest_compress_string_argument9  s    ÿrî   é   Úcmethodc           	   
   C   s  |   d¡j}tjddd�tdƒdddœg d	i d
df}|dkrPtd krPt d¡ n|dkrltj	d
 rlt d¡ |d | }|D ]‚}t
j||||fd� t|dƒ�}t|ƒ|ks°t‚W 5 Q R X t
 |¡}t|t|ƒƒsÖt‚t|tjƒròtj ||¡ q|||ks|t‚q|d S )Nr*   )rj   rj   Zf8r“   r<   r   ru   )rc   rE   r   r   g      ð?©rÓ   Úxzzlzma is support not availablerÅ   zlz4 is not installed.Ú.r+   ræ   )r,   r-   r   rl   ÚrangerÓ   ÚpytestÚskipr   r½   r   r.   rè   r   r2   r/   rU   r\   rV   rW   rX   )	r3   r(   rð   r4   ÚobjectsZdump_filenamere   rá   rv   r   r   r   Útest_joblib_compression_formatsD  s0         þ

rø   c              	   C   sH   t t | d¡ƒ�}| ¡ }W 5 Q R X t|dƒ�}| |¡ W 5 Q R X dS )zDecompress a gzip file.ræ   ÚwbN)r   rå   ÚGzipFileÚreadrè   Úwrite©Zsource_filenameZtarget_filenameÚfoÚbufr   r   r   Ú_gzip_file_decompressd  s    r   c              	   C   sH   t | dƒ�}t | ¡ ¡}W 5 Q R X t |dƒ�}| |¡ W 5 Q R X dS )zDecompress a zlib file.ræ   rù   N)rè   rC   Ú
decompressrû   rü   rý   r   r   r   Ú_zlib_file_decompressm  s    r  zextension,decompressú.zrÃ   c                 C   sH   d}|   d¡j}|| }t ||¡ |||ƒ t |¡}||ksDt‚d S )Nza string to persistr*   )r,   r-   r   r.   r/   r2   )r3   Ú	extensionr  re   Zfilename_rawZfilename_compressedrv   r   r   r   Ú'test_load_externally_decompressed_filesv  s    

r  zextension,cmethod)r  rC   )rÃ   rå   )rÄ   Úbz2)rÇ   rÓ   )rÆ   rò   )rÂ   únot-compressed)rÙ   r  c              	   C   s�   |dkrt d krt d¡ |  d¡j}d}|| }t ||¡ t|dƒ�}t|ƒ|ksZt	‚W 5 Q R X t 
|¡}t|t|ƒƒs€t	‚||ksŒt	‚d S )Nrñ   zlzma is missingr*   zobject to dumpræ   )rÓ   rõ   rö   r,   r-   r   r.   rè   r   r2   r/   rU   r\   )r3   r  rð   r4   re   Z
dump_fnamerá   rv   r   r   r   Ú%test_compression_using_file_extension‹  s    

r  c           	      C   sø   t j d¡dg}tjtjg}td k	r0|tjg7 }|  d¡j	}|D ]²}|D ]¨}||dƒ�}t
 ||¡ W 5 Q R X ||dƒ�}t
 |¡}W 5 Q R X t|dƒ�}t
 |¡}W 5 Q R X t|t jƒrØt j ||¡ t j ||¡ qH||ksät‚||ksHt‚qHq@d S )Nr’   ú	some datar*   rù   ræ   )r   rL   r  ÚBZ2Filerå   rú   rÓ   ÚLZMAFiler,   r-   r   r.   r/   rè   rU   rV   rW   rX   r2   )	r3   ÚobjsZfobjsr4   re   Zfobjrá   rv   Zobj_reloaded_2r   r   r   Útest_file_handle_persistence§  s$    r  c                  C   sd   t j d¡dg} | D ]J}t ¡ }t ||¡ t |¡}t|t jƒrRt j	 
||¡ q||kst‚qd S )Nr’   r	  )r   rL   ÚioÚBytesIOr   r.   r/   rU   rV   rW   rX   r2   )r  re   rá   rv   r   r   r   Útest_in_memory_persistenceÆ  s    
r  c              	   C   sp   t j d¡}|  d¡j}t|dƒ�}t ||¡ W 5 Q R X t|dƒ�}tj|dd�}W 5 Q R X t j 	||¡ d S )Nr’   r*   rù   ræ   rs   rq   )
r   rL   r,   r-   rè   r   r.   r/   rW   rX   )r3   re   r4   rá   rv   r   r   r   Ú!test_file_handle_persistence_mmapÓ  s    r  c              
   C   s°   t j d¡}|  d¡j}t|dƒ�}tj||dd� W 5 Q R X tt 	|d¡ƒ�X}t
tƒ�}tj|dd� W 5 Q R X t|ƒd	ks‚t‚t|d
 jƒd|ddœ ks¢t‚W 5 Q R X d S )Nr’   r*   rù   rä   r+   ræ   rs   rq   r   r   zP"%(fileobj)r" is not a raw file, mmap_mode "%(mmap_mode)s" flag will be ignored.)Úfileobjrr   )r   rL   r,   r-   rè   r   r.   r   rå   rú   r   r„   r/   rQ   r2   r~   rƒ   )r3   re   r4   rá   r…   r   r   r   Ú,test_file_handle_persistence_compressed_mmapá  s    
ÿÿr  c               	   C   st   t j d¡} t ¡ }t | |¡ ttƒ�}tj|dd� W 5 Q R X t	|ƒdksRt
‚t|d jƒdddi kspt
‚d S )Nr’   rs   rq   r   r   zuIn memory persistence is not compatible with mmap_mode "%(mmap_mode)s" flag passed. mmap_mode option will be ignored.rr   )r   rL   r  r  r   r.   r   r„   r/   rQ   r2   r~   rƒ   )re   rÿ   r…   r   r   r   Ú+test_file_handle_persistence_in_memory_mmapò  s    
þÿr  r   s   a little data as bytes.i'  z{}r”   zlatin-1za little data as bytes.za large data as bytes.)ZidsÚcompress_levelé	   c                 C   sD  |   d¡j}t|dƒ�¬}t|d|d��l}| ¡ s4t‚| |¡ | ¡ | ¡ ksRt‚tt	j
ƒ� | ¡  W 5 Q R X tt	j
ƒ� | ¡  W 5 Q R X W 5 Q R X |js¢t‚ttƒ� | ¡  W 5 Q R X W 5 Q R X t|dƒ�®}t|ƒ�Ž}| ¡ sêt‚| ¡ söt‚| ¡ | ¡ k�st‚| ¡ |k�st‚tt	j
ƒ� | ¡  W 5 Q R X | ¡ �sJt‚| d¡ | ¡ dk�sft‚W 5 Q R X |j�s|t‚W 5 Q R X t|d|d��}| ¡ �s¤t‚| |¡ W 5 Q R X t|dƒ�&}| ¡ |k�sÖt‚| ¡ �sät‚W 5 Q R X t|d|d�}| ¡ �s
t‚| |¡ | ¡  t|dƒ}| ¡ |k�s8t‚| ¡  d S )Nr*   rù   ©Úcompresslevelræ   r   )r,   r-   rè   r   Úwritabler2   rü   Úfilenor   r  ÚUnsupportedOperationÚ_check_can_readÚ_check_can_seekÚclosedr7   Ú_check_not_closedÚreadableÚseekablerû   Ú_check_can_writeÚseekÚtellÚclose)r3   r   r  r4   rá   Zfzr   r   r   Útest_binary_zlibfile  sX    ÿ




ÿ

r&  Ú	bad_valueé   rc   c              	   C   sL   |   d¡j}ttƒ�}t|d|d� W 5 Q R X t d |¡¡}| |¡ d S )Nr*   rù   r  zS'compresslevel' must be an integer between 1 and 9. You provided 'compresslevel={}')	r,   r-   r   r7   r   r´   rº   r>   r?   )r3   r'  r4   r@   Úpatternr   r   r   Ú+test_binary_zlibfile_bad_compression_levels<  s    
ÿr*  Úbad_moder   rp   r�   rE   c              	   C   s8   |   d¡j}ttƒ�}t||ƒ W 5 Q R X | d¡ d S )Nr*   zInvalid mode)r,   r-   r   r7   r   r?   )r3   r+  r4   r@   r   r   r   Ú"test_binary_zlibfile_invalid_modesF  s    
r,  Úbad_filec              	   C   s,   t tƒ�}t| dƒ W 5 Q R X | d¡ d S )Nræ   z1filename must be a str or bytes object, or a file)r   Ú	TypeErrorr   r?   )r-  r@   r   r   r   Ú*test_binary_zlibfile_invalid_filename_typeN  s    
r/  c                   @   s   e Zd Zdd„ ZdS )ÚSubArrayc                 C   s   t t | ¡ffS r   )Ú_load_sub_arrayr   Zasarrayr    r   r   r   Ú
__reduce__[  s    zSubArray.__reduce__N)r$   r%   r&   r2  r   r   r   r   r0  Y  s   r0  c                 C   s   t | jƒ}| |d d …< |S r   )r0  rJ   )ZarrÚdr   r   r   r1  ^  s    
r1  c                   @   s   e Zd ZdZdd„ ZdS )r^   z7A complex object containing numpy arrays as attributes.c                 C   s:   t jddd�| _t jddd�| _t jdddgdd�| _d S )	Nrj   r[   rk   Zint32rc   r<   g      4@rx   )r   r¥   r_   rl   r`   rY   ra   r    r   r   r   Ú__init__f  s    zComplexTestObject.__init__N)r$   r%   r&   Ú__doc__r4  r   r   r   r   r^   c  s   r^   c                 C   sJ   |   d¡j}tdƒ}t ||¡ t |¡}t|tƒs8t‚tj	 
||¡ d S )Nr*   )r<   )r,   r-   r0  r   r.   r/   rU   r2   r   rW   rX   )r3   r4   rc   Úcr   r   r   Útest_numpy_subclassl  s    
r7  c                 C   sX   |   d¡j}d}t |t|ƒ¡ t |¡|ks2t‚t ||¡ t t|ƒ¡|ksTt‚d S )Nr*   é{   )r,   r-   r   r.   r   r/   r2   )r3   r4   Úvaluer   r   r   Útest_pathlibv  s    r:  c                 C   s”   |   d¡j}t ddgddgg¡dd … tjddd�d d …d d…d d …f fD ]@}|jjr^t‚|jjrjt‚t	 
||¡ t	 |¡}tj ||¡ qNd S )	Nr*   r   rE   rB   rH   )r<   é2   é   ÚF)Úorder)r,   r-   r   Zasfortranarrayrl   rt   Úc_contiguousr2   Úf_contiguousr   r.   r/   rW   rX   )r3   r4   rY   Úarray_reloadedr   r   r   Ú"test_non_contiguous_array_pickling  s    "û
rB  c                 C   sD   |   d¡j}t d¡}tj||tjd� t |¡}tj	 
||¡ d S )Nr*   r<   ©Úprotocol)r,   r-   r   Úzerosr   r.   ÚpickleÚHIGHEST_PROTOCOLr/   rW   rX   )r3   r4   Ú
test_arrayrA  r   r   r   Útest_pickle_highest_protocol�  s
    

rI  c            
   
   C   sø   t  d¡} d}t tjtj¡}| |¡ | d¡ t |¡�²}| ¡ \}}| 	d¡�}t
 | |¡ W 5 Q R X | 	d¡�}t
 |¡}W 5 Q R X t j || ¡ t ¡ }	t
 | |	¡ | |	 ¡ ¡ | 	d¡�}t
 |¡}W 5 Q R X t j || ¡ W 5 Q R X d S )Nr<   )Ú	localhosti90  r   rù   ræ   )r   r¥   ÚsocketÚAF_INETÚSOCK_STREAMÚbindÚlistenÚcreate_connectionÚacceptÚmakefiler   r.   r/   rW   rX   r  r  ÚsendÚgetvalue)
rH  Z_ADDRZlistenerÚclientÚserverÚclient_addrZsfÚcfrA  Zbytes_to_sendr   r   r   Útest_pickle_in_socketŸ  s$    


rY  c                 C   s€   |   d¡j}tj}tj|dd�tj|dd�g}t ||¡ tj	|dd�}t
|d tjƒs\t‚|d j|ksnt‚tj ||¡ d S )Nú	test.mmaprm   rk   rp   rq   r   )r,   r-   rF   ÚALLOCATIONGRANULARITYr   rE  rl   r   r.   r/   rU   rZ   r2   ÚoffsetrW   rX   )r3   rª   r™   re   Zmemmapsr   r   r   Ú test_load_memmap_with_big_offset¿  s    r]  c                    sn   d}d‰G dd„ dt jƒ‰ G ‡ ‡fdd„dtƒ}t||ƒ ƒ t| jˆ ksNt‚t| jˆks`t‚t |¡ d S )Nú	test-nameztest-prefixc                   @   s   e Zd ZdS )z:test_register_compressor.<locals>.BinaryCompressorTestFileN©r$   r%   r&   r   r   r   r   ÚBinaryCompressorTestFileÔ  s   r`  c                       s   e Zd Z‡ ‡fdd„ZdS )z=test_register_compressor.<locals>.BinaryCompressorTestWrapperc                    s   t j| ˆ ˆd� d S )N©re   Úprefix©r   r4  r    ©r`  Zcompressor_prefixr   r   r4  Ù  s    ÿzFtest_register_compressor.<locals>.BinaryCompressorTestWrapper.__init__N©r$   r%   r&   r4  r   rd  r   r   ÚBinaryCompressorTestWrapper×  s   rf  )	r  ÚBufferedIOBaser   r   r   Úfileobj_factoryr2   rb  Úpop)r3   Úcompressor_namerf  r   rd  r   Útest_register_compressorÏ  s    ÿrk  Úinvalid_namec              	   C   s,   t tƒ�}t| d ƒ W 5 Q R X | d¡ d S )Nz"Compressor name should be a string)r   r7   r   r?   )rl  r@   r   r   r   Ú%test_register_compressor_invalid_nameè  s    
rm  c               	      sP   G dd„ dƒ‰ G ‡ fdd„dt ƒ} ttƒ�}td| ƒ ƒ W 5 Q R X | d¡ d S )Nc                   @   s   e Zd ZdS )zCtest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectNr_  r   r   r   r   ÚInvalidFileObjectó  s   rn  c                       s   e Zd Z‡ fdd„ZdS )zJtest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectWrapperc                    s   t j| ˆ dd� d S ©Ns   prefixra  rc  r    ©rn  r   r   r4  ÷  s    ÿzStest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectWrapper.__init__Nre  r   rp  r   r   ÚInvalidFileObjectWrapperö  s   rq  ÚinvalidzQCompressor 'fileobj_factory' attribute should implement the file object interface)r   r   r7   r   r?   )rq  r@   r   rp  r   Ú(test_register_compressor_invalid_fileobjð  s
    
rs  c                   @   s   e Zd Zdd„ ZdS )ÚAnotherZlibCompressorWrapperc                 C   s   t j| tdd� d S ro  )r   r4  r   r    r   r   r   r4    s    z%AnotherZlibCompressorWrapper.__init__Nre  r   r   r   r   rt    s   rt  c                   @   s   e Zd Zdd„ ZdS )Ú StandardLibGzipCompressorWrapperc                 C   s   t j| tjdd� d S ro  )r   r4  rå   rú   r    r   r   r   r4  
  s    z)StandardLibGzipCompressorWrapper.__init__Nre  r   r   r   r   ru    s   ru  c               	   C   s~   d} t | tƒ ƒ ttƒ�}t | tƒ ƒ W 5 Q R X | d | ¡¡ t | tƒ dd� | tks\t‚t|  j	t
jkspt‚t | ¡ d S )Nr^  z#Compressor '{}' already registered.T)Úforce)r   rt  r   r7   ru  r?   r>   r   r2   rh  rå   rú   ri  )rj  r@   r   r   r   Ú+test_register_compressor_already_registered  s    
ÿÿÿrw  c              	   C   sÞ   dd l }d}|tkst‚t| j|jjks.t‚|  d¡j}d}tj	|||d� t
|dƒ�}| ttƒ¡tkspt‚W 5 Q R X t |¡|ksŒt‚t 	||d ¡ t
|dƒ�}| ttƒ¡tks¾t‚W 5 Q R X t |¡|ksÚt‚d S )Nr   rÅ   r*   ú	test datar+   ræ   r³   )Z	lz4.framer   r2   rh  ÚframeZLZ4FrameFiler,   r-   r   r.   rè   rû   rQ   r   r/   )r3   rÅ   Z
compressorrª   r   rá   r   r   r   Útest_lz4_compression&  s      rz  c              	   C   st   |   d¡j}d}t}ttƒ�}tj||dd� W 5 Q R X | |¡ ttƒ�}t ||d ¡ W 5 Q R X | |¡ d S )Nz
test.nolz4rx  rÅ   r+   r³   )r,   r-   r   r   r7   r   r.   r?   )r3   rª   r   Úmsgr@   r   r   r   Ú test_lz4_compression_without_lz4=  s    


r|  rD  c                 C   s4  |   d¡j}tj d¡}tj|||d� tj|dd�}t|tj	ƒsFt
‚tj ||¡ |jjtj dksjt
‚|jjsvt
‚tj d¡tj d¡tj d¡tj d¡g}|   d¡j}tj|||d� tj|dd�}t|ƒD ]N\}}t|tj	ƒsìt
‚tj || |¡ |jjtj dk�st
‚|jjsÔt
‚qÔtjdtjd�tjd	tjd�tjd
tjd�tjdtjd�tjdtjd�tjdtjd�tjdtjd�tjdtjd�tjdtjd�dœ	}|   d¡j}tj|||d� tj|dd�}	|	 ¡ D ]T\}
}t|tj	ƒ�sôt
‚tj ||
 |¡ |jjtj dk�st
‚|jj�sÚt
‚�qÚd S )NrZ  rE   rC  rp   rq   r   z
test1.mmaprk   rB   r›   é   é   é   é   é   é   )	Za0Za1Za2Úa3Za4Za5Za6Za7Za8z
test2.mmap)r,   r-   r   rL   Zrandnr   r.   r/   rU   rZ   r2   rW   rX   Úctypesr   ZNUMPY_ARRAY_ALIGNMENT_BYTESrt   ZalignedrO   r¥   rm   Úitems)r3   rD  rª   rc   rZ   Z
array_listZ
l_reloadedÚidxZ
array_dictZ
d_reloadedÚkeyr   r   r   Útest_memmap_alignment_paddingQ  sT    ÿ
 

 
þÿ÷ÿrˆ  )N)…r5  r0   rR   rL   r´   r  rÝ   r‹   rå   rC   r  rF  rK  Ú
contextlibr   rF   Úpathlibr   rÓ   ÚImportErrorrõ   Zjoblib.test.commonr   r   r   r   r   r	   Zjoblib.testingr
   r   r   Zjoblibr   r   Zjoblib.testr   Zjoblib.numpy_pickle_utilsr   r   r   r   Zjoblib.compressorr   r   r   r   r   ZtypelistZ_noneÚappendr\   Ú_typeÚboolZ_boolr–   Ú_intÚfloatZ_floatÚcomplexZ_complexr~   Ú_stringÚ_tupleZ_listÚ_dictrQ   Z_builtinr   r   rx   r'   Z	_instanceÚ_objectr5   r9   r8   rA   ri   ro   rw   rz   r|   r†   r‘   rš   r®   rÁ   rÖ   rØ   râ   ré   r>   rì   rî   rø   r   r  r  r  r  r  r  r  r  r˜   Úencoder&  r*  r,  r/  rV   r0  r1  r^   r7  r:  rB  rI  rY  r]  rk  rm  rs  rt  ru  rw  rz  r|  ÚDEFAULT_PROTOCOLÚ	protocolsrG  rˆ  r   r   r   r   Ú<module>   s†  
	
















	2
	
1




S
 

(

ÿÿüÿ


		ÿÿ
ùþ






ÿÿþü4
	


	
		






