U
    ¾mœd�F  ã                   @   s2  d Z ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddl	Z	ddl
mZ ddlZddlZddlZddlZddlmZ ddlZddlZddlZddlmZmZ ddlmZ dd	lmZ ddlm  mZ  ddl!Z"dd
l!m#Z#m$Z$m%Z% ddl&m'Z( ddl)m  m*Z+ ddl,m-Z-m.Z. ej/dd�dd„ ƒZ0dd„ Z1e	j2 3e	j2 4e5¡dd¡Z6e e	j2 3e6dd¡¡Z7ej/e7d�dd„ ƒZ8ej9 :ddde;dƒe<dƒe =d¡eddddgƒe<dƒ >dd¡e<dƒ >dd¡ddd… e ?d ¡j@d!d"d#�e ?d ¡j@d!d$d#�e ?d ¡j@d!d"d#�dd…ddd…f g¡d%d&„ ƒZAd'd(„ ZBd)d*„ ZCd+d,„ ZDej9 :d-ejEeCd.d/�ejEe"jFd0d/�ejEeje"jFejGd1�d2d/�ejEeje"jFd3d1�d4d/�ejEeje"jFd5d1�d6d/�g¡ej9 :d7e"jFeCg¡d8d9„ ƒƒZHd:d;„ ZId<d=„ ZJej/d>d?„ ƒZKG d@dA„ dAƒZLG dBdC„ dCƒZMej9 :dDdEgdFe#dGdHdIgƒfdJe"jN OdGdHdIgdKdd"gg¡fg¡dLdM„ ƒZPdNdO„ ZQej9 :dPdQdRdSg¡dTdU„ ƒZRe  SdV¡dWdX„ ƒZTG dYdZ„ dZejUƒZVd[d\„ ZWd]d^„ ZXd_d`„ ZYdadb„ ZZej9 :dcdddedce dfdgdg¡dhg¡didj„ ƒZ[dkdl„ Z\dmdn„ Z]e j^dodp„ ƒZ_ej9 :dqej`ejGg¡drds„ ƒZadtdu„ ZbdS )va<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
é    )ÚarrayN)Úpartial)ÚPath)Úcatch_warnings)Úget_lzma_fileÚis_platform_little_endian)Úimport_optional_dependency)Úflatten_buffer)ÚIndexÚSeriesÚperiod_range)ÚDayÚMonthEndÚmodule)Úscopec               
   C   s0   ddl m}  tƒ � | ƒ W  5 Q R £ S Q R X d S )Nr   ©Úcreate_pickle_data)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r   © r   úT/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pandas/tests/io/test_pickle.pyÚcurrent_pickle_data8   s    r   c                 C   s†   t |tƒrt || ¡ d S | d¡r2t | |¡ nP|dkrb|tjkrT| tjks`t‚q‚| |ks‚t‚n t	td|› d�tj
ƒ}|| |ƒ d S )NZsp_Ú	timestampZassert_Z_equal)Ú
isinstancer
   ÚtmÚassert_index_equalÚ
startswithZassert_equalÚpdZNaTÚAssertionErrorÚgetattrZassert_almost_equal)ÚresultÚexpectedÚtypZ
comparatorr   r   r   Úcompare_elementD   s    


r!   ÚdataÚlegacy_pickleÚ*z*.pickle)Úparamsc                 C   s
   || j ƒS ©N)Úparam)ÚrequestÚdatapathr   r   r   r#   Y   s    s   123s   123456ÚIé   é   é   ÚB)r-   r,   é   )r-   é   ÚC)ÚorderÚFc                 C   s„   t | ƒ}t| ƒ d¡}||ks"t‚t| ttfƒr>|| ks€t‚nBt|tƒr€|jdksVt‚|jdksdt‚|j	snt‚|j
|jfks€t‚d S )NÚAr+   r.   )r	   Ú
memoryviewÚtobytesr   r   ÚbytesÚ	bytearrayÚndimÚformatÚ
contiguousÚshapeÚnbytes)r"   r   r   r   r   r   Útest_flatten_bufferc   s    

r>   c           	      C   sø  t ƒ st d¡ t | ¡}| ¡ D �]Î\}}| ¡ D �]º\}}|| | }|dk�r2|dk�r2t ||¡ |jj	|jj	ks~t
‚|jj	jrŒt
‚t |dk|dk¡ |jj	}|tdƒ tdƒksÀt
‚|tjdd� }t|tjƒsàt
‚|tjddd�ksöt
‚|tjdd	� }t|tjƒ�st
‚|tjddd
�k�sðt
‚q4|dk�r |dk�r t ||¡ t|j	tƒ�sdt
‚|j	tƒ k�svt
‚|jdk�s†t
‚t | d¡| d¡¡ q4|dk�rÂ|dk�rÂt ||¡ q4|dk�rä|dk�rät ||¡ q4t|||ƒ q4q"d S )Nz"known failure on non-little endianZseriesÚtsr   r+   r,   )Úhours)Údaysr@   )Únanoseconds)rA   rB   ÚindexZperiodÚM)Zdt_tzÚcatÚframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   ÚpytestÚskipr   Úread_pickleÚitemsr   Úassert_series_equalrC   Úfreqr   Ú	normalizer   Z	Timedeltar   r   r   ZfreqstrÚshiftÚassert_frame_equalr!   )	r#   r"   r    ÚdvÚdtr   r   rL   Úresr   r   r   Útest_pickles€   s<    

rS   c              	   C   s*   t |dƒ�}tj| |dd� W 5 Q R X d S ©NÚwbéÿÿÿÿ©Úprotocol©ÚopenÚpickleÚdump©ÚobjÚpathÚfhr   r   r   Úpython_pickler®   s    ra   c              
   C   s6   t | dƒ�"}| d¡ t |¡W  5 Q R £ S Q R X d S )NÚrbr   )rZ   Úseekr[   Úload)r_   r`   r   r   r   Úpython_unpickler³   s    
re   Úpickle_writerÚpython)ÚidZpandas_proto_defaultrW   Zpandas_proto_highestr0   Zpandas_proto_4é   Zpandas_proto_5Úwriterc                 C   sØ   | }|  ¡ D ]Æ\}}|  ¡ D ]´\}}t ¡ �ž}|||ƒ t |¡}	t|	||ƒ t|ƒ}	t|	||ƒ t|dd��}
|||ƒ |
 d¡ W 5 Q R X t|dd��}
t |
¡}	|
 d¡ W 5 Q R X t|	||ƒ W 5 Q R X qqd S )NrU   ©Úmoder   rb   )	rJ   r   Úensure_cleanr   rI   r!   re   rZ   rc   )r   rf   rj   r"   r    rP   rQ   r   r_   r   Úhandler   r   r   Útest_round_trip_current¹   s     




ro   c                  C   s(   t  ¡ } t  | jtj¡}t  | |¡ d S r&   )r   ÚmakeDataFrameÚround_trip_pathlibÚ	to_pickler   rI   rO   ©Údfr   r   r   r   Útest_pickle_path_pathlibã   s    ru   c                  C   s(   t  ¡ } t  | jtj¡}t  | |¡ d S r&   )r   rp   Zround_trip_localpathrr   r   rI   rO   rs   r   r   r   Útest_pickle_path_localpathé   s    rv   c                   C   s   dt  ¡ › d�S )NÚ__z	__.pickle)ÚuuidÚuuid4r   r   r   r   Úget_random_pathô   s    rz   c                   @   sX   e Zd ZejZdd„ Zdd„ Zej	 
ddddd	g¡d
d„ ƒZdd„ Zdd„ Zdd„ ZdS )ÚTestCompressionc           	   
   C   s`  |d krt  ||¡ d S |dkr.t |d¡}nð|dkrDt |d¡}nÚ|dkr€tj|dtjd��}| 	|t
j |¡¡ W 5 Q R X nž|dkrÚt|dƒ�@}tj|dd��&}| |t
j |¡¡}| ||¡ W 5 Q R X W 5 Q R X nD|d	krðtƒ |dƒ}n.|d
k�rtdƒ |d¡}nd|› �}t|ƒ‚|dk�r\t|dƒ�$}|� | 	| ¡ ¡ W 5 Q R X W 5 Q R X d S )NÚgzipÚwÚbz2Úzip©ÚcompressionÚtarrb   rk   ÚxzZzstdZ	zstandardrU   zUnrecognized compression type: )r   r‚   )ÚshutilÚcopyfiler|   rZ   r~   ÚBZ2FileÚzipfileÚZipFileÚZIP_DEFLATEDÚwriteÚosr_   ÚbasenameÚtarfileÚ
gettarinfoÚaddfiler   r   Ú
ValueErrorÚread)	ÚselfZsrc_pathÚ	dest_pathr�   Úfr`   r‚   ÚtarinfoÚmsgr   r   r   Úcompress_fileü   s2     "


zTestCompression.compress_filec                 C   s²   |}|d }|d }t  |¡�Š}t  |¡�t}t  ¡ }|j||d� t j||d��*}	t|dƒ�}
|
 |	 ¡ ¡ W 5 Q R X W 5 Q R X tj	|d d�}t  
||¡ W 5 Q R X W 5 Q R X d S )Nú.compressedú.rawr€   rU   )r   rm   rp   rr   Údecompress_filerZ   rŠ   r‘   r   rI   rO   )r’   r�   rz   ÚbaseÚpath1Úpath2Úp1Úp2rt   r”   r`   Údf2r   r   r   Útest_write_explicit  s    "z#TestCompression.test_write_explicitr�   Ú ÚNoneÚbadZ7zc              
   C   sJ   t jtdd��2 t |¡�}t ¡ }|j||d� W 5 Q R X W 5 Q R X d S )NzUnrecognized compression type)Úmatchr€   )rG   Zraisesr�   r   rm   rp   rr   )r’   r�   rz   r_   rt   r   r   r   Útest_write_explicit_bad/  s    z'TestCompression.test_write_explicit_badc                 C   s¾   |}|| }|d }| j  | ¡ ¡}t |¡�†}t |¡�p}t ¡ }	|	 |¡ tj||d��*}
t|dƒ�}| 	|
 
¡ ¡ W 5 Q R X W 5 Q R X tj|d d�}t |	|¡ W 5 Q R X W 5 Q R X d S )Nr™   r€   rU   )Ú_extension_to_compressionÚgetÚlowerr   rm   rp   rr   rš   rZ   rŠ   r‘   r   rI   rO   )r’   Úcompression_extrz   r›   rœ   r�   r�   rž   rŸ   rt   r”   r`   r    r   r   r   Útest_write_infer6  s    
"z TestCompression.test_write_inferc           
   
   C   s„   |}|d }|d }t  |¡�\}t  |¡�F}t  ¡ }|j|d d� | j|||d� tj||d�}	t  ||	¡ W 5 Q R X W 5 Q R X d S )Nr™   r˜   r€   )r   rm   rp   rr   r—   r   rI   rO   )
r’   r�   rz   r›   rœ   r�   rž   rŸ   rt   r    r   r   r   Útest_read_explicitL  s    z"TestCompression.test_read_explicitc              
   C   s�   |}|d }|| }| j  | ¡ ¡}t |¡�X}t |¡�B}t ¡ }	|	j|d d� | j|||d� t 	|¡}
t 
|	|
¡ W 5 Q R X W 5 Q R X d S )Nr™   r€   )r§   r¨   r©   r   rm   rp   rr   r—   r   rI   rO   )r’   rª   rz   r›   rœ   r�   r�   rž   rŸ   rt   r    r   r   r   Útest_read_infer^  s    
zTestCompression.test_read_inferN)Ú__name__Ú
__module__Ú__qualname__ÚicomZextension_to_compressionr§   r—   r¡   rG   ÚmarkÚparametrizer¦   r«   r¬   r­   r   r   r   r   r{   ù   s   
r{   c                   @   s*   e Zd Zej dddddg¡dd„ ƒZdS )	ÚTestProtocolrX   rV   r   r+   r,   c              	   C   sF   t  |¡�2}t  ¡ }|j||d� t |¡}t  ||¡ W 5 Q R X d S )NrW   ©r   rm   rp   rr   r   rI   rO   )r’   rX   rz   r_   rt   r    r   r   r   Ú	test_readx  s
    
zTestProtocol.test_readN)r®   r¯   r°   rG   r²   r³   r¶   r   r   r   r   r´   w  s   r´   Úpickle_fileÚexcolsztest_py27.pklÚaÚbÚcztest_mi_py27.pklr4   c                 C   s*   | ddd|ƒ}t  |¡}t |j|¡ d S )NÚior"   r[   )r   rI   r   r   Úcolumns)r)   r·   r¸   r_   rt   r   r   r   Útest_unicode_decode_error�  s    
r¾   c               
   C   sl   t  ¡ �Z} t  ¡ }t| dƒ�}| |¡ W 5 Q R X t| dƒ�}t |¡}W 5 Q R X t  ||¡ W 5 Q R X d S )NrU   rb   )r   rm   rp   rZ   rr   r   rI   rO   )r_   rt   r`   r   r   r   r   Útest_pickle_buffer_roundtripš  s    
r¿   Úmockurlzhttp://url.comzftp://test.comzhttp://gzip.comc              	      sp   dd„ }G dd„ dƒ‰ t  ¡ �H‰‡ ‡fdd„}t  ¡ }||ˆƒ |  d|¡ t |¡}t  ||¡ W 5 Q R X d S )Nc              	   S   s*   t |dƒ�}tj| |dd� W 5 Q R X d S rT   rY   r]   r   r   r   ra   ­  s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s:   e Zd Zddœdd„Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponseN©Úreturnc                 S   s.   t |dƒ| _d|kr ddi| _n
ddi| _d S )Nrb   r|   zContent-Encodingr¢   )rZ   ÚfileÚheaders)r’   r_   r   r   r   Ú__init__²  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r&   r   ©r’   r   r   r   Ú	__enter__¹  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |   ¡  d S r&   )Úclose)r’   Úargsr   r   r   Ú__exit__¼  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  ¡ S r&   )rÃ   r‘   rÆ   r   r   r   r‘   ¿  s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  ¡ S r&   )rÃ   rÈ   rÆ   r   r   r   rÈ   Â  s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.close)r®   r¯   r°   rÅ   rÇ   rÊ   r‘   rÈ   r   r   r   r   ÚMockReadResponse±  s
   rË   c                     s   ˆ ˆƒS r&   r   )rÉ   Úkwargs©rË   r_   r   r   Úmock_urlopen_readÇ  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rm   rp   Úsetattrr   rI   rO   )ZmonkeypatchrÀ   ra   rÎ   rt   r   r   rÍ   r   Útest_pickle_generalurl_read©  s    


rÐ   Zfsspecc               	   C   sD   t  ¡ �2 d} t  ¡ }| | ¡ t | ¡}t  ||¡ W 5 Q R X d S )Nzmemory://afilerµ   )rÀ   rt   r   r   r   r   Útest_pickle_fsspec_roundtripÑ  s    


rÑ   c                   @   s   e Zd Zddœdd„ZdS )ÚMyTzNrÁ   c                 C   s   d S r&   r   rÆ   r   r   r   rÅ   Ü  s    zMyTz.__init__)r®   r¯   r°   rÅ   r   r   r   r   rÒ   Û  s   rÒ   c                  C   sD   t td�tƒ f} t | ¡}t |d | d ¡ t|d tƒs@t‚d S )N)Zdtyper   r+   )r   ÚobjectrÒ   r   Úround_trip_picklerK   r   r   )r   r   r   r   r   Útest_read_pickle_with_subclassà  s    
rÕ   c              	   C   s–   t  ¡ }t  ¡ � }|j|| d� t|ƒ ¡ }W 5 Q R X t ¡ }|j|| d� | d¡ | 	¡ |ksn| dksnt
‚tj|| d�}| d¡ t  ||¡ dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r€   r   )r|   r   r‚   N)r   rp   rm   rr   r   Ú
read_bytesr¼   ÚBytesIOrc   Úgetvaluer   r   rI   rO   )r�   rt   r_   Ú	referenceÚbufferZread_dfr   r   r   Ú%test_pickle_binary_object_compressioné  s    


rÛ   c                 C   s8   | }|}dd„ }||ƒ ||j ƒ ||ƒ ||j ƒ d S )Nc                 S   s   t  | ¡}t  | |¡ d S r&   )r   rÔ   rO   )rF   Ú	unpickledr   r   r   Ú_test_roundtrip  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)ÚT)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdrF   rÝ   r   r   r   Ú+test_pickle_dataframe_with_multilevel_index  s    
rß   c                  C   sB   t dddd�} ttj t| ƒ¡| ƒ}t |¡}|jj	dks>t
‚d S )Nz1/1/2011z1/1/2012rD   )rL   )r   r   ÚnpÚrandomZrandnÚlenr   rÔ   rC   rL   r   )Zprngr?   Znew_tsr   r   r   Ú"test_pickle_timeseries_periodindex  s    
rã   Únamei	  g     Hˆ@iÑ  é   )r+   r,   c                 C   s$   t  t j| d�¡}|j| ks t‚d S )N)rä   )r   rÔ   ZmakeTimeSeriesrä   r   )rä   rÜ   r   r   r   Útest_pickle_preserve_name  s    ræ   c                 C   s   t  | ¡}t  || ¡ d S r&   ©r   rÔ   rK   )Zdatetime_seriesZunp_tsr   r   r   Útest_pickle_datetimes%  s    
rè   c                 C   s   t  | ¡}t  || ¡ d S r&   rç   )Zstring_seriesZ
unp_seriesr   r   r   Útest_pickle_strings*  s    
ré   c                  C   sf   t tdƒƒ d¡jdg } t | ¡}|jjd jdks:t	‚|jjd j
dksPt	‚t |dg | ¡ d S )NÚabcÚcategoryr   r+   )r+   T)r   ÚlistZastypeZilocr   rÔ   Z_mgrÚblocksr9   r   r<   rK   )ZserrR   r   r   r   Ú test_pickle_preserves_block_ndim/  s
    
rî   rX   c                 C   s@   t  tdƒ¡}t t|j| |d�tt j|d�¡}t ||¡ d S )Ni † )rX   r�   r€   )	r   Ú	DataFrameÚranger   rq   r   rr   rI   rO   )rX   r�   rt   r   r   r   r   Ú%test_pickle_big_dataframe_compression<  s    þrñ   c               	   C   sN   t j tdd¡} t| dƒ�}t |¡}W 5 Q R X tjg g d�}t	 
||¡ d S )Nz1.2.4zempty_frame_v1_2_4-GH#42345.pklrb   )rC   r½   )r‹   r_   ÚjoinÚlegacy_dirnamerZ   r[   rd   r   rï   r   rO   )r_   Úfdrt   r   r   r   r   Ú#test_pickle_frame_v124_unpickle_130G  s
    rõ   )cÚ__doc__r   r~   ÚdatetimeÚ	functoolsr   Úglobr|   r¼   r‹   Úpathlibr   r[   r„   r�   rx   Úwarningsr   r‡   Únumpyrà   rG   Zpandas.compatr   r   Zpandas.compat._optionalr   Zpandas.compat.compressorsr	   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsÚtdZpandasr   r
   r   r   Zpandas._testingZ_testingr   Zpandas.io.commonÚcommonr±   Zpandas.tseries.offsetsr   r   Zfixturer   r!   r_   rò   ÚdirnameÚ__file__ró   Úfilesr#   r²   r³   r8   r5   ÚPickleBufferÚcastZarangeZreshaper>   rS   ra   re   r'   rr   ÚHIGHEST_PROTOCOLro   ru   rv   rz   r{   r´   Z
MultiIndexZfrom_arraysr¾   r¿   rÐ   Z
skip_if_norÑ   ÚtzinforÒ   rÕ   rÛ   rß   rã   ræ   rè   ré   Zskip_array_manager_invalid_testrî   ÚDEFAULT_PROTOCOLrñ   rõ   r   r   r   r   Ú<module>   sÜ   



	$õþ
.þþøþ
~
þþþ

 ÿ
%
		 ÿ



