U
    ¾mœdXª  ã                   @   s®  d Z ddlZddlmZ ddlZddlZddlmZ ddlZ	ddl
Z
ddlmZ ddlmZ ddlmZmZ ddlm  mZ ddlZddlmZ ddlmZ dd	lmZmZm Z m!Z!m"Z" zddl#Z#d
Z$W n e%k
râ   dZ$Y nX zddl&Z&d
Z'W n e%k
�r   dZ'Y nX e
j(e
j)de
j*j+e' �p2edƒdkdd�d�e
j)de
j*j+e$ dd�d�gd�dd„ ƒZ,e
j(dd„ ƒZ-e
j(dd„ ƒZ.e
j(dd„ ƒZ/e
j(dd„ ƒZ0e
j(dd „ ƒZ1e
j(ej 2ej3j4¡ej 2ej3j5¡ej 2ej3j6¡ej 7d!d"¡ej 7d#d"¡ej 7d$d"¡ej 7d%d"¡gd�d&d'„ ƒZ8dEd)d*„Z9d+d,„ Z:d-d.„ Z;d/d0„ Z<d1d2„ Z=d3d4„ Z>d5d6„ Z?d7d8„ Z@d9d:„ ZAd;d<„ ZBG d=d>„ d>ƒZCG d?d@„ d@eCƒZDG dAdB„ dBeCƒZEG dCdD„ dDeCƒZFdS )Fz test parquet compat é    N)ÚBytesIO)Úcatch_warnings)Ú
get_option)Úis_platform_windows)Úpa_version_under7p0Úpa_version_under8p0)ÚVersion)ÚFastParquetImplÚPyArrowImplÚ
get_engineÚread_parquetÚ
to_parquetTFÚfastparquetúmode.data_managerÚarrayz4fastparquet is not installed or ArrayManager is used©Úreason)ZmarksÚpyarrowúpyarrow is not installed)Úparamsc                 C   s   | j S ©N©Úparam©Úrequest© r   úU/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pandas/tests/io/test_parquet.pyÚengine3   s    r   c                   C   s   t st d¡ dS )Nr   r   )Ú_HAVE_PYARROWÚpytestÚskipr   r   r   r   ÚpaH   s    
r!   c                   C   s*   t st d¡ ntdƒdkr&t d¡ dS )Nzfastparquet is not installedr   r   z.ArrayManager is not supported with fastparquetr   )Ú_HAVE_FASTPARQUETr   r    r   r   r   r   r   ÚfpO   s
    
r#   c                   C   s   t  dddgddœ¡S )Né   é   é   Úfoo©ÚAÚB)ÚpdÚ	DataFramer   r   r   r   Ú	df_compatX   s    r-   c               
   C   sD   t  tdƒttddƒƒtjdddd�dd	dgt jd
dd�dœ¡} | S )NÚabcr$   é   ç      @ç      @Úfloat64©ÚdtypeTFÚ20130101r&   ©Úperiods)ÚaÚbÚdÚeÚf)r+   r,   ÚlistÚrangeÚnpÚarangeÚ
date_range)Údfr   r   r   Údf_cross_compat]   s    úÿrC   c                   C   s’   t  tdƒdtjdgdd dgdddgddd	gttd
dƒƒt dd¡ d¡tjdddd�dtjdgdddgt jddd�t  	d¡t j
t  	d¡gdœ¡S )Nr.   r8   Úcó   fooó   bars   bazr'   ÚbarÚbazr$   r/   r&   é   Úu1r0   r1   r2   r3   ç       @ç      @TFr5   r6   Z20130103)ÚstringZstring_with_nanZstring_with_noneÚbytesÚunicodeÚintZuintÚfloatZfloat_with_nanÚboolÚdatetimeÚdatetime_with_nat)r+   r,   r=   r?   Únanr>   r@   ÚastyperA   Ú	TimestampZNaTr   r   r   r   Údf_fullo   s$    

ýôÿrX   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                 C   s   | j S r   r   r   r   r   r   Útimezone_aware_date_list‡   s    rY   r%   c
              	      s€   ˆp
ddi‰ˆpi ‰ˆdkr ˆ‰|r4|ˆd< |ˆd< ‡ ‡‡‡‡‡‡‡fdd„}
ˆdkrtt  ¡ �‰|
|	ƒ W 5 Q R X n|
|	ƒ dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    ÚcompressionNr   c              
      sV   t | ƒD ]H}ˆjˆfˆŽ tdd�� tˆfˆŽ}W 5 Q R X tjˆ|ˆˆˆ d� qd S )NT©Úrecord)Úcheck_namesÚ
check_likeÚcheck_dtype)r>   r   r   r   ÚtmÚassert_frame_equal)ÚrepeatÚ_Úactual©r_   r^   r]   rB   ÚexpectedÚpathÚread_kwargsÚwrite_kwargsr   r   ÚcompareÂ   s    ûz!check_round_trip.<locals>.compare)r`   Úensure_clean)rB   r   rg   ri   rh   rf   r]   r^   r_   rb   rj   r   re   r   Úcheck_round_trip–   s    "
rl   c                 C   s|   t rLddlm} |j| dd�}t|jjƒt|ƒks6t‚|jjt|ƒksxt‚n,ddl	m
} |j
| dd�}|jjj|ksxt‚dS )zÂCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   NF)Zvalidate_schemaZhive)Úpartitioning)r   Úpyarrow.parquetÚparquetZParquetDatasetÚlenZ
partitionsZpartition_namesÚAssertionErrorÚsetZpyarrow.datasetÚdatasetrm   ÚschemaÚnames)rg   rf   Úpqrs   Zdsr   r   r   Úcheck_partition_names×   s    
rw   c              	   C   s.   d}t jt|d�� t| ddƒ W 5 Q R X d S )Nz.engine must be one of 'pyarrow', 'fastparquet'©Úmatchr'   rG   )r   ÚraisesÚ
ValueErrorrl   )r-   Úmsgr   r   r   Útest_invalid_engineî   s    r}   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Núio.parquet.enginer   ©r+   Úoption_contextrl   )r-   r!   r   r   r   Útest_options_pyô   s    r�   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Nr~   r   r   )r-   r#   r   r   r   Útest_options_fpû   s    r‚   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Nr~   Úautor   )r-   r#   r!   r   r   r   Útest_options_auto  s    r„   c              	   C   s  t tdƒtƒst‚t tdƒtƒs$t‚t dd¡�< t tdƒtƒsDt‚t tdƒtƒsVt‚t tdƒtƒsht‚W 5 Q R X t dd¡�< t tdƒtƒs’t‚t tdƒtƒs¤t‚t tdƒtƒs¶t‚W 5 Q R X t dd¡�> t tdƒtƒsàt‚t tdƒtƒsòt‚t tdƒtƒ�st‚W 5 Q R X d S )Nr   r   r~   rƒ   )Ú
isinstancer   r
   rq   r	   r+   r€   )r#   r!   r   r   r   Útest_options_get_engine	  s    r†   c               	   C   s.  ddl m}  |  d¡}|  d¡}ts(dnttjƒt|ƒk }tsBdnttjƒt|ƒk }to\| }tof| }|�s*|�s*|r¨d|› d�}t	j
t|d�� td	ƒ W 5 Q R X n&d
}t	j
t|d�� td	ƒ W 5 Q R X |�rd|› d�}t	j
t|d�� td	ƒ W 5 Q R X n&d}t	j
t|d�� td	ƒ W 5 Q R X d S )Nr   )ÚVERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.rx   rƒ   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)Zpandas.compat._optionalr‡   Úgetr   r   r   Ú__version__r"   r   r   rz   ÚImportErrorr   )r‡   Z
pa_min_verZ
fp_min_verZhave_pa_bad_versionZhave_fp_bad_versionZhave_usable_paZhave_usable_fpry   r   r   r   Ú"test_get_engine_auto_error_message  s8    

ÿýÿý

r‹   c              	   C   sj   | }t  ¡ �T}|j||d d� t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W 5 Q R X d S )N©r   rZ   ©r   r8   r:   ©r   Úcolumns)r`   rk   r   r   ra   ©rC   r!   r#   rB   rg   Úresultr   r   r   Útest_cross_engine_pa_fpH  s    
r’   c              
   C   s€   | }t  ¡ �j}|j||d d� tdd��D t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W 5 Q R X W 5 Q R X d S )NrŒ   Tr[   r�   r8   r:   rŽ   )r`   rk   r   r   r   ra   r�   r   r   r   Útest_cross_engine_fp_paV  s    
r“   c                   @   s:   e Zd Zdd„ Zdd„ Zejjejddd�dd	„ ƒƒZ	d
S )ÚBasec              
   C   sB   t  ¡ �0}tj||d�� t|||d d� W 5 Q R X W 5 Q R X d S )Nrx   ©rZ   )r`   rk   r   rz   r   )ÚselfrB   r   ÚexcÚerr_msgrg   r   r   r   Úcheck_error_on_writee  s    
zBase.check_error_on_writec              
   C   s>   t  ¡ �,}t  |¡� t|||d d� W 5 Q R X W 5 Q R X d S )Nr•   )r`   rk   Úexternal_error_raisedr   )r–   rB   r   r—   rg   r   r   r   Úcheck_external_error_on_writek  s    
z"Base.check_external_error_on_writeúdhttps://raw.githubusercontent.com/pandas-dev/pandas/main/pandas/tests/io/data/parquet/simple.parquetT)ÚurlZcheck_before_testc                 C   s.   |dkrt  |¡ d}t|ƒ}t ||¡ d S )Nrƒ   rœ   )r   Úimportorskipr   r`   ra   )r–   r-   r   r�   rB   r   r   r   Útest_parquet_read_from_urlq  s    	
ÿzBase.test_parquet_read_from_urlN)
Ú__name__Ú
__module__Ú__qualname__r™   r›   r   ÚmarkÚnetworkr`   rŸ   r   r   r   r   r”   d  s   ûr”   c                   @   sÊ   e Zd Zdd„ Zdd„ Zej ddddd	g¡d
d„ ƒZdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zejjed d!�d"d#„ ƒZej d$d%d&d'd(d)d*d+d,d-g	¡d.d/„ ƒZdS )0Ú	TestBasicc              	   C   sJ   t  dddg¡ddt  d¡t dddg¡fD ]}d}|  ||t|¡ q,d S )Nr$   r%   r&   r'   r5   z+to_parquet only supports IO with DataFrames)r+   ÚSeriesrW   r?   r   r™   r{   )r–   r   Úobjr|   r   r   r   Ú
test_error…  s    ûzTestBasic.test_errorc                 C   s6   t  tdƒttddƒƒdœ¡}ddg|_t||ƒ d S )Nr.   r$   r/   ©rM   rP   r'   rG   )r+   r,   r=   r>   r�   rl   )r–   r   rB   r   r   r   Útest_columns_dtypes�  s    
zTestBasic.test_columns_dtypesrZ   NÚgzipÚsnappyÚbrotlic                 C   sP   |dkrt  d¡ n|dkr&t  d¡ t ddddgi¡}t||d|id� d S )	Nr¬   r­   r)   r$   r%   r&   rZ   ©ri   )r   rž   r+   r,   rl   )r–   r   rZ   rB   r   r   r   Útest_compression—  s    
zTestBasic.test_compressionc                 C   sJ   t  tdƒttddƒƒdœ¡}t  dtdƒi¡}t|||ddgid� d S )Nr.   r$   r/   r©   rM   r�   ©rf   rh   )r+   r,   r=   r>   rl   )r–   r   rB   rf   r   r   r   Útest_read_columns¢  s       ÿzTestBasic.test_read_columnsc                 C   s¦   |dk}t  ddddgi¡}t||ƒ dddgt jddd�td	ƒdddgg}|D ]2}||_t|t jƒrt|j d ¡|_t|||d
� qPdddg|_d|j_	t||ƒ d S )Nr   r)   r$   r%   r&   r/   r5   r6   r.   )r]   r   r'   )
r+   r,   rl   rA   r=   Úindexr…   ZDatetimeIndexÚ
_with_freqÚname)r–   r   r]   rB   Zindexesr²   r   r   r   Útest_write_index«  s     
üzTestBasic.test_write_indexc                 C   s>   |}t  ddddgi¡}t j dddg¡}||_t||ƒ d S )Nr)   r$   r%   r&   ©r8   r$   ©r8   r%   ©r9   r$   )r+   r,   Ú
MultiIndexÚfrom_tuplesr²   rl   )r–   r!   r   rB   r²   r   r   r   Útest_write_multiindexÃ  s
    zTestBasic.test_write_multiindexc                 C   s¢   |}t jdddd�}t jtj dt|ƒ d¡tdƒd�}t jj	d	d
g|gddgd�}|j
d d�}||fD ]4}||_t||ƒ t||dddgi|ddg d� qhd S )Nz01-Jan-2018z01-Dec-2018ZMS)Úfreqr%   r&   ÚABC©r�   ÚLevel1ÚLevel2ÚlevelÚdate)ru   r�   r)   r*   ©rh   rf   )r+   rA   r,   r?   ÚrandomÚrandnrp   r=   r¹   Zfrom_productÚcopyr²   rl   )r–   r!   r   ÚdatesrB   Zindex1Úindex2r²   r   r   r   Útest_multiindex_with_columnsÌ  s"    $
 ÿ
  
 
ÿz&TestBasic.test_multiindex_with_columnsc              	   C   sê   t  dddgdddgdœ¡}d dd	œ}|jd
d�}t||||d� t jdddgdddgdœdddgd�}t||||d� ddddddddgddddddddgg}t jttdƒƒdd„ tdƒD ƒdœ|d�}|jd
d�}t||||d� d S )Nr$   r%   r&   ÚqÚrÚs)r8   r9   F)rZ   r²   T)Údrop©ri   rf   ZzyxZwvuZtsr©r²   rG   rH   r'   ÚquxÚoneÚtwoé   c                 S   s   g | ]
}| ‘qS r   r   )Ú.0Úir   r   r   Ú
<listcomp>ö  s     z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>)rÑ   rÒ   )r+   r,   Zreset_indexrl   r=   r>   )r–   r   rB   ri   rf   Úarraysr   r   r   Útest_write_ignoring_indexÜ  s$    
 ÿþ ÿz#TestBasic.test_write_ignoring_indexc                 C   sZ   t j dddg¡}t jtj dd¡|d�}|dkrD|  ||td¡ n|d	krVt	||ƒ d S )
Nr¶   r·   r¸   r/   r&   r¾   r   úColumn name must be a stringr   )
r+   r¹   rº   r,   r?   rÄ   rÅ   r™   Ú	TypeErrorrl   )r–   r   Z
mi_columnsrB   r   r   r   Útest_write_column_multiindexü  s       ÿz&TestBasic.test_write_column_multiindexc              	   C   s˜   ddddddddgddddddddgg}t jtj dd¡|d�}d	d
g|j_|dkr‚ttj	ƒtdƒk rlt
}nt}|  |||d¡ n|dkr”t||ƒ d S )NrG   rH   r'   rÐ   r$   r%   rÓ   r¾   r¿   rÀ   r   z0.7.0zColumn namer   )r+   r,   r?   rÄ   rÅ   r�   ru   r   r   r‰   rÚ   r{   r™   rl   )r–   r   r×   rB   Úerrr   r   r   Ú&test_write_column_multiindex_nonstring  s    þz0TestBasic.test_write_column_multiindex_nonstringc              	   C   s^   |}ddddddddgddddddddgg}t jtj dd¡|d�}d	d
g|j_t||ƒ d S )NrG   rH   r'   rÐ   rÑ   rÒ   rÓ   r¾   Z	ColLevel1Z	ColLevel2)r+   r,   r?   rÄ   rÅ   r�   ru   rl   ©r–   r!   r   r×   rB   r   r   r   Ú#test_write_column_multiindex_string  s    þz-TestBasic.test_write_column_multiindex_stringc                 C   s>   |}ddddg}t jtj dd¡|d�}d|j_t||ƒ d S )	NrG   rH   r'   rÐ   rÓ   r/   r¾   Z	StringCol)r+   r,   r?   rÄ   rÅ   r�   r´   rl   rÞ   r   r   r   Útest_write_column_index_string*  s
    z(TestBasic.test_write_column_index_stringc                 C   sT   ddddg}t jtj dd¡|d�}d|j_|dkrF|  ||td	¡ n
t	||ƒ d S )
Nr$   r%   r&   r/   rÓ   r¾   ZNonStringColr   rÙ   )
r+   r,   r?   rÄ   rÅ   r�   r´   r™   rÚ   rl   )r–   r   r×   rB   r   r   r   Ú!test_write_column_index_nonstring6  s       ÿz+TestBasic.test_write_column_index_nonstringzminimum pyarrow not installedr   c           
      C   sÖ  dd l m} |dkr.tjjdd�}|j |¡ t t 	dddd gd¡t 	dddd gd	¡t 	d
ddd g¡t 	dddd g¡t 	ddddgd¡t 	dddd gd¡t 	dddd gd¡dœ¡}t
 ¡ �,}| ||¡ t||d�}t||dd�}W 5 Q R X |d
 jt d¡k�st‚t tj	dddd gdd�tj	dddd gdd�tj	d
ddd gdd�tj	dddd gdd�tj	ddddgdd�tj	dddd gdd�tj	dddd gdd�dœ¡}	|dk�rÆ|jddd �}|	jddd �}	t
 ||	¡ d S )!Nr   r   z.Fastparquet nullable dtype support is disabledr   r$   r%   r&   Zint64Zuint8r8   r9   rD   TFr/   ç      ð?rK   rL   Zfloat32r2   )r8   r9   rD   r:   r;   r<   Úgr�   Únumpy_nullable©r   Údtype_backendÚInt64r3   ÚUInt8rM   ÚbooleanZFloat32ÚFloat64)Zaxis)rn   ro   r   r£   ÚxfailÚnodeÚ
add_markerr   Útabler   r`   rk   Zwrite_tabler   r4   r?   rq   r+   r,   rÍ   ra   )
r–   r   r   rv   r£   rî   rg   Zresult1Zresult2rf   r   r   r   Útest_dtype_backendD  sF    ÿ÷ÿ
ùÿ
zTestBasic.test_dtype_backendr4   rç   rè   ré   Úobjectzdatetime64[ns, UTC]rQ   z	period[D]rê   rM   c                 C   sT   t  dt jg |d�i¡}d }|dkr<t  dt jg dd�i¡}t||ddi|d� d S )NÚvaluer3   rQ   rê   ræ   rä   rÃ   )r+   r,   r   rl   )r–   r!   r4   rB   rf   r   r   r   Útest_read_empty_arrayv  s$     ÿÿ ÿÿ   ÿzTestBasic.test_read_empty_array)r    r¡   r¢   r¨   rª   r   r£   Úparametrizer¯   r±   rµ   r»   rÉ   rØ   rÛ   rÝ   rß   rà   rá   Úskipifr   rï   rò   r   r   r   r   r¥   „  s<   

		 
1÷þr¥   c                   @   s²  e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	e
jjeƒ dd�e
j deejg¡dd„ ƒƒZdd„ Ze
jjdd„ ƒZe
jjdd„ ƒZe
jje d¡e
j ddgg g¡dd„ ƒƒƒZe d¡d d!„ ƒZe d¡d"d#„ ƒZd$d%„ Zd&d'„ Ze
jjded(d)„ gd*d+gd,�d-d.„ ƒZd/d0„ Zd1d2„ Z e d¡d3d4„ ƒZ!e d¡d5d6„ ƒZ"e d¡d7d8„ ƒZ#d9d:„ Z$d;d<„ Z%e d¡d=d>„ ƒZ&d?d@„ Z'dAdB„ Z(dCdD„ Z)dEdF„ Z*dGdH„ Z+dIS )JÚTestParquetPyArrowc                 C   sB   |}t jdddd�}| d ¡}||d< dd dg|d< t||ƒ d S )Nr5   r&   úEurope/Brussels©r7   ÚtzÚdatetime_tzTÚbool_with_none)r+   rA   r³   rl   )r–   r!   rX   rB   Údtir   r   r   Ú
test_basic™  s    
zTestParquetPyArrow.test_basicc                 C   s<   |}t jdddd�|d< t|||ddg dddgid	� d S )
Nr5   r&   rö   r÷   rù   rM   rP   r�   r°   )r+   rA   rl   )r–   r!   rX   rB   r   r   r   Útest_basic_subset_columns¤  s    

üz,TestParquetPyArrow.test_basic_subset_columnsc                 C   s:   |j |d�}t|tƒst‚t|ƒ}t|ƒ}t ||¡ d S )Nr�   )r   r…   rN   rq   r   r   r`   ra   )r–   r!   rX   Z	buf_bytesZ
buf_streamÚresr   r   r   Ú*test_to_bytes_without_path_or_buf_provided²  s
    z=TestParquetPyArrow.test_to_bytes_without_path_or_buf_providedc                 C   s8   t jt d¡ dd¡tdƒd� ¡ }|  ||td¡ d S )Né   r/   r&   Úaaar¾   zDuplicate column names found©	r+   r,   r?   r@   Zreshaper=   rÆ   r™   r{   ©r–   r!   rB   r   r   r   Útest_duplicate_columns½  s    $z)TestParquetPyArrow.test_duplicate_columnsc                 C   s:   t  dt jddd�i¡}tr,|  ||t¡ n
t||ƒ d S )Nr8   ú1 dayr&   r6   )r+   r,   Útimedelta_ranger   r›   ÚNotImplementedErrorrl   r  r   r   r   Útest_timedeltaÂ  s    z!TestParquetPyArrow.test_timedeltac                 C   s(   t  ddddgi¡}|  ||tj¡ d S )Nr8   r$   rK   )r+   r,   r›   r   ÚArrowExceptionr  r   r   r   Útest_unsupportedÉ  s    z#TestParquetPyArrow.test_unsupportedc                 C   s6   t jddt jd�}tj|dgd�}|  ||tj¡ d S )Nr%   é
   r3   Úfp16©Údatar�   )r?   r@   Úfloat16r+   r,   r›   r   r	  )r–   r!   r  rB   r   r   r   Útest_unsupported_float16Ð  s    z+TestParquetPyArrow.test_unsupported_float16zqPyArrow does not cleanup of partial files dumps when unsupported dtypes are passed to_parquet function in windowsr   Ú	path_typec              
   C   sx   t jddt jd�}tj|dgd�}t ¡ �D}||ƒ}t tj	¡� |j
||d� W 5 Q R X tj |¡rjt‚W 5 Q R X d S )Nr%   r  r3   r  r  )rg   r   )r?   r@   r  r+   r,   r`   rk   rš   r   r	  r   Úosrg   Úisfilerq   )r–   r!   r  r  rB   Zpath_strrg   r   r   r   Ú test_unsupported_float16_cleanup×  s    
z3TestParquetPyArrow.test_unsupported_float16_cleanupc                 C   sx   t  ¡ }t  tdƒ¡|d< t jddddd dgt  dddg¡d�|d< t jddddddgddd	gd
d�|d< t||ƒ d S )NZabcdefr8   rG   r'   rH   r3   r9   rD   r:   T)Ú
categoriesZordered)r+   r,   ÚCategoricalr=   ZCategoricalDtyperl   r  r   r   r   Útest_categoricalì  s    þ
  ÿ
z#TestParquetPyArrow.test_categoricalc                 C   s4   t  d¡}|jf |Ž}d|i}t||d||d� d S )NÚs3fsÚ
filesystemzpandas-test/pyarrow.parquet©rg   rh   ri   )r   rž   ZS3FileSystemrl   )r–   r-   Ús3_resourcer!   Ús3sor  Zs3Úkwr   r   r   Útest_s3_roundtrip_explicit_fsþ  s    
ûz0TestParquetPyArrow.test_s3_roundtrip_explicit_fsc                 C   s   d|i}t ||d||d� d S )NÚstorage_optionsz s3://pandas-test/pyarrow.parquetr  ©rl   )r–   r-   r  r!   r  r   r   r   Útest_s3_roundtrip  s    ûz$TestParquetPyArrow.test_s3_roundtripr  Úpartition_colr)   c              
   C   s^   |  ¡ }|r6| t |tj¡¡}d}||  |¡||< t|||dd|i|d |dœddd� d S )NÚcategoryzs3://pandas-test/parquet_dirr  )Úpartition_colsrZ   r  Tr$   )rf   rg   rh   ri   r^   rb   )rÆ   rV   ÚdictÚfromkeysr?   Zint32rl   )r–   r-   r  r!   r"  r  Zexpected_dfZpartition_col_typer   r   r   Útest_s3_roundtrip_for_dir  s(    ÿýôz,TestParquetPyArrow.test_s3_roundtrip_for_dirr   c                 C   s(   t ƒ }| |¡ t|ƒ}t ||¡ d S r   )r   r   r   r`   ra   )r–   r-   ÚbufferZdf_from_bufr   r   r   Útest_read_file_like_obj_support>  s    
z2TestParquetPyArrow.test_read_file_like_obj_supportc              	   C   sb   |  dd¡ |  dd¡ tjtdd�� tdƒ W 5 Q R X tjtdd�� | d¡ W 5 Q R X d S )NÚHOMEZTestingUserÚUSERPROFILEz.*TestingUser.*rx   z~/file.parquet)Zsetenvr   rz   ÚOSErrorr   r   )r–   r-   Zmonkeypatchr   r   r   Útest_expand_userE  s    z#TestParquetPyArrow.test_expand_userc                 C   s>   ddg}|}|j ||d d� t||ƒ t|ƒj|jks:t‚d S )NrR   rP   ©r$  rZ   ©r   rw   r   Úshaperq   )r–   Útmp_pathr!   rX   r$  rB   r   r   r   Útest_partition_cols_supportedN  s
    
z0TestParquetPyArrow.test_partition_cols_supportedc                 C   s@   d}|g}|}|j ||d d� t||ƒ t|ƒj|jks<t‚d S )NrR   r.  r/  )r–   r1  r!   rX   r$  Úpartition_cols_listrB   r   r   r   Útest_partition_cols_stringV  s    
z-TestParquetPyArrow.test_partition_cols_stringc                 C   s   | S r   r   )Úxr   r   r   Ú<lambda>`  ó    zTestParquetPyArrow.<lambda>rM   zpathlib.Path)Zidsc           	      C   s<   d}|g}|}||ƒ}|j ||d� t|ƒj|jks8t‚d S )Nr*   )r$  )r   r   r0  rq   )	r–   r1  r!   r-   r  r$  r3  rB   rg   r   r   r   Útest_partition_cols_pathlib_  s    z.TestParquetPyArrow.test_partition_cols_pathlibc                 C   s   t jg g d�}t||ƒ d S )N)r²   r�   )r+   r,   rl   r  r   r   r   Útest_empty_dataframem  s    z'TestParquetPyArrow.test_empty_dataframec                 C   sV   dd l }t dddgi¡}| |jd| ¡ d�g¡}| t¡}t||d|i|d� d S )Nr   r5  r$   )Útypert   rÎ   )	r   r+   r,   rt   ÚfieldZbool_rV   rR   rl   )r–   r!   r   rB   rt   Zout_dfr   r   r   Útest_write_with_schemar  s
    
z)TestParquetPyArrow.test_write_with_schemac                 C   sz   t  t jdddgdd�t jdddgdd�t jdd dgd	d�d
œ¡}t||ƒ t  dt jdddd gdd�i¡}t||ƒ d S )Nr$   r%   r&   rç   r3   ZUInt32r8   rD   rM   )r8   r9   rD   )r+   r,   r¦   rl   r  r   r   r   Ú test_additional_extension_arraysz  s    ýÿ
 z3TestParquetPyArrow.test_additional_extension_arraysc              	   C   sV   t  dt jdd dgdd�i¡}t  d|¡�" t||| d|› d�¡d� W 5 Q R X d S )	Nr8   rD   zstring[pyarrow]r3   Ústring_storagezstring[ú]©rf   )r+   r,   r¦   r€   rl   rV   )r–   r!   r>  rB   r   r   r   Ú test_pyarrow_backed_string_arrayŠ  s    z3TestParquetPyArrow.test_pyarrow_backed_string_arrayc                 C   sN   t  t j dddg¡t jdddd�t j t jdddd�¡d	œ¡}t||ƒ d S )
N)r   r$   )r$   r%   )r&   r/   z
2012-01-01r&   ÚD)r7   r¼   r/   )rD   r:   r;   )r+   r,   ZIntervalIndexrº   Úperiod_rangeZfrom_breaksrA   rl   r  r   r   r   Útest_additional_extension_types‘  s    ÿüÿ
z2TestParquetPyArrow.test_additional_extension_typesc                 C   s>   t s
d}nd}t dtjdddd�i¡}t||d|id	� d S )
Nz2.6z2.0r8   z
2017-01-01Z1nr  ©r¼   r7   Úversionr®   )r   r+   r,   rA   rl   )r–   r!   ÚverrB   r   r   r   Útest_timestamp_nanoseconds¡  s
    z-TestParquetPyArrow.test_timestamp_nanosecondsc                 C   sV   t s(|jtjjkr(|j tjj	dd�¡ d|g }t
j|d|id�}t||dd� d S )Nzitemporary skip this test until it is properly resolved: https://github.com/pandas-dev/pandas/issues/37286r   é   Úindex_as_col©r²   r  F)r_   )r   ÚtzinforS   ÚtimezoneÚutcrì   rí   r   r£   rë   r+   r,   rl   )r–   r   r!   rY   ÚidxrB   r   r   r   Útest_timezone_aware_index¬  s    ÿþÿÿ
z,TestParquetPyArrow.test_timezone_aware_indexc              	   C   s^   t  dttddƒƒi¡}t ¡ �$}| ||¡ t||dgdd�}W 5 Q R X t|ƒdksZt	‚d S )Nr8   r   r&   ©r8   z==r   F)ÚfiltersZuse_legacy_datasetr$   )
r+   r,   r=   r>   r`   rk   r   r   rp   rq   )r–   r!   rB   rg   r‘   r   r   r   Útest_filter_row_groupsÅ  s    
   ÿz)TestParquetPyArrow.test_filter_row_groupsc              	   C   s~   t jtj dd¡dddgd�}t ¡ �}| ||¡ t||ƒ}W 5 Q R X |rdt	|j
t jjjƒszt‚nt	|j
t jjjƒszt‚d S )Nr  r&   r)   r*   ÚCr¾   )r+   r,   r?   rÄ   rÅ   r`   rk   r   r   r…   Z_mgrÚcoreZ	internalsZArrayManagerrq   ZBlockManager)r–   r!   Zusing_array_managerrB   rg   r‘   r   r   r   Útest_read_parquet_managerÐ  s    
z,TestParquetPyArrow.test_read_parquet_managerc                 C   s´   dd l }|}tjdddd�}| d ¡}||d< dd dg|d< |j |¡}|jtjd	�}|d
  d¡|d
< |d  d¡|d< |d  t |j	ddd�¡¡|d< t
||ddi|d� d S )Nr   r5   r&   rö   r÷   rù   Trú   )Ztypes_mapperrS   ztimestamp[us][pyarrow]rT   Úus)Úunitrø   ræ   r   ©r   rh   rf   )r   r+   rA   r³   ÚTableZfrom_pandasZ	to_pandasZ
ArrowDtyperV   Ú	timestamprl   )r–   r!   rX   r   rB   rû   Zpa_tablerf   r   r   r   Ú&test_read_dtype_backend_pyarrow_configÜ  s*    
ÿÿüz9TestParquetPyArrow.test_read_dtype_backend_pyarrow_configc                 C   sn   t jdddgit jddgdd�dd	�}| ¡ }d
d l}t|jƒtdƒkrV|j d¡|_t	||ddi|d� d S )Nr8   r$   r%   r&   r/   Útest©r´   zint64[pyarrow])r²   r4   r   z11.0.0ræ   r   rY  )
r+   r,   ÚIndexrÆ   r   r   r‰   r²   rV   rl   )r–   r!   rB   rf   r   r   r   r   Ú,test_read_dtype_backend_pyarrow_config_indexù  s    
  ÿüz?TestParquetPyArrow.test_read_dtype_backend_pyarrow_config_indexc              	   C   s”   t  tdƒttddƒƒdœ¡}ddg|_t||ƒ ddg|_tjtdd	�� t||ƒ W 5 Q R X t	 	d
dddd¡t	 	d
dddd¡g|_t||ƒ d S )Nr.   r$   r/   r©   r   rE   rF   z|S3rx   éÛ  )
r+   r,   r=   r>   r�   rl   r   rz   r  rS   r  r   r   r   Útest_columns_dtypes_not_invalid	  s    


þz2TestParquetPyArrow.test_columns_dtypes_not_invalidc                 C   s*   t jt jdddgdd�d�}t||ƒ d S )Nr8   r9   rD   úcustom namer^  rÏ   ©r+   r,   r_  rl   r  r   r   r   Útest_empty_columns  s    z%TestParquetPyArrow.test_empty_columnsN),r    r¡   r¢   rü   rý   rÿ   r  r  r
  r  r   r£   rë   r   ró   ÚstrÚpathlibÚPathr  r  Ú
single_cpur  r!  ÚtdZ
skip_if_nor'  r)  r-  r2  r4  r8  r9  r<  r=  rA  rD  rH  rP  rS  rV  r\  r`  rb  re  r   r   r   r   rõ   ˜  sp   ý

þþ

	 
 ÿ





rõ   c                   @   s¬   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	e
jjdd„ ƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'S )(ÚTestParquetFastParquetc                 C   sF   |}t jdddd�}| d ¡}||d< t jddd�|d< t||ƒ d S )	Nr5   r&   z
US/Easternr÷   rù   r  r6   Ú	timedelta)r+   rA   r³   r  rl   )r–   r#   rX   rB   rû   r   r   r   rü   $  s    
z!TestParquetFastParquet.test_basicc                 C   s”   t  tdƒttddƒƒdœ¡}t}d}ddg|_|  ||||¡ ddg|_|  ||||¡ t d	dddd¡t d	dddd¡g|_|  ||||¡ d S )
Nr.   r$   r/   r©   rÙ   r   rE   rF   ra  )r+   r,   r=   r>   rÚ   r�   r™   rS   )r–   r#   rB   rÜ   r|   r   r   r   Útest_columns_dtypes_invalid-  s    

þz2TestParquetFastParquet.test_columns_dtypes_invalidc                 C   s<   t jt d¡ dd¡tdƒd� ¡ }d}|  ||t|¡ d S )Nr   r/   r&   r  r¾   z9Cannot create parquet dataset with duplicate column namesr  ©r–   r#   rB   r|   r   r   r   r  B  s    $z-TestParquetFastParquet.test_duplicate_columnsc                 C   sB   t  ddd dgi¡}t jddtjdgidd�}t|||dd� d S )	Nr8   TFrâ   g        r  r3   )rf   r_   )r+   r,   r?   rU   rl   ©r–   r#   rB   rf   r   r   r   Útest_bool_with_noneH  s    z*TestParquetFastParquet.test_bool_with_nonec                 C   sV   t  dt jdddd�i¡}|  ||td ¡ t  ddddgi¡}d}|  ||t|¡ d S )	Nr8   Z2013ÚMr&   rE  r$   rK   z"Can't infer object conversion type)r+   r,   rC  r™   r{   rn  r   r   r   r
  O  s
    z'TestParquetFastParquet.test_unsupportedc                 C   s&   t  dt  tdƒ¡i¡}t||ƒ d S )Nr8   r.   )r+   r,   r  r=   rl   )r–   r#   rB   r   r   r   r  Z  s    z'TestParquetFastParquet.test_categoricalc              	   C   sf   dt tddƒƒi}t |¡}t ¡ �(}|j||d dd� t||dgd�}W 5 Q R X t|ƒdksbt	‚d S )Nr8   r   r&   r$   )rZ   Zrow_group_offsetsrQ  )rR  )
r=   r>   r+   r,   r`   rk   r   r   rp   rq   )r–   r#   r:   rB   rg   r‘   r   r   r   rS  ^  s    

z-TestParquetFastParquet.test_filter_row_groupsc                 C   s    t ||dd|id |dœd� d S )Nz$s3://pandas-test/fastparquet.parquetr  )rZ   r  r  r   )r–   r-   r  r#   r  r   r   r   r!  f  s    ûz(TestParquetFastParquet.test_s3_roundtripc                 C   s\   ddg}|}|j |d|d d� tj |¡s.t‚dd l}| t|ƒd¡j}t	|ƒdksXt‚d S )NrR   rP   r   ©r   r$  rZ   r   Fr%   ©
r   r  rg   Úexistsrq   r   ZParquetFilerf  Zcatsrp   ©r–   r1  r#   rX   r$  rB   r   Zactual_partition_colsr   r   r   r2  q  s    üz4TestParquetFastParquet.test_partition_cols_supportedc                 C   sX   d}|}|j |d|d d� tj |¡s*t‚dd l}| t|ƒd¡j}t	|ƒdksTt‚d S )NrR   r   rr  r   Fr$   rs  ru  r   r   r   r4  �  s    üz1TestParquetFastParquet.test_partition_cols_stringc                 C   s\   ddg}|}|j |dd |d� tj |¡s.t‚dd l}| t|ƒd¡j}t	|ƒdksXt‚d S )NrR   rP   r   )r   rZ   Úpartition_onr   Fr%   rs  ru  r   r   r   Útest_partition_on_supported‘  s    üz2TestParquetFastParquet.test_partition_on_supportedc              	   C   sB   ddg}|}d}t jt|d�� |j|dd ||d� W 5 Q R X d S )NrR   rP   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datarx   r   )r   rZ   rv  r$  )r   rz   r{   r   )r–   r1  r#   rX   r$  rB   r|   r   r   r   Ú3test_error_on_using_partition_cols_and_partition_on¡  s    ÿûzJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_onc                 C   s"   t  ¡ }| ¡ }t|||d� d S )Nr@  )r+   r,   rÆ   rl   ro  r   r   r   r9  ´  s    z+TestParquetFastParquet.test_empty_dataframec                 C   s>   d|g }t j|d|id�}| ¡ }d|j_t|||d� d S )NrI  rJ  rK  r²   r@  )r+   r,   rÆ   r²   r´   rl   )r–   r#   rY   rO  rB   rf   r   r   r   rP  º  s
    
z0TestParquetFastParquet.test_timezone_aware_indexc                 C   sš   t  dddgi¡}t ¡ �v}| |¡ tjtdd��* t t	¡� t
|ddd� W 5 Q R X W 5 Q R X tjtdd�� t
|dd	d
� W 5 Q R X W 5 Q R X d S )Nr8   r$   r%   z!not supported for the fastparquetrx   r   T)r   Zuse_nullable_dtypesr   rå   )r+   r,   r`   rk   r   r   rz   r{   Zassert_produces_warningÚFutureWarningr   )r–   r#   rB   rg   r   r   r   Ú&test_use_nullable_dtypes_not_supportedÃ  s    

"z=TestParquetFastParquet.test_use_nullable_dtypes_not_supportedc              
   C   sb   t  d¡�N}t |¡ d¡ tjtdd�� t|dd� W 5 Q R X t |¡j	dd� W 5 Q R X d S )	Nútest.parquets   breakitÚ rx   r   r�   F)Ú
missing_ok)
r`   rk   rg  rh  Úwrite_bytesr   rz   Ú	Exceptionr   Úunlink)r–   rg   r   r   r   Ú$test_close_file_handle_on_read_errorÎ  s
    z;TestParquetFastParquet.test_close_file_handle_on_read_errorc              
   C   sp   t jddgddgdœd�}t d¡�6}t| ¡ dƒ�}| |¡ W 5 Q R X t||d�}W 5 Q R X t ||¡ d S )Nr   r$   r(   )r  r{  Úwbr�   )	r+   r,   r`   rk   ÚopenÚencoder   r   ra   )r–   r   rB   rg   r<   r‘   r   r   r   Útest_bytes_file_nameÖ  s    z+TestParquetFastParquet.test_bytes_file_namec              
   C   sf   d}t  dttddƒƒi¡}t d¡�6}| |¡ tjt	|d�� t
|dd� W 5 Q R X W 5 Q R X d S )	NzPdtype_backend numpy is invalid, only 'numpy_nullable' and 'pyarrow' are allowed.rP   r$   r/   ztmp.parquetrx   Únumpy)ræ   )r+   r,   r=   r>   r`   rk   r   r   rz   r{   r   )r–   r   r|   rB   rg   r   r   r   Útest_invalid_dtype_backendà  s    ÿ
z1TestParquetFastParquet.test_invalid_dtype_backendc                 C   sJ   t jt jdddgdd�d�}t jt jdddgdd�d�}t|||d� d S )Nr8   r9   rD   rc  r^  rÏ   r@  rd  ro  r   r   r   re  ë  s    z)TestParquetFastParquet.test_empty_columnsN)r    r¡   r¢   rü   rm  r  rp  r
  r  rS  r   r£   ri  r!  r2  r4  rw  rx  r9  rP  rz  r�  r…  r‡  re  r   r   r   r   rk  #  s(   	

	
rk  )	NNNNNTFTr%   )GÚ__doc__rS   Úior   r  rg  Úwarningsr   r†  r?   r   Zpandas._configr   Zpandas.compatr   Zpandas.compat.pyarrowr   r   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsrj  Zpandasr+   Zpandas._testingZ_testingr`   Zpandas.util.versionr   Zpandas.io.parquetr	   r
   r   r   r   r   r   rŠ   r   r"   Zfixturer   r£   rô   r   r!   r#   r-   rC   rX   ÚnowrM  rN  ÚminÚmaxÚstrptimerY   rl   rw   r}   r�   r‚   r„   r†   r‹   r’   r“   r”   r¥   rõ   rk  r   r   r   r   Ú<module>   s¸   

þþ ÿþøÿ





ùÿ
         ö
A+      