U
    ¾mœd¯F  ã                   @   sÞ  d dl mZ d dlmZ d dlmZ d dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ ejdd„ ƒZd	d
„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zej dddg¡dd„ ƒZdd„ Zdd„ Zdd„ Zd d!„ Zej dddg¡d"d#„ ƒZej dd d$d%d&g¡d'd(„ ƒZ ej dddd)g¡d*d+„ ƒZ!d,d-„ Z"ej d.dd)g¡d/d0„ ƒZ#ej d1d2d3g¡d4d5„ ƒZ$d6d7„ Z%d8d9„ Z&d:d;„ Z'ej d<d=d>d?g¡d@dA„ ƒZ(dBdC„ Z)ej dDdEdFg¡dGdH„ ƒZ*dIdJ„ Z+dKdL„ Z,dMdN„ Z-dOdP„ Z.dS )Qé    )ÚStringIO)ÚPath)ÚIteratorN)Ú	DataFrameÚ	read_json)Ú
JsonReaderc                  C   s(   t dddgdddgdœƒ} | jdd	d
�S )Né   é   é   é   é   é   ©ÚAÚBTÚrecords©ÚlinesÚorient)r   Úto_json)Údf© r   ú\/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pandas/tests/io/json/test_readlines.pyÚlines_json_df   s    r   c                  C   s8   t ddd�} tddgddggddgd�}t | |¡ d S )	Nz!{"a": 1, "b": 2}
{"b":2, "a" :1}
T©r   r   r	   ÚaÚb©Úcolumns©r   r   ÚtmÚassert_frame_equal)ÚresultÚexpectedr   r   r   Útest_read_jsonl   s    r$   c                 C   sB   t | ddddƒd|d�}tddd	gd
ddgdœƒ}t ||¡ d S )NÚioÚjsonÚdataúline_delimited.jsonT©r   Úenginer   r
   r   r	   r   r   ©r   r   r   )Údatapathr*   r"   r#   r   r   r   Útest_read_jsonl_engine_pyarrow   s    ýr-   c                 C   s’   |dkr$d}| j  tjj|td�¡ tddgddgdfgd	d
dgd�}|jddd�}t||d�}tdddgdddggd	d
dgd�}t	 
||¡ d S )NÚpyarrowzEPyarrow only supports a file path as an input and line delimited json©ÚreasonÚraisesr   r	   z
2020-03-05z2020-04-08T09:58:49+00:00ZhectorZaccountsÚdateÚnamer   Tr   r   ©r*   )ÚnodeÚ
add_markerÚpytestÚmarkÚxfailÚ
ValueErrorr   r   r   r    r!   )Úrequestr*   r0   r   Z	json_liner"   r#   r   r   r   Útest_read_datetime(   s    þþr<   c                  C   s|   d} t | ƒ} t| dd�}tddgddggddgd	�}t ||¡ d} t| dd�}tddgddggddgd	�}t ||¡ d S )
Nu5   {"a": "fooâ€�", "b": "bar"}
{"a": "foo", "b": "bar"}
Tr   u   fooâ€�ÚbarÚfoor   r   r   )r   r   r   r    r!   )r&   r"   r#   r   r   r   Útest_read_jsonl_unicode_chars<   s    r?   c                  C   sÚ   t ddgddggddgd�} | jddd�}d	}||ks:t‚t d
dgddggddgd�} | jddd�}d}||kstt‚t t|dd�| ¡ t ddgddggddgd�} | jddd�}d}||ksÂt‚t t|dd�| ¡ d S )Nr   r	   r   r   r   r   T©r   r   z{"a":1,"b":2}
{"a":1,"b":2}
zfoo}r=   zfoo"z/{"a":"foo}","b":"bar"}
{"a":"foo\"","b":"bar"}
r   zfoo\za\z4{"a\\":"foo\\","b":"bar"}
{"a\\":"foo\"","b":"bar"}
)r   r   ÚAssertionErrorr    r!   r   )r   r"   r#   r   r   r   Útest_to_jsonlN   s    rB   c                  C   sD   t ddgddggddgd�} | jddd� d	¡}d}||ks@t‚d S )
Nr   r	   r   r   r   r   Tr@   Ú
)r   r   ÚcountrA   )r   Zactual_new_lines_countZexpected_new_lines_countr   r   r   Útest_to_jsonl_count_new_linesc   s    rE   Ú	chunksizer   g      ð?c              	   C   sn   |dkr$d}| j  tjj|td�¡ tt|ƒdd�}tt|ƒd||d��}t 	|¡}W 5 Q R X t
 ||¡ d S )Nr.   úmPyarrow only supports a file path as an input and line delimited jsonand doesn't support chunksize parameter.r/   Tr   ©r   rF   r*   )r5   r6   r7   r8   r9   r:   r   r   ÚpdÚconcatr    r!   )r;   r   rF   r*   r0   Ú	unchunkedÚreaderÚchunkedr   r   r   Útest_readjson_chunksk   s    ÿ   ÿrN   c              
   C   sB   d}t jt|d��& tt| ƒdd|d��}W 5 Q R X W 5 Q R X d S )Nz*chunksize can only be passed if lines=True©ÚmatchFr	   rH   ©r7   r1   r:   r   r   )r   r*   ÚmsgÚ_r   r   r   Ú&test_readjson_chunksize_requires_lines‚   s       ÿrT   c              	   C   sž   |dkr"d}| j  tjj|d�¡ t dddœ¡}t|jddd	�ƒ}t	|dd
|d�}t|jddd	�ƒ}t	|dd
d|d��}t 
|¡}W 5 Q R X t ||¡ d S )Nr.   rG   )r0   r   r	   r   Tr   r   ÚSeries)r   Útypr*   )r   rV   rF   r*   )r5   r6   r7   r8   r9   rI   rU   r   r   r   rJ   r    Zassert_series_equal)r;   r*   r0   ÚsZstriorK   rL   rM   r   r   r   Útest_readjson_chunks_series‹   s$    ÿ    ÿrX   c              	   C   st   |dkr$d}| j  tjj|td�¡ tt|ƒdd|d��}t|ƒ}W 5 Q R X |d j	dks^t
‚|d	 j	d
kspt
‚d S )Nr.   rG   r/   Tr	   rH   r   ©r	   r	   r   )r   r	   )r5   r6   r7   r8   r9   r:   r   r   ÚlistÚshaperA   )r;   r   r*   r0   rL   Úchunksr   r   r   Útest_readjson_each_chunk£   s    ÿ   ÿr]   c              
   C   s¨   |dkr$d}| j  tjj|td�¡ t d¡�p}tdddgdd	d
gdœƒ}|j	|ddd� t
|dd|d��}t |¡}W 5 Q R X t
|d|d�}t ||¡ W 5 Q R X d S )Nr.   rG   r/   ú	test.jsonr   r	   r
   r   r   r   r   Tr   r   rH   r)   )r5   r6   r7   r8   r9   r:   r    Úensure_cleanr   r   r   rI   rJ   r!   )r;   r*   r0   Úpathr   rL   rM   rK   r   r   r   Útest_readjson_chunks_from_file¶   s    ÿra   c                 C   s˜   t  d¡�„}tdddgdddgdœƒ}|j|d	d
d� t|d dd	d	d	d	dd d d	| d d d�}|� | ¡  W 5 Q R X |jjjsŠt	d| › �ƒ‚W 5 Q R X d S )Nr^   r   r	   r
   r   r   r   r   Tr   r   ÚframeF)r   rV   ZdtypeZconvert_axesZconvert_datesZkeep_default_datesZprecise_floatZ	date_unitÚencodingr   rF   ÚcompressionÚnrowsz%didn't close stream with chunksize = )
r    r_   r   r   r   ÚreadZhandlesÚhandleÚclosedrA   )rF   r`   r   rL   r   r   r   Útest_readjson_chunks_closesÈ   s2    òÿþri   éÿÿÿÿgš™™™™™@r>   c              
   C   sB   d}t jt|d��& tt| ƒd||d��}W 5 Q R X W 5 Q R X d S )Nz"'chunksize' must be an integer >=1rO   TrH   rQ   )r   rF   r*   rR   rS   r   r   r   Útest_readjson_invalid_chunksizeä   s       ÿrk   r	   c              	   C   sh   d}t dddgdddgdœƒ}t|d	| d
�}| d k	rN|� t |¡}W 5 Q R X tj||d| › �d� d S )NzF

    {"A":1,"B":4}



    {"A":2,"B":5}







    {"A":3,"B":6}
    r   r	   r
   r   r   r   r   T©r   rF   zchunksize: )Úobj)r   r   rI   rJ   r    r!   )rF   ÚjÚorigÚtestr   r   r   Ú)test_readjson_chunks_multiple_empty_linesï   s    rq   c              
   C   s˜   |dkr$d}| j  tjj|td�¡ t d¡�`}| ddd„ ¡ t	|dd	d
��}| 
d¡ W 5 Q R X t||d�}tddgiƒ}t ||¡ W 5 Q R X d S )Nr.   rG   r/   r^   zlocale.getpreferredencodingc                 S   s   dS )NÚcp949r   )Údo_setlocaler   r   r   Ú<lambda>  ó    z'test_readjson_unicode.<locals>.<lambda>Úwzutf-8)rc   u6   {"Â£Â©ÂµÃ€Ã†Ã–ÃžÃŸÃ©Ã¶Ã¿":["Ð�Ð‘Ð’Ð“Ð”Ð°Ð±Ð²Ð³Ð´ê°€"]}r4   u   Â£Â©ÂµÃ€Ã†Ã–ÃžÃŸÃ©Ã¶Ã¿u   Ð�Ð‘Ð’Ð“Ð”Ð°Ð±Ð²Ð³Ð´ê°€)r5   r6   r7   r8   r9   r:   r    r_   ÚsetattrÚopenÚwriter   r   r!   )r;   Zmonkeypatchr*   r0   r`   Úfr"   r#   r   r   r   Útest_readjson_unicode	  s    ÿr{   re   c                 C   sJ   d}t |d| d�}tddddgdd	d
dgdœƒjd | … }t ||¡ d S )Nú[{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}T)r   re   r   r
   r   é   r	   r   r   é   r+   )r   r   Úilocr    r!   )re   r*   Újsonlr"   r#   r   r   r   Útest_readjson_nrows  s    (r�   znrows,chunksizerY   )r   r	   c           	   	   C   sˆ   |dkr$d}| j  tjj|td�¡ d}t|d|||d��}t |¡}W 5 Q R X t	ddd	d
gddddgdœƒj
d |… }t ||¡ d S )Nr.   rG   r/   r|   T)r   re   rF   r*   r   r
   r   r}   r	   r   r   r~   r+   )r5   r6   r7   r8   r9   r:   r   rI   rJ   r   r   r    r!   )	r;   re   rF   r*   r0   r€   rL   rM   r#   r   r   r   Útest_readjson_nrows_chunks)  s     ÿ    ÿ(r‚   c              	   C   s6   d}d}t jt|d�� t|dd| d� W 5 Q R X d S )Nr|   z&nrows can only be passed if lines=TruerO   Fr	   )r   re   r*   )r7   r1   r:   r   )r*   r€   rR   r   r   r   Ú"test_readjson_nrows_requires_linesA  s    rƒ   c           
   	   C   sÌ   |dkr$d}| j  tjj|td�¡ tddggddgdgd	�td
dggddgdgd	�tddggddgdgd	�g}|ddddƒ}t|ƒ ¡ }t	|dd|d��(}t
|ƒD ]\}}	t |	|| ¡ q¤W 5 Q R X d S )Nr.   rG   r/   r   r	   r   r   r   )r   Úindexr
   r   r   r   r%   r&   r'   r(   TrH   )r5   r6   r7   r8   r9   r:   r   r   Úas_urir   Ú	enumerater    r!   )
r;   r,   r*   r0   Zdf_list_expectedZos_pathÚfile_urlZ
url_readerr„   Zchuckr   r   r   Ú"test_readjson_lines_chunks_fileurlM  s    ÿýrˆ   c                  C   sN   dd } G dd„ dƒ}|| ƒ}t tt|ddd�ƒƒdks<t‚|jd	ksJt‚d S )
Nz\{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}
iè  c                   @   s0   e Zd Zddœdd„Zdd„ Zedœdd„ZdS )	z/test_chunksize_is_incremental.<locals>.MyReaderN)Úreturnc                 S   s   d| _ t|ƒ| _d S )Nr   )Ú
read_countr   Ústringio)ÚselfÚcontentsr   r   r   Ú__init__o  s    z8test_chunksize_is_incremental.<locals>.MyReader.__init__c                 W   s   |  j d7  _ | jj|Ž S ©Nr   )rŠ   r‹   rf   )rŒ   Úargsr   r   r   rf   s  s    z4test_chunksize_is_incremental.<locals>.MyReader.readc                 S   s   |  j d7  _ t| jƒS r�   )rŠ   Úiterr‹   )rŒ   r   r   r   Ú__iter__w  s    z8test_chunksize_is_incremental.<locals>.MyReader.__iter__)Ú__name__Ú
__module__Ú__qualname__rŽ   rf   r   r’   r   r   r   r   ÚMyReadern  s   r–   Téd   rl   r   é
   )ÚlenrZ   r   rA   rŠ   )r€   r–   rL   r   r   r   Útest_chunksize_is_incrementald  s    üÿrš   Úorient_Úsplitr„   Útablec              	   C   sF   t ddgddgdœƒ}d}tjt|d�� |jd| d� W 5 Q R X d S )	Nr   r	   r   r   ©Úcol1Úcol2úOmode='a' \(append\) is only supported whenlines is True and orient is 'records'rO   )Úmoder   ©r   r7   r1   r:   r   )r›   r   rR   r   r   r   Útest_to_json_append_orient€  s
    ÿr¤   c               	   C   sH   t ddgddgdœƒ} d}tjt|d�� | jddd	d
� W 5 Q R X d S )Nr   r	   r   r   rž   r¡   rO   Fr   ©r¢   r   r   r£   )r   rR   r   r   r   Útest_to_json_append_lines�  s
    ÿr¦   Úmode_ÚrÚxc              	   C   sP   t ddgddgdœƒ}d| › d�}tjt|d�� |j| d	d
d� W 5 Q R X d S )Nr   r	   r   r   rž   zmode=z@ is not a valid option.Only 'w' and 'a' are currently supported.rO   Fr   r¥   r£   )r§   r   rR   r   r   r   Útest_to_json_append_mode™  s
    
ÿrª   c               	   C   sž   t ddgddgdœƒ} t ddgdd	gdœƒ}t ddddgdddd	gdœƒ}t d
¡�@}| j|ddd� |j|dddd� t|dd�}t ||¡ W 5 Q R X d S )Nr   r	   r   r   rž   r
   r   ÚcÚdr^   Tr   r   r¥   r   ©r   r    r_   r   r   r!   )Údf1Údf2r#   r`   r"   r   r   r   Ú-test_to_json_append_output_consistent_columns¦  s    r°   c               	   C   sª   t ddgddgdœƒ} t ddgdd	gd
œƒ}t ddd d gddddgd d dd	gdœƒ}t d¡�B}| j|dddd� |j|dddd� t|dd�}t ||¡ W 5 Q R X d S )Nr   r	   r   r   rž   Úerz   ú!ú#©r    Úcol3)rŸ   r    rµ   r^   Tr   r¥   r   r­   )r®   Údf3r#   r`   r"   r   r   r   Ú/test_to_json_append_output_inconsistent_columns¸  s    


ýÿr·   c                  C   s(  t ddgddgdœƒ} t ddgdd	gdœƒ}t d
dgddgdœƒ}t dddgiƒ}t ddddd d d d gdddd	d
dd d gd d d d ddd d gd d d d d d ddgdœƒ ddi¡}t d¡�f}| j|dddd� |j|dddd� |j|dddd� |j|dddd� t|dd�}t ||¡ W 5 Q R X d S )Nr   r	   r   r   rž   r
   r   r«   r¬   r±   rz   r²   r³   r´   Úcol4TF)rŸ   r    rµ   r¸   Úfloatr^   r   r¥   r   ©r   Zastyper    r_   r   r   r!   ©r®   r¯   r¶   Zdf4r#   r`   r"   r   r   r   Ú,test_to_json_append_output_different_columnsÐ  s(    üÿùr¼   c                  C   s(  t ddgddgdœƒ} t ddgdd	gdœƒ}t d
dgddgdœƒ}t dddgiƒ}t ddd d d d d d gd d d
ddd	ddgd d ddd d d d gd d d d ddddgdœƒ ddi¡}t d¡�f}|j|dddd� |j|dddd� |j|dddd� | j|dddd� t|dd�}t ||¡ W 5 Q R X d S )Nr   r	   r   r   rž   r
   r   r«   r¬   r±   rz   r²   r³   r´   r¸   TF)r¸   r    rµ   rŸ   r¹   r^   r   r¥   r   rº   r»   r   r   r   Ú6test_to_json_append_output_different_columns_reorderedí  s(    üÿùr½   )/r%   r   Úpathlibr   Útypingr   r7   ZpandasrI   r   r   Zpandas._testingZ_testingr    Zpandas.io.json._jsonr   Zfixturer   r$   r-   r<   r?   rB   rE   r8   ZparametrizerN   rT   rX   r]   ra   ri   rk   rq   r{   r�   r‚   rƒ   rˆ   rš   r¤   r¦   rª   r°   r·   r¼   r½   r   r   r   r   Ú<module>   sX   


	







