U
    ¾mœd,&  ã                   @   s*  d Z ddlmZmZ ddlZddlZddlZddlZddl	Z	ddl
mZmZ ddlmZ e	j d¡Ze	j d¡Zdd„ Zed	d
„ ƒZee	j dddg¡e	j ddddg¡dd„ ƒƒƒZdd„ Zdd„ Ze	j ddi eddgiƒfdddieddgiƒfdd dgiedd!d"giƒfd#dgd$d%œeddgiƒfd#dgd&d%œedejdgiƒfg¡d'd(„ ƒZd)d*„ Ze	j d+d,d-d.g¡d/d0„ ƒZee	j d1d$d&g¡d2d3„ ƒƒZed4d5„ ƒZe	j dd6dd7d8d9g¡d:d;„ ƒZ ee	j dd6ddd<d=g¡d>d?„ ƒƒZ!ed@dA„ ƒZ"edBdC„ ƒZ#e	j d¡e	j dDdEdFg¡dGdH„ ƒƒZ$dS )IzZ
Tests encoding functionality during parsing
for all of the parsers defined in parsers.py
é    )ÚBytesIOÚTextIOWrapperN)Ú	DataFrameÚread_csvZpyarrow_skipZpyarrow_xfailc                 C   sL   d}| }t d |¡ƒ}|j|d|d�}tddggddgd	�}t ||¡ d S )
NÚcp1255u   ×©×œ×•×�:1234
562:123ú:)ÚsepÚencodingi2  é{   u   ×©×œ×•×�Ú1234©Úcolumns©r   Úencoder   r   ÚtmÚassert_frame_equal)Úall_parsersr	   ÚparserÚdataÚresultÚexpected© r   ú]/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pandas/tests/io/parser/test_encoding.pyÚtest_bytes_io_input   s    r   c                 C   s@   | }t d ¡ ƒ}|j|ddd d�}tddggƒ}t ||¡ d S )Nu   Å�aski, Jan;1ú;úutf-8)r   r	   Úheaderu   Å�aski, Jané   r   )r   r   r   r   r   r   r   r   Útest_read_csv_unicode%   s
    r   r   ú,ú	r	   úutf-16zutf-16lezutf-16bec              
   C   sÌ   | }d  d|¡}dt ¡ › d�}|ddœ}d}t |¡�Š}| |¡}t|dƒ�}	|	 |¡ W 5 Q R X tt	| |¡ƒ|d	��2}
|j
|fd
|i|—Ž}|j
|
fd
|i|—Ž}W 5 Q R X t ||¡ W 5 Q R X d S )Nz)skip this
skip this too
A,B,C
1,2,3
4,5,6r   Ú__z__.csvé   )r   Zskiprowsr   Úwb©r	   r	   )ÚreplaceÚuuidÚuuid4r   Úensure_cleanr   ÚopenÚwriter   r   r   r   )r   r   r	   r   r   ÚpathÚkwargsÚutf8Ú
bytes_dataÚfZbytes_bufferr   r   r   r   r   Útest_utf16_bom_skiprows/   s      û

 r1   c                 C   s6   t j |d¡}| }|j|ddd�}t|ƒdks2t‚d S )Nzutf16_ex.txtr!   r    )r	   r   é2   )Úosr,   Újoinr   ÚlenÚAssertionError)r   Úcsv_dir_pathr,   r   r   r   r   r   Útest_utf16_exampleL   s    r8   c                 C   sL   t j |d¡}| }|j|d dd�}| d¡}|d d }d}||ksHt‚d S )Núunicode_series.csvúlatin-1)r   r	   r   r   i`  u$   Ã� kÃ¶ldum klaka (Cold Fever) (1994))r3   r,   r4   r   Z	set_indexr6   )r   r7   r,   r   r   Úgotr   r   r   r   Útest_unicode_encodingS   s    
r<   zdata,kwargs,expectedza
1Úar   z"a"
1Ú	quotecharú"zb
1ÚnamesÚbÚ1ú
1T)r@   Úskip_blank_linesFc                    sx   | }d‰ d‰‡ ‡fdd„}|j dkrN|dkrN| dd¡rN|j tjjd	d
�¡ |j||ƒfdˆi|—Ž}t 	||¡ d S )Nu   ï»¿r   c                    s   ˆ |    ˆ¡}t|ƒS )N)r   r   )Ú_dataZbom_data©Úbomr.   r   r   Ú_encode_data_with_bomx   s    z,test_utf8_bom.<locals>._encode_data_with_bomZpyarrowrC   rD   TzPyarrow can't read blank lines)Úreasonr	   )
ÚengineÚgetÚnodeZ
add_markerÚpytestÚmarkZxfailr   r   r   )r   r   r-   r   Úrequestr   rH   r   r   rF   r   Útest_utf8_bom_   s    ÿþ
ýÿrP   c                 C   sL   t dgdgdœƒ}| }| |¡}d |¡}|jt|ƒ|d�}t ||¡ d S )Ng333333@Útest)Zmb_numZ	multibytezmb_num,multibyte
4.8,testr%   )r   Úformatr   r   r   r   r   )r   Ú	utf_valueÚencoding_fmtr   r   r	   r   r   r   r   r   Útest_read_csv_utf_aliasesŠ   s    

rU   zfile_path,encoding))Úior   Úcsvz	test1.csvr   ))rV   r   r   r9   r:   ))rV   r   r   zsauron.SHIFT_JIS.csvÚshiftjisc           
   	   C   sÐ   | }||Ž }|j ||d�}t||d��}|  |¡}|jr<t‚W 5 Q R X t ||¡ t|dd��}	|j |	|d�}|	jrxt‚W 5 Q R X t ||¡ t|ddd��}	|j |	|d�}|	jr¶t‚W 5 Q R X t ||¡ d S )Nr%   Úrb©Úmoder   )r[   Ú	buffering)r   r*   Úclosedr6   r   r   )
r   Ú	file_pathr	   Údatapathr   Zfpathr   Úfar   Zfbr   r   r   Útest_binary_mode_file_buffers–   s    
ra   Úpass_encodingc           	   	   C   sr   | }|  |¡}tddgiƒ}tjd|dd��<}| d¡ | d¡ |j||rP|nd d�}t ||¡ W 5 Q R X d S )	NZfooÚbarzw+T)r[   r	   Zreturn_filelikezfoo
barr   r%   )rR   r   r   r)   r+   Úseekr   r   )	r   rS   rT   rb   r   r	   r   r0   r   r   r   r   Útest_encoding_temp_file¶   s    


re   c              	   C   s~   | }d}d}d}t ||giƒ}t ¡ �N}| |› d|› � |¡¡ | d¡ |j||d�}t ||¡ |j	rpt
‚W 5 Q R X d S )Nz	shift-jisu	   ã�¦ã�™ã�¨u   ã�“ã‚€Ú
r   r%   )r   ÚtempfileÚNamedTemporaryFiler+   r   rd   r   r   r   r]   r6   )r   r   r	   Útitler   r   r0   r   r   r   r   Útest_encoding_named_temp_fileÇ   s    

rj   r   z	utf-16-bez	utf-16-lezutf-32c                 C   sR   d}t | | ¡ƒ}t|d| d�}tddgddgdd	ggd
dgd�}t ||¡ d S )Nu   a	b
ï¼šfoo	0
bar	1
baz	2r    )Ú	delimiterr	   u   ï¼šfoor   rc   r   Zbazr#   r=   rA   )r   r   r   )r	   r   Zencoded_datar   r   r   r   r   Ú%test_parse_encoded_special_charactersÜ   s
    "rl   r   r:   c              	   C   sp   | }t ddddgddddgd	d
ddgdœƒ}t ¡ �&}|j|d|d� |j||dd�}W 5 Q R X t ||¡ d S )NZRaphaelZ	DonatellozMiguel AngelZLeonardoÚredÚpurpleZorangeÚblueZsaizbo staffZnunchunkZkatana)ÚnameÚmaskZweaponF)Úindexr	   T)r	   Ú
memory_map)r   r   r)   Úto_csvr   r   )r   r	   r   r   ÚfileÚdfr   r   r   Útest_encoding_memory_mapê   s    


ýÿ
rw   c              	   C   sh   | }t dgd d�}d|jd< t d¡�*}|j|dddd	� |j|d
ddd�}W 5 Q R X t ||¡ d
S )zO
    Chunk splits a multibyte character with memory_map=True

    GH 43540
    Zaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaai   )r   u�   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaÄ…iÿ  zbug-gh43540.csvFr   ©rr   r   r	   NTÚc)r   rs   rJ   )r   Zilocr   r)   rt   r   r   )r   r   rv   ÚfnameÚdfrr   r   r   Ú test_chunk_splits_multibyte_charü   s    
r|   c              	   C   sÖ   g }d}d}d}t t|ƒt|ƒ|ƒD ]X}d dd„ t ||d ƒD ƒ¡d }z| d¡ W n tk
rp   Y q$Y nX | |¡ q$| }t|ƒ}t d	¡�,}	|j	|	d
d
dd� |j
|	ddddd�}
W 5 Q R X t ||
¡ dS )zg
    GH 43787

    Test correct handling of UTF-8 chars when memory_map=True and encoding is UTF-8
    é€   ú u   ð�‚€Ú c                 S   s   g | ]}t |ƒ‘qS r   )Úchr)Ú.0ry   r   r   r   Ú
<listcomp>  s     z,test_readcsv_memmap_utf8.<locals>.<listcomp>rf   r   zutf8test.csvFrx   NTry   )r   rs   rJ   r	   )ÚrangeÚordr4   r   ÚUnicodeEncodeErrorÚappendr   r   r)   rt   r   r   )r   ÚlinesÚline_lengthZ
start_charZend_charÚlnumÚliner   rv   rz   r{   r   r   r   Útest_readcsv_memmap_utf8  s.    "
    ÿr‹   r[   zw+bzw+tc              	   C   sh   | }d}d|krd}t j|d��$}| |¡ | d¡ | |¡}W 5 Q R X tg dgd�}t ||¡ d S )Ns   abcdÚtÚabcdrZ   r   r   )rg   ÚSpooledTemporaryFiler+   rd   r   r   r   r   )r   r[   r   ÚcontentÚhandlerv   r   r   r   r   Útest_not_readable.  s    

r‘   )%Ú__doc__rV   r   r   r3   rg   r'   ÚnumpyÚnprM   Zpandasr   r   Zpandas._testingZ_testingr   rN   ZusefixturesZskip_pyarrowZxfail_pyarrowr   r   Zparametrizer1   r8   r<   ÚnanrP   rU   ra   re   rj   rl   rw   r|   r‹   r‘   r   r   r   r   Ú<module>   sz   
	
ýöþ
ýþ

 ÿ



