U
    ¾mœd¯:  ã                   @  sN  d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZ	ddl
mZmZmZmZ ddlmZ ddlmZ ddlZdd	lmZ dd
lmZ dZdZdZdZddddddddddddddddgZdZd Zd!Z d"Z!d#e› d$e › d$e› d$e!› d%�	Z"d&e› d$e› d'�Z#d(Z$d)d*d+œd,d-„Z%d)d.œd/d0„Z&d1d2„ Z'd3d4„ Z(G d5d6„ d6eej)ƒZ*dS )7a-  
Read a SAS XPort format file into a Pandas DataFrame.

Based on code from Jack Cushman (github.com/jcushman/xport).

The file format is defined here:

https://support.sas.com/content/dam/SAS/support/en/technical-papers/record-layout-of-a-sas-version-5-or-6-data-set-in-sas-transport-xport-format.pdf
é    )Úannotations)Úabc)ÚdatetimeN)ÚCompressionOptionsÚDatetimeNaTTypeÚFilePathÚ
ReadBuffer)ÚAppender)Úfind_stack_level)Ú
get_handle)Ú
ReaderBasezPHEADER RECORD*******LIBRARY HEADER RECORD!!!!!!!000000000000000000000000000000  zKHEADER RECORD*******MEMBER  HEADER RECORD!!!!!!!000000000000000001600000000zPHEADER RECORD*******DSCRPTR HEADER RECORD!!!!!!!000000000000000000000000000000  zPHEADER RECORD*******OBS     HEADER RECORD!!!!!!!000000000000000000000000000000  ÚntypeZnhfunÚfield_lengthZnvar0ÚnameÚlabelZnformZnflZnum_decimalsZnfjZnfillZniformZniflZnifdZnposÚ_z‚Parameters
----------
filepath_or_buffer : str or file-like object
    Path to SAS file or object implementing binary read method.zâindex : identifier of index column
    Identifier of column that should be used as index of the DataFrame.
encoding : str
    Encoding for text data.
chunksize : int
    Read file `chunksize` lines at a time, returns iterator.zBformat : str
    File format, only `xport` is currently supported.z\iterator : bool, default False
    Return XportReader object for reading file incrementally.z#Read a SAS file into a DataFrame.

Ú
a  

Returns
-------
DataFrame or XportReader

Examples
--------
Read a SAS Xport file:

>>> df = pd.read_sas('filename.XPT')

Read a Xport file in 10,000 line chunks:

>>> itr = pd.read_sas('filename.XPT', chunksize=10000)
>>> for chunk in itr:
>>>     do_something(chunk)

z$Class for reading SAS Xport files.

zš

Attributes
----------
member_info : list
    Contains information about the file
fields : list
    Contains information about the variables in the file
zÊRead observations from SAS Xport file, returning as data frame.

Parameters
----------
nrows : int
    Number of rows to read from data file; if None, read whole
    file.

Returns
-------
A DataFrame.
Ústrr   )ÚdatestrÚreturnc                 C  s.   zt  | d¡W S  tk
r(   tj Y S X dS )z1Given a date in xport format, return Python date.z%d%b%y:%H:%M:%SN)r   ÚstrptimeÚ
ValueErrorÚpdZNaT)r   © r   úP/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pandas/io/sas/sas_xport.pyÚ_parse_date‘   s    r   )Úsc                 C  s@   i }d}|D ](\}}| ||| …   ¡ ||< ||7 }q|d= |S )a  
    Parameters
    ----------
    s: str
        Fixed-length string to split
    parts: list of (name, length) pairs
        Used to break up string, name '_' will be filtered from output.

    Returns
    -------
    Dict of name:contents of string at given location.
    r   r   )Ústrip)r   ÚpartsÚoutÚstartr   Úlengthr   r   r   Ú_split_lineš   s    
r"   c                 C  sT   |dkrPt  t| ƒt  d¡¡}t  d|› dd| › �¡}|j|d�}| |d< |S | S )Né   ZS8ÚSz,S©ÚdtypeÚf0)ÚnpÚzerosÚlenr&   Úview)ÚvecÚnbytesÚvec1r&   Zvec2r   r   r   Ú_handle_truncated_float_vec°   s    	r/   c           	      C  s  t  d¡}| j|d�}|d }|d }|d@ }t jt| ƒt jd�}d|t  |d@ ¡< d|t  |d	@ ¡< d
|t  |d@ ¡< ||L }||? |d@ dd
|  > B }|dM }||d? d@ d d> | d d> |d@ B O }t jt|ƒfdd�}||d< ||d< |jdd�}| d¡}|S )zf
    Parse a vector of float values representing IBM 8 byte floats into
    native 8 byte floats.
    z>u4,>u4r%   r'   Úf1iÿÿÿ é   i    é   i  @ é   i  € é   é   l   ÿß é   é   éA   iÿ  é   l        z>f8Zf8)	r(   r&   r+   r)   r*   Zuint8ÚwhereÚemptyZastype)	r,   r&   r.   Zxport1Zxport2Zieee1ÚshiftZieee2Zieeer   r   r   Ú_parse_float_vecÃ   s*    
		 ÿ
r=   c                   @  s’   e Zd ZeZd dddddœd	d
„Zddœdd„Zdd„ Zdd„ Zddœdd„Z	ddœdd„Z
d!ddœdd„Zdd„ Zeeƒd"dddœdd„ƒZdS )#ÚXportReaderNú
ISO-8859-1ÚinferzFilePath | ReadBuffer[bytes]z
str | Noner   ÚNone)Úfilepath_or_bufferÚencodingÚcompressionr   c                 C  sf   || _ d| _|| _|| _t|d|d|d�| _| jj| _z|  ¡  W n t	k
r`   |  
¡  ‚ Y nX d S )Nr   ÚrbF)rC   Zis_textrD   )Ú	_encodingÚ_lines_readÚ_indexÚ
_chunksizer   ÚhandlesÚhandlerB   Ú_read_headerÚ	ExceptionÚclose)ÚselfrB   ÚindexrC   Ú	chunksizerD   r   r   r   Ú__init__ÿ   s"    û
zXportReader.__init__)r   c                 C  s   | j  ¡  d S )N)rJ   rN   ©rO   r   r   r   rN     s    zXportReader.closec                 C  s   | j  d¡ ¡ S )NéP   )rB   ÚreadÚdecoderS   r   r   r   Ú_get_row  s    zXportReader._get_rowc              	   C  sB  | j  d¡ |  ¡ }|tkr4d|kr,tdƒ‚tdƒ‚|  ¡ }ddgddgd	dgd
dgddgg}t||ƒ}|d dkr|tdƒ‚t|d ƒ|d< || _|  ¡ }t|d d… ƒ|d< |  ¡ }|  ¡ }| t	¡}|t
k}	|rØ|	sàtdƒ‚t|dd… ƒ}
ddgddgddgddgd	dgd
dgddgg}t|  ¡ |ƒ}ddgd
dgddgddgg}| t|  ¡ |ƒ¡ t|d ƒ|d< t|d ƒ|d< || _dddœ}t|  ¡ dd… ƒ}|
| }|d �rÂ|d|d  7 }| j  |¡}g }d}t|ƒ|
k�rÌ|d |
… ||
d …  }}| d¡}t d|¡}ttt|ƒƒ}|d
= ||d   |d < |d! }|d  dk�rv|d"k �sb|dk�rvd#|› d$�}t|ƒ‚| ¡ D ]2\}}z| ¡ ||< W n tk
�r¬   Y nX �q~||d! 7 }||g7 }�qÖ|  ¡ }|tk�sætd%ƒ‚|| _|| _| j  ¡ | _|  ¡ | _ d&d'„ | jD ƒ| _!d(d'„ t"| jƒD ƒ}t# $|¡}|| _%d S ))Nr   z**COMPRESSED**z<Header record indicates a CPORT file, which is not readable.z#Header record is not an XPORT file.Úprefixr6   Úversionr#   ZOSr   Úcreatedé   zSAS     SAS     SASLIBz!Header record has invalid prefix.ÚmodifiedzMember header not foundéûÿÿÿéþÿÿÿÚset_nameZsasdatar   é(   ÚtypeÚnumericÚchar)r1   r2   é6   é:   rT   éŒ   z>hhhh8s40s8shhh2s8shhl52sr   r   r2   zFloating field width z is not between 2 and 8.zObservation header not found.c                 S  s   g | ]}|d    ¡ ‘qS )r   )rV   )Ú.0Úxr   r   r   Ú
<listcomp>…  s     z,XportReader._read_header.<locals>.<listcomp>c                 S  s,   g | ]$\}}d t |ƒ dt |d ƒ f‘qS )r   r$   r   )r   )rg   ÚiÚfieldr   r   r   ri   ˆ  s   ÿ)&rB   ÚseekrW   Ú_correct_line1r   r"   r   Ú	file_infoÚ
startswithÚ_correct_header1Ú_correct_header2ÚintÚupdateÚmember_inforU   r*   ÚljustÚstructÚunpackÚdictÚzipÚ
_fieldkeysÚ	TypeErrorÚitemsr   ÚAttributeErrorÚ_correct_obs_headerÚfieldsÚrecord_lengthÚtellÚrecord_startÚ_record_countÚnobsÚcolumnsÚ	enumerater(   r&   Ú_dtype)rO   Úline1Úline2Úfifrn   Úline3Úheader1Úheader2Z	headflag1Z	headflag2ZfieldnamelengthZmemrt   ÚtypesZ
fieldcountZ
datalengthZ	fielddatar   Z
obs_lengthZ
fieldbytesZfieldstructrk   ÚflÚmsgÚkÚvÚheaderZdtypelr&   r   r   r   rL   !  sž    ÿ"

ù	



þ
"


þ
zXportReader._read_headerzpd.DataFramec                 C  s   | j | jpdd�S )Nr1   ©Únrows)rU   rI   rS   r   r   r   Ú__next__�  s    zXportReader.__next__rr   c                 C  sÎ   | j  dd¡ | j  ¡ | j }|d dkr:tjdtƒ d� | jdkr\| j  | j¡ || j S | j  dd¡ | j  d¡}t	j
|t	jd�}t	 |dk¡}t|ƒdkr¦d}nd	t|ƒ }| j  | j¡ || | j S )
zÇ
        Get number of records in file.

        This is maybe suboptimal because we have to seek to the end of
        the file.

        Side effect: returns file position to record_start.
        r   r2   rT   zxport file may be corrupted.)Ú
stackleveli°ÿÿÿr%   l     @@€  r#   )rB   rl   r�   r‚   ÚwarningsÚwarnr
   r€   rU   r(   Ú
frombufferZuint64Zflatnonzeror*   )rO   Ztotal_records_lengthZlast_card_bytesZ	last_cardZixZtail_padr   r   r   rƒ   ’  s&    	þ

zXportReader._record_countc                 C  s   |dkr| j }| j|d�S )a  
        Reads lines from Xport file and returns as dataframe

        Parameters
        ----------
        size : int, defaults to None
            Number of lines to read.  If None, reads whole file.

        Returns
        -------
        DataFrame
        Nr”   )rI   rU   )rO   Úsizer   r   r   Ú	get_chunk¸  s    zXportReader.get_chunkc                 C  sl   |j dd�}|d dk|d dk@ |d dk@ }|d dk|d d	k@ |d d
kB |d dkB }||M }|S )Nzu1,u1,u2,u4r%   r0   r   Úf2Zf3r'   r8   éZ   é_   é.   )r+   )rO   r,   r’   ÚmissZmiss1r   r   r   Ú_missing_doubleÉ  s    $
ÿ
þÿzXportReader._missing_doublez
int | None)r•   r   c                   sj  |d krˆ j }t|ˆ j ˆ j ƒ}|ˆ j }|dkr>ˆ  ¡  t‚ˆ j |¡}tj	|ˆ j
|d�}i }tˆ jƒD ]®\}}|dt|ƒ  }	ˆ j| d }
|
dkrÊt|	ˆ j| d ƒ}	ˆ  |	¡}t|	ƒ}tj||< n@ˆ j| d dk�r
dd	„ |	D ƒ}ˆ jd k	�r
‡ fd
d	„|D ƒ}| ||i¡ qjt |¡}ˆ jd k�rLt tˆ jˆ j| ƒ¡|_n| ˆ j¡}ˆ  j|7  _|S )Nr   )r&   Úcountr   r   rb   r   rc   c                 S  s   g | ]}|  ¡ ‘qS r   )Úrstrip©rg   Úyr   r   r   ri   ë  s     z$XportReader.read.<locals>.<listcomp>c                   s   g | ]}|  ˆ j¡‘qS r   )rV   rF   r¥   rS   r   r   ri   î  s     )r„   ÚminrG   r€   rN   ÚStopIterationrB   rU   r(   rš   r‡   r†   r…   r   r   r/   r¢   r=   ÚnanrF   rs   r   Z	DataFramerH   ÚIndexÚrangerP   Z	set_index)rO   r•   Z
read_linesÚread_lenÚrawÚdataZdf_dataÚjrh   r,   r   r¡   r’   Zdfr   rS   r   rU   Ô  s:    


zXportReader.read)Nr?   Nr@   )N)N)Ú__name__Ú
__module__Ú__qualname__Ú_xport_reader_docÚ__doc__rR   rN   rW   rL   r–   rƒ   rœ   r¢   r	   Ú_read_method_docrU   r   r   r   r   r>   ü   s       ún&r>   )+r´   Ú
__future__r   Úcollectionsr   r   rv   r˜   Únumpyr(   Zpandas._typingr   r   r   r   Zpandas.util._decoratorsr	   Zpandas.util._exceptionsr
   Zpandasr   Zpandas.io.commonr   Zpandas.io.sas.sasreaderr   rm   rp   rq   r~   rz   Z_base_params_docZ_params2_docZ_format_params_docZ_iterator_docZ_read_sas_docr³   rµ   r   r"   r/   r=   ÚIteratorr>   r   r   r   r   Ú<module>   sz   	ÿÿÿÿðþýüûýü	9