U
    ¾mœdüi  ã                   @  s(  d Z ddlmZ ddlmZ ddlmZmZ ddlZddlm	Z	 ddl
ZddlmZmZmZ ddlmZmZ ddlZdd	lmZmZ dd
lmZ ddlmZmZmZmZmZ ddlm Z m!Z! ddl"m#  m$  m%Z& ddl'm(Z( dddœdd„Z)ddddœdd„Z*G dd„ dƒZ+G dd„ de(ej,ƒZ-dS )a�  
Read SAS7BDAT files

Based on code written by Jared Hobbs:
  https://bitbucket.org/jaredhobbs/sas7bdat

See also:
  https://github.com/BioStatMatt/sas7bdat

Partial documentation of the file format:
  https://cran.r-project.org/package=sas7bdat/vignettes/sas7bdat.pdf

Reference for binary data compression:
  http://collaboration.cmc.ec.gc.ca/science/rpn/biblio/ddj/Website/articles/CUJ/1992/9210/ross/ross.htm
é    )Úannotations)Úabc)ÚdatetimeÚ	timedeltaN)Úcast)ÚCompressionOptionsÚFilePathÚ
ReadBuffer)ÚEmptyDataErrorÚOutOfBoundsDatetime)Ú	DataFrameÚisna)Ú
get_handle)Úread_double_with_byteswapÚread_float_with_byteswapÚread_uint16_with_byteswapÚread_uint32_with_byteswapÚread_uint64_with_byteswap)ÚParserÚget_subheader_index)Ú
ReaderBaseÚfloatÚstr©Zsas_datetimeÚunitc                 C  sV   t | ƒrtjS |dkr,tdddƒt| d� S |dkrJtdddƒt| d� S tdƒ‚d S )NÚsé¨  é   )ÚsecondsÚd)Údayszunit must be 'd' or 's')r   ÚpdZNaTr   r   Ú
ValueErrorr   © r#   úO/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pandas/io/sas/sas7bdat.pyÚ_parse_datetime<   s    r%   z	pd.Series)Úsas_datetimesr   Úreturnc                 C  sJ   zt j| |dd�W S  tk
rD   | jt|d�}tt j|ƒ}| Y S X dS )aÉ  
    Convert to Timestamp if possible, otherwise to datetime.datetime.
    SAS float64 lacks precision for more than ms resolution so the fit
    to datetime.datetime is ok.

    Parameters
    ----------
    sas_datetimes : {Series, Sequence[float]}
       Dates or datetimes in SAS
    unit : {str}
       "d" if the floats represent dates, "s" for datetimes

    Returns
    -------
    Series
       Series of datetime64 dtype or datetime.datetime.
    z
1960-01-01)r   Úorigin©r   N)r!   Úto_datetimer   Úapplyr%   r   ÚSeries)r&   r   Zs_seriesr#   r#   r$   Ú_convert_datetimesJ   s    r-   c                   @  sX   e Zd ZU ded< ded< ded< ded< ded< ded	< ddddddd
dœdd„ZdS )Ú_ColumnÚintÚcol_idústr | bytesÚnameÚlabelÚformatÚbytesÚctypeÚlengthÚNone)r0   r2   r3   r4   r6   r7   r'   c                 C  s(   || _ || _|| _|| _|| _|| _d S ©N)r0   r2   r3   r4   r6   r7   )Úselfr0   r2   r3   r4   r6   r7   r#   r#   r$   Ú__init__l   s    
z_Column.__init__N)Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r;   r#   r#   r#   r$   r.   d   s   
r.   c                   @  sâ  e Zd ZU dZded< ded< dSd	d
d
ddd
d
dddœ	dd„Zddœdd„Zddœdd„Zddœdd„Zddœdd„Z	ddœdd„Z
ddœdd „Zddd!œd"d#„Zdddd$œd%d&„Zddd'œd(d)„Zddd*d+œd,d-„Zddœd.d/„Zd
dœd0d1„Zddœd2d3„Zddœd4d5„Zdddd+œd6d7„Zdddd+œd8d9„Zdddd+œd:d;„Zdddd+œd<d=„Zdddd+œd>d?„Zdddd+œd@dA„Zdddd+œdBdC„Zdddd+œdDdE„ZdTdddFœdGdH„ZdIdJ„ ZddœdKdL„ZdMdN„ ZdOd*dPœdQdR„Z dS )UÚSAS7BDATReaderaË  
    Read SAS files in SAS7BDAT format.

    Parameters
    ----------
    path_or_buf : path name or buffer
        Name of SAS file or file-like object pointing to SAS file
        contents.
    index : column identifier, defaults to None
        Column to use as index.
    convert_dates : bool, defaults to True
        Attempt to convert dates to Pandas datetime values.  Note that
        some rarely used SAS date formats may be unsupported.
    blank_missing : bool, defaults to True
        Convert empty strings to missing values (SAS uses blanks to
        indicate missing character variables).
    chunksize : int, defaults to None
        Return SAS7BDATReader object for iterations, returns chunks
        with given number of lines.
    encoding : str, 'infer', defaults to None
        String encoding acc. to Python standard encodings,
        encoding='infer' tries to detect the encoding from the file header,
        encoding=None will leave the data in binary format.
    convert_text : bool, defaults to True
        If False, text variables are left as raw bytes.
    convert_header_text : bool, defaults to True
        If False, header text, including column names, are left as raw
        bytes.
    r/   Ú_int_lengthzbytes | NoneÚ_cached_pageNTÚinferzFilePath | ReadBuffer[bytes]Úboolz
int | Nonez
str | Noner   r8   )	Úpath_or_bufÚconvert_datesÚblank_missingÚ	chunksizeÚencodingÚconvert_textÚconvert_header_textÚcompressionr'   c
           
   	   C  sú   || _ || _|| _|| _|| _|| _|| _d| _d| _g | _	g | _
g | _g | _g | _d | _g | _g | _g | _d| _d| _d| _t|dd|	d�| _| jj| _| j| j| j| j| j| j| j| jd g	| _ z|  !¡  |  "¡  W n t#k
rô   |  $¡  ‚ Y nX d S )Nzlatin-1ó    r   ÚrbF)Zis_textrL   )%ÚindexrF   rG   rH   rI   rJ   rK   Údefault_encodingrL   Úcolumn_names_rawÚcolumn_namesÚcolumn_formatsÚcolumnsÚ%_current_page_data_subheader_pointersrB   Ú_column_data_lengthsÚ_column_data_offsetsÚ_column_typesÚ_current_row_in_file_indexZ_current_row_on_page_indexr   ÚhandlesÚhandleÚ_path_or_bufÚ_process_rowsize_subheaderÚ_process_columnsize_subheaderÚ_process_subheader_countsÚ_process_columntext_subheaderÚ_process_columnname_subheaderÚ#_process_columnattributes_subheaderÚ_process_format_subheaderÚ_process_columnlist_subheaderÚ_subheader_processorsÚ_get_propertiesÚ_parse_metadataÚ	ExceptionÚclose)
r:   rE   rO   rF   rG   rH   rI   rJ   rK   rL   r#   r#   r$   r;   ¡   sX       ÿ
÷zSAS7BDATReader.__init__z
np.ndarray)r'   c                 C  s   t j| jt jd�S )z5Return a numpy int64 array of the column data lengths©Údtype)ÚnpÚasarrayrV   Úint64©r:   r#   r#   r$   Úcolumn_data_lengthsà   s    z"SAS7BDATReader.column_data_lengthsc                 C  s   t j| jt jd�S )z0Return a numpy int64 array of the column offsetsrj   )rl   rm   rW   rn   ro   r#   r#   r$   Úcolumn_data_offsetsä   s    z"SAS7BDATReader.column_data_offsetsc                 C  s   t j| jt  d¡d�S )zj
        Returns a numpy character array of the column types:
           s (string) or d (double)
        ZS1rj   )rl   rm   rX   rk   ro   r#   r#   r$   Úcolumn_typesè   s    zSAS7BDATReader.column_typesc                 C  s   | j  ¡  d S r9   )rZ   ri   ro   r#   r#   r$   ri   ï   s    zSAS7BDATReader.closec                 C  s
  | j  d¡ | j  d¡| _| jdttjƒ… tjkr<tdƒ‚|  tj	tj
¡}|tjkrtd| _d| _tj| _tj| _nd| _tj| _tj| _d| _|  tjtj¡}|tjkr²tj}nd}|  tjtj¡}|dkrâd	| _tjd
k| _nd| _tjdk| _|  tjtj¡d }|tj k�r6tj | | _!| j"dk�rD| j!| _"nd|› d�| _!t#dddƒ}|  $tj%| tj&¡}|t'j(|dd� | _)|  $tj*| tj+¡}|t'j(|dd� | _,|  -tj.| tj/¡| _0| j  | j0d ¡}|  j|7  _t| jƒ| j0k�rðtdƒ‚|  -tj1| tj2¡| _3d S )Nr   i   z'magic number mismatch (not a SAS file?)Té   Fé   ó   ú<Úbigú>ÚlittlerC   zunknown (code=ú)r   r   r   r)   z*The SAS7BDAT file appears to be truncated.)4r\   ÚseekÚreadrB   ÚlenÚconstÚmagicr"   Ú_read_bytesZalign_1_offsetZalign_1_lengthZu64_byte_checker_valueÚU64rA   Zpage_bit_offset_x64Ú_page_bit_offsetZsubheader_pointer_length_x64Ú_subheader_pointer_lengthZpage_bit_offset_x86Zsubheader_pointer_length_x86Zalign_2_offsetZalign_2_lengthZalign_1_checker_valueZalign_2_valueZendianness_offsetZendianness_lengthÚ
byte_orderÚsysÚ	byteorderÚneed_byteswapZencoding_offsetZencoding_lengthZencoding_namesZinferred_encodingrI   r   Ú_read_floatZdate_created_offsetZdate_created_lengthr!   Zto_timedeltaZdate_createdZdate_modified_offsetZdate_modified_lengthZdate_modifiedÚ
_read_uintZheader_size_offsetZheader_size_lengthÚheader_lengthZpage_size_offsetZpage_size_lengthÚ_page_length)r:   ÚbufZalign1ÚepochÚxr#   r#   r$   rf   ò   sj    



 ÿ ÿ ÿ ÿzSAS7BDATReader._get_propertiesr   c                 C  s(   | j | jpdd�}|jr$|  ¡  t‚|S )Nr   )Únrows)r|   rH   Úemptyri   ÚStopIteration)r:   Údar#   r#   r$   Ú__next__8  s
    zSAS7BDATReader.__next__)ÚoffsetÚwidthc                 C  sR   | j d k	st‚|dkr&t| j || jƒS |dkr>t| j || jƒS |  ¡  tdƒ‚d S )Nrt   rs   zinvalid float width)rB   ÚAssertionErrorr   r‡   r   ri   r"   ©r:   r”   r•   r#   r#   r$   rˆ   @  s      ÿ  ÿzSAS7BDATReader._read_float)r”   r•   r'   c                 C  s‚   | j d k	st‚|dkr&|  |d¡d S |dkr>t| j || jƒS |dkrVt| j || jƒS |dkrnt| j || jƒS |  ¡  tdƒ‚d S )Nr   r   é   rt   rs   zinvalid int width)	rB   r–   r€   r   r‡   r   r   ri   r"   r—   r#   r#   r$   r‰   O  s.      ÿ  ÿ  ÿzSAS7BDATReader._read_uint)r”   r7   c                 C  sB   | j d k	st‚|| t| j ƒkr0|  ¡  tdƒ‚| j ||| … S )NzThe cached page is too small.)rB   r–   r}   ri   r"   ©r:   r”   r7   r#   r#   r$   r€   c  s
    zSAS7BDATReader._read_bytesr1   )r”   r7   r'   c                 C  s   |   |  ||¡ d¡¡S )Nó     )Ú_convert_header_textr€   Úrstripr™   r#   r#   r$   Ú_read_and_convert_header_textj  s    ÿz,SAS7BDATReader._read_and_convert_header_textc                 C  sN   d}|sJ| j  | j¡| _t| jƒdkr(qJt| jƒ| jkr@tdƒ‚|  ¡ }qd S )NFr   z2Failed to read a meta data page from the SAS file.)r\   r|   r‹   rB   r}   r"   Ú_process_page_meta)r:   Údoner#   r#   r$   rg   o  s    zSAS7BDATReader._parse_metadatac                 C  sZ   |   ¡  tjtjtjg }| j|kr,|  ¡  | jtjk}| jtjk}t|pV|pV| j	g kƒS r9   )
Ú_read_page_headerr~   Úpage_meta_typesZpage_amd_typeÚpage_mix_typeÚ_current_page_typeÚ_process_page_metadataÚpage_data_typerD   rU   )r:   ÚptZis_data_pageZis_mix_pager#   r#   r$   rž   y  s    
ÿýz!SAS7BDATReader._process_page_metac                 C  s^   | j }tj| }|  |tj¡tj@ | _tj| }|  |tj¡| _	tj
| }|  |tj¡| _d S r9   )r‚   r~   Zpage_type_offsetr‰   Zpage_type_lengthZpage_type_mask2r£   Zblock_count_offsetZblock_count_lengthZ_current_page_block_countZsubheader_count_offsetZsubheader_count_lengthÚ_current_page_subheaders_count)r:   Ú
bit_offsetZtxr#   r#   r$   r    †  s    
ÿ

 ÿz SAS7BDATReader._read_page_headerc                 C  s  | j }t| jƒD �]}tj| }|| j|  }|  || j¡}|| j7 }|  || j¡}|| j7 }|  |d¡}|d7 }|  |d¡}|dks|tjkr’q|  	|| j¡}	t
|	ƒ}
| j|
 }|d k�r|tjdfk}|tjk}| jrô|rô|rô| j ||f¡ n|  ¡  td|	› �ƒ‚q|||ƒ qd S )Nr   r   zUnknown subheader signature )r‚   Úranger§   r~   Zsubheader_pointers_offsetrƒ   r‰   rA   Ztruncated_subheader_idr€   r   re   Zcompressed_subheader_idZcompressed_subheader_typerL   rU   Úappendri   r"   )r:   r¨   Úir”   Ztotal_offsetZsubheader_offsetZsubheader_lengthZsubheader_compressionZsubheader_typeZsubheader_signatureZsubheader_indexZsubheader_processorÚf1Úf2r#   r#   r$   r¤   “  s>    


ÿþ


ÿÿz%SAS7BDATReader._process_page_metadatac                 C  sÒ   | j }|}|}| jr&|d7 }|d7 }n|d7 }|d7 }|  |tj|  |¡| _|  |tj|  |¡| _|  |tj|  |¡| _	|  |tj
|  |¡| _tj| }|  || |¡| _|  |d¡| _|  |d¡| _d S )Niª  iÂ  ib  iz  r˜   )rA   r�   r‰   r~   Zrow_length_offset_multiplierZ
row_lengthZrow_count_offset_multiplierÚ	row_countZcol_count_p1_multiplierÚcol_count_p1Zcol_count_p2_multiplierÚcol_count_p2Z'row_count_on_mix_page_offset_multiplierZ_mix_page_row_countÚ_lcsÚ_lcp)r:   r”   r7   Úint_lenZ
lcs_offsetZ
lcp_offsetZmxr#   r#   r$   r]   ¾  s8    
þþ ÿ ÿ
z)SAS7BDATReader._process_rowsize_subheaderc                 C  sT   | j }||7 }|  ||¡| _| j| j | jkrPtd| j› d| j› d| j› d�ƒ d S )Nz Warning: column count mismatch (z + z != z)
)rA   r‰   Úcolumn_countr¯   r°   Úprint)r:   r”   r7   r³   r#   r#   r$   r^   Ü  s    ÿz,SAS7BDATReader._process_columnsize_subheaderc                 C  s   d S r9   r#   r™   r#   r#   r$   r_   ç  s    z(SAS7BDATReader._process_subheader_countsc           	      C  sš  || j 7 }|  |tj¡}|  ||¡}|d|…  d¡}| j |¡ t| jƒdk�r–d}tj	D ]}||kr\|}q\|| _
|| j 8 }|d }| jr”|d7 }|  || j¡}| d¡}|dkrðd| _|d }| jrÐ|d7 }|  || j¡}|d| j… | _nŒ|tjk�r4|d	 }| j�r|d7 }|  || j¡}|d| j… | _nH| jdk�r|d| _|d }| j�r^|d7 }|  || j¡}|d| j… | _t| d
ƒ�r–|  | j¡| _d S )Nr   rš   r   rM   é   rt   ó    é    é(   Úcreator_proc)rA   r‰   r~   Ztext_block_size_lengthr€   rœ   rQ   rª   r}   Zcompression_literalsrL   r�   r²   r±   rº   Zrle_compressionÚhasattrr›   )	r:   r”   r7   Ztext_block_sizerŒ   Z	cname_rawZcompression_literalÚclZoffset1r#   r#   r$   r`   ê  sN    



z,SAS7BDATReader._process_columntext_subheaderc                 C  sÒ   | j }||7 }|d|  d d }t|ƒD ]¢}|tj|d   tj }|tj|d   tj }|tj|d   tj }|  |tj¡}	|  |tj	¡}
|  |tj
¡}| j|	 }||
|
| … }| j |  |¡¡ q*d S )Nr˜   é   rs   r   )rA   r©   r~   Zcolumn_name_pointer_lengthZ!column_name_text_subheader_offsetZcolumn_name_offset_offsetZcolumn_name_length_offsetr‰   Z!column_name_text_subheader_lengthZcolumn_name_offset_lengthZcolumn_name_length_lengthrQ   rR   rª   r›   )r:   r”   r7   r³   Zcolumn_name_pointers_countr«   Ztext_subheaderZcol_name_offsetZcol_name_lengthÚidxÚ
col_offsetZcol_lenZname_rawÚcnamer#   r#   r$   ra     sD    ÿþÿÿþÿÿþÿ ÿ ÿ
z,SAS7BDATReader._process_columnname_subheaderc           
      C  sÞ   | j }|d|  d |d  }t|ƒD ]²}|| tj ||d   }|d|  tj ||d   }|d|  tj ||d   }|  ||¡}	| j |	¡ |  |tj	¡}	| j
 |	¡ |  |tj¡}	| j |	dkrÒdnd¡ q&d S )Nr˜   r½   rs   r   ó   dó   s)rA   r©   r~   Zcolumn_data_offset_offsetZcolumn_data_length_offsetZcolumn_type_offsetr‰   rW   rª   Zcolumn_data_length_lengthrV   Zcolumn_type_lengthrX   )
r:   r”   r7   r³   Zcolumn_attributes_vectors_countr«   Zcol_data_offsetZcol_data_lenZ	col_typesrŽ   r#   r#   r$   rb   8  s*    ÿÿþ
ýÿÿz2SAS7BDATReader._process_columnattributes_subheaderc                 C  s   d S r9   r#   r™   r#   r#   r$   rd   R  s    z,SAS7BDATReader._process_columnlist_subheaderc                 C  sx  | j }|tj d|  }|tj d|  }|tj d|  }|tj d|  }|tj d|  }|tj d|  }	|  |tj	¡}
t
|
t| jƒd ƒ}|  |tj¡}|  |tj¡}|  |tj¡}t
|t| jƒd ƒ}|  |tj¡}|  |	tj¡}| j| }|  |||| … ¡}| j| }|  |||| … ¡}t| jƒ}t|| j| ||| j| | j| ƒ}| j |¡ | j |¡ d S )Né   r   )rA   r~   Z)column_format_text_subheader_index_offsetZcolumn_format_offset_offsetZcolumn_format_length_offsetZ(column_label_text_subheader_index_offsetZcolumn_label_offset_offsetZcolumn_label_length_offsetr‰   Z)column_format_text_subheader_index_lengthÚminr}   rQ   Zcolumn_format_offset_lengthZcolumn_format_length_lengthZ(column_label_text_subheader_index_lengthZcolumn_label_offset_lengthZcolumn_label_length_lengthr›   rT   r.   rR   rX   rV   rS   rª   )r:   r”   r7   r³   Ztext_subheader_formatZcol_format_offsetZcol_format_lenZtext_subheader_labelZcol_label_offsetZcol_label_lenrŽ   Z
format_idxZformat_startZ
format_lenZ	label_idxZlabel_startZ	label_lenZlabel_namesZcolumn_labelZformat_namesZcolumn_formatZcurrent_column_numberÚcolr#   r#   r$   rc   V  s`    ÿÿ ÿ ÿ ÿ ÿ
ÿ
ÿ
ú	z(SAS7BDATReader._process_format_subheader)r�   r'   c                 C  sò   |d kr| j d k	r| j }n|d kr(| j}t| jƒdkrF|  ¡  tdƒ‚|dkr`| j| jkr`tƒ S t|| j| j ƒ}| j 	d¡}| j 	d¡}t
j||ftd�| _t
j|d| ft
jd�| _d| _t| ƒ}| |¡ |  ¡ }| jd k	rî| | j¡}|S )Nr   zNo columns to parse from filerÁ   rÂ   rj   rs   )rH   r®   r}   rX   ri   r
   rY   r   rÄ   Úcountrl   r�   ÚobjectÚ_string_chunkZzerosZuint8Ú_byte_chunkÚ_current_row_in_chunk_indexr   r|   Ú_chunk_to_dataframerO   Z	set_index)r:   r�   ÚndÚnsÚpÚrsltr#   r#   r$   r|   �  s*    

zSAS7BDATReader.readc                 C  s¦   g | _ | j | j¡| _t| jƒdkr(dS t| jƒ| jkrf|  ¡  dt| jƒd›d| jd›d�}t|ƒ‚|  ¡  | j	t
jkr‚|  ¡  | j	t
jt
jt
jg kr¢|  ¡ S dS )Nr   Tz-failed to read complete page from file (read r   z of z bytes)F)rU   r\   r|   r‹   rB   r}   ri   r"   r    r£   r~   r¡   r¤   r¥   r¢   Ú_read_next_page)r:   Úmsgr#   r#   r$   rÐ   ¬  s$    ÿþzSAS7BDATReader._read_next_pagec                 C  st  | j }| j}t|| |ƒ}i }d\}}t| jƒD �]*}| j| }| j| dkrÚ| j|d d …f j| jd d�}	t	j
|	tj|d�||< | jrÐ| j| tjkr®t|| dƒ||< n"| j| tjkrÐt|| dƒ||< |d7 }q0| j| dk�r<t	j
| j|d d …f |d	�||< | j�r2| jd k	�r2|  || j¡||< |d7 }q0|  ¡  td
t| j| ƒ› �ƒ‚q0t|| j|dd�}
|
S )N)r   r   rÁ   r   rj   )rk   rO   r   r   rÂ   )rO   zunknown column type F)rT   rO   Úcopy)rÊ   rY   r©   r´   rR   rX   rÉ   Úviewr„   r!   r,   rl   Zfloat64rF   rS   r~   Zsas_date_formatsr-   Zsas_datetime_formatsrÈ   rJ   rI   Ú_decode_stringr   ri   r"   Úreprr   )r:   ÚnÚmZixrÏ   ÚjsZjbÚjr2   Zcol_arrZdfr#   r#   r$   rË   Å  s2    
 
 
z"SAS7BDATReader._chunk_to_dataframec                 C  s   |  | jp| j¡S r9   )ÚdecoderI   rP   ©r:   Úbr#   r#   r$   rÔ   ä  s    zSAS7BDATReader._decode_stringr5   )rÜ   r'   c                 C  s   | j r|  |¡S |S d S r9   )rK   rÔ   rÛ   r#   r#   r$   r›   ç  s    
z#SAS7BDATReader._convert_header_text)NTTNNTTrC   )N)!r<   r=   r>   Ú__doc__r?   r;   rp   rq   rr   ri   rf   r“   rˆ   r‰   r€   r�   rg   rž   r    r¤   r]   r^   r_   r`   ra   rb   rd   rc   r|   rÐ   rË   rÔ   r›   r#   r#   r#   r$   r@      sN   
        ö ?F
+-!7r@   ).rÝ   Ú
__future__r   Úcollectionsr   r   r   r…   Útypingr   Únumpyrl   Zpandas._typingr   r   r	   Zpandas.errorsr
   r   Zpandasr!   r   r   Zpandas.io.commonr   Zpandas.io.sas._byteswapr   r   r   r   r   Zpandas.io.sas._sasr   r   Zpandas.io.sas.sas_constantsÚioÚsasZsas_constantsr~   Zpandas.io.sas.sasreaderr   r%   r-   r.   ÚIteratorr@   r#   r#   r#   r$   Ú<module>   s&   