U
    Ãmœd]_  ã                   @  sì   d Z ddlmZ ddlmZ ddlmZ ddlZddl	m
Z
mZmZmZ ddlm  mZ ddlmZmZ ddlmZ d	d
„ Zdd„ Zdd„ ZG dd„ dƒZG dd„ deƒZG dd„ deƒZdd„ Zdd„ Zd dd„Zdd„ Z d!dd„Z!dS )"zr
Base tools for handling various kinds of data structures, attaching metadata to
results, and doing data cleaning
é    )Úannotations)Úlmap)ÚreduceN)Ú	DataFrameÚSeriesÚisnullÚ
MultiIndex)Úcache_readonlyÚcache_writable)ÚMissingDataErrorc                 C  s,   t  | ¡jdkr(t  | ¡ ¡ jdkr(d S d S ©Né   )ÚnpÚasarrayÚndimÚsqueeze©Úx© r   úN/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/statsmodels/base/data.pyÚ_asarray_2dcolumns   s    $r   c                 C  sB   t  | ¡} | jdkr$| dd…df } t jt| ƒdd�dd…df S )zy
    Makes sure input is an array and is 2d. Makes sure output is 2d. True
    indicates a null in the rows of 2d x.
    r   N©Zaxis)r   r   r   Úanyr   r   r   r   r   Ú_asarray_2d_null_rows   s    

r   c                  G  s0   t | ƒdkr| dggf7 } dd„ }t|| ƒ ¡ S )z¸
    Returns a boolean array which is True where any of the rows in any
    of the _2d_ arrays in arrs are NaNs. Inputs can be any mixture of Series,
    DataFrames or array_like.
    r   Fc                 S  s0   t | dƒo| jtko| }t t| ƒ|t|ƒB ¡S )NÚdtype)Úhasattrr   Úboolr   Ú
logical_orr   )r   ÚyZx_is_boolean_arrayr   r   r   Ú_nan_row_maybe_two_inputs.   s    

ÿz,_nan_rows.<locals>._nan_row_maybe_two_inputs)Úlenr   r   )Zarrsr   r   r   r   Ú	_nan_rows%   s    r!   c                   @  sJ  e Zd ZdZdZdZdCdd„Zdd„ Zdd	„ Zd
d„ Z	e
dd„ ƒZe
dd„ ƒZe
dd„ ƒZdd„ Zeƒ dd„ ƒZeƒ ddœdd„ƒZedd„ ƒZejdd„ ƒZedd„ ƒZejdd„ ƒZed d!„ ƒZd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zd*d+„ ZdDd-d.„Zd/d0„ Zd1d2„ Zd3d4„ Z d5d6„ Z!d7d8„ Z"d9d:„ Z#d;d<„ Z$d=d>„ Z%d?d@„ Z&dAdB„ Z'dS )EÚ	ModelDatazi
    Class responsible for handling input data and extracting metadata into the
    appropriate form
    NÚnonec           	      K  sþ   t  |¡st  |¡r(ddlm} t|ƒ‚d|kr<| d¡| _d|krP| d¡| _|dkrª| j|||f|Ž\}}|| _	| j
 |¡ | j| _| j| _|  | j| j¡\| _| _n,| j
 |¡ || _|| _|  ||¡\| _| _d | _d| _|  |¡ |  ¡  i | _d S )Nr   )Úrecarray_exceptionÚdesign_infoÚformular#   )Ú	data_utilZ_is_recarrayÚstatsmodels.tools.sm_exceptionsr$   ÚNotImplementedErrorÚpopr%   r&   Úhandle_missingZmissing_row_idxÚ__dict__ÚupdateÚendogÚ
orig_endogÚexogÚ	orig_exogÚ_convert_endog_exogÚ	const_idxÚ
k_constantÚ_handle_constantÚ_check_integrityÚ_cache)	Úselfr.   r0   ÚmissingÚhasconstÚkwargsr$   ZarraysZnan_idxr   r   r   Ú__init__>   s6    ÿÿ
zModelData.__init__c                 C  s0   ddl m } || jƒ}d|kr,|d= d|d< |S )Nr   )Úcopyr%   TÚrestore_design_info)r=   r,   )r8   r=   Údr   r   r   Ú__getstate__\   s    
zModelData.__getstate__c           
      C  sÔ   d|krÄddl m}m} g }z|d }W n& tk
rN   |d  |d ¡}Y nX dD ]X}z"||d ||d	d
�\}}W  q¶W qT t|fk
rª }	 z| |	¡ W 5 d }	~	X Y qTX qT|d ‚|j| _|d= | j 	|¡ d S )Nr>   r   )Ú	dmatricesÚ
PatsyErrorÚframer/   r1   )é   é   r   r   é   r&   Z	dataframe)Zeval_envÚreturn_typeéÿÿÿÿ)
ZpatsyrA   rB   ÚKeyErrorÚjoinÚ	NameErrorÚappendr%   r,   r-   )
r8   r?   rA   rB   ÚexcÚdataÚdepthÚ_ZdesignÚer   r   r   Ú__setstate__d   s(    ÿ


zModelData.__setstate__c                 C  s¼  |dks| j d kr"d| _d | _�n–d}tj| j dd�}t |¡ ¡ sLtdƒ‚tj| j dd�}t 	||k¡d  
¡ }|j| _| jdkr°| j d d …|f  ¡ dkrªt|ƒ| _nd}n¢| jdk�r@g }|D ]B}| j d d …|f  ¡ }|dkrüd| _t|ƒ| _ �qR| |¡ qÄt |¡dk}	|	 ¡ �r:d| _t||	 ¡  ƒ| _nd}n| jdk�rRd}n |�r¬|�s¬t t | j jd ¡| j f¡}
tj |
¡}tj | j ¡}t||kƒ| _d | _n|�r¸d| _d S )NFr   r   zexog contains inf or nansr   T)r0   r4   r3   r   ÚmaxÚisfiniteÚallr   ÚminÚwherer   ÚsizeZmeanÚintrL   Úarrayr   ZargmaxZcolumn_stackZonesÚshapeZlinalgZmatrix_rank)r8   r:   Zcheck_implicitZexog_maxZexog_minr3   ÚvaluesÚidxÚvalueÚposZaugmented_exogZ	rank_augmZ	rank_origr   r   r   r5   }   sP    



ÿzModelData._handle_constantc                 C  s   || S ©Nr   ©Úclsr   Únan_maskr   r   r   Ú
_drop_nansº   s    zModelData._drop_nansc                 C  s   || d d …|f S r`   r   ra   r   r   r   Ú_drop_nans_2d¾   s    zModelData._drop_nans_2dc                   s¤  g }ˆ  dd¡}|dk	r4d}g }|dkrd|dg7 }n0|dk	rN||f}ddg}n|f}dg}|dg7 }d}	g }
tˆƒ�r*ˆ ¡ D ]ª\}}|dksœt |¡dkr¨||g7 }q~|jdkrÎ|t |¡f7 }||g7 }q~| ¡ jdkrø|t |¡f7 }||g7 }q~|jdk�r |	t |¡f7 }	|
|g7 }
q~td	ƒ‚q~|dk	�rÔ|‰d}|�rzt|Ž }|j	d ˆj	d k�rhtd
ƒ‚|ˆ  }ˆ|O ‰|	�rüt|	ƒ}|j	d ˆj	d k�r¦tdƒ‚|dk	�rÀ||ˆ  O }n
|ˆ  }ˆ|O ‰n(t|Ž ‰|	�rütˆdd…df f|	 Ž ‰t 
ˆ¡�s†tt||ƒƒ}|	�r0| tt|
|	ƒƒ¡ |�rN| ‡fdd„|D ƒ¡ |dk	�r~| d|i¡ |dk	�r~| d|i¡ |g fS |dk�rœtdƒ‚�n|dk�r”ˆ ‰‡ ‡fdd„}‡ ‡fdd„}tt|t||ƒƒƒ}|dk	�r>|dk	�r| }ˆ  ||¡}|dk	�rˆ  ||¡}| d|i¡ |dk	�r>| d|i¡ |	�r^| tt|
t||	ƒƒƒ¡ |�r|| ‡fdd„|D ƒ¡ |t ˆ ¡d  ¡ fS td| ƒ‚dS )zu
        This returns a dictionary with keys endog, exog and the keys of
        kwargs. It preserves Nones.
        Úmissing_idxNr   r0   r.   r   r   rD   z6Arrays with more than 2 dimensions are not yet handledzBShape mismatch between endog/exog and extra arrays given to model.zEShape mismatch between endog/exog and extra 2d arrays given to model.c                   s   i | ]}|ˆ   |d ¡“qS r`   ©Úget©Ú.0Úk©r;   r   r   Ú
<dictcomp>  s   ÿ z,ModelData.handle_missing.<locals>.<dictcomp>Úraisez!NaNs were encountered in the dataZdropc                   s   ˆ   | ˆ¡S r`   )rd   r   ©rb   rc   r   r   Ú<lambda>"  ó    z*ModelData.handle_missing.<locals>.<lambda>c                   s   ˆ   | ˆ¡S r`   )re   r   ro   r   r   rp   #  rq   c                   s   i | ]}|ˆ   |d ¡“qS r`   rg   ri   rl   r   r   rm   6  s   ÿ z missing option %s not understood)r*   r    Úitemsr   r   r   r   Ú
ValueErrorr!   r[   r   ÚdictÚzipr-   r   r   rd   rW   Útolist)rb   r.   r0   r9   r;   Znone_array_namesrf   ÚcombinedZcombined_namesZcombined_2dZcombined_2d_namesÚkeyZvalue_arrayZupdated_row_maskZcombined_nansZcombined_2d_nansZ	drop_nansZdrop_nans_2dr   )rb   r;   rc   r   r+   Â   s²    










ÿ








ÿÿzModelData.handle_missingc                 C  sT   |   |¡}d }|d k	rL|  |¡}|jdkr:|d d …d f }|jdkrLtdƒ‚||fS )Nr   rD   zexog is not 1d or 2d)Ú	_get_yarrÚ	_get_xarrr   rs   )r8   r.   r0   ZyarrZxarrr   r   r   r2   =  s    



zModelData._convert_endog_exogc                 C  s>   | j }|  |¡}|st| jƒ}t|ƒdkr2|d S t|ƒS d S )Nr   r   )r/   Ú
_get_namesÚ_make_endog_namesr.   r    Úlist)r8   r.   Úynamesr   r   r   r~   K  s    

zModelData.ynameszlist[str] | None)Úreturnc                 C  s2   | j }|d k	r.|  |¡}|s&t| jƒ}t|ƒS d S r`   )r1   r{   Ú_make_exog_namesr0   r}   )r8   r0   Úxnamesr   r   r   r�   W  s    

zModelData.xnamesc                 C  s   | j p
| jS r`   )Ú_param_namesr�   ©r8   r   r   r   Úparam_namesa  s    zModelData.param_namesc                 C  s
   || _ d S r`   )r‚   )r8   r\   r   r   r   r„   f  s    c                 C  s   | j dk	r| j S | jS )zÔ
        Labels for covariance matrices

        In multidimensional models, each dimension of a covariance matrix
        differs from the number of param_names.

        If not set, returns param_names
        N)Ú
_cov_namesr„   rƒ   r   r   r   Ú	cov_namesj  s    
zModelData.cov_namesc                 C  s
   || _ d S r`   )r…   )r8   r^   r   r   r   r†   y  s    c                 C  s.   | j }|d k	r|  |¡}n| j}|  |¡}|S r`   )r1   Ú_get_row_labelsr/   )r8   r0   Ú
row_labelsr.   r   r   r   rˆ   ~  s    
zModelData.row_labelsc                 C  s   d S r`   r   ©r8   Zarrr   r   r   r‡   ˆ  s    zModelData._get_row_labelsc                 C  st   t |tƒr2t |jtƒr&dd„ |jD ƒS t|jƒS n>t |tƒrP|jrJ|jgS d S n z
|jjW S  t	k
rn   Y nX d S )Nc                 S  s    g | ]}d   dd„ |D ƒ¡‘qS )rP   c                 s  s   | ]}|r|V  qd S r`   r   )rj   Úlevelr   r   r   Ú	<genexpr>�  s      z2ModelData._get_names.<locals>.<listcomp>.<genexpr>)rJ   )rj   Úcr   r   r   Ú
<listcomp>�  s   ÿz(ModelData._get_names.<locals>.<listcomp>)
Ú
isinstancer   Úcolumnsr   r}   r   Únamer   ÚnamesÚAttributeErrorr‰   r   r   r   r{   ‹  s    
ÿ

zModelData._get_namesc                 C  sZ   t  |¡rt  |¡}t |¡}t|ƒdkrR|jdkr8|S |jdkrRt | ¡ g¡S | ¡ S r   )r'   Ú_is_structured_ndarrayÚstruct_to_ndarrayr   r   r    r   r   )r8   r.   r   r   r   ry      s    




zModelData._get_yarrc                 C  s   t  |¡rt  |¡}t |¡S r`   )r'   r“   r”   r   r   )r8   r0   r   r   r   rz   ¬  s    

zModelData._get_xarrc                 C  s*   | j d k	r&t| j ƒt| jƒkr&tdƒ‚d S )Nz+endog and exog matrices are different sizes)r0   r    r.   rs   rƒ   r   r   r   r6   ±  s    
zModelData._check_integrityr�   c                 C  sÀ   |dkr|   |¡S |dkr$|  |¡S |dkr6|  |¡S |dkrH|  |¡S |dkrZ|  |¡S |dkrl|  |¡S |dkr€|  ||¡S |dkr”|  ||¡S |d	kr¦|  |¡S |d
kr¸|  	|¡S |S d S )Nr�   ÚrowsZcovÚdatesZ
columns_eqZcov_eqZgeneric_columnsZgeneric_columns_2dr~   Zmultivariate_confint)
Úattach_columnsÚattach_rowsÚ
attach_covÚattach_datesÚattach_columns_eqÚattach_cov_eqÚattach_generic_columnsÚattach_generic_columns_2dÚattach_ynamesÚattach_mv_confint)r8   ÚobjÚhowr‘   r   r   r   Úwrap_output¶  s*    







zModelData.wrap_outputc                 C  s   |S r`   r   ©r8   Úresultr   r   r   r—   Î  s    zModelData.attach_columnsc                 C  s   |S r`   r   r¤   r   r   r   r›   Ñ  s    zModelData.attach_columns_eqc                 C  s   |S r`   r   r¤   r   r   r   r™   Ô  s    zModelData.attach_covc                 C  s   |S r`   r   r¤   r   r   r   rœ   ×  s    zModelData.attach_cov_eqc                 C  s   |S r`   r   r¤   r   r   r   r˜   Ú  s    zModelData.attach_rowsc                 C  s   |S r`   r   r¤   r   r   r   rš   Ý  s    zModelData.attach_datesc                 C  s   |S r`   r   r¤   r   r   r   r    à  s    zModelData.attach_mv_confintc                 O  s   |S r`   r   ©r8   r¥   Úargsr;   r   r   r   r�   ã  s    z ModelData.attach_generic_columnsc                 O  s   |S r`   r   r¦   r   r   r   rž   æ  s    z#ModelData.attach_generic_columns_2dc                 C  s   |S r`   r   r¤   r   r   r   rŸ   é  s    zModelData.attach_ynames)Nr#   N)r�   N)(Ú__name__Ú
__module__Ú__qualname__Ú__doc__r‚   r…   r<   r@   rR   r5   Úclassmethodrd   re   r+   r2   r
   r~   r�   Úpropertyr„   Úsetterr†   r	   rˆ   r‡   r{   ry   rz   r6   r£   r—   r›   r™   rœ   r˜   rš   r    r�   rž   rŸ   r   r   r   r   r"   6   sX   
=


z
	




	
r"   c                   @  s   e Zd Zdd„ ZdS )Ú	PatsyDatac                 C  s   |j jS r`   )r%   Úcolumn_namesr‰   r   r   r   r{   î  s    zPatsyData._get_namesN)r¨   r©   rª   r{   r   r   r   r   r¯   í  s   r¯   c                      s¨   e Zd ZdZd!‡ fdd„	Ze‡ fdd„ƒZe‡ fdd„ƒZ‡ fd	d
„Zdd„ Z	dd„ Z
d"dd„Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Z‡  ZS )#Ú
PandasDataz^
    Data handling class which knows how to reattach pandas metadata to model
    results
    Nc                   sV   t  |¡}|d kr|nt  |¡}|jtks<|d k	rD|jtkrDtdƒ‚tt| ƒ ||¡S )NzRPandas data cast to numpy dtype of object. Check input data with np.asarray(data).)r   r   r   Úobjectrs   Úsuperr±   r2   ©r8   r.   r0   ©Ú	__class__r   r   r2   ø  s
    
zPandasData._convert_endog_exogc                   s.   t |ttfƒr|j| S tt| ƒ ||¡S d S r`   )rŽ   r   r   Úlocr³   r±   rd   ra   rµ   r   r   rd     s    
zPandasData._drop_nansc                   s<   t |ttfƒr&|j| jd d …|f S tt| ƒ ||¡S d S r`   )rŽ   r   r   r·   r³   r±   re   ra   rµ   r   r   re     s    zPandasData._drop_nans_2dc                   sV   | j | j }}|d k	rDt|dƒrDt|dƒrD| j j | jj¡sDtdƒ‚tt| ƒ ¡  d S )NÚindexz.The indices for endog and exog are not aligned)	r/   r1   r   r¸   Úequalsrs   r³   r±   r6   r´   rµ   r   r   r6     s    ÿÿþzPandasData._check_integrityc                 C  s*   z|j W S  tk
r$   | jj  Y S X d S r`   )r¸   r’   r/   r‰   r   r   r   r‡     s    zPandasData._get_row_labelsc                 C  s   t | |d ƒ}t||d�S )N©r¸   )Úgetattrr   )r8   r¥   r‘   r°   r   r   r   r�      s    z!PandasData.attach_generic_columnsc                 C  s.   |p|}t | |d ƒ}t | |d ƒ}t|||d�S ©N©r¸   r�   )r»   r   )r8   r¥   ZrownamesZcolnamesr   r   r   rž   %  s    z$PandasData.attach_generic_columns_2dc                 C  s*   |j dkrt|| jd�S t|| jd�S d S )Nr   rº   )r   r   r„   r   r¤   r   r   r   r—   +  s    
zPandasData.attach_columnsc                 C  s   t || j| jd�S r¼   )r   r�   r~   r¤   r   r   r   r›   4  s    zPandasData.attach_columns_eqc                 C  s   t || j| jd�S r¼   )r   r†   r¤   r   r   r   r™   7  s    zPandasData.attach_covc                 C  s   t || j| jd�S r¼   )r   r~   r¤   r   r   r   rœ   :  s    zPandasData.attach_cov_eqc                 C  s€   |  ¡ }tj| jdd�jd }|dkrB|j|fkrB|d d d …f }|jdk rVt|ƒ}nt|ƒ}| j|_| j	t
|ƒ d … |_|S )Nr   ©Zndminr   rD   )r   r   rZ   r~   r[   r   r   r   r�   rˆ   r    r¸   )r8   r¥   ÚsqueezedÚk_endogÚoutr   r   r   r˜   =  s    

zPandasData.attach_rowsc                 C  s|   |  ¡ }tj| jdd�jd }|dkrH|j|fkrHt |¡d d d …f }|jdk r`t|| jd�S t	t |¡| j| jd�S d S )Nr   r¾   r   rD   rº   r½   )
r   r   rZ   r~   r[   r   r   r   Zpredict_datesr   )r8   r¥   r¿   rÀ   r   r   r   rš   M  s    

þzPandasData.attach_datesc                 C  s   t | d¡| jddgd�S )N)rH   rD   ÚlowerÚupperr½   )r   Zreshaper†   r¤   r   r   r   r    Z  s    
þzPandasData.attach_mv_confintc                 C  s2   |  ¡ }|jdk r t|| jd�S t|| jd�S d S )NrD   )r�   )r�   )r   r   r   r~   r   )r8   r¥   r¿   r   r   r   rŸ   _  s    
zPandasData.attach_ynames)N)N)r¨   r©   rª   r«   r2   r¬   rd   re   r6   r‡   r�   rž   r—   r›   r™   rœ   r˜   rš   r    rŸ   Ú__classcell__r   r   rµ   r   r±   ò  s$   		
	r±   c                 C  s<   | j dks| jd dkr dg}ndd„ t| jd ƒD ƒ}|S )Nr   r   c                 S  s   g | ]}d |d  ‘qS )zy%dr   r   ©rj   Úir   r   r   r�   l  s     z%_make_endog_names.<locals>.<listcomp>)r   r[   Úrange)r.   r~   r   r   r   r|   h  s    r|   c                 C  sh   |   d¡}|dk ¡ rF| ¡ }dd„ td| jd ƒD ƒ}| |d¡ ndd„ td| jd d ƒD ƒ}|S )Nr   c                 S  s   g | ]}d | ‘qS ©zx%dr   rÅ   r   r   r   r�   w  s     z$_make_exog_names.<locals>.<listcomp>r   Úconstc                 S  s   g | ]}d | ‘qS rÈ   r   rÅ   r   r   r   r�   z  s     )Úvarr   ZargminrÇ   r[   Úinsert)r0   Zexog_varr3   Z
exog_namesr   r   r   r€   q  s    
r€   r#   c                 K  sH   t | |ƒ}|dkr0t| |d�}| |¡ |d fS |j| |fd|i|—ŽS )Nr#   )r.   r0   r9   )Úhandle_data_class_factoryrt   r-   r+   )r.   r0   r9   r;   ÚklassZret_dictr   r   r   r+     s    

r+   c                 C  sd   t  | |¡rt}nNt  | |¡r$t}n<t  | |¡r6t}n*t  | |¡rHt}ntdt	| ƒt	|ƒf ƒ‚|S )z
    Given inputs
    z%unrecognized data structures: %s / %s)
r'   Z_is_using_ndarray_typer"   Z_is_using_pandasr±   Z_is_using_patsyr¯   Z_is_using_ndarrayrs   Útype)r.   r0   rÍ   r   r   r   rÌ   ˆ  s    ÿrÌ   c                 K  sR   t | ttfƒrt | ¡} t |ttfƒr0t |¡}t| |ƒ}|| f|||dœ|—ŽS )N)r0   r9   r:   )rŽ   r}   Útupler   r   rÌ   )r.   r0   r9   r:   r;   rÍ   r   r   r   Úhandle_data›  s    


ÿrÐ   )Nr#   )r#   N)"r«   Ú
__future__r   Zstatsmodels.compat.pythonr   Ú	functoolsr   Únumpyr   Zpandasr   r   r   r   Zstatsmodels.tools.dataZtoolsrN   r'   Zstatsmodels.tools.decoratorsr	   r
   r(   r   r   r   r!   r"   r¯   r±   r|   r€   r+   rÌ   rÐ   r   r   r   r   Ú<module>   s,      :v	
	