U
    Ãmœdª1  ã                   @   sÐ   d dl Zd dlZd dlZd dlmZ d dlmZ	 d dl
mZmZ d dlZzd dlmZ W n ek
rl   Y nX dZerŒd dlmZ edƒZndZdd„ Zd	d
„ Zdd„ ZG dd„ dƒZG dd„ dƒZdd„ ZdS )é    N)Úmice)Úassert_equalÚassert_allcloseF)ÚPdfPagesztest_mice.pdfc                 C   s   t r|  |¡ d S ©N)Ú
pdf_outputZsavefig)ÚpdfÚfig© r
   ú_/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/statsmodels/imputation/tests/test_mice.pyÚclose_or_save   s    r   c                   C   s   t rt ¡  d S r   )r   r   Úcloser
   r
   r
   r   Úteardown_module   s    r   c                  C   sr  t j d¡ d} d}t jj| |fd�}|dd…df |dd…df  d|dd…df   |dd…d	f< |dd…d	f  t jj| d�7  < d|dd…df d	k |dd…df< | d¡t jj| d� }t |¡}d
d„ td|d ƒD ƒ|_||d< t j	|j
d	d…df< t j	|j
d	d…df< t j	|j
ddd…df< t j	|j
ddd…df< t j	|j
dd…df< t j	|j
ddd…df< |S )z0
    Create a data set with missing values.
    iÃ…  éÈ   é   ©ÚsizeNé   é   é   r   c                 S   s   g | ]}d | ‘qS )úx%dr
   )Ú.0Úkr
   r
   r   Ú
<listcomp>4   s     zgendat.<locals>.<listcomp>Úyé;   Úx1é'   Úx2é
   é   Úx3é   é1   é   Úx4é(   é,   Úx5é   éc   )ÚnpÚrandomÚseedÚnormalÚsumÚpdÚ	DataFrameÚrangeÚcolumnsÚnanÚloc)ÚnÚpZexogZendogÚdfr
   r
   r   Úgendat"   s$    @"$
r9   c                   @   s|   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Ze	j
jdd„ ƒZe	j
jdd„ ƒZe	j
jdd„ ƒZe	j
jdd„ ƒZdS )ÚTestMICEDatac              
      sÞ  t ƒ ‰ ˆ  ¡ }t ˆ ¡}t ˆ ¡‰ˆ j\}}tˆjd t	 
d¡ƒ tˆjd t	 
dd¡ƒ tˆjd t	 
d¡ƒ tˆjd t	 
ddd	¡ƒ tˆjd t	 t	 
d¡t	 
d
dd	¡t	 
dd¡f¡ƒ t‡fdd„ˆjD ƒ‡ fdd„ˆ D ƒƒ tdƒD ]l}ˆ ¡  tˆjjd |ƒ tˆjjd |ƒ t|| ˆj| ƒ t‡fdd„ˆjD ƒ‡ fdd„ˆ D ƒƒ qîd}tˆjd |ƒ tˆjddddddgƒ ˆ ˆjk	�s˜t‚ˆ d¡\}}}	}
}tt|ƒdƒ t|jddgƒ t|	jddgƒ d S )Nr   é<   r   r   r&   r!   r   r)   r   é   c                    s   g | ]}t ˆ j| ƒ‘qS r
   ©ÚsetÚdata©r   Úcol©Úimp_datar
   r   r   V   s     z-TestMICEData.test_default.<locals>.<listcomp>c                    s   g | ]}t ˆ |  ¡ ƒ‘qS r
   ©r>   Zdropnar@   ©r8   r
   r   r   W   s     r$   r   r   c                    s   g | ]}t ˆ j| ƒ‘qS r
   r=   r@   rB   r
   r   r   ^   s     c                    s   g | ]}t ˆ |  ¡ ƒ‘qS r
   rD   r@   rE   r
   r   r   _   s     zx1 ~ x2 + x3 + x4 + x5 + yr(   r%   r   é¾   é   )r9   Úcopyr0   Únotnullr   ÚMICEDataÚshaper   Úix_missr+   ÚarangeZix_obsZconcatenater   r?   r2   Ú
update_allÚconditional_formulaÚ_cycle_orderÚAssertionErrorZget_split_dataÚlen)ÚselfÚorigÚmxÚnrowÚncolr   ÚfmlZ	endog_obsZexog_obsZ	exog_missZpredict_obs_kwdsZpredict_miss_kwdsr
   )r8   rC   r   Útest_defaultE   sJ    




þÿÿÿÿ zTestMICEData.test_defaultc              
   C   s–   t ƒ }t t|ƒ¡|d< |j d¡|d< t |¡}t 	dd¡�N t
jdd��6}t
 d¡ | ¡  dd	„ |D ƒ}t|ƒd
ks~t‚W 5 Q R X W 5 Q R X dS )z�Test that MICEData does not throw a SettingWithCopyWarning when imputing (https://github.com/statsmodels/statsmodels/issues/5430)ÚintcolZint32zmode.chained_assignmentÚwarnT)ÚrecordÚalwaysc                 S   s   g | ]}d |j kr|‘qS )z\pandas\)Úfilename)r   Úwr
   r
   r   r      s     
 z<TestMICEData.test_settingwithcopywarning.<locals>.<listcomp>r   N)r9   r+   rM   rR   rZ   Zastyper   rJ   r0   Zoption_contextÚwarningsÚcatch_warningsÚsimplefilterrN   rQ   )rS   r8   ZmiceDataÚwsr
   r
   r   Útest_settingwithcopywarningo   s    

z(TestMICEData.test_settingwithcopywarningc                 C   sj   t ƒ }t |¡}g }tdƒD ]4}| ¡ }t|tjƒs8t‚t	|j
|j
ƒ | |¡ q|d |d ksft‚d S )Nr   r   r   )r9   r   rJ   r2   Únext_sampleÚ
isinstancer0   r1   rQ   r   rK   Úappend)rS   r8   rC   Zall_xÚjÚxr
   r
   r   Útest_next_sample‚   s    
zTestMICEData.test_next_samplec           	      C   s¤   t ƒ }| ¡ }t |¡}|j\}}dD ]`}tj||d�}tdƒD ]D}| ¡  t	|j
jd |ƒ t	|j
jd |ƒ t|| |j
| ƒ q@q&t	|jdddd	d
dgƒ d S )N©ZgaussianÚboot©Úperturbation_methodr   r   r   r(   r!   r%   r   r   r   )r9   rH   r0   rI   rK   r   rJ   r2   rN   r   r?   r   rP   )	rS   r8   rT   rU   rV   rW   Z	pert_methrC   r   r
   r
   r   Útest_pertmeth’   s    

zTestMICEData.test_pertmethc              	      sh  t j d¡ d}t jj|d�}t jj|d�}t jj|d�t  |¡ }t jj|d�}t  ||k ||¡}t  ||kdd¡}t ||||dœ¡}t j	|j
dd…d	f< t j	|j
dd…d
f< t j	|j
dd…df< t j	|j
dd…df< ddlm}	 g ‰ ‡ fdd„}
dD ]P}tj|||
d�}|jd	d|	d
t d
¡iddi|d� | ¡ }t|tjƒsút‚qútdd„ ˆ D ƒƒ�sdt‚d S )Ni&"  é,  r   r   r   )ÚtimeÚstatusr   r   r   r&   rq   rr   r)   é2   r   r;   r   )ÚPHRegc                    s   ˆ   | jj¡ d S r   )rg   r?   rK   )Úimp©Úhistr
   r   Úcb¼   s    z#TestMICEData.test_phreg.<locals>.cbrk   )rn   Zhistory_callbackz0 + x1 + x2Z	pred_typeÚhr)Úmodel_classÚ	init_kwdsZpredict_kwdsrn   c                 S   s   g | ]}|d k‘qS ))i+  r   r
   )r   Úvalr
   r
   r   r   É   s     z+TestMICEData.test_phreg.<locals>.<listcomp>)r+   r,   r-   r.   ZexponentialÚexpÚwherer0   r1   r4   r5   Z&statsmodels.duration.hazard_regressionrt   r   rJ   Úset_imputerZPatsyFormulare   rf   rQ   Úall)rS   r6   r   r   Z
event_timeZobs_timerq   rr   r8   rt   rx   ÚpmÚidatari   r
   rv   r   Ú
test_phreg§   s4    
ýzTestMICEData.test_phregc                 C   s˜  ddl m} ddlm} tƒ }| ¡ }t |¡}|j\}}t	 
|¡}| dd¡ | dd¡ |jdtjd	tj ¡ id
� | ¡  t|jjd |ƒ t|jjd |ƒ t|| |j| ƒ tddƒD ]�}	|	dk�rtt|jd tjƒdƒ tt|jd jtjjƒdƒ tt|jd |ƒdƒ qÂtt|jd|	  tjƒdƒ tt|jd|	  |ƒdƒ qÂd}
t|jd |
ƒ d}
t|jd |
ƒ t|jddddddgƒ d S )Nr   ©ÚRegressionResultsWrapper©ÚGLMResultsWrapperr   zx3 + x4 + x3*x4r   zx4 + I(x5**2)r!   Úfamily)rz   r{   r   rG   r$   Tr   zx1 ~ x3 + x4 + x3*x4zx4 ~ x1 + x2 + x3 + x5 + yr%   r(   r   )Ú#statsmodels.regression.linear_modelr…   Ú+statsmodels.genmod.generalized_linear_modelr‡   r9   rH   r0   rI   rK   r   rJ   r   ÚsmÚGLMÚfamiliesÚBinomialrN   r   r?   r   r2   rf   Úmodelsrˆ   ÚresultsÚOLSrO   rP   )rS   r…   r‡   r8   rT   rU   rV   rW   rC   rh   rX   r
   r
   r   Útest_set_imputerË   s8    



ÿ
zTestMICEData.test_set_imputerc                 C   sZ   t ƒ }t |¡}dD ]@}dD ]6}dD ],}t ¡  |j|||d�}tt|ƒ |ƒ  q$qqd S )N)ÚpatternÚraw©FT)Ú	row_orderÚhide_complete_rowsÚcolor_row_patterns)r9   r   rJ   ÚpltÚclfZplot_missing_patternr   r   )rS   Úclose_figuresr8   rC   r–   r—   r˜   r	   r
   r
   r   Útest_plot_missing_patternò   s    
þ
z&TestMICEData.test_plot_missing_patternc                 C   s`   t ƒ }t |¡}| ¡  t ¡  dD ]6}|jdd|d�}| ¡ d  d¡ t	t
|ƒ |ƒ  q$d S )Nr•   r   r%   ©Úplot_pointsr   Úplot_bivariate)r9   r   rJ   rN   r™   rš   rŸ   Úget_axesÚ	set_titler   r   ©rS   r›   r8   rC   rž   r	   r
   r
   r   Útest_plot_bivariate  s    

z TestMICEData.test_plot_bivariatec                 C   s^   t ƒ }t |¡}| ¡  t ¡  dD ]4}|jd|d�}| ¡ d  d¡ t	t
|ƒ |ƒ  q$d S )Nr•   r%   r�   r   Zplot_fit_scatterplot)r9   r   rJ   rN   r™   rš   Zplot_fit_obsr    r¡   r   r   r¢   r
   r
   r   Útest_fit_obs  s    

zTestMICEData.test_fit_obsc                 C   sZ   t ƒ }t |¡}| ¡  t ¡  dD ]0}| d¡}| ¡ d  d¡ t	t
|ƒ |ƒ  q$d S )Nr•   r%   r   Úplot_imputed_hist)r9   r   rJ   rN   r™   rš   r¥   r    r¡   r   r   r¢   r
   r
   r   Útest_plot_imputed_hist!  s    


z#TestMICEData.test_plot_imputed_histN)Ú__name__Ú
__module__Ú__qualname__rY   rd   rj   ro   rƒ   r’   ÚpytestÚmarkZ
matplotlibrœ   r£   r¤   r¦   r
   r
   r
   r   r:   C   s   *$'


r:   c                   @   s<   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zejj	d	d
„ ƒZ
dS )ÚTestMICEc                 C   sJ   t ƒ }t |¡}t dtj|¡}| dd¡}t|jtj	ƒs>t
‚| ¡ }d S )Núy ~ x1 + x2 + x1:x2r   r$   )r9   r   rJ   ÚMICEr‹   r‘   ÚfitÚ
issubclassÚ	__class__ZMICEResultsrQ   Úsummary)rS   r8   rC   ÚmiÚresultZsmrr
   r
   r   Ú	test_MICE2  s    
zTestMICE.test_MICEc                 C   sV   t ƒ }t |¡}t dtj|¡}ddlm} tdƒD ]}| 	¡ }t
|j|ƒs4t‚q4d S )Nr­   r   r„   r$   )r9   r   rJ   r®   r‹   r‘   r‰   r…   r2   re   r°   r±   rQ   )rS   r8   rC   r³   r…   rh   ri   r
   r
   r   Ú
test_MICE1?  s    
zTestMICE.test_MICE1c                 C   s6   t ƒ }tj|dd�}|jdddddœd� | ¡  d S )	Nrl   rm   r   zx2 + yr   r   )ÚalphaZL1_wt)Zfit_kwds)r9   r   rJ   r   rN   )rS   r8   ru   r
   r
   r   Útest_MICE1_regularizedL  s    zTestMICE.test_MICE1_regularizedc                 C   sv   ddl m} tƒ }t |¡}tjdtj|dtj 	¡ id�}t
dƒD ].}| ¡ }t||ƒs\t‚t|jtjj	ƒsBt‚qBd S )Nr   r†   zx3 ~ x1 + x2rˆ   )r{   r$   )rŠ   r‡   r9   r   rJ   r®   r‹   rŒ   r�   rŽ   r2   re   rf   rQ   rˆ   )rS   r‡   r8   rC   r³   rh   ri   r
   r
   r   Ú
test_MICE2T  s    
ÿzTestMICE.test_MICE2c                 C   sú   t j d¡ t jjdd�}t jjdd�}|| t jjdd� }t j|dd…< t j|dd …< t |||dœ¡}t |¡}tj	dt
j|d	d
�}| dd	¡}t  dddg¡}t|j|dd� t  dddg¡}	t|j|	dd� t  dddg¡}
t|j|
dd� d S )Ni9  rp   r   r   éd   éú   )r   r   r   zy ~ x1 + x2r"   )Zn_skipr   gÍl¡užÂÆ?g7¨»º|N¼?gçùZöÒ?gñhãˆµøä>)ZatolgmCœ¹MÙ¡¿g×Ê$¸Ëî?g Â,Yºçï?gk�‹(õ~á¿g)K¥†Ó�.@gß½'ÿ(:+@)r+   r,   r-   r.   r4   r0   r1   r   rJ   r®   r‹   r‘   r¯   Zasarrayr   Zfrac_miss_infoÚparamsÚtvalues)rS   r   r   r   r8   r‚   r³   r´   Zfmir¼   r½   r
   r
   r   Út_est_combineb  s     
zTestMICE.t_est_combineN)r§   r¨   r©   rµ   r¶   r¸   r¹   rª   r«   Zslowr¾   r
   r
   r
   r   r¬   0  s   r¬   c                  C   sÜ   t j d¡ t t j dd¡¡} ddddg| _t j| jd< t j| jd	d
gdf< t	 
| ¡}| ¡  t|j ¡ j ¡ dƒ t jg t jd�t jd	gt jd�t jd	d
gt jd�t jg t jd�dœ}|D ]}t|j| || ƒ q¾d S )Nr   rs   r   Úvar1Úvar2Úvar3Úvar4)r   r   r   r$   r   )Zdtype)r¿   rÀ   rÁ   rÂ   )r+   r,   r-   r0   r1   Zrandr3   r4   Zilocr   rJ   rN   r   r?   ZisnullÚvaluesr/   ÚarrayZint64rL   )r?   Zdata_imprL   r   r
   r
   r   Útest_micedata_miss1z  s    
ýrÅ   )Únumpyr+   Zpandasr0   rª   Zstatsmodels.imputationr   Zstatsmodels.apiÚapir‹   Znumpy.testingr   r   r`   Zmatplotlib.pyplotZpyplotr™   ÚImportErrorr   Zmatplotlib.backends.backend_pdfr   r   r   r   r9   r:   r¬   rÅ   r
   r
   r
   r   Ú<module>   s,   
! nJ