U
    Ãmœd¼-  ã                   @   sæ   d Z ddlZddlmZmZ ddlZddlm	Z	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ G d	d
„ d
ee	ƒZG dd„ dee
ƒZG dd„ deeƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ ZdS )!z>
Created on Wed May 23 12:53:27 2018

Author: Josef Perktold

é    N)Úassert_allcloseÚassert_equal)ÚPoissonÚLogit)ÚGLM©Úfamily)ÚPenalizedMixin)ÚVariableScreeningc                   @   s   e Zd ZdS )ÚPoissonPenalizedN©Ú__name__Ú
__module__Ú__qualname__© r   r   ú^/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/statsmodels/base/tests/test_screening.pyr      s   r   c                   @   s   e Zd ZdS )ÚLogitPenalizedNr   r   r   r   r   r      s   r   c                   @   s   e Zd ZdS )ÚGLMPenalizedNr   r   r   r   r   r      s   r   c            	      C   sÚ   t j d¡ d\} }d}t j | |¡dt j | d¡d   d d }|d9 }|| d	¡ | d	¡ }d|d d …d	f< t  |¡}d	d
dddg}dt  d|d ¡ ||< t  |¡}| 	|¡}t  
|¡}t j |¡}||||fS )NéÙ ©éd   éô  é   ç      ð?é   ç      à?é   ç333333ó?r   r   é,  é�  é›  )ÚnpÚrandomÚseedÚrandÚmeanÚstdÚzerosÚarangeÚsqrtÚdotÚexpÚpoisson©	ÚnobsÚk_varsÚ	k_nonzeroÚxÚbetaÚidx_nonzero_trueÚlinpredÚmuÚyr   r   r   Ú_get_poisson_data    s*    ÿÿÿ



r7   c                  C   sR  t j d¡ tƒ \} }}}t| ƒ}dd„ |D ƒ}d|d< tj|| |dgd�}tj|d d …|f |d�}t| |ƒ ¡ }|j	|d	< t
| t  |¡|d
 d�}	t|	ƒ}
|d d …dd …f }|
j|dd�}tt  |j¡|ƒ dd„ |jD ƒ}d|d< |jj|d� |j ¡  t|jjd dƒ tj|jj	|dd�}|j|dd�}t|d	 |d dd� d S )Nr   c                 S   s   g | ]}d | ‘qS ©zvar%4dr   ©Ú.0Úiir   r   r   Ú
<listcomp><   s     z*test_poisson_screening.<locals>.<listcomp>Úconstr   Útrue©ÚindexÚcolumns©rA   Úoracler   ©Ú
pen_weightr   é
   ©Úmaxiterc                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   M   s     ©ZxnameÚ	convergedTÚfinal©r@   ÚnameÚouter©ÚhowçñhãˆµøÔ>©Zatol)r!   r"   r#   r7   ÚlenÚpdÚ	DataFramer   ÚfitÚparamsr   Úonesr
   Úscreen_exogr   ÚsortÚidx_nonzeroÚresults_finalÚsummaryÚresults_penÚmle_retvalsÚSeriesÚjoinr   ©r6   r1   r3   r2   r.   Úxnames_trueÚ
parametersÚxframe_trueÚ
res_oracleÚmod_initialÚscreenerÚexog_candidatesÚ
res_screenÚxnamesÚpsr   r   r   Útest_poisson_screening5   s0    ÿ

rm   c                     s¸  t j d¡ d\‰‰t j ˆˆd ¡dt j ˆd¡d   d d } | d9 } | |  d¡ |  d¡ } t  t  ˆ¡| f¡} dt  dˆd ¡ }t  	|¡}|  
|¡}t  |¡}t j |¡}| d d …dd …f  d¡d d …d f ‰ | ‰‡ ‡‡‡fd	d
„}t  ˆ¡}d|d ˆd …< t  t  ˆ¡|f¡}dD ]Ž}t||d d …d |…f ˆd d�}	t|	ƒ}
d|
_|
 |ƒ ¡}ddddg}t|j|ƒ t jddgddgddgddggt jd�}t|j|ƒ �q$d S )Nr   )r   r   r   r   r   r   r   r   c                  3   s    d} d}t |ƒD ]Š}dˆ  tj ˆ| ¡ dtj ˆd¡d   d d }|d9 }|ˆd k r|ˆd d …|d f |d d …d	f< || d
¡ | d
¡ }|V  qd S )Nr   é   gš™™™™™©?r   r   r   r   r   rF   r   )Úranger!   r"   r$   r%   r&   )r/   Z	n_batchesÚir1   ©Úcommonr0   r.   Z	x_nonzeror   r   Úexog_iteratorp   s    ÿÿÿ z+test_screen_iterated.<locals>.exog_iterator©r   r   r   rD   é   Zvar0_10Zvar1_10Zvar2_10Zvar3_10rF   é   )Zdtype)r!   r"   r#   r$   r%   r&   Zcolumn_stackrX   r(   r)   r*   r+   r,   Úsumr   r
   Ú	k_max_addZscreen_exog_iteratorr   Zexog_final_namesÚarrayZint64Zidx_nonzero_batches)r1   r2   r4   r5   r6   rs   ÚdummyZ	exog_keepÚkrg   rh   rK   ÚnamesZidx_fullr   rq   r   Útest_screen_iterated[   sL    ÿÿÿ


&
"
ýýr}   c                  C   sN  t ƒ \} }}}t| ƒ}dd„ |D ƒ}d|d< tj|| |dgd�}tj|d d …|f |d�}t| |t ¡ d� ¡ }|j|d	< t| t	 
|¡t ¡ d�}	t|	ƒ}
|d d …d
d …f }|
j|dd�}tt	 |j¡|ƒ dd„ |jD ƒ}d|d< |jj|d� |j ¡  t|jjd dƒ tj|jj|dd�}|j|dd�}t|d	 |d dd� d S )Nc                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   •   s     z-test_glmpoisson_screening.<locals>.<listcomp>r=   r   r>   r?   rB   r   rC   r   rF   rG   c                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   ¥   s     rI   rJ   TrK   rL   rN   rO   rQ   rR   )r7   rS   rT   rU   r   r   r   rV   rW   r!   rX   r
   rY   r   rZ   r[   r\   r]   r^   r_   r`   ra   r   rb   r   r   r   Útest_glmpoisson_screening�   s*    

r~   c            	      C   sö   t j d¡ d\} }d}t j | |¡dt j | d¡d   d d }|d9 }|| d	¡ | d	¡ }d|d d …d	f< t  |¡}d	d
dddg}dt  d|d ¡d  ||< t  |¡}| 	|¡}ddt  
| ¡  }t j t|ƒ¡|k  t¡}||||fS )Nr   )r   r   r   ç{®Gáz„?r   r   r   r   r   r   r   r   r    r   )r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   rS   ZastypeÚintr-   r   r   r   Ú_get_logit_data³   s*    ÿÿÿ


r�   c               	   C   sš  t ƒ \} }}}t| ƒ}t|d dd�}dd„ |D ƒ}d|d< tj|| |dgd	�}tj|d d …|f |d
�}t| |ƒ ¡ }	|	j|d< t| t	 
|¡|d d�}
t|
f|Ž}d|_|d d …dd …f }|j|dd�}t	 |jj¡dk}tt	 |j| ¡|ƒ t	 dddddddg¡}tt	 |j¡|ƒ dd„ |jD ƒ}d|d< |jj|d� |j ¡  t|jjd dƒ tj|jj|dd�}||d< t|d |d dd� d S ) Ngffffffæ?çü©ñÒMbP?)rE   Úthreshold_trimc                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   Ï   s     z(test_logit_screening.<locals>.<listcomp>r=   r   r>   r?   rB   rC   r   rD   ru   r   rG   çš™™™™™¹?éJ   r   é£   r   r   r    c                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   å   s     rI   rJ   TrK   rL   ç{®Gázt?rR   )r�   rS   ÚdictrT   rU   r   rV   rW   r   r!   rX   r
   rx   rY   Úabsr\   r   rZ   r[   ry   r]   r^   r_   r`   r   )r6   r1   r3   r2   r.   Úscreener_kwdsrc   rd   re   rf   rg   rh   ri   rj   ÚmaskZidx_rrk   rl   r   r   r   Útest_logit_screeningÈ   s8    ÿ

rŒ   c                  C   s^  t ƒ \} }}}t| ƒ}t|d ddd�}dd„ |D ƒ}d|d< tj|| |d	gd
�}tj|d d …|f |d�}t| |t ¡ d� ¡ }	|	j	|d< t| t
 |¡t ¡ d�}
t|
f|Ž}d|_|d d …dd …f }|j|dd�}|j |j dd„ |jD ƒ}d|d< |jj|d� |j ¡  t|jjd dƒ tj|jj	|dd�}||d< t|d |d dd� d S )Nç      è?r‚   úmodel.score_factor©rE   rƒ   Zranking_attrc                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   þ   s     z+test_glmlogit_screening.<locals>.<listcomp>r=   r   r>   r?   rB   r   rC   rF   r   ru   rG   c                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<     s     rI   rJ   TrK   rL   r‡   rR   )r�   rS   rˆ   rT   rU   r   r   ZBinomialrV   rW   r!   rX   r
   rx   rY   r[   r\   r]   r^   r   r_   r`   r   )r6   r1   r3   r2   r.   rŠ   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   r   r   r   Útest_glmlogit_screeningõ   s8    
ÿÿ

r�   c                  C   sÜ   t j d¡ d\} }d}t j | |¡dt j | d¡d   d d }|d9 }|| d	¡ | d	¡ }d|d d …d	f< t  |¡}d	dd
ddg}dt  d|d ¡ ||< t  |¡}| 	|¡}|dt j 
t|ƒ¡  }||||fS )Nr   r   r   r   r   r   r   r   r   r   r   r    r   r„   )r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   ZrandnrS   )r.   r/   r0   r1   r2   r3   r4   r6   r   r   r   Ú_get_gaussian_data#  s(    ÿÿÿ


r‘   c                  C   sŒ  t ƒ \} }}}t| ƒ}| |  d¡ } t|d ddd�}dd„ |D ƒ}d|d< tj|| |d	gd
�}tj|d d …|f |d�}t| |t ¡ d� 	¡ }	|	j
|d< dD ]æ}
t| |d d …d |
…f t ¡ d�}t|f|Ž}|d d …|
d …f }|j|dd�}tt |j¡|ƒ dd„ |jD ƒ}d|d< |jj|d� |j ¡  t|jjd dƒ tj|jj
|dd�}|j|dd�}t|d |d dd� |d= q d S )Nr   r�   r‚   rŽ   r�   c                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   C  s     z.test_glmgaussian_screening.<locals>.<listcomp>r=   r>   r?   rB   r   rC   rt   ru   rG   c                 S   s   g | ]}d | ‘qS r8   r   r9   r   r   r   r<   T  s     rI   rJ   TrK   rL   rN   rO   gñhãˆµøä>rR   )r‘   rS   r%   rˆ   rT   rU   r   r   ZGaussianrV   rW   r
   rY   r   r!   rZ   r[   r\   r]   r^   r_   r`   ra   r   )r6   r1   r3   r2   r.   rŠ   rc   rd   re   rf   Zk_keeprg   rh   ri   rj   rk   rl   r   r   r   Útest_glmgaussian_screening8  s:    
ÿÿ
"
r’   )Ú__doc__Únumpyr!   Znumpy.testingr   r   ZpandasrT   Z#statsmodels.discrete.discrete_modelr   r   Z+statsmodels.genmod.generalized_linear_modelr   Zstatsmodels.genmod.familiesr   Zstatsmodels.base._penalizedr	   Zstatsmodels.base._screeningr
   r   r   r   r7   rm   r}   r~   r�   rŒ   r�   r‘   r’   r   r   r   r   Ú<module>   s(   &5#-.