U md @sddlZddlZddlmZddlZddlmZddlm Z ddlm Z ddlm Z ddl m Z ddlmZdd lmZd d Zejd d dgddZddZejd d dgddZejdejejejgejdejejejgddZejdejejejgddZddZddZdd Zd!d"Zejjd#d$d%d&gd'd(d&gge d)d%d&gd*d(d&ggej d+d,d-gd.d/d-gge!d0ej d+d%d-gd.ej"d-gge!d0ej d+d%d-gd.e#d1d-gge!d0ej dd%d-gd.d(d-gge!d0ej dd%dgd.ej"dgge!d0ej dd%dgd.e#d1dgge!d0gd2d3d4d5d6d7d8d9gd:d;d<Z$ejd d dgejd=d>d?gejd@ddAgdBdCZ%ejd=d>d?gejdDd(d&gd%d&gd(d&ggdd%d%gdddgdd%d%ggfdEd.gdFd.gdGd+gdFd.ggdddddgddddd%gdd%dddggfgdHdIZ&dJdKZ'ejd@dLdAdgejdMdLdAdgdNdOZ(ejdPdQdRgejd#d%d(ge dSdTggdUdVZ)ejdPdQdRgdWdXZ*ejjdYd'd&gd$d&ggd'd$gd&ggej+fe d%d(gdZd(ggd%dZgd(ggej,fej d,d-gd/d-gge!d0d,d/gd-ggej+fe d,d-gd/d-ggd,d/gd-ggej-fe d%d(gej"d(ggd%ej"gd(ggej.fej d,ej"gdej"gge!d0d,dgej"ggej+fej d,e#d1gde#d1gge!d0d,dge#d1ggej+fgd2d3d4d[d\d]d^gd:d_d`Z/ejd d dgejjdaej d.d+gge!d0j0ej d.dbgge!d0j0d.d+dcggej+fej d%d(ggddd0j0ej d%deggddd0j0d%d(dZggej1fej d.d+gge!d0j0ej d.dbgge!d0j0e d.d+dcggej+fej dd.gge!d0j0ej dd+gge!d0j0dd.dfgge!fej d.d+gge!d0j0ej d.ej"gge!d0j0d.d+dfgge!fej d.dgge!d0j0ej d.ej"gge!d0j0d.ddfgge!fej d.ej"gge!d0j0ej d.dgge!d0j0d.ej"dfgge!fgd4d3dgdhdidjdkgd:dldmZ2dndoZ3dpdqZ4drdsZ5ejjdtdAdudvgfdLdudwdvgfdcd(d+gdxdygfgdAdzd{gd:d|d}Z6d~dZ7ejjd#d'd(d&gd$d%d&gge d)d(d&gdd%d&ggej d.d/d-gd+d,d-gge!d0gd2d3d4gd:ddZ8ejjdaej d.d+gge!d0j0ej d.dbgge!d0j0d.d+dcggej+fej d%d(ggddd0j0ej d%deggddd0j0d%d(dZggej1fej d.d+gge!d0j0ej d.dbgge!d0j0e d.d+dcggej+fgd4d3dgd:ddZ9ddZ:ddZ;ejde#eddZ?ddZ@ddZAddZBddZCejdej"de#d1gddZDejd@d'dZgd'dZdd.ggddZEejjdd?d>gddgd:ejjd@dAd.d(d+ggdAd{gd:ddZFejdeegddZGejddd(iddiddid(dddeddgejddd.d+dcdbgggddZHejd@dLdAd+ggddZIejd@d.gdbggddZJejdddZiddiddiddiddidZdddeddgddZKejd@dAd+ggddZLejd@d.gdbggddZMdd„ZNejddZd%dddeigddĄZOddƄZPddȄZQddʄZRdd̄ZSdd΄ZTejddd%dМgdd҄ZUejdd(dZdМgddԄZVddքZWejddddddddgejddddgddZXddZYejdej"dgddZZddZ[ejd dd gejdddgddZ\ejd d dgddZ]ejd d dgddZ^ejd d dgddZ_ddZ`ejdej"dgddZaejdddgejdej"dgddZbejjdaej d.ej"gge!d0j0ej d.d+gge!d0j0ej d.ej"dbge!d0gej+fej d.ej"gge!d0j0ej d.d+gge!d0j0ej d.ej"dbge!d0gej+fej dej"ggejd0j0ej dSggejd0j0e ddTej"ggejfgdddgd:ddZcejde dej"dSggj0e dej"dggj0e dTggfe ddTdSggj0e dddggj0e ej"ggfej dcej"d+gge!d0j0e dej"dggj0ej dbgge!d0fej dcd.d+gge!d0j0e dddggj0ej ej"gge!d0fgddZdd d Zed d Zfejd dd/ggej dd/ggdd0ej dd/ggdd0gejdd,d/ggej d,d/ggdd0ej d,d/ggdd0gddZgddZhddZiddZjejdd?d>gddZkejdej d.gdgge!d0dgej"gej"ggejldgdgdgge!d0fej ej"gdgd.gge!d0dgej"gej"ggejldgej"gej"gge!d0fgdd Zmd!d"Znd#d$Zod%d&Zpd'd(Zqd)d*ZrdS(+Nsparse)NotFittedError)assert_array_equal)assert_allclose)_convert_container) is_scalar_nan) OneHotEncoder)OrdinalEncodercCstdddgdddgg}t}tdd}||}||}|jdksLt|jdksZtt|shtt|rvtt| dd dd d gd dd dd ggt| |dS) NrF sparse_outputr ?) nparrayr fit_transformshapeAssertionErrorrissparsertoarray)XZ enc_sparseZ enc_denseX_trans_sparseZ X_trans_denserb/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/sklearn/preprocessing/tests/test_encoders.py!test_one_hot_encoder_sparse_denses   rhandle_unknownignoreinfrequent_if_existc Cstdddgdddgdddgg}tdddgg}tdd}||tjtdd ||W5QRXt|d}|||}t || td d d d d d d ggt ||dS) Nrr r r errorr Found unknown categoriesmatchrr) rrr fitpytestraises ValueError transformcopyrrrr rX2ohZ X2_passedrrr#test_one_hot_encoder_handle_unknown(s"     r2c CsLtdgdgg}tddgd}d}tjt|d||W5QRXdS)Nab categorieszqThis OneHotEncoder instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.r')rrr r*r+rr-)rencmsgrrrtest_one_hot_encoder_not_fitted@s r9c Cstddddgd}tddgd}t|d}|||}t||tddddgdd ddggt||dS) NZ11111111Z22Z333Z4444)r Z55555r%rr) rrreshaper r)r.rr-rr/rrr+test_one_hot_encoder_handle_unknown_stringsLs   r< output_dtype input_dtypecCstjddgg|dj}tjddgddgg|d}td|d}t|||t||||td|dd}t|||t||||dS)Nrr dtypeauto)r6r@F)r6r@r) rasarrayTr rrrr)r-)r>r=r X_expectedr1rrrtest_one_hot_encoder_dtype_s rEcCstd}|ddgddgd}tjddddgddddgg|d}t|d}t|||t| | ||t|d d }t|||t| | ||dS) Npandasr3r4r r ABrr?F)r@r) r* importorskip DataFramerrr rrrr)r-)r=pdX_dfrDr1rrr!test_one_hot_encoder_dtype_pandasns "  rNcCst}dddddgdddddgdd d d dgdd dd dgg}|||}tdddddddddddddddg||ddd d!d"g}td#d$d%d&d'd(d)d*d+d,d-d.d/d0d1g|tjtd2d3|ddgW5QRXdS)4NMaler girlr r Female) 3boy [Z x0_FemaleZx0_MaleZx1_1Zx1_41Zx1_51Zx1_91Zx2_boyZx2_girlZx3_1Zx3_2Zx3_12Zx3_21Zx4_3Zx4_10Zx4_30onetwothreefourfiveZ one_FemaleZone_MaleZtwo_1Ztwo_41Ztwo_51Ztwo_91Z three_boyZ three_girlZfour_1Zfour_2Zfour_12Zfour_21Zfive_3Zfive_10Zfive_30z!input_features should have lengthr')r r)get_feature_names_outrr*r+r,)r7r feature_namesZfeature_names2rrr"test_one_hot_encoder_feature_names~sb     racCs\t}tjddggtdj}|||}tddg||jdgd}tdd g|dS) Nuc❤t1Zdat2r?u x0_c❤t1Zx0_dat2un👍me)Zinput_featuresun👍me_c❤t1u n👍me_dat2)r rrobjectrCr)r_r)r7rr`rrr*test_one_hot_encoder_feature_names_unicodes rccCstddggj}t}|jddddggd|dddddggksLt||j dksdt|jdddddggd||j d kstdS) Nr r rr r5r6)r r#r#r) rrrCr set_params get_paramsrrrr)rr1rrrtest_one_hot_encoder_set_paramssrfcCsNtdd}||}tddd}||}t||t|sFt|S)NrAr5Fr6r)r rrrrZisspmatrix_csrr)rr7ZXtr1ZXtr2rrrcheck_categorical_onehots    rhrdefr 7abcr rSrr4rHcatr3rIr?nanmixednumericrbz mixed-nanzmixed-float-nanz mixed-Nonezmixed-None-nanzmixed-None-float-nan)ZidscCstt|dddgf}t|ddgddggtt|ddddgf}t|ddddgddddggtdd|}t|dddddgdddddggdS)Nrr rAr5)rhrrrr rr)rXtrrrrtest_one_hot_encoders rqsparse_FTdropfirstc Csdddgdddgdddgg}t||d}||}tj|td}t|||ddgddgddgg}t|d |d }||}t|}t||||dkrdddgdddgdddgg}t||ddgddgd dd ggd }||}tj|td}d|d<t|||ddgddgddgg}t|ddgd d gg|d}||}tj|td}d|d<d|dddf<t|||tdddgdddgg}td}t j t |d||W5QRXdS)Nrkr rjrir r rrsr?rA)rr6rs68)rr r6)r r )rr6r r rr)Shape of the passed X data is not correctr') r rrrrbrinverse_transformreescaper*r+r,)r rrrsrr7X_trexpr8rrrtest_one_hot_encoder_inversesH        rz X, X_transrZr[r\c CsJt|d|}d}|r"t|d}tjt|d||W5QRXdS)zCheck that `inverse_transform` raise an error with unknown samples, no dropped feature, and `handle_unknow="error`. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/14934 rzqSamples \[(\d )*\d\] can not be inverted when drop=None and handle_unknown='error' because they contain all zerosrr'N)r r)rr*r+r,rz)rX_transrrr7r8rrr?test_one_hot_encoder_inverse_transform_raise_error_with_unknown1s rcCsJtjddgddgddggtd}tddd }||}t|||dS) NrOr rQr r r? if_binaryFrsr)rrrbr rrrz)roher}rrr&test_one_hot_encoder_inverse_if_binaryQs   rr reset_dropcCstjddgddgddggtd}t|dd}||||}|}|j|d t| ||t |||t||dS) NrOr rQr r r?Frrs) rrrbr r)r-r_rdrrzr)rsrrrr}r`rrrtest_one_hot_encoder_drop_resetXs     rmethodr)r@@c Cs6t}d}tjt|dt|||W5QRXdS)N'Expected 2D array, got 1D array insteadr')r r*r+r,getattr)rrr1r8rrrtest_X_is_not_1Dgsrc CsRtd}|ddddg}t}d}tjt|dt|||W5QRXdS)NrFr r#rr')r*rJSeriesr r+r,r)rrLrr1r8rrrtest_X_is_not_1D_pandasqs  rzX, cat_exp, cat_dtyper stringz missing-floatzmissing-np.nan-objectzmissing-float-nan-objectcCs||dddfD]}tdd}||t|jts:tt|j|D]l\}}|}t|drt|dsrt|dd|ddkstn||kstt |j |sFtqFqdS)Nr:rAr5) r r) isinstance categories_listrziptolistrr issubdtyper@)rZcat_exp cat_dtypeXir7resr~Zres_listrrrtest_one_hot_encoder_categories|s#   rzX, X2, cats, cat_dtypedcint64r#zz object-stringzobject-string-nonezobject-string-nanzobject-None-and-nanzobject-nan-and-Nonec Cst|d}tdddgdddgg}t|||t|jdt|dksVt|j d t|dkstt|j dj |kstt|d}t j tdd||W5QRXt||d}tdddgdddgg}t||||dS)Nr5rrrr&r'r6r )r rrrrrrr6rrrr@r*r+r,r)r-)rr0catsrr r7r~rrr)test_one_hot_encoder_specified_categoriess:   rc CsDtjddggtdj}tdddggd}tdddgdddgg}t||||t| |||j d dddgkst t |j djtjst td d ggj}td d d ggd}d }tjt|d | |W5QRXtd d tjggj}td tjd ggd}tjt|d | |W5QRXdS)Nr3r4r?rr5rrrr r r z%Unsorted categories are not supportedr')rrrbrCr rr)r-rrrrrrr@object_r*r+r,rm)rr7r~r8rrr(test_one_hot_encoder_unsorted_categoriess rc Cstjddgddggtdj}tdddgdddggd}td d d d d d gd d d d d d gg}t||||jd dddgkst t |jdj tj st |jd dddgkst t |jdj tj st dS) Nr3r4rr r?rr r5rr)rrrbrCr rrrrrrrr@rrr7r~rrr7test_one_hot_encoder_specified_categories_mixed_columnss&rcCsLtd}|ddgddgd}t|}t|ddddgddddggdS)NrFr3r4r r rGr)r*rJrKrhr)rLrMrprrrtest_one_hot_encoder_pandas s rzdrop, expected_namesx0_cZx2_bZx1_2x0_bZx2_abinaryZmanualcCs>dddgdddgg}t|d}|||}t||dS)Nrr r3r4r)r r)r_r)rsZexpected_namesrrr`rrr'test_one_hot_encoder_feature_names_drop)s   rcCsddgddgddgg}tddddgddddgddddgg}tddg}td d d }||}t|j|t||d d gdd gdd gg}tddgddgddgg}tddg}td d d }||}t|j|t||dS)NrSyesnorYrrrrFrtruer3false)rrr rr drop_idx_r)rexpectedZexpected_drop_idxrresultrrr*test_one_hot_encoder_drop_equals_if_binary;s         rrcCsXt}tjdddgdddggdd}t|||dtdd}t|||dS)Nrr rr?float64)r rrrrZastyperrrrtest_ordinal_encoderSs  rzobject-string-catc Cst|d}tdgdgg}t|||t|jdt|dksJt|jd t|dksht|jdj |ks|tt|d}t j t dd||W5QRXdS)Nr5rrrr&r')r rrrrrr6rrrr@r*r+r,r))rr0rrr7r~rrr)test_ordinal_encoder_specified_categoriesds  rc Csdddgdddgg}t}||}tj|td}t|||tddddgddddgg}td}t j t |d ||W5QRXdS) Nrkr rjrir r?rryr') r rrrrbrrzr{r|r*r+r,)rr7r}r~r8rrrtest_ordinal_encoder_inverses  rcCstddd}tjddgddgdd ggtd }tjdd gd dgddggtd }||||}tjd dgddgddggdd }t||||}tjddgddgddggtd }t||dS)Nuse_encoded_valuer unknown_valuer3xr4yrrr?ZxyZblar r rr)r rrrbr)r-rrz)r7X_fitr X_trans_encr~ X_trans_invinv_exprrr+test_ordinal_encoder_handle_unknowns_strings         rr@cCstddd}tjddgddgdd gg|d }tjdd gd dgddgg|d }||||}tjddgddgd d ggdd }t||||}tjddgddgddggtd }t||dS)Nrrr r r r?rVrr)r rrr)r-rrzrb)r@r7rrrr~rrrrr,test_ordinal_encoder_handle_unknowns_numerics         rcCs`tdtjd}tdgdgdgg}|||dgdgdgg}t|dgdgtjggdS)Nrrr r r r#r)r rrmrr)r-r)r7rrrrr(test_ordinal_encoder_handle_unknowns_nans  rc CsNtdtjtd}tdgdgdgg}tjtdd||W5QRXdS)Nr)r rr@r r r z'dtype parameter should be a float dtyper') r rrmintrr*r+r,r))r7rrrr8test_ordinal_encoder_handle_unknowns_nan_non_float_dtypesrc Cs\tjdddddggtdj}dddg}t|d}d}tjt|d||W5QRXdS)NZLowZMediumZHighr?r5z*Shape mismatch: if categories is an array,r') rrrbrCr r*r+r,r))rrr7r8rrr+test_ordinal_encoder_raise_categories_shapes   rcstddtjddddgddddggdd}tjddgd d ggd dtjddgd d ggddtd d gddggtddgddggtjdd gd d ggddfD]BtfddtdDstt |qddgd d ggtfddtdDstt |dd gd d ggtfddtdDsltt |dS)NrAr5rrrr?r r r r#rr3r4rrabcdrbcsg|]}j|jjkqSrrr@.0irr7rr sz'test_encoder_dtypes..cs"g|]}tj|jtjqSr)rrrr@integerrr7rrrscsg|]}j|jdkqS)rbrrrrrrs) r rrr)allrangerrr-r)r~rrrtest_encoder_dtypess& "      rcstd}tddtjddddddgddddddggdd}|jdd gd d gd d gddd}|tfddtd Dst t  | ||dd gddgddgd}|dj |dj |dj g|tfddtd Dst t  | |dS)NrFrAr5rrrr?r r r r#rr)rHrICrcsg|]}j|jdkqS)rrrrrrrsz.test_encoder_dtypes_pandas..r3r4rrrHrIrcs g|]}j|j|kqSrrrZX_typer7rrr s)r*rJr rrrKr)rrrrr-rr@)rLr~rrrrtest_encoder_dtypes_pandass  "  "rcCs*t}ddgddgg}tj|j|dS)NrOr rQr )r rtestingZassert_no_warningsr)r7rrrrtest_one_hot_encoder_warning sr missing_valuec Csdddd|g}t|d}ddddd gddd dd gdddd|gg}||}d d d d d gd d d d d gd d d d d gg}t|||j|kstd d t|j|jD}| |}t j |t d}t |drzt|dd|ddt |dstt |dstt|ddddf|ddddft|dddf|dddft |dsftt |dstnt||t||dS)NrirVr rwrrkr rjr3r rcSsg|]\}}||qSrr)rrlfeaturerrrr!sz4test_one_hot_encoder_drop_manual..r?r:)r:r:)r rrrrsrrrrrzrrrbr) rZ cats_to_dropr7rZtransr~Z dropped_catsZ X_inv_transZX_arrayrrr test_one_hot_encoder_drop_manuals2    (   *" rrRc CsNt|d}d}tjt|d(|dddgdddgdd d ggW5QRXdS) Nrz-`drop` should have length equal to the numberr'rkr rjrir r ;)r r*r+r,r))rsr7err_msgrrrtest_invalid_drop_length8s rdensityrZdensecCst|d}t||d}dddgdddgg}||||t|j|j|dkrbt|jd n0t||j|jD]\}}}|t||ksrtqrt|jt j st|jj t kstdS) Nrrurr r3r r4rtr) r r)rrrrrrrrZndarrayr@rb)rrsZohe_baseZohe_testrZdrop_catZdrop_idxZcat_listrrrtest_categories@s     rEncodercCsd|dkstdS)NZ categoricalZX_types)Z _get_tagsr)rrrr"test_encoders_has_categorical_tagsTsrkwargsmax_categories min_frequency g(\?r)rrrVr6rAc Cstdgddgddgddgdgj}tf|d d d ||}t|jdddggdgdgdgdgd gg}td dgdd gdd gdd gdd gg}||}t||dddgdgdD}| |}t||| } tddg| dS)zpTest that different parameters for combine 'a', 'c', and 'd' into the infrequent category works as expected.r3rr4rrrSrr r"F)r6r rer rcSsg|] }|gqSrrrcolrrrrwsz2test_ohe_infrequent_two_levels..infrequent_sklearnr#rx0_infrequent_sklearnN rrrCr r)rinfrequent_categories_r-rrzr_) rr6X_trainrX_testrr expected_invX_invr`rrrtest_ohe_infrequent_two_levelsYs(2(    rcCstdgddgddgddgdgj}td d d |d |}|jd |jd dksdttdgdgg}||}t d gdgg|| }t dg|| |}t dgdgg|dS)z3Test two levels and dropping the frequent category.r3rr4rrrSrr r"Fr r rrrsrr rrN) rrrCr r)rrrr-rr_rrz)rsrrrrr` X_inverserrr,test_ohe_infrequent_two_levels_drop_frequents"2   rc Csztdgddgddgddgdgj}td d d |d }d |dd}tjt|d||W5QRXdS)z_Test two levels and dropping any infrequent category removes the whole infrequent category.r3rr4rrrSrr r"Fr rUnable to drop category r( from feature 0 because it is infrequentr'NrrrCr r*r+r,r)rsrrr8rrr5test_ohe_infrequent_two_levels_drop_infrequent_errorss2rrgQ?g{Gz?rc Cstdgddgddgddgdgj}tfd d d ||}t|jddggdgdgdgdgd gg}td ddgddd gdd dgddd gddd gg}||}t||dgdgdgdgdgg}| |}t||| }tdddg|dS)zkTest that different parameters for combing 'a', and 'd' into the infrequent category works as expected.r3rr4rrrSrr r"Fr rrr rrrrrNr) rrrrrrrrr`rrr test_ohe_infrequent_three_levelss022    rc Cstdgddgddgddgdgj}td d d|d |}tdgdgdgg}td d gd d gd d gg|||jdd|d}tj t |d|dgdgg}W5QRXtd d gd d gg|dS)z5Test three levels and dropping the frequent category.r3rr4rrrSrr r"Frrr r!r%r&r'rN) rrrCr r)rr-rdr*warns UserWarning)rsrrrr8rrrr.test_ohe_infrequent_three_levels_drop_frequents 2"r c Csztdgddgddgddgdgj}td d d|d }d |d d}tjt|d||W5QRXdS)z7Test three levels and dropping the infrequent category.r3rr4rrrSrr r"Frrrrr'Nrrrrr7test_ohe_infrequent_three_levels_drop_infrequent_errorss2r c Cstdgddgddgddgdgj}td d dd |}t|jddggdgdgdgdgg}td d d gd d d gd d d gd d d gg}||}t||dgg}d}t j t |d||W5QRXdS)zmTest that different parameters for combining 'a', and 'd' into the infrequent category works as expected.r3rr4rrrSrr r$F)r rrr rbadz.Found unknown categories \['bad'\] in column 0r'N) rrrCr r)rrr-rr*r+r,)rrrrrr8rrr(test_ohe_infrequent_handle_unknown_errors"2*  r cCstjdgddgdgtdj}tfddddggd d d ||}dgdgdgdgdgg}td d gd d gd d gd d gd d gg}||}t||dddgg}dgdgg}|D].}|j|d|td gd gg||qdS)zG'a' is the only frequent category, all other categories are infrequent.r3rrrYr?rrr4Fr"r6rr r rrtrrN) rrrbrCr r)r-rrd)rrrrrrZdropsrsrrr5test_ohe_infrequent_two_levels_user_cats_one_frequents&" (    rcCstjdgddgddgddgdgtd j}tddddggd d d d |}t|jdddggdgdgdgdgdgg}tddgddgddgddgddgg}||}t ||dddgdgdD}| |}t||dS)zFTest that the order of the categories provided by a user is respected.r3rr4rrrSrr r?Fr"r r6rr rrr rcSsg|] }|gqSrrrrrrrEsz.rr#N rrrbrCr r)rrr-rrzrrrrrrrrrr(test_ohe_infrequent_two_levels_user_cats/s(( (   rc Cstjdgddgddgddgdgtd j}tddddggd d dd |}t|jddggdgdgdgdgd gg}tdddgdddgdddgdddgdddgg}||}t ||dgdgdgdgdgg}| |}t||dS)zTest that the order of the categories provided by a user is respected. In this case 'c' is encoded as the first category and 'b' is encoded as the second one.r3rr4rrrSrr r?Fr"rrrr rNrrrrr*test_ohe_infrequent_three_levels_user_catsJs2( 2   rc Cstjdddddddddg dddddddddg f}tdddd}||ddgddgg}||}t|ddddgddddggdS) zaTest infrequent categories where feature 0 has infrequent categories, and feature 1 does not.rr r r rF)rrsrN)rc_r r)r-r)rrrrrrrtest_ohe_infrequent_mixedns 2  rc Cstjdddddddddg dddddddddg dddddddddg f}tdddd }||}t|jdddgt|jdddgt|jdd |}td d d dddddg|ddddddddgddddddddgddddddddgddddddddgddddddddgddddddddgddddddddgddddddddgddddddddgg }t||dddgdddgg}| |}ddddddddgddddddddgg}t||| |}tj ddd gddd ggt d}t||tdddd  |}tjtdd| |W5QRXdddgdddgg}| |}ddddddddgddddddddgg}t||| |}tj dddgdddggt d}t||d S)z?Test infrequent categories with feature matrix with 3 features.rr r r rrSrAr"r6rr NZx0_0Zx0_3rZx1_0Zx1_5Zx1_infrequent_sklearnZx2_0Zx2_1r#rr?r$r&r')rrr rrrrr_rr-rzrrbr)r*r+r,) rrrr`rr X_test_transrrrrr'test_ohe_infrequent_multiple_categoriess   (   ( rc Cs(td}|jdddddddddg dddd d d dddg d d d gd}tdddd}||}t|jdddgt|jdddd gddddddgddddddgddddddgddddddgddddddgddddddgddddddgddddddgddddddgg }t|||jddgdd gd d d gd}ddddddgddddddgg}| |}t||| |}t j ddgddggt d}t|||jddgd dgd d d gd}| |}ddddddgddddddgg}t||| |}t j ddgddggt d}t||dS)zHTest infrequent categories with a pandas dataframe with multiple dtypes.rFr3frr4rr rrSrV)strrrrcolumnsrAr"rr rr?N)r*rJrKr rrrrrr-rzrrrb) rLrrrrrrrrrrr.test_ohe_infrequent_multiple_categories_dtypess\            rrX)rrcCsptdgddgddgddgdgj}tfd d d |}|||dgg}t|d ggd S),All user provided categories are infrequent.r3rr4rrrSrr r"Frr N)rrrCr r)r-rrrrrrrr$test_ohe_infrequent_one_level_errorss2 r!cCsftjdgdgtdj}tfddddggdd d ||}|dgdgg}t|d gd ggd S) rrr r?rrr3r4Fr"r r N)rrrbrCr r)r-rr rrr5test_ohe_infrequent_user_cats_unknown_training_errors*s r"c CsHddgddgddgg}d}tjt|dtdd |W5QRXdS) NrOr rQr r z'`sparse` was renamed to `sparse_output`r'Fr)r*r FutureWarningr r))rr8rrr&test_one_hot_encoder_sparse_deprecated;sr$zinput_dtype, category_dtypeZOOZOUZUOZUUSOZSUZSS array_typerrZ dataframec Cstjdgdgg|d}tjddg|dg}t|dd|}tdgdgdgdgg||d}||}tddgddgddgddgg}t||t|d|} | |}tdgdgdgdgg}t||d S) a"Check that encoding work with object, unicode, and byte string dtypes. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/15616 https://github.com/scikit-learn/scikit-learn/issues/15726 https://github.com/scikit-learn/scikit-learn/issues/19677 r4r3r?Frgrr r5N) rrr r)rr-rr r) r>Zcategory_dtyper&rr6rrrroerrrtest_encoders_string_categoriesDs  "  r(c Cshtjdgdggdd}tjddgddg}t|dd}td}tjt|d ||W5QRXd S) zCheck that this mixture of predefined categories and X raises an error. Categories defined as bytes can not easily be compared to data that is a string. r4r3Ur?SFrgzjIn column 0, the predefined categories have type 'bytes' which is incompatible with values of type 'str_'.r'N) rrr r{r|r*r+r,r))rr6rr8rrr$test_mixed_string_bytes_categoricalscs r+cCsPtjdd|d|ggtdj}tddd|}|}t|ddd |gdS) Nr3r4r?Fr!rr Zx0_arZx0_)rrrbrCr r)r_r)rrrnamesrrr)test_ohe_missing_values_get_feature_namesxsr.c Cstd}|jddddgtjdddtjgtddd d gd }tdd ddd ddgd ddd dddgddd ddd dgd dddddd gg}t|}t||dS) NrFdogrlr rr#r?)col1col2r0r1rr ) r*rJrKrrrmfloatrhr)rLdfexpected_df_transrprrr%test_ohe_missing_value_support_pandass    r5 pd_nan_typepd.NAznp.nanc Cstd}|dkr|jntj}|d|jdd|ddgddi}td d d d gd d d d gd d d d gd d d d gd d d d gg}td |d }| |}t ||t |j d kst t|j d dd dddgt|j d d st dS)NrFr7r0rr3r4categoryr?rr Fr,r:)r*rJNArrmrKrrr rrlenrrrisnan)r6r rLpd_missing_valuer3r4rdf_transrrr1test_ohe_missing_value_support_pandas_categoricals*         r>c Csddgddgddgg}tdd|d}||}tdddgdddgdddgg}t||d d gg}tdddgg}d }tjt|d ||}W5QRXt||| |}t |tjddggt d dS)zZCheck drop='first' and handle_unknown='ignore'/'infrequent_if_exist' during transform.r3rr4r r rtFrsrr rr tFound unknown categories in columns \[0, 1\] during transform. These unknown categories will be encoded as all zerosr'r?N r rrrrr*rrr-rzrrbr rrrrDrwarn_msgrrrr/test_ohe_drop_first_handle_unknown_ignore_warnss.     rDc Csddgddgddgg}tdd|d}||}tddddgddddgddddgg}t||d d gg}tddddgg}d }tjt|d ||}W5QRXt||| |}t |tjdd ggt dd S)zDCheck drop='if_binary' and handle_unknown='ignore' during transform.r3rr4r r rFr?rr r@r'Nr?rArBrrr3test_ohe_drop_if_binary_handle_unknown_ignore_warnss.        rEc Csddgddgddgg}tdd|ddgddggd}||d dgg}tddgg}d }tjt|d ||}W5QRXt||d S) znCheck drop='first' and handle_unknown='ignore'/'infrequent_if_exist' during fit with categories passed in.r3rr4r r rtF)rsrr r6rzqFound unknown categories in columns \[0\] during transform. These unknown categories will be encoded as all zerosr'N) r r)rrr*rrr-r)r rrrrDrCrrrr'test_ohe_drop_first_explicit_categoriess  rFc CsXttjdddggj}ttjd}dtj}tjt|d| |W5QRXdS)zDTest ordinal encoder with nan passthrough fails when dtype=np.int32.rrr?zdThere are missing values in features \[0\]. For OrdinalEncoder to encode missing values with dtype: r'N) rrrmrCr int32r*r+r,r))rr'r8rrrBtest_ordinal_encoder_passthrough_missing_values_float_errors_dtypes   rHencoded_missing_valuercCstjtjdddggtjdj}t|d|}t|jdks@t t |jdddtjg| |}t ||gdgdgdgg| |}t ||dS) z.Test ordinal encoder with nan on float dtypes.rrr?rIr rrN) rrrmrrCr r)r:rrrr-rz)rIrr'rrrrr5test_ordinal_encoder_passthrough_missing_values_float,s  rKc Cs$td}|dkr|jntj}|d|jdd|ddgddi}t|d |}t |j d ksbt t |j d d d dddgt |j d dst ||}t|dgdg|gdgdgg||}|jdkst t |d dd fddgt |d d d fddgt |ds t d S)z0Check ordinal encoder is compatible with pandas.rFr7r0rr3r4r8r?rJr rNr r:@rr)rr r rx)r*rJr9rrmrKrr r)r:rrrr;r-rrzr)r6rIrLr<r3r'r=rrrr=test_ordinal_encoder_missing_value_support_pandas_categorical>s$   rMrLzobject-None-missing-valuezobject-nan-missing_valueznumeric-missing-valuec Cstt|d}tdgtjgg}t||||jdj|ksBtt|d}t j t dd| |W5QRXdS)z.Test ordinal encoder for specified categories.r5rrr&r'N) r rrrmrrrr@rr*r+r,r))rr0rrr'r~rrr=test_ordinal_encoder_specified_categories_missing_passthrough_s&  rNzX, expected_X_trans, X_testrrcCs8tddd}||}t||t||dggdS)z>Test the interaction between missing values and handle_unknownrr:rgN)r rrr-)rZexpected_X_transrr'rrrr/test_ordinal_encoder_handle_missing_and_unknowns   rOc Cstdddgdddgg}t|}t}d}tjt|d||W5QRXtjt|d| |W5QRX| |}t|}tjt|d| |W5QRXdS)zCheck that we raise proper error with sparse input in OrdinalEncoder. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/19878 r r r rz6A sparse matrix was passed, but dense data is requiredr'N) rrrZ csr_matrixr r*r+ TypeErrorr)rrz)rZX_sparseencoderrrrrrrtest_ordinal_encoder_sparses   rRc Cstddddddgddtjf}tdddggddd }||tdddggd d }tjtd d ||W5QRXdS)zCheck OrdinalEncoder.fit works with unseen category when `handle_unknown="use_encoded_value"`. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/19872 rr r rNr:rr)r6r rr$rr&r')rrZnewaxisr r)r*r+r,)rr'rrr-test_ordinal_encoder_fit_with_unseen_categorys$  rSrZAAOr)rcCs4tddd}||||}t|ddggdS)zChecks that `OrdinalEncoder` transforms string dtypes. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/19872 rirrN)r r)r-r)rrr7rrrr1test_ordinal_encoder_handle_unknown_string_dtypess   rUcCsftddddgdd}t|}t|jtj|ddj| |}t|dgd gd gdggd S) zCheck that `OrdinalEncoder` accepts Python integers that are potentially larger than 64 bits. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/20721 l HP 1&l H]viel :?i}GalIRK2e6kr:r r)Zaxisr r N) rrr;r r)rrsortrCr-)rrQrrrr#test_ordinal_encoder_python_integers  rWcCsLtd}dddg}|jdddgg|d}t|}|}t||d S) z-Check feature names out is same as the input.rFr4rr3r r r rN)r*rJrKr r)r_r)rLr-rr7Zfeature_names_outrrr.test_ordinal_encoder_features_names_out_pandass    rXcCstjdgdgtjggtd}tdtjdd|}||}t|dgdgdggtjd gtjggtd}||}t|tjgdgg||}|ddd kst t |ddst d S) zECheck interactions between encode_unknown and missing value encoding.r3r4r?rr rrIrr rN) rrrmrbr r)r-rrzrr;)rr'rrr X_roundtriprrr0test_ordinal_encoder_unknown_missing_interactions    r\ with_pandasc Cstjddgddgdtjggtd}d}|rPtd}|j|d d gd }|d }n|d }tdd}tjt |d| |W5QRXdS)zXCheck OrdinalEncoder errors when encoded_missing_value is used by an known category.r3r/r4rlrr?zTencoded_missing_value \(1\) is already used to encode a known category in features: rFletterZpetrz \['pet'\]z\[1\]r rJr'N) rrrmrbr*rJrKr r+r,r))r]r error_msgrLr'rrr0test_ordinal_encoder_encoded_missing_value_error8s"   r`z4X_train, X_test_trans_expected, X_roundtrip_expected1c Cstdtjtjd|}tdgtjgdgg}||}t||||}|jd}t |D]V}||df} ||df} | dkr| dkst q`t | rt | st q`| | ks`t q`dS)znCheck transform when unknown_value and encoded_missing_value is nan. Non-regression test for #24082. rrZrar4rN) r rrmr)rr-rrzrrrrr;) rZX_test_trans_expectedZX_roundtrip_expectedr'rrr[Z n_samplesrZ expected_valvalrrr9test_ordinal_encoder_unknown_missing_interaction_both_nanRs(       rcc Cstd}|ddgddgd}t}|jddd}tjt|d ||W5QRXtd d jd d}td d jdd}||}||}t| |t | |j d S)z*Check OneHotEncoder works with set_output.rFr3r4r r rGr-z*Pandas output does not support sparse datar'FrdefaultN) r*rJrKr set_outputr+r,rrto_numpyrr_r)rLrMrr(Z ohe_defaultZ ohe_pandas X_defaultX_pandasrrrtest_one_hot_encoder_set_outputs    rjcCsttd}|ddgddgd}tjdd}tjdd}||}||}t||t| |j d S) z+Check OrdinalEncoder works with set_output.rFr3r4r r rGrerdN) r*rJrKr rfrrrgrr_r)rLrMZ ord_defaultZ ord_pandasrhrirrrtest_ordinal_set_outputs   rkcCszdddddgddgg}t|d}|ddggt|t|jksFtt|jD]$\}}|jtksftt|||qPd S) zjCheck that the categories_ dtype is `object` for string categories Regression test for gh-25171. asZmmasZeasZrasZacsra2r5N) r r)r:rr enumerater@rbr)r6r7nrlrrr test_predefined_categories_dtypes rpcCs~tjdgdgtjggtd}tdd|}t|dgdgdggtddd |}td gg}||}t|dggd S) zBCheck missing value or unknown encoding can equal the cardinality.r/rlr?r rJr rrrsnakeN) rrrmrbr rrr)r-)rrr7rrrr1test_ordinal_encoder_missing_unknown_encoding_maxs rrcCstjdgddgddgddgddgdgtdj}tdd d d |}t|d d ddg|jd|j ddkst tjdgddgddgdgtdj}tdd dd |}t|dg|jd|j ddkst tjdgddgddgddgddgdgtdj}tdd dgd |}t|dd ddg|jd|j ddks|t tdd dd |}t|dd d ddg|j dkst dS)zkCheck drop_idx is defined correctly with infrequent categories. Non-regression test for gh-25550. r3r r4r#rrrr?Frt)rrrsrZx0_dZx0_errrSrrN) rrrbrCr r)rr_rrr)rrrrr#test_drop_idx_infrequent_categoriess<2 ,2  rs)sr{numpyrZscipyrr*Zsklearn.exceptionsrZsklearn.utils._testingrrrZ sklearn.utilsrZsklearn.preprocessingr r rmarkZ parametrizer2r9r<rGZfloat32rrErNrarcrfrhrrbrmr2rqrrrrrrrrZstr_Zfloat_rrCrrrrrrrrrrrrrrrrrrrrrrrrrrrr r r rrrrrrr!r"r$r(r+r.r5r>rDrErFrHrKrMrNrOrRrSrUrWrXr\r`rBrcrjrkrprrrsrrrrs             <       /0&     & &* !       -8               $           $[A       % $     "       !