U md ~%@s,ddlmZddlZddlZddlZddlmZmZddl m Z m Z m Z e ejdeejjZeejjZejddd~d d Zejddd d ZejddddZejddddZejdejejjejjddddejjejjddddgejejddddZejejejejjddejejjddfejjejjddddejjejjddddejjejjddddejjejjddddgdejjdddejjdddejjejjejjejjejjdddddZ ejddddZ!ejejej"ejjej"ejjfejjejjddddejjejjddddejjejjddddejjejjddddgdejjejjejjejjejjdddd d!Z#ejejejjejjddddejjejjddddejjejjddddejjejjddddgdejjejjejjejjejjejjd"ddd#d$Z$ed%d&Z%ejdd'd(d)Z&ejd*ejejjejjddddejjejjddddejjejjddddejjejjddddgdejjdddejjejjejj'ejjd+d,d-d.Z(ed/d0Z)ed1d2Z*edd4d5Z+ed6d7Z,ed8d9Z-ejd*ejejjejjddddejjejjddddejjejjddddejjejjddddgdd'd:d;Z.edd,d?d@Z0ej1dd'dAdBZ2edCdDZ3edEdFZ4edGdHZ5edIdJZ6edKdLZ7edMdNZ8edOdPZ9edQdRZ:ejdejjejjejjejj'ejjdSd,dTdUZ;ej1dddVdWdXZed^d_Z?ed`daZ@ejdejjejjejjejj'ejjdbd,dcddZAej1dddVdedfZBedgdhZCdidjZDeeCfdkdlZEeddmdnZFedodpZGedqdrZHejddsdtddvdwZIejddsdtddxdyZJe&e&e(e)e)e)e*e*e*e*e+e-e.e,e/e4e3e5e6e7e8e9e:e?eEeEeFeFeFeFe@eGeGeHeHeHdz$ZKd{ZLe(ejMd|e(ejMd|e;ee:0yE>cCs$t||}|||t|kSN)npabs)abZrtolatoldiffrK/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pynndescent/sparse.pyisclosesrcCs@t|}ttjdtjd|dd|ddkf}||S)NZdtype)r sort concatenateonesZbool_)Zarrauxflagrrr arr_uniques .rcCs:|jddkr|S|jddkr$|Stt||fSdSNr)shaperr r)ar1ar2rrr arr_union&s r#cCs:t||f}||dd|dd|ddkS)Nrr)r rr)r!r"rrrr arr_intersect2sr$zi4(i4[:],i4[:])r)readonly)i1i2)localsc Cs|jddks|jddkr dSd}d}|jdd}|jdd}||}||}d}||kr|d7}||kr|d7}||}nq||kr|d7}||}qqqX||kr||kr|d7}||}qX||kr||kr|d7}||}qXqqX|SNrrr ) r!r"r&r'Zlimit1Zlimit2j1j2resultrrrfast_intersection_size:s6    r.) result_ind result_datavalr&r'r+r,)fastmathr(r c Cs|jd|jd}tj|tjd}tj|tjd}d}d}d} ||jdkr(||jdkr(||} ||} | | kr||||} | dkr| || <| || <| d7} |d7}|d7}q@| | kr||} | dkr| || <| || <| d7} |d7}q@||} | dkr| || <| || <| d7} |d7}q@||jdkrv||} ||} | dkrj| || <| || <| d7} |d7}q(||jdkr||} ||} | dkr| || <| || <| d7} |d7}qv|d| }|d| }||fSNrrr)r r zerosint32float32) ind1data1ind2data2Z result_sizer/r0r&r'nnzr+r,r1rrr sparse_sumpsb           r<cCst|||| Sr )r<)r7r8r9r:rrr sparse_diffsr=)r1r&r'r+r,c Cstjjtjj}tjjtjj}d}d}||jdkr||jdkr||}||} || kr||||} | dkr|||| |d7}|d7}q,|| kr|d7}q,|d7}q,||fSr)) numbatypedListZ empty_listtypesr5r6r append) r7r8r9r:r/r0r&r'r+r,r1rrr sparse_muls$     rC)r-r1r&r'r+r,c Cs|jd}|jd}d}d}d}||} ||} | | kr||||} || 7}|d7}||krd|S||} |d7}||kr|S||} q0| | kr|d7}||kr|S||} q0|d7}||kr|S||} q0|S)Nrrr*) r7r8r9r:Zdim1Zdim2r-r&r'r+r,r1rrrsparse_dot_products8     rEc Cst||}tj|jdtjd}tj|jdtjd}d}d}d} ||jdkr*||jdkr*||} ||} | | kr||||} | dkr|||| <|||| <| d7} |d7}|d7}qB| | kr||} | dkr|||| <| d7} |d7}qB||} | dkr |||| <| d7} |d7}qB||jdkrl||} | dkr`|||| <| d7} |d7}q*||jdkr||} | dkr|||| <| d7} |d7}ql|d| }|d| }||fSr3)r#r r4r r6) r7r8r9r:r/Z result_data1Z result_data2r&r'r;r+r,r1rrr dense_union>sV                  rF)r2cCsDt||||\}}d}t|jdD]}|||d7}q$t|S)NrDr)r=ranger r sqrtr7r8r9r:_aux_datar-irrrsparse_euclideanxs rNz#f4(i4[::1],f4[::1],i4[::1],f4[::1]))rLr-rdimrM)r2r(c CsDt||||\}}d}t|}t|D]}|||||7}q&|SNrD)r=lenrH) r7r8r9r:rKrLr-rOrMrrrsparse_squared_euclideans  rRcCs@t||||\}}d}t|jdD]}|t||7}q$|SNrDrr=rHr r rrJrrrsparse_manhattans rUcCsBt||||\}}d}t|jdD]}t|t||}q$|SrS)r=rHr maxr rrJrrrsparse_chebyshevs rW@c CsLt||||\}}d}t|jdD]}|t|||7}q$|d|S)NrDr?rT) r7r8r9r:prKrLr-rMrrrsparse_minkowskis r[cCs$t||||djd}t||Sr)r=r float)r7r8r9r: n_features num_not_equalrrrsparse_hammingsr_cCs~t|}t|}t||||\}}d|tj}t||||\}} t| } t|| ||\} } d} | D] } | | 7} ql| S)NrYrD)r rr<Zastyper6r=rC)r7r8r9r:Z abs_data1Z abs_data2Z denom_inds denom_dataZ numer_inds numer_datarKZval_datar-r1rrrsparse_canberras    rbc Csvt||||\}}t|}|jddkr.dSt|}|dkrDdSt||||\}}t|}t|}t||SNrrD)r<r rr sumr=r\) r7r8r9r:rKr` denominatorra numeratorrrrsparse_bray_curtiss     rgcCsBt||}|jd|jd|}|dkr.dSt|||SdSrcr.r r\r7r8r9r: num_equal num_non_zerorrrsparse_jaccards  rl)rkrjcCsNt||}|jd|jd|}|dkr.dS|dkr:tSt|| SdSrc)r.r FLOAT32_MAXr log2rirrrsparse_alternative_jaccards rocCsdtd| S)NrYrX)pow)vrrrcorrect_alternative_jaccardsrrcCs6t||}|jd|jd|}||}t||Srrhr7r8r9r:r] num_true_truerkr^rrrsparse_matchings rucCsJt||}|jd|jd|}||}|dkr6dS|d||SdS)NrrDrXr.r r7r8r9r:rtrkr^rrr sparse_dice#s  rxcCsRt||}|jd|jd|}||}|dkr6dSt|||||SdSrcrhrsrrrsparse_kulsinski/s rycCs:t||}|jd|jd|}||}d|||SNrrXrvrsrrrsparse_rogers_tanimoto=s r{cCsl|jd|jdkr&t||kr&dSt||}|t|dkkrX|t|dkkrXdSt|||SdSrc)r r allr.rdr\)r7r8r9r:r]rtrrrsparse_russellraoFs " $r}cCs:t||}|jd|jd|}||}d|||Srzrvrsrrrsparse_sokal_michenerSs r~cCsJt||}|jd|jd|}||}|dkr6dS|d||SdS)NrrDg?rvrwrrrsparse_sokal_sneath\s  rc Cstt||||\}}d}t|}t|}|D] } || 7}q*|dkrL|dkrLdS|dks\|dkr`dSd|||SdSNrDrY)rCr) r7r8r9r:rKrLr-norm1norm2r1rrr sparse_cosinehs r)r-norm_xnorm_yrOrMc Cst||||\}}d}t|}t|}t|} t| D]} ||| 7}q6|dkr\|dkr\dS|dksl|dkrptS|dkr|tS|||}t|SdSrP)rCrrQrHrmr rn) r7r8r9r:rKrLr-rrrOrMrrrsparse_alternative_cosinezs   r)r2r cCs2tdt|dds|dkrdSdtd| SdSNrDgHz>)rrYrX)rrrpdrrr!sparse_correct_alternative_cosinesrcCst||||}d|S)NrY)rEr7r8r9r:r-rrr sparse_dotsrr-cCs*t||||}|dkrtSt| SdSrP)rErmr rnrrrrsparse_alternative_dotsrcCsTd}d}d}|jddkr,|jddkr,dS|jddksH|jddkrLdSt|jdD]}|||7}qZt|jdD]}|||7}qz||}||}tj|jdtjd} tj|jdtjd} t|jdD]}|||| |<qt|jdD]}|||| |<qtt| d||jd|d} tt| d||jd|d} t|| || \} }t| }|D]}||7}q~t|jdD]$}|||kr|| ||8}qt|jdD]$}|||kr|| ||8}qt ||}|||||jd7}| dkr2| dkr2dS|dkr@dSd|| | SdS)NrDrrYrrG) r rHr emptyr6rIrrCsetr#)r7r8r9r:r]Zmu_xZmu_yZ dot_productrMZ shifted_data1Z shifted_data2rrZ dot_prod_indsZ dot_prod_dataZcommon_indicesr1Z all_indicesrrrsparse_correlationsT     rc Cst||||\}}d}t|}t|}t||} |D]} |t| 7}q<|dkrd|dkrddS|dkst|dkrxdS|| krdStd|| SdSr)rCr rdrI) r7r8r9r:aux_indsrLr-rrZsqrt_norm_prodr1rrrsparse_hellingers  r)r- l1_norm_x l1_norm_yrOrMc Cst||||\}}d}t|}t|}t|} t| D]} |t|| 7}q:|dkrf|dkrfdS|dksv|dkrztS|dkrtSt|||}t|SdSrS)rCr rdrQrHrIrmrn) r7r8r9r:rrLr-rrrOrMrrrsparse_alternative_hellinger s    rcCs8tdt|dds|dkrdStdtd| SdSr)rrr rIrprrrr$sparse_correct_alternative_hellinger)srcCst||k Sr )r r6)xyrrrdummy_ground_metric1srcstfdd}|S)aGenerate a "ground_metric" suitable for passing to a ``sparse_kantorovich`` distance function. This should be a metric that, given indices of the data, should produce the ground distance between the corresponding vectors. This allows the construction of a cost_matrix or ground_distance_matrix between sparse samples on the fly -- without having to compute an all pairs distance. This is particularly useful for things like word-mover-distance. For example, to create a suitable ground_metric for word-mover distance one would use: ``wmd_ground_metric = create_ground_metric(word_vectors, cosine)`` Parameters ---------- ground_vectors: array of shape (n_features, d) The set of vectors between which ground_distances are measured. That is, there should be a vector for each feature of the space one wishes to compute Kantorovich distance over. metric: callable (numba jitted) The underlying metric used to cpmpute distances between feature vectors. Returns ------- ground_metric: callable (numba jitted) A ground metric suitable for passing to ``sparse_kantorovich``. cs||Sr r)Zindex1index2ground_vectorsmetricrr ground_metricSsz+create_ground_metric..ground_metric)r>njit)rrrrrrcreate_ground_metric6srcCsht|jd|jdf}t|jdD]2}t|jdD]}||||||||f<q:q(t|||Sr)r rr rHr)r7r8r9r:rZ cost_matrixrMjrrrsparse_kantorovichZs rcCsd}d}d}d}d} d} d} t|} t|} dd}||jdkrF| |jdkrF||}|| }||kr||||7}| ||| 7} | || | 7} || | |}|}|d7}| d7} q8||kr ||||7}| ||| 7} || | |}|}|d7}q8||||7}| || | 7} || | |}|}| d7} q8||jdkr||}||||7}| ||| 7} || | |}|}|d7}qF| |jdkr|| }||||7}| || | 7} || | |}|}| d7} qt|d|S)NrDrcSstt||Sr )r powerr)rrZrrrqz'sparse_wasserstein_1d..rrY)r rdr r)r7r8r9r:rZr-Zold_inddeltar&r'Zcdf1Zcdf2rrrr+r,rrrsparse_wasserstein_1des^         rcCst||||\}}t||Sr )rFrr7r8r9r:Z dense_data1Z dense_data2rrr sparse_jensen_shannon_divergencesrcCst||||\}}t||Sr )rFrrrrrsparse_symmetric_kl_divergencesrF)parallelr rYc Cst|jdD]}||dfg} ||dfg} td|jdD] } ||| fdkr^qfd} tt| D]} | | }||||| f|||| fd}||||| f|||| fd}|||||d}|||||d}|||||}| | tkrn|||| fkrnt||krnd} qpranger rHrQ FLOAT32_EPSrrBr inf)indicesZ distances data_indices data_indptr data_datadist rng_stateprune_probabilityrMZ new_indicesZ new_distancesrrkcZfrom_ind from_dataZto_indto_datarrrr diversifysD   rc Cs|jdd} t| D]~} ||| || d} ||| || d} t| } tj| jdtjd}td| jdD]}| |}t|D]}| |}||dkr| |}| |}|||||d}|||||d}|||||d}|||||d}|||||}| |tkr|| |krt ||krd||<q~qq~t| jdD],}| |}||dkrjd||| |<qjqdS)Nrrr) r r>rr ZargsortrZint8rHrr)Z graph_indptrZ graph_indicesZ graph_datarrrrrrZn_nodesrMZcurrent_indicesZ current_dataorderZretainedidxrrlrZqZ from_indsrZto_indsrrrrr diversify_csrs6     r)$ euclideanl2Z sqeuclideanZ manhattanl1ZtaxicabZ chebyshevZlinfZlinftyZ linfinityZ minkowskiZcanberraZ braycurtishammingjaccardZdicematching kulsinskirogerstanimoto russellrao sokalmichenerZ sokalsneathcosine correlationrZ wassersteinZwasserstein_1dzwasserstein-1dkantorovich-1dr hellingerzjensen-shannonZjensen_shannonz symmetric-klZ symmetric_klZsymmetric_kullback_liebler)rrrrrrr)rZ correction)rrrdotrr)r r )rX)r)rY)rY)O __future__rlocalenumpyr r>Zpynndescent.utilsrrZpynndescent.distancesrrr setlocale LC_NUMERICZfinfor6ZepsrrVrmrrrr#r$rAr5ZArrayZuint16r.Tupler<r=ZListTyperCrErFrNZintprRrUrWr[r_rbrgrlroZ vectorizerrrurxryr{r}r~rrrrrrrrrrrrrrrrrrZsparse_named_distancesZsparse_need_n_featuresrIZ!sparse_fast_distance_alternativesrrrrs.         )   ?     ' 9                             <     $  ?   6 8,