U
    ÅmœdYS  ã                   @   s¸  d Z ddlZddlZddlmZ ddlmZmZm	Z	m
Z
 ddlZddlZddlZddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ d1eeeef eee eee	e df eee	e df eee edœ	dd„Zd2dd„Zdd„ Zdd„ Z d3dd„Z!dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd „ Z&d!d"„ Z'd#d$„ Z(d%d&„ Z)d'd(„ Z*d)d*„ Z+d4eeeef eee	e e
eef edf ee	e e
eef df ee,eeeedf ee, ed.œd/d0„Z-dS )5z)Exporting to formats for other software.
é    N)ÚPath)ÚUnionÚOptionalÚIterableÚMapping)ÚAnnData)Úis_categorical_dtypeé   )Ú_get_mean_var)ÚNeighborsViewÚn_countsF)	ÚadataÚproject_dirÚembedding_methodÚsubplot_nameÚcell_groupingsÚcustom_color_tracksÚtotal_counts_keyÚneighbors_keyÚ	overwritec	              	      sd  |dkrd}|| j krtdƒ‚||  ¡ krzd| |  ¡ krDd| }n6|| j krnd| d | j | d d  }ntd| ƒ‚| j| ‰tˆƒ‰|dkršˆjnˆ| }	|	jd	d	d
� td|	› �ƒ d	}
dddddg}t‡fdd„|D ƒƒ�r|�st	 
ˆ› d�¡ d}
nt	 
dˆ› d�¡ | jdk	�r<| jj ¡ }t| jjƒ}n| j ¡ }t| jƒ}|| jk�rnt | j| ¡}n| d¡j}|
�rðt||ˆd ƒ t|ˆd ƒ t|ˆd ƒ ˆd  d¡� }|D ]}| |d ¡ �qÀW 5 Q R X t ˆd |¡ i }i }|dk�r:| jD ].}t| j| ƒ�rdd„ | j| D ƒ||< �qnzt|tƒ�rL|g}|D ]b}|| jk�rtt	 
d|›d�¡ n<t| j| ƒ�rždd„ | j| D ƒ||< nt	 
d|›d�¡ �qP|dk�rò| jD ]*}t| j| ƒ�sÄt | j| ¡||< �qÄnvt|tƒ�r|g}|D ]^}|| jk�r,t	 
d |›d�¡ n8t| j| ƒ�sRt | j| ¡||< nt	 
d |›d!�¡ �qt |jd" ¡|d#< t ||	d$ ƒ i }t!|||ƒ}t"||ƒ}t#|	d% |ƒ i }t$||ƒ}t%|	d& |ƒ t&| |ƒ}t'|	d' |jd" |ƒ t(|	d( |ƒ tj|	d) t )|jd" ¡d*d+� t *|	d, t )|jd" ¡¡ ˆˆ +d"¡ddd…f  ‰ˆt d-d-g¡ˆ ,d"¡ ddd…f  t d.d/g¡ddd…f  ‰tj|	d0 t -t )|jd" ¡dd…df ˆf¡d1d+� d2|  ¡ k�ròtj.|	d3 | jd2 |d4� d5| j k�r`t | j| j d5 d6  j/j0¡‰ | j| j d5 d6  j/j1}‡ ‡fd7d„t2t3|ƒƒD ƒ}t4| ||	d8 ƒ dS )9aÆ      Exports to a SPRING project directory [Weinreb17]_.

    Visualize annotation present in `adata`. By default, export all gene expression data
    from `adata.raw` and categorical and continuous annotations present in `adata.obs`.

    See `SPRING <https://github.com/AllonKleinLab/SPRING>`__ or [Weinreb17]_ for details.

    Parameters
    ----------
    adata
        Annotated data matrix: `adata.uns['neighbors']` needs to
        be present.
    project_dir
        Path to directory for exported SPRING files.
    embedding_method
        Name of a 2-D embedding in `adata.obsm`
    subplot_name
        Name of subplot folder to be created at `project_dir+"/"+subplot_name`
    cell_groupings
        Instead of importing all categorical annotations when `None`,
        pass a list of keys for `adata.obs`.
    custom_color_tracks
        Specify specific `adata.obs` keys for continuous coloring.
    total_counts_key
        Name of key for total transcript counts in `adata.obs`.
    overwrite
        When `True`, existing counts matrices in `project_dir` are overwritten.

    Examples
    --------
    See this `tutorial <https://github.com/scverse/scanpy_usage/tree/master/171111_SPRING_export>`__.
    NÚ	neighborszRun `sc.pp.neighbors` first.ZX_Ú_ÚparamsZlayoutz.Run the specified embedding method `%s` first.T)ÚparentsÚexist_okzWriting subplot to zcounts_norm_sparse_genes.hdf5zcounts_norm_sparse_cells.hdf5zcounts_norm.npzztotal_counts.txtz	genes.txtc                 3   s   | ]}ˆ |   ¡ V  qd S )N)Úis_file)Ú.0Úf)r   © úR/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/scanpy/external/exporting.pyÚ	<genexpr>p   s     z!spring_project.<locals>.<genexpr>z} is an existing SPRING folder. A new subplot will be created, but you must set `overwrite=True` to overwrite counts matrices.FzOverwriting the files in Ú.é   ÚwÚ
c                 S   s   g | ]}t |ƒ‘qS r   ©Ústr©r   Úxr   r   r   Ú
<listcomp>˜   s     z"spring_project.<locals>.<listcomp>zCell grouping z is not in adata.obsc                 S   s   g | ]}t |ƒ‘qS r   r%   r'   r   r   r   r)       s     z is not a categorical variablezCustom color track z is not a continuous variabler   ZUniformzcolor_data_gene_sets.csvzcolor_stats.jsonzcategorical_coloring_data.jsonzgraph_data.jsonz	edges.csvzcell_filter.txtz%i)Úfmtzcell_filter.npyiè  éÈ   i8ÿÿÿzcoordinates.txtz%i,%.6f,%.6fZX_pcazintermediates.npz)ZEpcaÚtotal_countsÚpagaÚgroupsc                    s&   g | ]}ˆˆ |kd d …f   d¡‘qS )Nr   )Úmean©r   Úi)ÚclustsÚcoordsr   r   r)   ë   s     zPAGA_data.json)5ÚunsÚ
ValueErrorZ	obsm_keysZobsmr   ÚparentÚmkdirÚprintÚallÚloggÚwarningÚrawÚXÚtocscÚlistZ	var_namesÚobsÚnpÚarrayÚsumÚA1Úwrite_hdf5_genesÚwrite_hdf5_cellsÚwrite_sparse_npzÚopenÚwriteZsavetxtr   Ú
isinstancer&   ÚzerosÚshapeÚ_write_color_tracksÚ_get_color_stats_genesÚ_get_color_stats_customÚ_write_color_statsÚ_build_categ_colorsÚ_write_cell_groupingsÚ
_get_edgesÚ_write_graphÚ_write_edgesZarangeÚsaveÚminZptpZhstackZsavez_compressedÚcatÚcodesÚ
categoriesÚrangeÚlenÚ_export_PAGA_to_SPRING)r   r   r   r   r   r   r   r   r   Zsubplot_dirZwrite_counts_matricesZbase_dir_filelistÚEÚ	gene_listr,   ÚoÚgZcategorical_extrasZcontinuous_extrasZobs_nameÚcolor_statsÚcategorical_coloring_dataÚedgesZuniq_clustsÚpaga_coordsr   )r2   r3   r   r   Úspring_project   sü    .


ÿþýÿÿ
ÿûÿ




ÿ


ÿ
 ÿ ÿ
 &ÿÿ$ýýrf   c                 C   sH   t | |ƒ}d|kr|d }n|d }| ¡ }dd„ t|j|jƒD ƒ}|S )NZ	distancesÚconnectivitiesc                 S   s   g | ]\}}||f‘qS r   r   ©r   r1   Újr   r   r   r)   û   s     z_get_edges.<locals>.<listcomp>)r   ZtocooÚzipÚrowÚcol)r   r   r   Úmatrixrd   r   r   r   rS   ô   s    

rS   c           
      C   s®   |   ¡ } t |d¡}| d¡}| d¡}| jd |jd< | jd |jd< t|ƒD ]P\}}| dd…|f j ¡ }t	 
|¡d }	||	 }|j||d	� |j||	d	� qP| ¡  dS )
zMSPRING standard: filename = main_spring_dir + "counts_norm_sparse_genes.hdf5"r#   ÚcountsÚcell_ixr   Úncellsr"   ÚngenesN©Údata)r>   Úh5pyÚFileÚcreate_grouprL   ÚattrsÚ	enumerateÚAÚsqueezerA   ÚnonzeroÚcreate_datasetÚclose)
r^   r_   ÚfilenameÚhfÚcounts_groupZ	cix_groupÚiGra   rn   ro   r   r   r   rE      s    

rE   c                 C   s¸   |   ¡ } t |d¡}| d¡}| d¡}| jd |jd< | jd |jd< t| jd ƒD ]T}| |dd…f j ¡ }t	 
|¡d }|| }|jt|ƒ|d	� |jt|ƒ|d	� qV| ¡  dS )
zMSPRING standard: filename = main_spring_dir + "counts_norm_sparse_cells.hdf5"r#   rn   Úgene_ixr   rp   r"   rq   Nrr   )Ztocsrrt   ru   rv   rL   rw   r[   ry   rz   rA   r{   r|   r&   r}   )r^   r~   r   r€   Z	gix_groupZiCrn   r‚   r   r   r   rF     s    

rF   c                 C   s   |   ¡ } tjj|| |d� dS )z@SPRING standard: filename = main_spring_dir + "/counts_norm.npz")Ú
compressedN)r>   ÚscipyÚsparseZsave_npz)r^   r~   rƒ   r   r   r   rG   ,  s    rG   c                 C   sJ   dd„ t |ƒD ƒ}dd„ |D ƒ}||dœ}t| dƒ tj|ddd�¡ d S )	Nc                 S   s   g | ]}t |ƒt |ƒd œ‘qS ))ÚnameÚnumber©Úintr0   r   r   r   r)   3  s     z _write_graph.<locals>.<listcomp>c                 S   s$   g | ]\}}t |ƒt |ƒd dœ‘qS )r   )ÚsourceÚtargetZdistancerˆ   rh   r   r   r   r)   4  s     )ÚnodesÚlinksr#   é   )ú,z: )ÚindentÚ
separators)r[   rH   rI   ÚjsonÚdumps)r~   Zn_nodesrd   rŒ   Úoutr   r   r   rT   2  s    
rT   c              	   C   s>   t | dƒ�*}|D ]}| d|d |d f ¡ qW 5 Q R X d S )Nr#   z%i;%i
r   r"   )rH   rI   )r~   rd   r   Úer   r   r   rU   9  s    rU   c                 C   sf   g }|   ¡ D ].\}}|d d dd„ |D ƒ¡ }||g7 }qt|dd„ d�}t|dƒ d |¡¡ d S )	Nr�   c                 S   s   g | ]}d | ‘qS )z%.3fr   r'   r   r   r   r)   B  s     z'_write_color_tracks.<locals>.<listcomp>c                 S   s   |   d¡d S )Nr�   r   )Úsplit)r(   r   r   r   Ú<lambda>D  ó    z%_write_color_tracks.<locals>.<lambda>)Úkeyr#   r$   )ÚitemsÚjoinÚsortedrH   rI   )ZctracksÚfnamer”   r†   ZscoreÚliner   r   r   rM   ?  s    rM   c                 C   s4   t tjt tj | ¡d d… ¡d td�ƒ}d| S )Né   éÿ   ©Zdtypez#%02x%02x%02x)ÚtuplerA   rB   ÚpltÚcmZjetr‰   )ÚfracÚrgbr   r   r   Ú_frac_to_hexH  s    ,r§   c              
   C   s0  t |ƒ\}}tj|jtd�}t ||dk ¡||dk< | d¡ ¡ j}| 	d¡ ¡ j}d}d| d |jd  }	tj|jd td�}
t
|jd ƒD ]˜}|j|d  |j|  }||	krît |j|j| |j|d  … dd|	 |  ¡|
|< nd|
|< ttt|| || || || |
| fƒƒ| || < q’| S )Nr¡   r   gfffffæX@éd   g      Y@r"   )r
   rA   rK   rL   ÚfloatÚsqrtrW   ZtodenserD   Úmaxr[   ZindptrÚ
percentilers   r¢   Úmap)rb   r^   r_   ZmeansZ	variancesZstdevsZminsZmaxesZpctlZpctl_nZpctlsr�   Z	n_nonzeror   r   r   rN   M  s(     ÿ
&ÿrN   c                 C   sR   |  ¡ D ]D\}}tttt |¡t |¡t |¡t |¡t 	|d¡fƒƒ| |< q| S )Néc   )
rš   r¢   r­   r©   rA   r/   ZstdrW   r«   r¬   )rb   Zcustom_colorsÚkÚvr   r   r   rO   e  s    ,þÿ
rO   c              	   C   s0   t | dƒ�}| tj|ddd�¡ W 5 Q R X d S ©Nr#   rŽ   T)r�   Ú	sort_keys©rH   rI   r’   r“   )r~   rb   r   r   r   r   rP   p  s    rP   c                    sB   |  ¡ D ]4\}‰ ‡ fdd„tttˆ ƒƒƒD ƒ}|ˆ dœ| |< q| S )Nc                    s*   i | ]"\}}|t t|ƒttˆ ƒƒ ƒ“qS r   )r§   r©   r\   Úset)r   r1   Úl©Úlabelsr   r   Ú
<dictcomp>w  s   ÿ z'_build_categ_colors.<locals>.<dictcomp>)Úlabel_colorsZ
label_list)rš   rx   r?   r´   )rc   r   r¯   r¹   r   r¶   r   rQ   u  s    
þþrQ   c              	   C   s0   t | dƒ�}| tj|ddd�¡ W 5 Q R X d S r±   r³   )r~   rc   r   r   r   r   rR   ‚  s    ÿrR   c              
      s   | j d d }| j| jj}dd„ |D ƒ}t| j |d  ƒ}| j| jjj‰ ‡ fdd„tt|ƒƒD ƒ}|d | j krˆt| j |d  ƒ}n*dd l	}	|	j
j | |¡ t| j |d  ƒ}| j d d	  ¡ \}
}t | j d d	 j¡d
 }t|ƒd t|ƒkrüd}nt|ƒt|ƒ d  }t|ƒdt|ƒ k �r.d}nt|ƒt|ƒ d  }g }ttt|ƒƒ|||||ƒD ].\}}}}}}| |t|ƒ||||dœ¡ �qbg }t|
||ƒD ]>\}}}||k �r¢||k�r¢| t|ƒt|ƒt|ƒdœ¡ �q¢|t |¡dœ}|||dœ}dd l}|j|t|dƒdd� d S )Nr-   r.   c                 S   s   g | ]}t |ƒ‘qS r   )r?   )r   Úxyr   r   r   r)   �  s     z*_export_PAGA_to_SPRING.<locals>.<listcomp>Z_sizesc                    s(   g | ] }d d„ t  ˆ |k¡d D ƒ‘qS )c                 S   s   g | ]}t |ƒ‘qS r   rˆ   )r   ri   r   r   r   r)   ’  s     z5_export_PAGA_to_SPRING.<locals>.<listcomp>.<listcomp>r   )rA   r{   r0   ©Zclus_labelsr   r   r)   ‘  s    Ú_colorsr   rg   rŸ   é   r¨   )ÚindexÚsizeÚcolorZcoordinatesÚcellsr†   )rŠ   r‹   Úweight)Zmin_edge_weightZmax_edge_weight)rŒ   r�   Úedge_weight_metar#   rŽ   )r�   )r4   r@   rX   rZ   r?   rY   Úvaluesr[   r\   Zscanpy.plotting.utilsZplottingÚutilsZ,add_colors_for_categorical_sample_annotationr{   rA   rª   rs   rœ   rj   Úappendr‰   r©   r«   r’   ÚdumprH   )r   re   ZoutpathZ	group_keyÚnamesr3   ÚsizesZcell_groupsÚcolorsZscanpyÚsourcesÚtargetsÚweightsZmin_edge_weight_viewZmin_edge_weight_saverŒ   r1   r†   rº   rÀ   r¿   rÁ   r�   rŠ   r‹   rÂ   rÃ   Z	PAGA_datar’   r   r»   r   r]   ‰  sn    

ÿ ÿ
     ÿúÿÿþr]   ©ÚlouvainZpercent_mitoZn_genesr   rÏ   é2   )r   Údata_dirÚ	data_nameÚembedding_keysÚ
annot_keysÚcluster_fieldÚ	nb_markerÚskip_matrixÚhtml_dirÚportÚdo_debugc                 C   s˜   zddl m} W n  tk
r0   t d¡ ‚ Y nX t|ƒ}| |
¡ |j| |||||||dd�	 |dk	r”t|ƒ}|j||dd� |	dk	r”| 	||	¡ dS )aÏ      Export adata to a UCSC Cell Browser project directory. If `html_dir` is
    set, subsequently build the html files from the project directory into
    `html_dir`. If `port` is set, start an HTTP server in the background and
    serve `html_dir` on `port`.

    By default, export all gene expression data from `adata.raw`, the
    annotations `louvain`, `percent_mito`, `n_genes` and `n_counts` and the top
    `nb_marker` cluster markers. All existing files in data_dir are
    overwritten, except `cellbrowser.conf`.

    See `UCSC Cellbrowser <https://github.com/maximilianh/cellBrowser>`__ for
    details.

    Parameters
    ----------
    adata
        Annotated data matrix
    data_dir
        Path to directory for exported Cell Browser files.
        Usually these are the files `exprMatrix.tsv.gz`, `meta.tsv`,
        coordinate files like `tsne.coords.tsv`,
        and cluster marker gene lists like `markers.tsv`.
        A file `cellbrowser.conf` is also created with pointers to these files.
        As a result, each adata object should have its own project_dir.
    data_name
        Name of dataset in Cell Browser, a string without special characters.
        This is written to `data_dir/cellbrowser.conf`.
        Ideally this is a short unique name for the dataset,
        like `"pbmc3k"` or `"tabulamuris"`.
    embedding_keys
        2-D embeddings in `adata.obsm` to export.
        The prefix `X_` or `X_draw_graph_` is not necessary.
        Coordinates missing from `adata` are skipped.
        By default (or when specifying `'all'` or `None`), these keys are tried:
        [`"tsne"`, `"umap"`, `"pagaFa"`, `"pagaFr"`, `"pagaUmap"`, `"phate"`,
        `"fa"`, `"fr"`, `"kk"`, `"drl"`, `"rt"`, `"trimap"`].
        For these, default display labels are automatically used.
        For other values, you can specify a mapping from coordinate name to
        display label, e.g. `{"tsne": "t-SNE by Scanpy"}`.
    annot_keys
        Annotations in `adata.obsm` to export.
        Can be a mapping from annotation column name to display label.
        Specify `None` for all available columns in `.obs`.
    skip_matrix
        Do not export the matrix.
        If you had previously exported this adata into the same `data_dir`,
        then there is no need to export the whole matrix again.
        This option will make the export a lot faster,
        e.g. when only coordinates or meta data were changed.
    html_dir
        If this variable is set, the export will build html
        files from `data_dir` to `html_dir`, creating html/js/json files.
        Usually there is one global html output directory for all datasets.
        Often, `html_dir` is located under a webserver's (like Apache)
        htdocs directory or is copied to one.
        A directory `html_dir`/`project_name` will be created and
        an index.html will be created under `html_dir` for all subdirectories.
        Existing files will be overwritten.
        If do not to use html_dir,
        you can use the command line tool `cbBuild` to build the html directory.
    port
        If this variable and `html_dir` are set,
        Python's built-in web server will be spawned as a daemon in the
        background and serve the files under `html_dir`.
        To kill the process, call `cellbrowser.cellbrowser.stop()`.
    do_debug
        Activate debugging output

    Examples
    --------
    See this
    `tutorial <https://github.com/scverse/scanpy_usage/tree/master/181126_Cellbrowser_exports>`__.
    r   Nz[The package cellbrowser is not installed. Install with 'pip install cellbrowser' and retry.)ZcoordFieldsZ
metaFieldsZclusterFieldrÖ   Z
skipMatrixÚdoDebug)rÛ   )
Zcellbrowser.cellbrowserÚcellbrowserÚImportErrorr:   Úerrorr&   ÚsetDebugZscanpyToCellbrowserÚbuildZserve)r   rÑ   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   Úcbr   r   r   rÜ   ×  s2    ]ÿ
÷rÜ   )NNNr   NF)N)F)NrÎ   rÏ   rÐ   FNNF).Ú__doc__r’   Úloggingr:   Úpathlibr   Útypingr   r   r   r   ÚnumpyrA   Zscipy.sparser„   rt   Zmatplotlib.pyplotZpyplotr£   Zanndatar   Zpandas.api.typesr   Zpreprocessing._utilsr
   Ú_utilsr   r&   Úboolrf   rS   rE   rF   rG   rT   rU   rM   r§   rN   rO   rP   rQ   rR   r]   r‰   rÜ   r   r   r   r   Ú<module>   s‚         ÷
÷ a

	R        ð
ð