U md @sdZddlmZddlmZmZddlZddlmZddl Z ddl Z ddl Z ddl m Z mZmZmZmZmZddlZddlZddlmZddlZddlZddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'm(Z)m*Z*m+Z+m,Z,ddl-Z-ddl.Z.dd l/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7dd l8m9Z9dd l:m;Z;dd lddl?m@Z@ddlAmBZBddlCmDZDmEZEmFZFmGZGddlHmIZIddlJmKZKeLe,e*e+ZMeMNdeOdZPe#de0dZQe jRGdddZSe jRGdddeeZTdddddZUe%d d d!d"d#ZVe%ddd!d$d#ZVd%d%d!d&d#ZVdd'd(d)d*ZWe%dd,dd d-d.d/ZXe%ddddd-d0d/ZXdd2dd%d-d3d/ZXd4d5ZYd6ddd7d8ZZe@eKd9eKd:d;d<dd6d d?d d@ddAdBdCZ[d d dDdEdFZ\dGdGdGdGdHdIdJdKdLdM Z]eLe]^Z_d?dNdOdPdQZ`e@eKd:d;dRd6dSdSdTdUdVZadWd'dDdXdYZbe%d+d+d+d+d+dZd6d dSd?dd[dSd@d\d] d^d_Zce%d+d+d+d+d+d+d`d6d dSd?ddadSd@dbd] dcd_Zce%d+d+d+d+d+d+d`d6d dSd?dddSd@ddd] ded_Zce@eKd:dfdRddd1dgddd`d6d dSd?dddSd@ddd] dhd_ZcGdidjdjeeZdGdkdldledZeGdmdndnedZfGdodpdpZgGdqdrdrZhdsddtdudvdwZid6dd!dxdyZjd6d ddzd{d|Zke jld}d~ddZmdddddddZnddddddZodS)zCommon IO api utilities) annotations)ABCabstractmethodN) defaultdict)BufferedIOBaseBytesIO RawIOBaseStringIO TextIOBase TextIOWrapper)Path) IOAnyAnyStr DefaultDictGenericHashableLiteralMappingSequenceTypeVarcastoverload)urljoinurlparse uses_netloc uses_params uses_relative) BaseBufferCompressionDictCompressionOptionsFilePath ReadBuffer ReadCsvBufferStorageOptions WriteBuffer) get_lzma_file)import_optional_dependency)BZ2File)doc)find_stack_level)is_bool is_file_like is_integer is_list_like) MultiIndex) _shared_docsz^[A-Za-z][A-Za-z0-9+\-+.]*:// BaseBufferT)boundc@s>eZdZUdZded<ded<ded<ded<d Zd ed <d S) IOArgsz? Return value of io/common.py:_get_filepath_or_buffer. str | BaseBufferfilepath_or_bufferstrencodingmoder compressionFbool should_closeN)__name__ __module__ __qualname____doc____annotations__r<rBrBI/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/pandas/io/common.pyr4Ss r4c@sneZdZUdZded<ded<ejedZded<d Z d ed <d d ddZ dd ddZ dd dddZ dS) IOHandlesau Return value of io/common.py:get_handle Can be used as a context manager. This is used to easily close created buffers and to handle corner cases when TextIOWrapper is inserted. handle: The file handle to be used. created_handles: All file handles that are created by get_handle is_wrapped: Whether a TextIOWrapper needs to be detached. z IO[AnyStr]handlerr:)default_factoryzlist[IO[bytes] | IO[str]]created_handlesFr; is_wrappedNonereturncCs\|jr8t|jtst|j|j|j|j|jD] }| q>g|_d|_dS)z Close all created buffers. Note: If a TextIOWrapper was inserted, it is flushed and detached to avoid closing the potentially user-created buffer. FN) rH isinstancerEr AssertionErrorflushdetachrGremoveclose)selfrErBrBrCrQus    zIOHandles.closezIOHandles[AnyStr]cCs|SNrBrRrBrBrC __enter__szIOHandles.__enter__r)argsrKcGs |dSrS)rQ)rRrVrBrBrC__exit__szIOHandles.__exit__N) r=r>r?r@rA dataclassesfieldlistrGrHrQrUrWrBrBrBrCrD`s  rDobjectr;)urlrKcCst|tsdSt|jtkS)z Check to see if a URL has a valid protocol. Parameters ---------- url : str or unicode Returns ------- isurl : bool If `url` has a valid protocol return True otherwise False. F)rLr7 parse_urlscheme _VALID_URLSr\rBrBrCis_urls rar7)r6rKcCsdSrSrBr6rBrBrC _expand_usersrccCsdSrSrBrbrBrBrCrcszstr | BaseBufferTcCst|trtj|S|S)a] Return the argument with an initial component of ~ or ~user replaced by that user's home directory. Parameters ---------- filepath_or_buffer : object to be converted if possible Returns ------- expanded_filepath_or_buffer : an expanded filepath or the input if not expandable )rLr7ospath expanduserrbrBrBrCrcs  rI)headerrKcCs|dkr dSt|r2tt|}|dkr.tddSt|ddr|tt|}ttt|s^tdtdd|DrxtddSt |rt d tddS) NrzUPassing negative integer to header is invalid. For no header, use header=None insteadF)Z allow_setsz*header must be integer or list of integerscss|]}|dkVqdS)rNrB).0irBrBrC sz&validate_header_arg..z8cannot specify multi-index header with negative integerszPassing a bool to header is invalid. Use header=None for no header or header=int or list-like of ints to specify the row(s) making up the column names) r-rint ValueErrorr.rallmapanyr+ TypeError)rgrBrBrCvalidate_header_args*   rq.r!)r6convert_file_likerKcCsdSrSrBr6rrrBrBrCstringify_pathsrtcCsdSrSrBrsrBrBrCrtsFzFilePath | BaseBufferTcCs2|st|rtt|St|tjr*|}t|S)a Attempt to convert a path-like object to a string. Parameters ---------- filepath_or_buffer : object to be converted Returns ------- str_filepath_or_buffer : maybe a string version of the object Notes ----- Objects supporting the fspath protocol (python 3.6+) are coerced according to its __fspath__ method. Any other object is passed through unchanged, which includes bytes, strings, buffers, or anything else that's not even path-like. )r,rr2rLrdPathLike __fspath__rcrsrBrBrCrts    cOsddl}|jj||S)z` Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of the stdlib. rN)urllib.requestrequesturlopen)rVkwargsurllibrBrBrCrysryzFilePath | BaseBuffercCs$t|to"tt|o"|d S)zR Returns true if the given URL looks like something fsspec can handle )zhttp://zhttps://)rLr7r;_RFC_3986_PATTERNmatch startswithr`rBrBrC is_fsspec_urls    rstorage_optionscompression_optionsr6)rrutf-8rr r$)r6r8r:r9rrKc Cst|}t|\}}t||}|rJt|drJd|krJtjdttdd}t||d}d|kr|dkr|d krtj|d |t td|}d |krd|kr|d7}t |t r.t |r.|pi}d dl }|jj||d }t|2} | jdd} | dkrddi}t| } W5QRXt| ||d|dSt|rXt |t sHt|dr`|dd}|drx|dd}td} g} z&tdd dlm}m}||tg} Wntk rYnXz$| j |fd|i|pi }Wn^t!| k rD|dkrddi}nt|}d|d<| j |fd|i|p6i }YnXt|||d|dS|rft"dt |t t#t$j$frtt%|||d|dSt|dst|dsdt&|}t"|t|||d|dS)a% If the filepath_or_buffer is a url, translate and return the buffer. Otherwise passthrough. Parameters ---------- filepath_or_buffer : a url, filepath (str, py.path.local or pathlib.Path), or buffer {compression_options} .. versionchanged:: 1.4.0 Zstandard support. encoding : the encoding to use to decode bytes, default is 'utf-8' mode : str, optional {storage_options} .. versionadded:: 1.2.0 ..versionchange:: 1.2.0 Returns the dataclass IOArgs. writebzDcompression has no effect when passing a non-binary object as input.) stacklevelN)methodw)bz2xz)zutf-16zutf-32z( will not write the byte order mark for tr)headerszContent-EncodinggziprT)r6r8r:r<r9zs3a://zs3://zs3n://fsspecZbotocore) ClientErrorNoCredentialsErrorr9Zanonz?storage_options passed with file object or non-fsspec file pathFreadz)Invalid file path or buffer object type: )'rtget_compression_methodinfer_compressionhasattrwarningswarnRuntimeWarningr*dictUnicodeWarningrLr7rarwrxRequestryrgetrrr4rrMr~replacer'Zbotocore.exceptionsrrPermissionError ImportErroropentuplerlbytesmmaprctype)r6r8r:r9rcompression_methodZ fsspec_moder{req_inforeqcontent_encodingreaderrZerr_types_to_retry_with_anonrrZfile_objmsgrBrBrC_get_filepath_or_buffers"                r)rerKcCsddlm}td||S)z converts an absolute native path to a FILE URL. Parameters ---------- path : a path in native format Returns ------- a valid FILE URL r) pathname2urlzfile:)rwrr)rerrBrBrCfile_path_to_urls rtarrrziprzstd) .tar.tar.gz.tar.bz2.tar.xz.gz.bz2.zip.xzz.zstz"tuple[str | None, CompressionDict])r:rKc Cs`t|trPt|}z|d}WqXtk rL}ztd|W5d}~XYqXXni}|}||fS)a Simplifies a compression argument to a compression method string and a mapping containing additional arguments. Parameters ---------- compression : str or mapping If string, specifies the compression method. If mapping, value at key 'method' specifies compression method. Returns ------- tuple of ({compression method}, Optional[str] {compression arguments}, Dict[str, Any]) Raises ------ ValueError on mapping missing 'method' key rz.If mapping, compression must have key 'method'N)rLrrpopKeyErrorrl)r:compression_argsrerrrBrBrCrs r)r str | None)r6r:rKcCs|dkr dS|dkrZt|dd}t|ts.dStD]\}}||r6|Sq6dS|tkrf|Sddgtt}d|d|}t |dS)a/ Get the compression method for filepath_or_buffer. If compression='infer', the inferred compression method is returned. Otherwise, the input compression method is returned unchanged, unless it's invalid, in which case an error is raised. Parameters ---------- filepath_or_buffer : str or file handle File path or object. {compression_options} .. versionchanged:: 1.4.0 Zstandard support. Returns ------- string or None Raises ------ ValueError on invalid compression specified. NZinferT)rrUnrecognized compression type: z Valid compression types are ) rtrLr7extension_to_compressionitemslowerendswith_supported_compressionssortedrl)r6r: extensionZvalidrrBrBrCrs    rz Path | strcCs&t|j}|s"td|ddS)z Check if parent directory of a file exists, raise OSError if it does not Parameters ---------- path: Path or str Path to check parent directory of z1Cannot save file into a non-existent directory: ''N)r parentis_dirOSError)rerrBrBrCcheck_parent_directoryMs r)r8r: memory_maperrorsrzLiteral[False]zIOHandles[bytes]) path_or_bufr9r8r:ris_textrrrKcCsdSrSrBrr9r8r:rrrrrBrBrC get_handle[s r)r8r:rrrrz Literal[True]zIOHandles[str]cCsdSrSrBrrBrBrCrjs z!IOHandles[str] | IOHandles[bytes]cCsdSrSrBrrBrBrCrys rTcCs|pd}|pd}t||r*d|kr*|d7}t|t|trHt|t|||||d}|j} t| |\} }} t| t} t |j } | d}d|kr| rt t| |rP|dkr|j dd |_ n |dkrd|j kr|j d7_ |d kr2t| trtjf| |j d | } ntjf| |j d | } n|d krVt| fd|j i| } n|dkrt| |j f| } | jj dkr4| | | j} t| dkr| j| } n$| std|ntd| nZ|dkr| d|j t| trtfd| i| } ntfd| i| } t| ts4td| jj kr4| | | j}t|dkr| j|d}|dk st|} n$|std|ntd|n|dkrt| |j } nl|dkr"td}d|j krd|j f| i}nd|j!f| i}|j| fd|j i|} nd|}t|t| trDt| | nNt| tr|j"rd|j krt| |j |j"|d d} n t| |j } | | d }|s|j d!krt| t#rt$| |j"d"} n~|rN|s|st| |j rNt%| d#rt%| d$rt%| d%st&| } t'| |j"|d d} | | t|jtpJ|j( }d|j kr|t%| d&s|t)d't*|jd(| +|j(rt|jtrt| |jt,| | ||j d)S)*a Get file handle for given path/buffer and mode. Parameters ---------- path_or_buf : str or file handle File path or object. mode : str Mode to open path_or_buf with. encoding : str or None Encoding to use. {compression_options} .. versionchanged:: 1.0.0 May now be a dict with key 'method' as compression mode and other keys as compression options if compression mode is 'zip'. .. versionchanged:: 1.1.0 Passing compression options as keys in dict is now supported for compression modes 'gzip', 'bz2', 'zstd' and 'zip'. .. versionchanged:: 1.4.0 Zstandard support. memory_map : bool, default False See parsers._parser_params for more information. Only used by read_csv. is_text : bool, default True Whether the type of the content passed to the file/buffer is string or bytes. This is not the same as `"b" not in mode`. If a string content is passed to a binary file/buffer, a wrapper is inserted. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. storage_options: StorageOptions = None Passed to _get_filepath_or_buffer .. versionchanged:: 1.2.0 Returns the dataclass IOHandles rstrictr)r8r:r9rrrrrr1r)filenamer9)fileobjr9rr9rzZero files found in ZIP file z9Multiple files found in ZIP file. Only one file per ZIP: rnamerrNz Zero files found in TAR archive zDMultiple files found in TAR archive. Only one file per TAR archive: r zstandardZdctxZcctxr)r8rnewlineFrb)r8readablewritableseekablerz1Expected file path name or file-like object, got z type)rErGrHr:)-_is_binary_modecodecslookuprLr7 lookup_errorrr6_maybe_memory_maprr:rrr9rrGzipFile_BZ2File _BytesZipFilebufferappendnamelistlenrrl setdefault _BytesTarFilerMgetnames extractfiler&r'ZstdDecompressorZZstdCompressorr8r _BytesIOWrapperr _IOWrapperr r<rprreverserD)rr9r8r:rrrrZioargsrEhandlesZis_pathrZ zip_namesfilesfilerZ open_argsrrHrBrBrCrs>6                              cs8eZdZdZeddddZddfdd ZZS)_BufferedWriterz Some objects do not support multiple .write() calls (TarFile and ZipFile). This wrapper writes to the underlying buffer on close. rIrJcCsdSrSrBrTrBrBrCwrite_to_buffersz_BufferedWriter.write_to_bufferc sP|jr dS|r8|d|j|W5QRXn |jtdS)Nr)closedgetvalueseekrrrQsuperrT __class__rBrCrQs  z_BufferedWriter.close)r=r>r?r@rrrQ __classcell__rBrBrrCrsrcsXeZdZdddddddfdd Zd d d d d ZddddZddddZZS)rNrrzLiteral[('r', 'a', 'w', 'x')]z-ReadBuffer[bytes] | WriteBuffer[bytes] | NonerI)rr9r archive_namerKc s<t||_||_tjjf||||d||_dS)N)rr9r) r__init__rrtarfileTarFiler extend_moder)rRrr9rrrzrrBrCrs z_BytesTarFile.__init__r7)r9rKcCsP|dd}|dkr|S|jdk rLt|jj}|dkrL|d|dd}|S)Nrr1r)rrr:r)rrr suffix)rRr9rrBrBrCrs   z_BytesTarFile.extend_moderJcCsP|jdkrdSt|j}|jdkr.|djS|jdkrJ|ddjS|jS)z If an explicit archive_name is not given, we still want the file inside the zip file not to be named something.tar, because that causes confusion (GH39465). Nrr1)rrr)rr r with_suffixrRrrBrBrCinfer_filenames     z_BytesTarFile.infer_filenamecCs>|jp|pd}tj|d}t||_|j||dS)Nrr) rrrTarInforrsizeraddfile)rRrtarinforBrBrCrs z_BytesTarFile.write_to_buffer)NrNN)r=r>r?rrrrrrBrBrrCrs rcsFeZdZddddddfdd Zdd d d Zdd d d ZZS)rNz1FilePath | ReadBuffer[bytes] | WriteBuffer[bytes]r7rrI)rr9rrKc s@t|dd}||_|dtjtj||f||_dS)Nrr1r:) rrrrrzipfile ZIP_DEFLATEDZipFiler)rRrr9rrzrrBrCrs   z_BytesZipFile.__init__rJcCs@t|jjtjtfrr?rrrrrBrBrrCrs rc@sTeZdZdddddZdddd Zd d d d Zd d ddZd d ddZdS)rrrI)rrKcCs ||_dSrS)r)rRrrBrBrCrsz_IOWrapper.__init__r7rcCs t|j|SrSgetattrr)rRrrBrBrC __getattr__sz_IOWrapper.__getattr__r;rJcCst|jdr|jSdS)NrT)rrrrTrBrBrCr"s  z_IOWrapper.readablecCst|jdr|jSdS)NrT)rrrrTrBrBrCr's  z_IOWrapper.seekablecCst|jdr|jSdS)NrT)rrrrTrBrBrCr,s  z_IOWrapper.writableN)r=r>r?rrrrrrBrBrBrCrs rc@s@eZdZdddddddZddd d Zdd d dddZdS)rrzStringIO | TextIOBaser7rI)rr8rKcCs||_||_d|_dS)N)rr8overflow)rRrr8rBrBrCr5sz_BytesIOWrapper.__init__)attrcCs t|j|SrSr)rRrrBrBrCr>sz_BytesIOWrapper.__getattr__z int | Noner)nrKcCst|jdk st|j||j}|j|}|dksH|dksH|t|krRd|_|S|d|}||d|_|SdS)Nrr)rrMrencoder8rr)rRr bytestringZcombined_bytestringZ to_returnrBrBrCrAs  z_BytesIOWrapper.readN)r)r)r=r>r?rrrrBrBrBrCr2s rr5z/tuple[str | BaseBuffer, bool, list[BaseBuffer]])rErrKc Csg}|t|dpt|tM}|s*|||fStt|}t|trRt|d}||zt t j | dt j d}W5t|D] }|qzX|||gfS)zTry to memory map file/buffer.filenorr)access) rrLr7rr#rrreversedrQrrr ACCESS_READ)rErrwrappedrBrBrCrOs&       rc CsHd}t|}t|ts|Sztj|}Wnttfk rBYnX|S)zTest whether file exists.F)rtrLr7rdreexistsrprl)r6rrBrBrC file_existsqs r)rEr9rKcCsVd|ksd|krd|kStjtjtjf}tt||r:dSt|tpTdt|d|kS)z+Whether the handle is opened in binary moderrFr9) r StreamWriter StreamReaderStreamReaderWriter issubclassrrL_get_binary_io_classesr)rEr9Z text_classesrBrBrCrsrztuple[type, ...]rJc CsHttf}tddd}|dk rD|d}|t|f7}W5QRX|S)z!IO classes that that expect bytesrignore)rNr)rrr'r stream_readerr)Zbinary_classesrrrBrBrCr#s  r#zSequence[Hashable] | MultiIndexzbool | Sequence[int] | None)columns index_colrKcsDdksttrgtt|o@t|t o@tfdd|DS)a Check whether or not the `columns` parameter could be converted into a MultiIndex. Parameters ---------- columns : array-like Object which may or may not be convertible into a MultiIndex index_col : None, bool or list, optional Column or columns to use as the (possibly hierarchical) index Returns ------- bool : Whether or not columns could become a MultiIndex Nc3s$|]}|tkrt|tVqdSrS)rZrLr)rhcr'rBrCrjs z+is_potential_multi_index..)rLr;rr/rm)r&r'rBr)rCis_potential_multi_indexs r*zSequence[Hashable])namesis_potential_multiindexrKcCst|}tt}t|D]\}}||}|dkr|d||<|rpt|tsNt|dd|dd|f}n|d|}||}q(|||<|d||<q|S)a: Rename column names if duplicates exist. Currently the renaming is done by appending a period and an autonumeric, but a custom pattern may be supported in the future. Examples -------- >>> dedup_names(["x", "y", "x", "x"], is_potential_multiindex=False) ['x', 'y', 'x.1', 'x.2'] rrNr.)rZrrk enumeraterLrrM)r+r,countsricolZ cur_countrBrBrC dedup_namess " r1).).)F)rNrN)N)pr@ __future__rabcrrr collectionsrrX functoolsriorrrr r r rrdpathlibr rertypingr rrrrrrrrrrr urllib.parserrr]rrrrr Zpandas._typingrrr r!r"r#r$r%Z pandas.compatr&Zpandas.compat._optionalr'Zpandas.compat.compressorsr(rZpandas.util._decoratorsr)Zpandas.util._exceptionsr*Zpandas.core.dtypes.commonr+r,r-r.Zpandas.core.indexes.apir/Zpandas.core.shared_docsr0setr_discardcompiler|r2 dataclassr4rDrarcrqrtryrrrrvaluesrrrrrrrrrrrrr lru_cacher#r*r1rBrBrBrCs    8(           ,"   3 #6$$$$7$"