U md@sdZddlZddlZddlZddlZddlmZddlZddlZ ddl Z ddl m Z mZddlmZmZmZmZmZmZmZmZmZddlmZddlmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-dd l.m/Z/d d Z0d d Z1ddZ2ddZ3GdddeZ4ddZ5GdddZ6ddZ7ee ddZ8ee ddZ9ee ddZ:ee ddZ;Gdd d Z|t|q||q|S)zyConstruct lists from cell arrays (loaded as numpy ndarrays), recursing into items if they contain mat_struct objects.)r#r!append_matstruct_to_dictr*_inspect_cell_array)r%Z elem_listZsub_elemr(r(r)r-ps  r-cCsTi}|jD]D}|j|}t|tr0t|||<q t|rFt|||<q |||<q |S)z/Construct nested dicts from mat_struct objects.) _fieldnames__dict__r#r!r,r*r-)Zmatobjdfr'r(r(r)r,~s    r,cCsJ|D]@}t||tr(t||||<qt||rt||||<q|S)z,Convert mat objects in dict to nested dicts.)r#r!r,r*r-)r0keyr(r(r)_simplify_cellss  r3c sbeZdZdZedfdd ZddZd d Zd d Zd dZ dddZ dddZ ddZ Z S)MatFile5Readera Reader for Mat 5 mat files Adds the following attribute to base class uint16_codec - char codec to use for uint16 char arrays (defaults to system default codec) Uses variable reader that has the following stardard interface (see abstract class in ``miobase``:: __init__(self, file_reader) read_header(self) array_from_header(self) and added interface:: set_stream(self, stream) read_full_tag(self) NFTc s>t||||||||| | s(t} | |_d|_d|_dS)zInitializer for matlab 5 file format reader %(matstream_arg)s %(load_args)s %(struct_arg)s uint16_codec : {None, string} Set codec to use for uint16 char arrays (e.g., 'utf-8'). Use system default codec if None N)super__init__sysgetdefaultencoding uint16_codec _file_reader_matrix_reader) self mat_stream byte_orderZ mat_dtypeZ squeeze_meZchars_as_stringsZmatlab_compatibleZstruct_as_record verify_compressed_data_integrityr9simplify_cells __class__r(r)r6s  zMatFile5Reader.__init__cCs4|jd|jd}|jd|dkr0dp2dS)z3 Guess byte order. Sets stream pointer to 0~rsIM<>)r=seekread)r<mir(r(r)guess_byte_orders   zMatFile5Reader.guess_byte_ordercCsdi}t|jdd}t|j|}|dd|d<|dd?}|dd@}d ||f|d <|S) z Read in mat 5 file header dtypes file_header descriptions Z __header__versionz%d.%d __version__)rr>r r=itemstrip)r<ZhdictZ hdr_dtypehdrZv_majorZv_minorr(r(r)read_file_headers   zMatFile5Reader.read_file_headercCst||_t||_dS)za Run when beginning read of variables Sets up readers from parameters in `self` N)rr:r;r<r(r(r)initialize_reads zMatFile5Reader.initialize_readcCs|j\}}|dkstd|j|}|tkrbt|j|}|j||j }|j\}}nd}|j|j|t kst d||j |}||fS)a Read header, return header, next position Header has to define at least .name and .is_global Parameters ---------- None Returns ------- header : object object that can be passed to self.read_var_array, and that has attributes .name and .is_global next_position : int position in stream of next variable rzDid not read any bytesFz$Expecting miMATRIX type here, got %d) r:Z read_full_tag ValueErrorr=tellrr"r;Z set_streamr?r TypeErrorZ read_header)r<mdtype byte_countZnext_posstreamZcheck_stream_limitheaderr(r(r)read_var_headers    zMatFile5Reader.read_var_headercCs|j||S)a Read array, given `header` Parameters ---------- header : header object object with fields defining variable header process : {True, False} bool, optional If True, apply recursive post-processing during loading of array. Returns ------- arr : array array with post-processing applied or not according to `process`. )r;Zarray_from_header)r<r^processr(r(r)read_var_arrayszMatFile5Reader.read_var_arrayc Cst|tr|g}n|dk r"t|}|jd||}g|d<|sj|\}}|j dkrjdn |j d}||krt j d|t dd|d krd }d }nd }|dk r||kr|j|qFz|||}WnDtk r}z$t j d ||ftddd|}W5d}~XYnX|j||||<|jrF|d||dk rF||t|dkrFqjqF|jrzt|S|SdS)z get variables from stream as dictionary variable_names - optional list of variable names to get If variable_names is None, then get all variables in file Nr __globals__Nonelatin1zDuplicate variable name "%s" in stream - replacing previous with new Consider mio5.varmats_from_mat to split file into single variable filesrD) stacklevel__function_workspace__FTz&Unreadable variable "%s", because "%s"zRead error: %s)r#strlistr=rGrWrU end_of_streamr_namedecodewarningswarnrrar Warning is_globalr+removelenr@r3) r<Zvariable_namesmdictrT next_positionrkr`reserrr(r(r) get_variables&s^        zMatFile5Reader.get_variablescCs|jd||g}|s|\}}|jdkrBdn |jd}|dkrZd}|j |}|j rrd}nt |j d}||||f|j|q |S) z list variables from stream rNrcrdrfrgZlogicalunknown)r=rGrWrUrjr_rkrlr;Zshape_from_header is_logicalr getmclassr+)r<varsrTrtrkshapeinfor(r(r)list_variables`s    zMatFile5Reader.list_variables) NFFTFTTNF)T)N)__name__ __module__ __qualname____doc__rr6rJrUrWr_rarwr __classcell__r(r(rAr)r4s&&  #  :r4c Cst|}|dttddj}||}|d|||}g}| s|}| \}}|j dkrzdn |j d}||||} || } t } | || | | d||| fqT|S)a Pull variables out of mat 5 file as a sequence of mat file objects This can be useful with a difficult mat file, containing unreadable variables. This routine pulls the variables out in raw form and puts them, unread, back into a file stream for saving or reading. Another use is the pathological case where there is more than one variable of the same name in the file; this routine returns the duplicates, whereas the standard reader will overwrite duplicates in the returned dictionary. The file pointer in `file_obj` will be undefined. File pointers for the returned file-like objects are set at 0. Parameters ---------- file_obj : file-like file object containing mat file Returns ------- named_mats : list list contains tuples of (name, BytesIO) where BytesIO is a file-like object containing mat file contents as for a single variable. The BytesIO contains a string with the original header and a single var. If ``var_file_obj`` is an individual BytesIO instance, then save as a mat file with something like ``open('test.mat', 'wb').write(var_file_obj.read())`` Examples -------- >>> import scipy.io BytesIO is from the ``io`` module in Python 3, and is ``cStringIO`` for Python < 3. >>> mat_fileobj = BytesIO() >>> scipy.io.savemat(mat_fileobj, {'b': np.arange(10), 'a': 'a string'}) >>> varmats = varmats_from_mat(mat_fileobj) >>> sorted([name for name, str_obj in varmats]) ['a', 'b'] rrKrLNrcrd)r4rGrritemsizerHrWrUrYrjr_rkrlrwriter+) Zfile_objrdrZhdr_lenZraw_hdrrtZ named_matsZstart_positionrTrkr\Zvar_strZout_objr(r(r)varmats_from_matys,)         rc@seZdZdZdS)EmptyStructMarkerz= Class to indicate presence of empty matlab struct on output N)rrrrr(r(r(r)rsrcCst|tjr|S|dkrdSt|do8t|do8t|d}t|tjrHn*|srt|drrtdd|jD}d}|rg}g}|D]:\}}t|tr|d d kr| t|t f| |q|rt t |g|St St|}|jjt tjfkr|jd kr||krdS|S) a Convert input object ``source`` to something we can write Parameters ---------- source : object Returns ------- arr : None or ndarray or EmptyStructMarker If `source` cannot be converted to something we can write to a matfile, return None. If `source` is equivalent to an empty dictionary, return ``EmptyStructMarker``. Otherwise return `source` converted to an ndarray with contents for writing to matfile. Nkeysvaluesitemsr/css$|]\}}|ds||fVqdS)_N) startswith).0r2valuer(r(r) s zto_writeable..TrZ _0123456789r()r#r$r%hasattrZgenericdictr/rrhr+objectarraytuplerZ asanyarraydtypetypeZobject_r})sourceZ is_mappingrrfieldrnarrr(r(r) to_writeables@      rrKrLZtag_fullZ tag_smalldataZ array_flagsc@seZdZdZedeZeed<ddZ ddZ dd Z d,d d Z d dZ ddZd-ddZddZddZddZddZd.ddZd d!Zd"d#Zd$d%Zd&d'Zd(d)Zd*d+Zd S)/ VarWriter5z% Generic matlab matrix writing class r(r[cCs0|j|_|j|_|j|_|j|_d|_d|_dS)NF) file_streamunicode_stringslong_field_namesoned_as _var_name_var_is_global)r<Z file_writerr(r(r)r6s zVarWriter5.__init__cCs|j|jdddS)NForder)rrtobytesr<arrr(r(r) write_bytes szVarWriter5.write_bytescCs|j|dSN)rr)r<sr(r(r) write_string szVarWriter5.write_stringNcCsj|dkrt|jjdd}|jjtkr4|}|j|j}|dkrX| |||n| |||dS)z write tag and data Nr) rrrh byteorderrbyteswapZ newbyteorderr&rwrite_smalldata_elementwrite_regular_element)r<rr[r\r(r(r) write_elements   zVarWriter5.write_elementcCs:tdt}|d>||d<|jdd|d<||dS)Nr(Zbyte_count_mdtyperrdata)r$zeros NDT_TAG_SMALLrr)r<rr[r\tagr(r(r)rs z"VarWriter5.write_smalldata_elementcCsTtdt}||d<||d<|||||d}|rP|jdd|dS)Nr(r[r\rO)r$r NDT_TAG_FULLrrr)r<rr[r\rZbc_mod_8r(r(r)r%s   z VarWriter5.write_regular_elementFrc Cs|j}|j}|j|_||jtdt }t |d<d|d<|d>|d>B|d>B} || d>B|d<||d <||| tj |d d t |}|d kr||td n | |td |_d|_dS)a Write header for given data options shape : sequence array shape mclass - mat5 matrix class is_complex - True if matrix is complex is_logical - True if matrix is logical nzmax - max non zero elements for sparse arrays We get the name and the global flag from the object, and reset them to defaults after we've used them r(Z data_typerOr\rDrZ flags_classnzmaxi4rrfrFN)rrrrYZ _mat_tag_posrmat_tagr$rNDT_ARRAY_FLAGSrrrZasarrayrr) r<r}r{ is_complexryrrkrpafflagsr(r(r) write_header1s$      zVarWriter5.write_headercCsX|j}|j|||d}|dkr2td||jd<||j|j|dS)NrOlz-Matrix too large to save with Matlab 5 formatr\)rrYrGr rr)r< start_poscurr_posr\r(r(r)update_matrix_tag\s     zVarWriter5.update_matrix_tagcCs||_||_||dS)a Write variable at top level of mat file Parameters ---------- arr : array_like array-like object to create writer for name : str, optional name as it will appear in matlab workspace default is empty string is_global : {False, True}, optional whether variable will be global on load into matlab N)rrr)r<rrkrpr(r(r) write_topgszVarWriter5.write_topcCs|j}tj|r.||||dSt|}|dkrRtd|t |ft |t rh| |nt |t r|tdnn|tkr|n\|jjr||nH|jjr||n4|jjdkr|jrd}nd}|||n ||||dS)z Write `arr` to stream at top and sub levels Parameters ---------- arr : array_like array-like object to create writer for Nz'Could not convert %s (type %s) to arrayzCannot write matlab functions)USUTF8ascii)rrYscipysparseissparse write_sparserrrZrr#r write_objectrr rwrite_empty_structrfields write_structZ hasobject write_cellskindr write_char write_numeric)r<rZ mat_tag_posrcodecr(r(r)r{s6              zVarWriter5.writecCs|jjdk}|jjdk}zt|jjdd}WnBtk rr|rP|d}n|r`|d}n |d}t}YnX|jt||j |||d|r| |j | |j n | |dS)NcbrZc128i1Zf8)rry) rrrrhKeyErrorastyperrrrrrealimag)r<rZimagfZlogifr{r(r(r)rs(       zVarWriter5.write_numericrcCs|jdkst|dkrJdt|jdg}||t||tddSt |}|j }||t|j j dkr|jrt |}tjdt|||jd}||}tjt|fd |d}|j|td dS) z5 Write string array `arr` with given `codec` rrfrrDNrr(r}rbufferZS1r[)r&r$allmaxndimrrrrr r}rrprodr%r TcopyrRencoderrr)r<rrr}Zn_charsZst_arrstr(r(r)rs(     zVarWriter5.write_charcCs|}||jjdk}|jjdk}|j}|jt||jt|||dkrNdn|d| |j d| |j d| |j j|r| |j jdS)z Sparse matrices are 2D rrrr)rryrrN)ZtocscZ sort_indicesrrZnnzrrrrrindicesrZindptrrrr)r<rArryZnzr(r(r)rs   zVarWriter5.write_sparsecCs<|t||jtt|d}|D]}||q(dS)Nr)rrrrr$ atleast_2dflattenr)r<rrelr(r(r)rs zVarWriter5.write_cellscCs<|dt|tjdtjd|tjgtjddS)N)rrrr)rrrr$rZint32Zint8rVr(r(r)rs zVarWriter5.write_empty_structcCs"|t||jt||dSr)rrrr _write_itemsrr(r(r)r szVarWriter5.write_structcCsdd|jjD}tdd|Dd}|jr2dp4d}||krNtd|d|tj|gdd |jtj|d |d td t | d }|D]}|D]}| ||qqdS) NcSsg|] }|dqSrr()rr1r(r(r) sz+VarWriter5._write_items..cSsg|] }t|qSr()rr)r fieldnamer(r(r)rsr@ z+Field names are restricted to %d charactersrrzS%drr) rdescrrrrXrr$rrrrr)r<r fieldnameslength max_lengthrrr1r(r(r)rs zVarWriter5._write_itemscCs<|t||jt|jtj|jddtd| |dS)zmSame as writing structs, except different mx class, and extra classname element after header rrrN) rrrrrr$r classnamerrrr(r(r)r szVarWriter5.write_object)N)FFr)r)rrrrr$rrrrr6rrrrrrrrrrrrrrrrrr(r(r(r)rs0    + ( +rc@s0eZdZdZed ddZddZd d d ZdS) MatFile5Writerz Class for writing mat5 files FNrowcCs:||_||_||_|r||_ng|_||_||_d|_dS)a< Initialize writer for matlab 5 format files Parameters ---------- %(do_compression)s %(unicode_strings)s global_vars : None or sequence of strings, optional Names of variables to be marked as global for matlab %(long_fields)s %(oned_as)s N)rdo_compressionr global_varsrr_matrix_writer)r<rrrrrrr(r(r)r6.szMatFile5Writer.__init__cCsXtdt}dtjtf|d<d|d<tjddtdd|d <|j | dS) Nr(z0MATLAB 5.0 MAT-file Platform: %s, Created on: %srMrNZS2iIMrZ endian_test) r$r NDT_FILE_HDRosrktimeasctimer%Zuint16rrr)r<rTr(r(r)write_file_headerKs   z MatFile5Writer.write_file_headerc Cs|dkr|jdk}|r"|t||_|D]\}}|ddkrJq4||jk}|jrt}||j_|j || d|t | }tdt}t|d<t||d<|j||j|q4|j || d|q4dS)a Write variables in `mdict` to stream Parameters ---------- mdict : mapping mapping with method ``items`` returns name, contents pairs where ``name`` which will appear in the matlab workspace in file load, and ``contents`` is something writeable to a matlab file, such as a NumPy array. write_header : {None, True, False}, optional If True, then write the matlab file header before writing the variables. If None (the default) then write the file header if we are at position 0 in the stream. By setting False here, and setting the stream position to the end of the file, you can append variables to a matlab file Nrrrdr(r[r\)rrYrrrrrrrrrzlibcompressgetvaluer$emptyrrrrrr) r<rsrrkvarrpr]Zout_strrr(r(r) put_variablesVs(     zMatFile5Writer.put_variables)FFNFr)N)rrrrrr6rr r(r(r(r)r+s  r)>rrrr7riorrmnumpyr$Z scipy.sparserZ_byteordercodesrrZ_miobaserrrr r r r r rZ _mio5_utilsrZ _mio5_paramsrrrrrrrrrrrrrrrrr r!Z_streamsr"r*r-r,r3r4rrrrrrrrrr(r(r(r)s<I , P  dE41