U md@sdZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z ddlmZmZmZddlmZmZddlmZmZdd lmZdd lmZddlZddlmmZGd d d e ZGd ddeZGdddeZ GdddeZ!GdddZ"GdddZ#GdddeZ$GdddeZ%GdddeZ&ej'(de)dfej*dfej+dfe,dfej-dfej.dfej/dd d!d"fej/d#d d!d"fej/dd$d"fej/d#d$d"fe/d"fg d%d&Z0Gd'd(d(eZ1e 2d)ej3j4kd*Gd+d,d,eZ5e 2d)ej3j4kd*Gd-d.d.eZ6e 2d/ej3j4kd0Gd1d2d2eZ7e 2d3ej3j4kd4Gd5d6d6eZ8e 2d7ej3j4kd8Gd9d:d:eZ9e 2d;ej3j4kd<Gd=d>d>eZ:e 2d?ej3j4kd@GdAdBdBeZ;GdCdDdDeZGdIdJdJeZ?GdKdLdLeZ@GdMdNdNeZAGdOdPdPeZBGdQdRdReZCGdSdTdTeZDGdUdVdVeZEGdWdXdXeZFGdYdZdZeZGGd[d\d\eZHGd]d^d^eZIGd_d`d`eZJej'jKejjLdakdbdcZMGdddedeeZNGdfdgdgeZOGdhdidieZPGdjdkdkeZQGdldmdmeZRe SejjLdnkdodpdqZTe SejjLdrkp|ejjLdsko|ejjLdtkdudvdwZUdxdyZVdzd{ZWd|d}ZXd~dZYddZZddZ[ej'j\ejjLdkddcddZ]ddZ^GdddeZ_GdddeZ`dS)z Dataset testing operations. Tests all dataset operations, including creation, with the exception of: 1. Slicing operations for read and write, handled by module test_slicing 2. Type conversion for read and write (currently untested) N)utTestCase)get_data_file_path)FileGroupDataset)is_empty_dataspaceproduct)h5fh5t)H5pyDeprecationWarning)versionc@seZdZddZddZdS) BaseDatasetcCst|d|_dS)Nw)rmktempfselfrP/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/h5py/tests/test_dataset.pysetUp'szBaseDataset.setUpcCs|jr|jdSN)rcloserrrrtearDown*szBaseDataset.tearDownN)__name__ __module__ __qualname__rrrrrrr&src@seZdZdZddZdS)TestReprz5 Feature: repr(Dataset) behaves sensibly cCs<|jdd}|t|t|j|t|tdS)z( repr() works on live and dead datasets fooN)rcreate_datasetassertIsInstancereprstrrrdsrrrtest_repr_open4s zTestRepr.test_repr_openN)rrr__doc__r(rrrrr/src@steZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ e eed dddZddZdS)TestCreateShapez@ Feature: Datasets can be created from a shape only cCs |jdd}||jddS)z Create a scalar dataset rrNrr" assertEqualshaperdsetrrrtest_create_scalarBsz"TestCreateShape.test_create_scalarcCs |jdd}||jddS)z Create a size-1 dataset rrNr+r.rrrtest_create_simpleGsz"TestCreateShape.test_create_simplecCs |jdd}||jddS)z+ Create a size-1 dataset with integer shaperrr1Nr+r.rrrtest_create_integerLsz#TestCreateShape.test_create_integercCsX|jdd}||jd||jd|jdd}||jd||jddS) Create an extended dataset r?r6bar <Nrr"r,r-sizer.rrrtest_create_extendedQs z$TestCreateShape.test_create_extendedcCsX|jdd}||jd||jd|jdd}||jd||jddS)r4rr6r5r7r8r;Nr<r.rrrtest_create_integer_extendedZs z,TestCreateShape.test_create_integer_extendedcCs&|jdd}||jtddS)) Confirm that the default dtype is float rr5z=f4Nrr"r,dtypenpr.rrrtest_default_dtypecsz"TestCreateShape.test_default_dtypec Cs&|t|jdW5QRXdS)z Missing shape raises TypeError rN assertRaises TypeErrorrr"rrrrtest_missing_shapehs z"TestCreateShape.test_missing_shapecCsF|jjddtjd}tdkr2tdt||j tjdS)r@rr5rB)ppc64lez(Storage of long double deactivated on %sN) rr"rCZ longdoubleplatformmachinepytestxfailr,rBr.rrrtest_long_doublems z TestCreateShape.test_long_double complex256zNo support for complex256cCs0|jjddtdd}||jtddS)r@rr5rPrIN)rr"rCrBr,r.rrrtest_complex256ts zTestCreateShape.test_complex256cCs<|jdd}||jd|jdd}||jddS)Nsfoor1sbar/baz)r+rr/dset2rrrtest_name_bytes{szTestCreateShape.test_name_bytesN)rrrr)r0r2r3r>r?rDrHrOrskipIfhasattrrCrQrUrrrrr*<s   r*c@sxeZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ ddZ ddZ ddZddZddZdS)TestCreateDatazA Feature: Datasets can be created from existing data cCs0tdd}|jjd|d}||j|jdS)z- Create a scalar dataset from existing array rrrdataNrConesrr"r,r-rrZr/rrrr0s z!TestCreateData.test_create_scalarcCs0tdd}|jjd|d}||j|jdS)z/ Create an extended dataset from existing data r5rrrYNr[r]rrrr>s z#TestCreateData.test_create_extendedcCs4|jjdddd}||tj|d|jkdS)z1 Create dataset with missing intermediate groups z /foo/bar/bazr:r:z.check_vlen_asciiaabcrYbdefcdrINrr"rCarrayobject)rr}rrrtest_create_bytestrings z%TestCreateData.test_create_bytestringcCs4|jjdtjddgddd}|j|tjdddS) Nr~rrZS3rIrYrdrz)rr"rCrrxr r{r.rrrtest_create_np_sszTestCreateData.test_create_np_scszfdd}|jjddd|jjdddgd|jjddgdggd|jjd tjddgtd ddS) Ncsj|tjdddSry)rxr CSET_UTF8r|rrrcheck_vlen_utf8sz;TestCreateData.test_create_strings..check_vlen_utf8r~abcrYrdefrrrIr)rrrrrtest_create_stringss z"TestCreateData.test_create_stringsc Cs8|t$|jjdtjddgdddW5QRXdS)Nr~rrZU3rIrY)rFrGrr"rCrrrrrtest_create_np_us zTestCreateData.test_create_np_ucCs*|jjddd|t|jdjdS)NrrrI)rr"r`r rlrrrr test_empty_create_via_None_shapesz/TestCreateData.test_empty_create_via_None_shapecCs2|jjdtjddd|t|jdjdS)NrrrIrY)rr"r_Emptyr`r rlrrrr!test_empty_create_via_Empty_classsz0TestCreateData.test_empty_create_via_Empty_classc Cs2|t|jjddtddW5QRXdS)Nr7r!rdre)rFrkrr"rCrfrrrrtest_create_incompatible_datas z,TestCreateData.test_create_incompatible_dataN)rrrr)r0r>rarirmrxrrrrrrrrrrrrXs   rXc @seZdZdZejdddejddejddfddejddejd d fd d ejd d d d fejd d d dffddejdd ddfejd d fgddZ ddZ ddZ ddZ ddZ d S)TestReadDirectlyzI Feature: Read data directly from Dataset into a Numpy array +source_shape,dest_shape,source_sel,dest_seldrr:2r;FZNrbr:rr r9rRr9rdc Csltjt|dd|}|jd||d}tj|ddd}|} ||| |<||||tj || dS)Nint64rIr/rY) rCrfr rhr"fullcopy read_directtestingassert_array_equal) r writable_file source_shape dest_shape source_seldest_selZ source_valuesr/arrexpectedrrrtest_read_directs  z!TestReadDirectly.test_read_directcCsN|jddtjdddd}tjddd}||tj|tjddddS)Nr/r:r:rrIrY)r"rCrfr\rrrrrr/rrrr test_no_sels zTestReadDirectly.test_no_selc CsV|jddd}tdd}tt(||tjddtjddW5QRXdS NZedsetrrIrrr:rr;)r"rCr\rMraisesrGrs_)rr empty_dsetrrrr test_emptys  zTestReadDirectly.test_emptyc Cs>|jdddd}td}tt||W5QRXdSNr/rrrIr"rCr\rMrrGrrrrrtest_wrong_shapes  z!TestReadDirectly.test_wrong_shapec CsB|jdddd}tjddd}tt||W5QRXdSNr/r^rrIF)orderrrrrrtest_not_c_contiguouss z&TestReadDirectly.test_not_c_contiguous)rrrr)rMmark parametrizerCrrrrrrrrrrrs.$ rc @seZdZdZejdddejddejddfddejddejd d fd d ejd d d d fejd d d dffddejdd ddfejd d fgddZ ddZ ddZ ddZ d S)TestWriteDirectlyz> Feature: Write Numpy array directly into Dataset rrrr:rr;rrNrrrrrrRr9rdc Csj|jd|ddd}tt||}tj|ddd}||||<||||tj|dd|dS)Nr/int32rrB fillvaluerI) r"rCrfr rhr write_directrr) rrrrrrr/rrrrrtest_write_directs  z#TestWriteDirectly.test_write_directc CsP|jddd}tt.|tdtjddtjddW5QRXdSr)r"rMrrGrrCr\r)rrrrrrr!s zTestWriteDirectly.test_emptyc Cs>|jdddd}td}tt||W5QRXdSrr"rCr\rMrrGrrrrrr&s  z"TestWriteDirectly.test_wrong_shapec CsB|jdddd}tjddd}tt||W5QRXdSrrrrrrr,s z'TestWriteDirectly.test_not_c_contiguous) rrrr)rMrrrCrrrrrrrrrr s.$ rc@sPeZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ dS)TestCreateRequirezS Feature: Datasets can be created only if they don't exist in the file cCs.|jddd}||t||jddS)z& Create new dataset with no conflicts rrcrN)rrequire_datasetr#rr,r-r.rrr test_create9s zTestCreateRequire.test_createcCs0|jddd}|jddd}|||dS)z) require_dataset yields existing dataset rrcrNrrr,rSrrrtest_create_existing?sz&TestCreateRequire.test_create_existingcCs|jddd}|jddd}||||jddd}|jddd}||||jddd}|jddd}|||dS) z; require_dataset with integer shape yields existing datasetrr:rr7rZbazsbazNrrSrrrtest_create_1DEs  z TestCreateRequire.test_create_1Dc Cs:|jddd|t|jdddW5QRXdS)z6 require_dataset with shape conflict yields TypeError rrcr)r:r!Nrr"rFrGrrrrrtest_shape_conflictSs z%TestCreateRequire.test_shape_conflictc Cs6|jd|t|jdddW5QRXdS)z< require_dataset with object type conflict yields TypeError rrcrN)rZ create_grouprFrGrrrrrtest_type_conflictYs  z$TestCreateRequire.test_type_conflictc Cs:|jddd}|t|jdddW5QRXdS)zL require_dataset with dtype conflict (strict mode) yields TypeError rrcrZS10Nrr.rrrtest_dtype_conflict_s z%TestCreateRequire.test_dtype_conflictcCs4|jddd}|jjddddd}|||dS)z* require_dataset with exactly dtype match rrcrTexactN)rr"rr,rSrrrtest_dtype_exactfsz"TestCreateRequire.test_dtype_exactcCsH|jddd}|jjddddd}|||||jtddS)zJ require_dataset with convertible type succeeds (non-strict mode) rrci4i2FrN)rr"rr,rBrCrSrrrtest_dtype_closems z"TestCreateRequire.test_dtype_closeN) rrrr)rrrrrrrrrrrrr3src@sXeZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ ddZ dS)TestCreateChunkedzL Feature: Datasets can be created by manually specifying chunks cCs$|jjdddd}||jddS)z Create via chunks tuple rrrr-chunksNrr"r,rr.rrrtest_create_chunks|sz$TestCreateChunked.test_create_chunkscCs$|jjdddd}||jddS)z Create via chunks integer rrr:rrNrr.rrrtest_create_chunks_integersz,TestCreateChunked.test_create_chunks_integerc Cs,|t|jjddddW5QRXdS)z& Illegal chunk size raises ValueError rrrrNrFrkrr"rrrrtest_chunks_mismatchs z&TestCreateChunked.test_chunks_mismatchc Cs.|t|jjdddddW5QRXdS)z3 Chunked format required for given storage options rrrF)r-maxshaperNrrrrrtest_chunks_falses z#TestCreateChunked.test_chunks_falsec Cs,|t|jjddddW5QRXdS)z> Attempting to create chunked scalar dataset raises TypeError rr)rrNrErrrrtest_chunks_scalars z$TestCreateChunked.test_chunks_scalarcCs6|jjdddd}||jt|t|jddS)z Auto-chunking of datasets r)rrTrrRN)rr"r#rtupler,lenr.rrrtest_auto_chunkssz"TestCreateChunked.test_auto_chunkscCs&|jjddddd}||jddS)z7 Auto-chunking with pathologically large element sizes rrdZ S100000000Tr-rBrr1Nrr.rrrtest_auto_chunks_abusesz(TestCreateChunked.test_auto_chunks_abusecCs|jjddtjdd}d|ddddf<|t|ddddfdkd |d<|t|dd kd |d d <|t|d d d kdS) z+ Test scalar assignment of chunked dataset r)rdrr)rrrrr:rN( rrR)rr"rCrr`allr.rrrtest_scalar_assignments " z(TestCreateChunked.test_scalar_assignmentc Cs\tjtdd}|jjddddW5QRXtjtdd}|jjddddW5QRXd S) z, Auto-chunking of empty datasets not allowedr)matchrZS100T)rBrrrBrN)rMrrGrr")rerrrrrtest_auto_chunks_no_shapesz+TestCreateChunked.test_auto_chunks_no_shapeN) rrrr)rrrrrrrrrrrrrrvsrc@s8eZdZdZddZddZddZdd Zd d Zd S) TestCreateFillvaluez> Feature: Datasets can be created with fill value cCs6|jjdddd}||dd||dddS)z- Fill value is reflected in dataset contents rrg@rrrN)rr"r,r.rrrtest_create_fillvalsz'TestCreateFillvalue.test_create_fillvalcCs4|jjdddd}||jd||jtjdS)z( Fill value is recoverable via property rrg@rN)rr"r,rZassertNotIsInstancerCndarrayr.rrr test_propertysz!TestCreateFillvalue.test_propertycCs |jdd}||jddS)z0 .fillvalue property works correctly if not set rrrN)rr"r,rr.rrrtest_property_nonesz&TestCreateFillvalue.test_property_nonecCsVtddg}tjd|dd}|jjdd||d}||j|||d |d S) z& Fill value works with compound types )r~f4)ri8r1rIrrrrr!N)rCrBr\rr"r,rZassertAlmostEqual)rdtvr/rrr test_compounds z!TestCreateFillvalue.test_compoundc Cs2|t|jjddddgdd}W5QRXdS)z$ Bogus fill value raises ValueError rrr~i)rr*rNrr.rrrtest_excs   zTestCreateFillvalue.test_excN) rrrr)rrrrrrrrrrs rz dt,expectedutf-8rencodingrvasciircCs"|jdd|d}|j|kstdS)NrrrI)r"rrq)rrrr/rrrtest_get_unset_fill_valuesr c@seZdZdZddZdS)TestCreateNamedTypezC Feature: Datasets created from an existing named type cCsntd|jd<|jjdd|jdd}||jtd||j|jdj||jdS)z7 Named type object works and links the dataset to type Zf8typexrrIN) rCrBrr"r,rlrnr`Z committedr.rrr test_nameds zTestCreateNamedType.test_namedN)rrrr)rrrrrr sr gzipzDEFLATE is not installedc@s0eZdZdZddZddZddZdd Zd S) TestCreateGzipz= Feature: Datasets created with gzip compression cCs4|jjddddd}||jd||jddS)z# Create with explicit gzip options rrrbrr compressioncompression_optsNrr"r,rrr.rrr test_gzips  zTestCreateGzip.test_gzipcCs2|jjdddd}||jd||jddS)z+ Create with implicit gzip level (level 4) rrrrr!Nrr.rrrtest_gzip_implicitsz!TestCreateGzip.test_gzip_implicitc Cs|jjdddd}||jd||jdtjjj}z8t tjj_| t |jjdddd}W5QRXW5|tjj_XdS)z. Create with gzip level by specifying integer rrrrrN) rr"r,rrr__hldataset_LEGACY_GZIP_COMPRESSION_VALSrrFrk)rr/original_compression_valsrrrtest_gzip_numbers    zTestCreateGzip.test_gzip_numberc Cs|ttf|jjddddW5QRX|t|jjddddW5QRX|t|jjdddddW5QRXdS) z= Illegal gzip level (explicit or implicit) raises ValueError rrrrrN)rFrk RuntimeErrorrr"rrrr test_gzip_exc#s   zTestCreateGzip.test_gzip_excN)rrrr)rrrr#rrrrrs rc@s eZdZdZddZddZdS)TestCreateCompressionNumberz? Feature: Datasets created with a compression code cCs`tjjj}z(ttjj_|jjddtjjdd}W5|tjj_X| |j d| |j ddS)z^ Create with compression number of gzip (h5py.h5z.FILTER_DEFLATE) and a compression level of 7rr)rrrrN) r_rrrrrr"h5zFILTER_DEFLATEr,rr)rrr/rrrtest_compression_number5s   z3TestCreateCompressionNumber.test_compression_numberc Cs|t}|jjddddW5QRX|dt|j|t}|jjddddW5QRX|dt|jtjj j }zrrrrr=sr= scaleoffsetzSCALEOFFSET is not installedc@sHeZdZdZddZddZddZdd Zd d Zd d Z ddZ dS)TestCreateScaleOffsetz? Feature: Datasets can use the scale/offset filter c Cs.|t|jjddtdd}W5QRXdS)z[ Ensure that a scale factor is required for scaleoffset compression of floating point data rrTrBr?NrFrkrr"floatr.rrr test_float_fails_without_optionss z6TestCreateScaleOffset.test_float_fails_without_optionsc Cs.|t|jjddtdd}W5QRXdS)z# Check when scaleoffset is negetiverrgrANrBr.rrrtest_non_integers z&TestCreateScaleOffset.test_non_integerc Cs.|t|jjddtdd}W5QRXdS)z% Check when dtype is unsupported typerrTrAN)rFrGrr"boolr.rrrtest_unsupport_dtypes z*TestCreateScaleOffset.test_unsupport_dtypecCsd}d}dd|}tjj|d|}|jjd|t|d}|jdk sJt||d <|jj}|j t |d |_|jdd }|j ||d| d ||k rtdS) z2 Scaleoffset filter works for floating point data r!ri,rr:g?rrAN.r) precision)rCrandomrandrr"rCr?rqfilenamerr_rrgr)rZscalefacr-ranger0r/rMr1rrr test_floats  z TestCreateScaleOffset.test_floatcCsd}d}tjjdd|d|d}|jjd|tdd }|jd k sDt||d <|jj}|j t |d |_|jdd }| ||d S) zB Scaleoffset filter works for integer data with default precision rrHrrRrr=rTrAN.rI) rCrKrandintrr"intr?rqrMrr_rrgrnbitsr-r0r/rMr1rrrtest_ints zTestCreateScaleOffset.test_intcCsd}d}tjjdd||d}|jjd|t|d}||jdk||d<|jj}|j t |d |_|jdd}| ||d S) D Scaleoffset filter works for integer data with specified precision rrHrrRrPrrA.rIN) rCrKrQrr"rRr`r?rMrr_rrgrSrrrtest_int_with_minbitss z+TestCreateScaleOffset.test_int_with_minbitscCsd}d}tjjdd|dd|d}|jjd|t|d}||jdk||d <|jj}|j t |d |_|jdd }||k rt d S) rVrrHrrRrrPrrA.rIN)rCrKrQrr"rRr`r?rMrr_rrrqrSrrrtest_int_with_minbits_lossys z1TestCreateScaleOffset.test_int_with_minbits_lossyN) rrrr)rDrErGrOrUrWrXrrrrr@sr@c@s@eZdZdZddZddZddZdd Zd d Zd d Z dS) TestExternalzB Feature: Datasets with the external storage property c Csd}tj|}|}|dtjfg}|jjd||j|dd}||d<|jdk sTt t |d}| }W5QRX|| kst t jjd krt|j}t|jjj} || kst dS) z' Create and access an external dataset r9rrrz ${ORIGIN}rBexternal efile_prefix.Nrbrr:r)rCrKrr UNLIMITEDrr"rBr\rqopenreadtobytesr_rhdf5_version_tuplepathlibPathrlget_access_plistget_efile_prefixdecodeas_posixrMparent) rr-r0ext_filer\r/fidcontentsr]rkrrr test_contentss   zTestExternal.test_contentsc Csd}tj|}|}tj|dtjfg}|jj d||j |tj |d}||d<|j dk sdt t|d}|}W5QRX||kst tjjdkrt|j}t|j} || kst |jjd||j tj |d } | j dk st | d |kdS) z< Create and access an external dataset using an efile_prefixrZrrr[.Nr^r_)r]r)rCrKrospathbasenamer r`rr"rBdirnamer\rqrarbrcr_rrdrerfrlrgrhrirjrkr) rr-r0rlr\r/rmrnr]rkrTrrrtest_contents_efile_prefix*s"     z'TestExternal.test_contents_efile_prefixcCs|jjdd|ddS)z/ External argument may be a file name str only rrZr\N)rr"rrrrr test_name_strHszTestExternal.test_name_strcCs |jjddt|ddS)z0 External argument may be a file name path only rrZruN)rr"rerfrrrrrtest_name_pathMs  zTestExternal.test_name_pathcsN|d}tfddt|D}|jjdd|d}t|j|ksJtdS)z9 External argument may be an iterable of multiple tuples rc3s|]}|ddfVqdS)Nr.0rrlrr Xsz/TestExternal.test_iter_multi..ZpoorZruN)riterrNrr"rr\rq)rNr\r/rr{rtest_iter_multiSs zTestExternal.test_iter_multic Csd}|}t|gft|dgft|dtjgft|fgft|dfgft|dtjdfgft|ddfgffD]0\}}|||jjd||dW5QRXqfdS)z" Test with invalid external lists rZrz h5f.UNLIMITEDrruN)rrGr r`rkrFrr")rr-rlexc_typer\rrr test_invalid\s    zTestExternal.test_invalidN) rrrr)rortrvrwrrrrrrrY s rYc@s2eZdZdZd ddZddZddZd d Zd S) TestAutoCreatezP Feature: Datasets auto-created from data produce the correct types TcCsB|j}|t|tjj||||r>|s>t dSr) rlrnr,rr_r rprrrsrq)rr'ruvariablerwrrrassert_string_typets  z!TestAutoCreate.assert_string_typecCsxd|jd<||jdtjjddg|jd<||jdtjjtjddgtjd|jd<||jdtjjdS) z Datasets may be resized up to maxshape using integer maxshaperrrrr)rbNrr.rrrtest_resize_1Ds zTestResize.test_resize_1Dc Cs6|jjdddd}|t|dW5QRXdS)z. Resizing past maxshape triggers an exception rrrr)rrN)rr"rF Exceptionrr.rrrtest_resize_overs zTestResize.test_resize_overc Cs2|jdd}|t|dW5QRXdS)z/ Resizing non-chunked dataset raises TypeError rrrN)rr"rFrGrr.rrrtest_resize_nonchunkeds z!TestResize.test_resize_nonchunkedcCs2|jjdddd}|jddd||jdd S) z Resize specified axis rrrrrrZaxisrN)rr"rr,r-r.rrrtest_resize_axisszTestResize.test_resize_axisc Cs:|jjdddd}|t|jdddW5QRXdS) z Illegal axis raises ValueError rrrrrrRrN)rr"rFrkrr.rrr test_axis_exc s zTestResize.test_axis_exccCs2|jjdddd}||jd||jddS)z? Allow zero-length initial dims for unlimited axes (issue 111) r)r)rNrN)rr"r,r-rr.rrr test_zero_dimszTestResize.test_zero_dimN) rrrr)rrrrrrrrrrrrrrs  rc@s eZdZdZddZddZdS) TestDtypezD Feature: Dataset dtype is available as .dtype property cCs(|jddd}||jtddS)z Retrieve dtype from dataset rr|S10NrAr.rrr test_dtypeszTestDtype.test_dtypecCs<tdtjfdtjfg}|jdd|}||j|dS)z7 Retrieve dtype from complex float16 dataset (gh-2156) rIrrrN)rCrBfloat16rr"r,)rZ complex32r/rrrtest_dtype_complex32#szTestDtype.test_dtype_complex32N)rrrr)rrrrrrrsrc@s eZdZdZddZddZdS)TestLenzG Feature: Size of first axis is available via Python's len cCs"|jdd}|t|ddS)z Python len() (under 32 bits) r)8rrN)rr"r,rr.rrrtest_len1szTestLen.test_lenc Csj|jdd}||jdtjdkrF|tt|W5QRXn|t|d||ddS)z Python len() vs Dataset.len() r)rirN) rr"r,r-sysmaxsizerF OverflowErrorrr.rrr test_len_big6s  zTestLen.test_len_bigN)rrrr)rrrrrrr+src@s eZdZdZddZddZdS)TestIterz; Feature: Iterating over a dataset yields rows cCsXtjdddd}|jjd|d}t||D]$\}}|t|d|||q.dS) z& Iterating over a dataset yields rows rbrrIrcrrYrdN) rCrfrhrr"zipr,rrg)rrZr/rrrrr test_iterHs zTestIter.test_iterc Cs8|jjddd}|tdd|DW5QRXdS)z0 Iterating over scalar dataset raises TypeError rr)r-cSsg|]}|qSrrryrrr Tsz-TestIter.test_iter_scalar..N)rr"rFrGr.rrrtest_iter_scalarPs zTestIter.test_iter_scalarN)rrrr)rrrrrrrBsrc@seZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ ddZ ddZ ddZddZddZddZddZd d!Zd"d#Zd$S)% TestStringszr Feature: Datasets created with vlen and fixed datatypes correctly translate to and from HDF5 cCsntjdd}|jjdd|d}|j}|t|tjj || tjj t |j }||jddS)z3 Vlen bytes dataset maps to vlen ascii in the file r r rrrIN)r_ string_dtyperr"rlrnr,rr rprrr{check_string_dtyperBrrrr'rwZ string_inforrrr^s   zTestStrings.test_vlen_bytescCsptjdd}d}|jjdd||d}||jdd|||jdd|||jdj|dS) & Vlen bytes dataset handles fillvalue r r barrrrrNr_rrr"r,asstrrirrrZ fill_valuer'rrrtest_vlen_bytes_fillvaluehs  z%TestStrings.test_vlen_bytes_fillvaluecCsjt}|jjdd|d}|j}|t|tjj || tjj t |j }||jddS)z5 Vlen unicode dataset maps to vlen utf-8 in the file rrrIrN)r_rrr"rlrnr,rr rprrrrrBrrrrrrqs  zTestStrings.test_vlen_unicodecCstt}d}|jjdd||d}||jdd|d||jdd|||jdj|ddS)( Vlen unicode dataset handles fillvalue bárrrrrrN)r_rrr"r,encoderrrrrrtest_vlen_unicode_fillvalue{s z'TestStrings.test_vlen_unicode_fillvaluecCstd}|jjdd|d}|j}|t|tj j | | || d||tj jt|j}||jd||jddS)zK Fixed-length bytes dataset maps to fixed-length ascii in the file rrrrIr:r N)rCrBrr"rlrnr,rr_r rpZ assertFalsersrtrrr{rrrvrrrrtest_fixed_asciis   zTestStrings.test_fixed_asciicCsrtjddd}d}|jjdd||d}||jdd|||jdd|||jdj|d S) rr r:rrrrrrNrrrrrtest_fixed_bytes_fillvalues z&TestStrings.test_fixed_bytes_fillvaluec Cstjddd}|jjdd|d}|j}||tjj d}| d|d<||d <||g|d d <t j ||gt d|d d <t j | dgd |d|d d <|t t j ||gdd|d d<W5QRXt j|dd t j | dgd dddS)NrrrrrrIcùrrrRr!r9Ur:S)r_rrr"rlrnr,rrr rrrCrrrFrGrr)rrr'rwsrrrtest_fixed_utf8s " $zTestStrings.test_fixed_utf8cCsztjddd}dd}|jjdd||d}||jdd|||jdd|d||jdj|d S) rrr:rrrrrrN) r_rrrr"r,rrirrrrrtest_fixed_utf_8_fillvalues   z&TestStrings.test_fixed_utf_8_fillvaluec Cs6td}|t|jjdd|d}W5QRXdS)zA Fixed-length unicode datasets are unsupported (raise TypeError) z|U10rrrIN)rCrBrFrGrr"rrr'rrrtest_fixed_unicodes  zTestStrings.test_fixed_unicodecCsRtjdd}|jjdd|d}d}||d<|d}|t|t|||dS) zN writing and reading to vlen bytes dataset preserves type and content r r rrrIHellorN)r_rrr"r,rbytesrrr'rZoutrrrtest_roundtrip_vlen_bytess z%TestStrings.test_roundtrip_vlen_bytescCsRtd}|jjdd|d}d}||d<|d}|t|tj|||dS)z[ Writing to and reading from fixed-length bytes dataset preserves type and content rrrrIrrN)rCrBrr"r,rrrrrrtest_roundtrip_fixed_bytess z&TestStrings.test_roundtrip_fixed_bytescCs^t}|jjdd|d}d}||d<||dt|d}||t|||dS)NrrrIfàilter) r_rrr"r#rrr%r,rrrrtest_retrieve_vlen_unicodes  z&TestStrings.test_retrieve_vlen_unicodec Cs|jjddtd}d}||d<|d}|t|d}W5QRX||dddd||d d||d t |t j |dd t j |gtdt j t |dd t j |gtddS) NrrrIrrr ignoreZfiltelatin-1r:r)rr"r_rrrFUnicodeDecodeErrorr,rrrCrrrrasarray)rr'rZZstrwrap1rrrr test_asstrs"  zTestStrings.test_asstrcCstjdd}|jjdd|d}d}tj|d|d|d<||dtj| d}||t | ||| | d d dd | | d d|tj | dd tj|gtddS)NrrzrrrIrrrr rrrr)r_rrr"rCrrr#bytes_rr%r,rrrrrrrrtest_asstr_fixeds    zTestStrings.test_asstr_fixedc CsBtd}|jjdd|d}d}|t||d<W5QRXdS)zGEncoding error when writing a non-ASCII string to an ASCII vlen datasetr rrrIrrN)r_rrr"rFUnicodeEncodeError)rrr'rZrrrtest_unicode_write_error s   z$TestStrings.test_unicode_write_errorcCs\t}|jjdd|d}dtdd}||d<|d}|t|t|||dS) zJ Writing valid utf-8 byte strings to a unicode vlen dataset is OK rrrIrrutf8rN) r_rrr"rrr,rrrrrrtest_unicode_write_bytessz$TestStrings.test_unicode_write_bytescCsVtd}|jjdd|d}d}||d<|d}|t|t|||ddS)z: Writing an ascii str to ascii vlen dataset is OK r rrrIz ASCII stringrN)r_rrr"r,rrrrrrrtest_vlen_bytes_write_ascii_strs z+TestStrings.test_vlen_bytes_write_ascii_strN)rrrr)rrrrrrrrrrrrrrrrrrrrrrWs$          rc@s(eZdZdZddZddZddZdS) TestCompoundz: Feature: Compound types correctly round-trip c Cstdtjfdtjfdtjfdtjfdtjfdtjfdtjfg}tjd|d }|jD]}tjdd ||<qV||jd <|jd d }| t ||k| |j|jd S)z: Compound types are read back in correct order (issue 236)weightZcputimeZwalltimeZparents_offsetZ n_parentsstatus endpoint_typer7rIrtest.N) rCrBfloat64Zuint32uint8rfieldsrKrr`rr,)rrr0keyoutdatarrrtest_rt0s   zTestCompound.test_rtcCstdtjdffdtjfg}tjd|d}|jD] }tjj||jdd||<q2|jj dd|d}|jD]}||||<ql|jdd }| t ||k| |j|jdS) NrrdrrrIrPrr.) rCrBr r rr rKr-rr"r`rr,)rrr0r r'r rrr test_assignDs  zTestCompound.test_assigncCs@tdtjfdtjfdtjfg}tjd|d}|jD]}tjdd||<q6||jd<tj|jdddgdd|ddgtj|jdddd|dtjt |jdddg|ddgtdtj fg}tjtj |jddg|d|dg |t |jdddks)rr"rCrfr,rrr.rrrtest_astype_wrapper_len.sz"TestAstype.test_astype_wrapper_lencCsN|jjdddd}td|d<tj|ddd}||tjddddSr>)rr"rCrfrrrgr?rrrtest_astype_wrapper_asarray3sz&TestAstype.test_astype_wrapper_asarrayN)rrrr)r@rArBrrrrr=$sr=c@seZdZdZddZdS)TestScalarCompoundzl Retrieval of a single field from a scalar compound dataset should strip the field info cCs:tdg}|jjdd|d}||djtddS)NrrrrIr~r)rCrBrr"r,)rrr/rrrtest_scalar_compoundAs z'TestScalarCompound.test_scalar_compoundN)rrrr)rDrrrrrC:srCc@sfeZdZddZddZddZddZd d Zdd d ZddZ ddZ ddZ ddZ ddZ dS)TestVlencCsVtt}|jjdd|d}td|d<td|d<dddg|d<td|d<||dtd||dtd||dtdddg||dtdtjtdtd gt d|dd<||dtd||dtd ttdtdg|dd<||dtd||dtddS) Nvlenr rIrdrrrRrr!) r_ vlen_dtyperRrr"rCrfrgrrrrrrrUIs  &"zTestVlen.test_intcCs6tt}|jjdd|d}|jdd|djdS)NrFr1rIvlen2r)r_rGrRrr"rBrrrrtest_reuse_from_other[s zTestVlen.test_reuse_from_othercCsldtfdttfg}|jjdd|d}|jj}|jt|d|_|jdd|jdddjdS)Nr~rrFr1rIrHr) rRr_rGrr"rMrrrB)rrr'fnamerrrtest_reuse_struct_from_other`s  z%TestVlen.test_reuse_struct_from_othercCsNtt}|jjdd|d}tddg|d<tdg|d<ddd g|d<||dtddg||dtdg||dtddd gtjd d d d dgtdgt d|dd<||dtd||dtdttd ddgtdddgg|dd<||dtd ||dtd dS)NrFrrIgffffff?g333333?rrrRrdg?g?g@g@r!r皙@皙?) r_rGrRrr"rCrrgrfrrrrr test_converths  *zTestVlen.test_convertcCstt}|jjdd|d}td|d<tjtdtdgtdtdggtd|ddddf<ttdtdgtdtdgg|ddddf<dS)NrF)rRrRrIrr)rdrR) r_rGrRrr"rCrfrrrrrr test_multidimys zTestVlen.test_multidimrFc Csht|}|jj|d|d}tjdddg|d}tjdddd d g|d}tjd d d gtdd}||}dddddg} tj| |d} dddd ddg} tj| |d} ||d<||d <||d <| |d<| |d<|||d|||d |||d || |d|| |dtjddg|d} | dd|d<|| |d|j |j dS)z Helper for testing various vlen numpy data types. :param np_dt: Numpy datatype to test :param dataset_name: String name of the dataset to create for testing. rrIr5g@g>@g33333Y@g i@gfffffX@g%grrrRrrg @rgriri'rrdr!g333333?rLN) r_rGrr"rCrrBrrgr/r) rnp_dtZ dataset_namerr'Zarray_0Zarray_1Zarray_2Zcasted_array_2Zlist_3Z list_array_3Zlist_4Z list_array_4rrr_help_float_testings2   zTestVlen._help_float_testingcCstd}||dS)NrrCrBrQrrPrrrtest_numpy_float16s zTestVlen.test_numpy_float16cCstd}||dS)Nfloat32rRrSrrrtest_numpy_float32s zTestVlen.test_numpy_float32cCstd}||dS)Nr rRrSrrrtest_numpy_float64_from_dtypes z&TestVlen.test_numpy_float64_from_dtypecCstj}||dSr)rCr rQrSrrrtest_numpy_float64_2szTestVlen.test_numpy_float64_2c Cs"|jjddtddtdddddddg}|ddd|jdd <t|jdd |dddkst|jdd d |ddd|jjd dtd dtdd dddddg}|ddd|jd d <t|jd d |dddkst|jd d d |ddddS)z4Test that non-contiguous arrays are stored correctlyncrrFrITFNrRrz != Znc2Zint8r!rrrrdr)rr"r_rGrCrrrq)rrrrrrtest_non_contiguous_arrayssFz#TestVlen.test_non_contiguous_arraysN)rF)rrrrUrIrKrNrOrQrTrVrWrXrZrrrrrEHs 0rEc@seZdZddZddZdS) TestLowOpencCs|jdd}|j}dS)z Test H5Dget_access_plist rr N)rr"rlrg)rr'Zp_listrrrtest_get_access_listsz TestLowOpen.test_get_access_listcCsHtjtjj}|jdd}~tj|jjd|}| |tjj dS)z# Test the dapl keyword to h5d.open rrxN) r_Zh5pr8ZDATASET_ACCESSrr"Zh5drarlr#Z DatasetID)rZdaplr/dsidrrr test_dapls zTestLowOpen.test_daplN)rrrr\r_rrrrr[sr[)rr:rz#chunk info requires HDF5 >= 1.10.5c Cs@ddlm}|}t|d(}|jdddddd |ddd<W5QRX|dt|d }|dj}|d ks~tt d D]`}t t t |dd }||}|j|kst|jdkst|jdk st|jdkstq|d }|jd kst|jdkst|jdk s"t|jdks2tW5QRXdS)NrBytesIOrrrr^rr-rrBrrIrr:r))iorar_rr"seekrlget_num_chunksrqrNrrCrZ unravel_indexget_chunk_info chunk_offset filter_mask byte_offsetr=Zget_chunk_info_by_coord)rabuffoutfinr'joffsetsirrrtest_get_chunk_detailss*      rp)rrrd)rr:r:)rr:czJchunk iteration requires HDF5 1.10.10 and later 1.10, or 1.12.3 and laterc sddlm}|}t|d(}|jdddddd |dd d <W5QRX|dt|d `}|dj}|}|d kstit |D]}| |}||j <qfd d}| |W5QRXd S)z%H5Dchunk_iter() for chunk informationrr`rrrr^rrbrNrIrcsN|j}|j|jkst|j|jks*t|j|jks:t|j|jksJtdSr)rgrqrhrir=)Z chunk_infoZknowncirrcallbacks  z!test_chunk_iter..callback) rcrar_rr"rdrlrerqrNrfrgZ chunk_iter)rarjrr^Z num_chunksrmrortrrrrtest_chunk_iters         rucCs.|jddd}|jdkst|jdks*tdS)Nr+rrI)r"r-rqr)rr'rrrtest_empty_shapesrvc Csddlm}|}t|d}|jdddW5QRX|dt|dD}|djdksbt|dj dksxt|dj dkstW5QRXdS)Nrr`rr+r rIrI) rcrar_rr"rdrrqrlZ get_offsetZget_storage_size)rarjrkrlrrrtest_zero_storage_size%s  rwcCsttjjttjjdg}|jd|tjd}|jttjksHttj |ddtj |tjd||dd<tj |ddtj |tjddS)Nrrr6rI) rCZiinformaxr"Zuint64rBrqrrr)rrZr'rrrtest_python_int_uint643s  " rycCs>|jddtjd}tjjddd}||ddddd gd f<dS) NrZ)rrxrRrI)rrdrRrPrrRr!.)r"rCr rKrQ)rrblockrrrtest_setitem_fancy_indexingAsr|c Cs0ttd}|dddks"tW5QRXdS)Nzvlen_string_dset.h5ZDS1rsPartingrrrqrrrrtest_vlen_spacepadHsrc Cs0ttd}|dddks"tW5QRXdS)Nzvlen_string_dset_utc.h5Zds1rs2009-12-20T10:16:18.662409Zr}r~rrrtest_vlen_nulltermMsr)rr:rdz>Appears you cannot pass an unknown filter id for HDF5 < 1.10.3cCs0d}|jddtj|dd}t||jks,tdS)NrZr^T)r-rBrZallow_unknown_filter)r"rCr r%_filtersrq)rZfake_filter_idr'rrrtest_allow_unknown_filterRsrc Csddlm}|}t|dV}|jdddddd d d }|j}|jd d |ksdt |dkspt W5QRX| dt|dT}|j ddddddd}|j}|jd d |kst |dkst W5QRXd S)z2Chunk cache configuration for individual datasets.rr`rr)r:r)rr!r rM)r-rrB rdcc_nbytesrdcc_w0 rdcc_nslotsrN)rrrMrI0q= ףp?)r-rBrrr)rrr) rcrar_rr"rlrgZget_chunk_cacheZ get_cacherqrdr)rarjrkr'Zds_chunk_cacherlrrrtest_dset_chunk_cache`s8  rc@s eZdZdZddZddZdS)TestCommutativez| Test the symmetry of operators, at least with the numpy types. Issue: https://github.com/h5py/h5py/issues/1947 cCsd}|jjd|ttjj|d}t|d}t||k||kksHtt||k||kksbtd}t ||}t||k||kkstt||k||kkstdS)z Create a h5py dataset, extract one element convert to numpy Check that it returns symmetric response to == and != rrrrBrZrgMbP?N) rr"rCrCrKrLr rrqZnanmax)rr-r/valdeltaZnvalrrrtest_numpy_commutative{s  z&TestCommutative.test_numpy_commutativecCsTd}|jjd|ttjj|d}td}||k||kks s      FY0(CC&  ,'   qc,DTO 0     -