Skip to content

h5

h5 ⚓︎

HDF5 functions, and utils, and generators, OH MY!

Attributes⚓︎

h5.H5pyCompression = Literal['gzip', 'lzf', 'szip'] module-attribute ⚓︎

h5py mode strings (taken from h5py docstrings)): 

r        Readonly, file must exist (default)
r+       Read/write, file must exist
w        Create file, truncate if exists
w- or x  Create file, fail if exists
a        Read/write if exists, create otherwise

Classes⚓︎

h5.FileInfo(groups: Dict[str, GroupInfo], attrs: Dict[str, Any], datasets: Dict[str, DatasetInfo], fspath: str, fssize: int, key: str, h5type: Literal['file']) dataclass ⚓︎

Bases: GroupLikeInfo

Functions⚓︎
h5.FileInfo.from_h5py_file(h5py_group: h5py.File) -> FileInfo classmethod ⚓︎

could do with dict-comprehension, but not readable (imo)

datasets_and_groups = {
    obj.name: H5Group.from_h5py_group(obj)
    if isinstance(obj, Group)
    else H5Dataset.from_h5py_dataset(obj)
    for obj in h5py_group.values()
}
Source code in libs/h5/h5/_info.py 
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
@classmethod
def from_h5py_file(cls, h5py_group: h5py.File) -> FileInfo:
    """

    could do with dict-comprehension, but  not readable (imo)

    ```
    datasets_and_groups = {
        obj.name: H5Group.from_h5py_group(obj)
        if isinstance(obj, Group)
        else H5Dataset.from_h5py_dataset(obj)
        for obj in h5py_group.values()
    }
    ```
    """
    datasets = {}
    groups = {}
    for key, value in h5py_group.items():
        if isinstance(value, h5py.Group):
            groups[key] = GroupInfo.from_h5py_group(value)
        elif isinstance(value, h5py.Dataset):
            datasets[key] = DatasetInfo.from_h5py_dataset(value)
        else:
            raise TypeError(f"Unknown type: {type(value)}")
    attrs = dict(h5py_group.attrs)
    key = h5py_group.name
    fssize = path.getsize(h5py_group.file.filename)
    return cls(
        h5type="file",
        key=key,
        groups=groups,
        attrs=attrs,
        datasets=datasets,
        fspath=h5py_group.file.filename,
        fssize=fssize,
    )

Functions⚓︎

h5.as_h5py_obj(obj: T_FsPathOrGroupLike) -> Union[File, Group] ⚓︎

Convert a path or h5py object to an h5py object

Source code in libs/h5/h5/core.py 
146
147
148
149
150
def as_h5py_obj(obj: T_FsPathOrGroupLike) -> Union[File, Group]:
    """Convert a path or h5py object to an h5py object"""
    if is_fspath(obj):
        return File(obj, "r")
    return obj

h5.attrs(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, HDF5-attr)

Source code in libs/h5/h5/core.py 
449
450
451
452
453
454
455
456
def attrs(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, HDF5-attr)"""
    if isinstance(h5_obj, (Path, str)):
        yield from attrs_gen_from_fspath(h5_obj, h5_path=h5_path)
    else:
        yield from h5py_obj_attrs_gen(h5_obj, h5_path=h5_path)

h5.attrs_dict(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Dict[str, Dict[str, Union[str, npt.NDArray[Any], int, float]]] ⚓︎

Load an HDF5 file from a fspath into a dictionary

Given a fspath this method loads an HDF5 file into a dictionary where the key => value pairs are the HDF5-path => HDF5-dataset for the file. This method relies on the h5_dataset_gen that is in this very module to generate tuples of the form (HDF5-path, HDF5-dataset).

Parameters:

Name Type Description Default
h5_obj str

Filepath to an HDF5 format file

 required
h5_path str

HDF5 path to recurse down from

 ''

Returns:

Type Description
Dict[str, Dict[str, Union[str, NDArray[Any], int, float]]]

Dictionary with key => value paris of HDF5-path => HDF5-dataset
Source code in libs/h5/h5/core.py 
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
def attrs_dict(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Dict[str, Dict[str, Union[str, npt.NDArray[Any], int, float]]]:
    """Load an HDF5 file from a fspath into a dictionary

    Given a fspath this method loads an HDF5 file into a dictionary where the
    key => value pairs are the HDF5-path => HDF5-dataset for the file. This
    method relies on the h5_dataset_gen that is in this very module to
    generate tuples of the form (HDF5-path, HDF5-dataset).

    Args:
        h5_obj (str): Filepath to an HDF5 format file
        h5_path (str): HDF5 path to recurse down from

    Returns:
        Dictionary with key => value paris of HDF5-path => HDF5-dataset

    """
    if isinstance(h5_obj, (Path, str)):
        with File(str(h5_obj), mode="r") as h5file:
            return attrs_dict(h5_obj=h5file, h5_path=h5_path)
    return {
        k: cast(H5pyAttributesDict, {**v})
        for k, v in h5py_obj_attrs_gen(h5_obj, h5_path=h5_path)
    }

h5.attrs_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, HDF5-attr)

Source code in libs/h5/h5/core.py 
459
460
461
462
463
def attrs_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, HDF5-attr)"""
    return attrs(h5_obj, h5_path)

h5.attrs_gen_from_fspath(fspath: FsPath, h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath FsPath

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, AttributeManager]]

Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples
Source code in libs/h5/h5/core.py 
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
def attrs_gen_from_fspath(
    fspath: FsPath, h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (FsPath): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples

    """
    with File(str(fspath), mode="r") as f:
        yield from h5py_obj_attrs_gen(f, h5_path)

h5.datasets(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, Dataset)

Source code in libs/h5/h5/core.py 
432
433
434
435
436
437
438
439
def datasets(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Return a generator that yields tuples with: (HDF5-path, Dataset)"""
    if isinstance(h5_obj, (str, Path, PathLike)):
        yield from datasets_gen_from_fspath(str(h5_obj), h5_path=h5_path)
    else:
        yield from h5py_obj_dataset_gen(h5_obj, h5_path=h5_path)

h5.datasets_dict(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]] ⚓︎

Load an HDF5 file from a fspath into a dictionary

Given a fspath this method loads an HDF5 file into a dictionary where the key => value pairs are the HDF5-path => HDF5-dataset for the file. This method relies on the h5_dataset_gen that is in this very module to generate tuples of the form (HDF5-path, HDF5-dataset).

Parameters:

Name Type Description Default
h5_obj str

Filepath to an HDF5 format file

 required
h5_path str

Path to start from. Defaults to "".

 ''

Returns:

Type Description
Dict[str, Union[NDArray[Any], int8, float64]]

Dictionary with key => value paris of HDF5-path => HDF5-dataset
Source code in libs/h5/h5/core.py 
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
def datasets_dict(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]]:
    """Load an HDF5 file from a fspath into a dictionary

    Given a fspath this method loads an HDF5 file into a dictionary where the
    key => value pairs are the HDF5-path => HDF5-dataset for the file. This
    method relies on the h5_dataset_gen that is in this very module to
    generate tuples of the form (HDF5-path, HDF5-dataset).

    Args:
        h5_obj (str): Filepath to an HDF5 format file
        h5_path (str, optional): Path to start from. Defaults to "".

    Returns:
        Dictionary with key => value paris of HDF5-path => HDF5-dataset

    """
    if isinstance(h5_obj, (Path, str)):
        with File(h5_obj, mode="r") as h5file:
            datasets_dict = {
                h5_path: h5_dataset[()]
                for h5_path, h5_dataset in h5py_obj_dataset_gen(h5file, h5_path)
            }
            return datasets_dict
    else:
        return {
            h5_path: h5_dataset[()]
            for h5_path, h5_dataset in h5py_obj_dataset_gen(h5_obj, h5_path)
        }

h5.datasets_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, Dataset)

Source code in libs/h5/h5/core.py 
442
443
444
445
446
def datasets_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Return a generator that yields tuples with: (HDF5-path, Dataset)"""
    yield from datasets(h5_obj, h5_path)

h5.datasets_gen_from_fspath(fspath: str, h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath str

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, Dataset]]

Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples
Source code in libs/h5/h5/core.py 
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
def datasets_gen_from_fspath(
    fspath: str, h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (str): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples

    """
    with File(fspath, mode="r") as h5_obj:
        yield from h5py_obj_dataset_gen(h5_obj, h5_path=h5_path)

h5.datasets_keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str] ⚓︎

Return a list of all keys/paths for an h5py object.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group, FsPath]

h5-h5py group object

 required

Returns:

Type Description
List[str]

Generator that yields tuples; (h5-path, h5py.Dataset)
Source code in libs/h5/h5/core.py 
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
def datasets_keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str]:
    """Return a list of all keys/paths for an h5py object.

    Args:
        h5py_obj: h5-h5py group object

    Returns:
        Generator that yields tuples; (h5-path, h5py.Dataset)

    """
    if is_group(h5py_obj):
        keys = []

        def _fn(key: str, value: Union[File, Group, Dataset]) -> None:
            if is_h5py_dataset(value):
                keys.append(_leading_slash(key))

        h5py_obj.visititems(_fn)
        return keys
    with File(str(h5py_obj), mode="r") as f:
        return datasets_keys_list(f)

h5.fmt_h5_path(head: str, tail: str) -> str cached ⚓︎

Format function for HDF5-path-strings

Example

fmt_h5_path("foo", "bar") '/foo/bar'

Source code in libs/h5/h5/core.py 
134
135
136
137
138
139
140
141
142
143
@lru_cache(maxsize=128)
def fmt_h5_path(head: str, tail: str) -> str:
    """Format function for HDF5-path-strings

    Example:
        >>> fmt_h5_path("foo", "bar")
        '/foo/bar'

    """
    return _ensure_single_leading_slash(f"{head}/{tail}")

h5.groups(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, h5py.Group)

Source code in libs/h5/h5/core.py 
466
467
468
469
470
471
472
473
def groups(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, h5py.Group)"""
    if isinstance(h5_obj, (Path, str)):
        yield from groups_gen_from_fspath(str(h5_obj), h5_path=h5_path)
    else:
        yield from h5py_obj_groups_gen(h5_obj, h5_path=h5_path)

h5.groups_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, h5py.Group)

Source code in libs/h5/h5/core.py 
476
477
478
479
480
def groups_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, h5py.Group)"""
    yield from groups(h5_obj=h5_obj, h5_path=h5_path)

h5.groups_gen_from_fspath(fspath: FsPath, h5_path: str = '') -> Iterable[Tuple[str, Group]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath FsPath

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, Group]]

Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples
Source code in libs/h5/h5/core.py 
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
def groups_gen_from_fspath(
    fspath: FsPath, h5_path: str = ""
) -> Iterable[Tuple[str, Group]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (FsPath): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples

    """
    with File(str(fspath), mode="r") as f:
        yield from h5py_obj_groups_gen(f, h5_path)

h5.h5_attrs_dict(fspath: str, h5_path: str = '') -> Dict[str, H5pyAttributesDict] ⚓︎

Alias for h5.attrs_dict

Source code in libs/h5/h5/legacy.py 
54
55
56
57
@_h5_deprecated
def h5_attrs_dict(fspath: str, h5_path: str = "") -> Dict[str, H5pyAttributesDict]:
    """Alias for h5.attrs_dict"""
    return attrs_dict(h5_obj=fspath, h5_path=h5_path)

h5.h5_attrs_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Alias for h5.datasets_gen

Source code in libs/h5/h5/legacy.py 
68
69
70
71
72
73
@_h5_deprecated
def h5_attrs_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Alias for h5.datasets_gen"""
    return attrs_gen(h5_obj=h5_obj, h5_path=h5_path)

h5.h5_attrs_gen_from_fspath(fspath: FsPath, h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath FsPath

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, AttributeManager]]

Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples
Source code in libs/h5/h5/legacy.py 
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
@_h5_deprecated
def h5_attrs_gen_from_fspath(
    fspath: FsPath, h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (FsPath): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples

    """
    return attrs_gen_from_fspath(fspath, h5_path)

h5.h5_datasets_dict(fspath: str, h5_path: str = '') -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]] ⚓︎

Alias for h5.datasets_dict

Source code in libs/h5/h5/legacy.py 
60
61
62
63
64
65
@_h5_deprecated
def h5_datasets_dict(
    fspath: str, h5_path: str = ""
) -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]]:
    """Alias for h5.datasets_dict"""
    return datasets_dict(h5_obj=fspath, h5_path=h5_path)

h5.h5_datasets_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Alias for h5.datasets_gen

Source code in libs/h5/h5/legacy.py 
110
111
112
113
114
115
@_h5_deprecated
def h5_datasets_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Alias for h5.datasets_gen"""
    return datasets_gen(h5_obj=h5_obj, h5_path=h5_path)

h5.h5_datasets_gen_from_fspath(fspath: str, h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath str

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, Dataset]]

Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples
Source code in libs/h5/h5/legacy.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
@_h5_deprecated
def h5_datasets_gen_from_fspath(
    fspath: str, h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (str): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples

    """
    yield from datasets_gen_from_fspath(fspath, h5_path=h5_path)

h5.h5py_obj_attrs_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Recursive h5py.AttributeManager generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[Tuple[str, AttributeManager]]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
def h5py_obj_attrs_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[Tuple[str, AttributeManager]]:
    """Recursive h5py.AttributeManager generator.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield (h5py_obj.name, h5py_obj.attrs)

    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            (fmt_h5_path(h5_path, key), item.attrs)
            for key, item in h5py_obj.items()
            if isinstance(item, (Dataset, Group))
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_attrs_gen(item, fmt_h5_path(h5_path, key), root=False)
            for key, item in h5py_obj.items()
            if isinstance(item, Group)
        ),
    )

h5.h5py_obj_dataset_gen(h5py_obj: Union[File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Recursive h5 dataset generator.

Given an h5 group, which is what one gets after loading an h5 file via h5py, this function yields tuples containing (1.) a path (h5_path) to a dataset in the group, and (2.) the dataset itself as a numpy array.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''

Returns:

Type Description
Iterable[Tuple[str, Dataset]]

Generator that yields tuples; (h5-path, h5py.Dataset)
Source code in libs/h5/h5/core.py 
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
def h5py_obj_dataset_gen(
    h5py_obj: Union[File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Recursive h5 dataset generator.

    Given an h5 group, which is what one gets after loading an h5 file
    via h5py, this function yields tuples containing (1.) a path (h5_path) to
    a dataset in the group, and (2.) the dataset itself as a numpy array.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")

    Returns:
        Generator that yields tuples; (h5-path, h5py.Dataset)

    """
    return chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset
            (item.name, item) for item in h5py_obj.values() if isinstance(item, Dataset)
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_dataset_gen(item, h5_path or item.name)
            for item in h5py_obj.values()
            if isinstance(item, Group)
        ),
    )

h5.h5py_obj_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[Tuple[str, Union[Dataset, Group, File]]] ⚓︎

Recursive h5 dataset/group generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[Tuple[str, Union[Dataset, Group, File]]]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
def h5py_obj_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[Tuple[str, Union[Dataset, Group, File]]]:
    """Recursive h5 dataset/group generator.


    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield (h5py_obj.name, h5py_obj)
    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            (fmt_h5_path(h5_path, key), item) for key, item in h5py_obj.items()
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_gen(item, item.name, root=False)
            for key, item in h5py_obj.items()
            if isinstance(item, Group)
        ),
    )

h5.h5py_obj_groups_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[Tuple[str, Group]] ⚓︎

Recursive h5 groups generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[Tuple[str, Group]]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
def h5py_obj_groups_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[Tuple[str, Group]]:
    """Recursive h5 groups generator.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield h5py_obj.name, h5py_obj
    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            (fmt_h5_path(h5_path, key), item)
            for key, item in h5py_obj.items()
            if isinstance(item, (Group, File))
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_groups_gen(item, fmt_h5_path(h5_path, key), root=False)
            for key, item in h5py_obj.items()
            if isinstance(item, (Group, File))
        ),
    )

h5.h5py_obj_keys_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[str] ⚓︎

Recursive h5 dataset generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[str]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def h5py_obj_keys_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[str]:
    """Recursive h5 dataset generator.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield h5py_obj.name
    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            item.name for item in h5py_obj.values()
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_keys_gen(item, h5_path or item.name, root=False)
            # fmt_h5_path(h5_path, key))
            for item in h5py_obj.values()
            if isinstance(item, Group)
        ),
    )

h5.is_dataset(obj: Any) -> TypeGuard[Dataset] ⚓︎

h5py.Dataset type guard

Source code in libs/h5/h5/core.py 
118
119
120
def is_dataset(obj: Any) -> TypeGuard[Dataset]:
    """h5py.Dataset type guard"""
    return is_h5py_dataset(obj)

h5.is_file(obj: Any) -> TypeGuard[File] ⚓︎

h5py.File type guard

Source code in libs/h5/h5/core.py 
113
114
115
def is_file(obj: Any) -> TypeGuard[File]:
    """h5py.File type guard"""
    return is_h5py_file(obj)

h5.is_group(obj: Any) -> TypeGuard[Group] ⚓︎

h5py.Group type guard

Source code in libs/h5/h5/core.py 
104
105
106
def is_group(obj: Any) -> TypeGuard[Group]:
    """h5py.Group type guard"""
    return is_h5py_group(obj)

h5.is_h5py_dataset(obj: Any) -> TypeGuard[Dataset] ⚓︎

h5py.Dataset type guard

Source code in libs/h5/h5/core.py 
 99
100
101
def is_h5py_dataset(obj: Any) -> TypeGuard[Dataset]:
    """h5py.Dataset type guard"""
    return isinstance(obj, Dataset)

h5.is_h5py_file(obj: Any) -> TypeGuard[File] ⚓︎

h5py.File type guard

Source code in libs/h5/h5/core.py 
94
95
96
def is_h5py_file(obj: Any) -> TypeGuard[File]:
    """h5py.File type guard"""
    return isinstance(obj, File)

h5.is_h5py_group(obj: Any) -> TypeGuard[Group] ⚓︎

h5py.Group type guard

Source code in libs/h5/h5/core.py 
89
90
91
def is_h5py_group(obj: Any) -> TypeGuard[Group]:
    """h5py.Group type guard"""
    return isinstance(obj, Group)

h5.is_hdf5(path: FsPath) -> bool ⚓︎

Check if a file is an HDF5 file

Source code in libs/h5/h5/core.py 
84
85
86
def is_hdf5(path: FsPath) -> bool:
    """Check if a file is an HDF5 file"""
    return bool(_is_hdf5(str(path)))

h5.keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str] ⚓︎

Return a list of all keys/paths for an h5py object.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group, FsPath]

h5-h5py group object

 required

Returns:

Type Description
List[str]

Generator that yields tuples; (h5-path, h5py.Dataset)
Source code in libs/h5/h5/core.py 
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
def keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str]:
    """Return a list of all keys/paths for an h5py object.

    Args:
        h5py_obj: h5-h5py group object

    Returns:
        Generator that yields tuples; (h5-path, h5py.Dataset)

    """
    if is_group(h5py_obj):
        keys = [h5py_obj.name]
        h5py_obj.visit(lambda key: keys.append(_leading_slash(key)))
        return keys
    with File(str(h5py_obj), mode="r") as f:
        return keys_list(f)

Modules⚓︎

h5.__about__ ⚓︎

Package metadata/info

h5.__main__ ⚓︎

pkg entry ~ python -m h5

h5.cli ⚓︎

h5.cli

Classes⚓︎
h5.cli.H5CliConfig(datasets: bool, attributes: bool, groups: bool, include: Optional[Tuple[str, ...]] = None, exclude: Optional[Tuple[str, ...]] = None) dataclass ⚓︎
Functions⚓︎
h5.cli.H5CliConfig.filter_is_none() -> bool ⚓︎

Check if filter is None

Source code in libs/h5/h5/cli.py 
88
89
90
91
92
def filter_is_none(self) -> bool:
    """Check if filter is None"""
    return (self.include is None or self.include == ("**/*",)) and (
        self.exclude is None
    )
Functions⚓︎
h5.cli.cli(debug: bool = False) -> None ⚓︎

h5 command line interface

Source code in libs/h5/h5/cli.py 
126
127
128
129
130
131
132
133
@click.group(
    context_settings={"help_option_names": ["-h", "--help"]},
)
@click.option("--debug", envvar="DGPYDEBUG", is_flag=True, default=False)
def cli(debug: bool = False) -> None:
    """h5 command line interface"""
    if debug:
        click.echo("h5-debug: on")
h5.cli.lsd(fspath: str, json_: bool = False, include: Tuple[str, ...] = ('**/*'), exclude: Optional[Tuple[str, ...]] = None) -> None ⚓︎

List datasets

Source code in libs/h5/h5/cli.py 
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
@cli.command(help="List datasets", name="lsd")
@click.argument(
    "fspath",
    type=click.Path(exists=True),
)
@click.option(
    "-j",
    "--json",
    "json_",
    is_flag=True,
    default=False,
    help="Output JSON",
)
@click.option(
    "-i",
    "--include",
    "include",
    default=("**/*",),
    help="include pattern(s)",
    multiple=True,
)
@click.option(
    "-e",
    "--exclude",
    "exclude",
    default=(),
    help="exclude pattern(s)",
    multiple=True,
)
def lsd(
    fspath: str,
    json_: bool = False,
    include: Tuple[str, ...] = ("**/*",),
    exclude: Optional[Tuple[str, ...]] = None,
) -> None:
    """List datasets"""
    cfg = H5CliConfig.from_cli(
        datasets=True, attributes=False, groups=False, include=include, exclude=exclude
    )
    file_keys = _keys(fspath, cfg)
    if json_:
        _print_json(file_keys)
    else:
        console.print("\n".join(file_keys))
h5.cli.lsg(fspath: str, json_: bool = False, include: Tuple[str, ...] = ('**/*'), exclude: Optional[Tuple[str, ...]] = None) -> None ⚓︎

List groups

Source code in libs/h5/h5/cli.py 
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
@cli.command(help="List groups", name="lsg")
@click.argument(
    "fspath",
    type=click.Path(exists=True),
)
@click.option(
    "-j",
    "--json",
    "json_",
    is_flag=True,
    default=False,
    help="Output JSON",
)
@click.option(
    "-i",
    "--include",
    "include",
    default=("**/*",),
    help="include pattern(s)",
    multiple=True,
)
@click.option(
    "-e",
    "--exclude",
    "exclude",
    default=(),
    help="exclude pattern(s)",
    multiple=True,
)
def lsg(
    fspath: str,
    json_: bool = False,
    include: Tuple[str, ...] = ("**/*",),
    exclude: Optional[Tuple[str, ...]] = None,
) -> None:
    """List groups"""
    cfg = H5CliConfig.from_cli(
        datasets=False, attributes=False, groups=True, include=include, exclude=exclude
    )
    file_keys = _keys(fspath, cfg)
    if json_:
        _print_json(file_keys)
    else:
        console.print("\n".join(file_keys))
Modules⚓︎

h5.core ⚓︎

HDF5 functions, and utils, and generators, OH MY!

Functions⚓︎
h5.core.as_h5py_obj(obj: T_FsPathOrGroupLike) -> Union[File, Group] ⚓︎

Convert a path or h5py object to an h5py object

Source code in libs/h5/h5/core.py 
146
147
148
149
150
def as_h5py_obj(obj: T_FsPathOrGroupLike) -> Union[File, Group]:
    """Convert a path or h5py object to an h5py object"""
    if is_fspath(obj):
        return File(obj, "r")
    return obj
h5.core.attrs(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, HDF5-attr)

Source code in libs/h5/h5/core.py 
449
450
451
452
453
454
455
456
def attrs(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, HDF5-attr)"""
    if isinstance(h5_obj, (Path, str)):
        yield from attrs_gen_from_fspath(h5_obj, h5_path=h5_path)
    else:
        yield from h5py_obj_attrs_gen(h5_obj, h5_path=h5_path)
h5.core.attrs_dict(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Dict[str, Dict[str, Union[str, npt.NDArray[Any], int, float]]] ⚓︎

Load an HDF5 file from a fspath into a dictionary

Given a fspath this method loads an HDF5 file into a dictionary where the key => value pairs are the HDF5-path => HDF5-dataset for the file. This method relies on the h5_dataset_gen that is in this very module to generate tuples of the form (HDF5-path, HDF5-dataset).

Parameters:

Name Type Description Default
h5_obj str

Filepath to an HDF5 format file

 required
h5_path str

HDF5 path to recurse down from

 ''

Returns:

Type Description
Dict[str, Dict[str, Union[str, NDArray[Any], int, float]]]

Dictionary with key => value paris of HDF5-path => HDF5-dataset
Source code in libs/h5/h5/core.py 
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
def attrs_dict(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Dict[str, Dict[str, Union[str, npt.NDArray[Any], int, float]]]:
    """Load an HDF5 file from a fspath into a dictionary

    Given a fspath this method loads an HDF5 file into a dictionary where the
    key => value pairs are the HDF5-path => HDF5-dataset for the file. This
    method relies on the h5_dataset_gen that is in this very module to
    generate tuples of the form (HDF5-path, HDF5-dataset).

    Args:
        h5_obj (str): Filepath to an HDF5 format file
        h5_path (str): HDF5 path to recurse down from

    Returns:
        Dictionary with key => value paris of HDF5-path => HDF5-dataset

    """
    if isinstance(h5_obj, (Path, str)):
        with File(str(h5_obj), mode="r") as h5file:
            return attrs_dict(h5_obj=h5file, h5_path=h5_path)
    return {
        k: cast(H5pyAttributesDict, {**v})
        for k, v in h5py_obj_attrs_gen(h5_obj, h5_path=h5_path)
    }
h5.core.attrs_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, HDF5-attr)

Source code in libs/h5/h5/core.py 
459
460
461
462
463
def attrs_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, HDF5-attr)"""
    return attrs(h5_obj, h5_path)
h5.core.attrs_gen_from_fspath(fspath: FsPath, h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath FsPath

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, AttributeManager]]

Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples
Source code in libs/h5/h5/core.py 
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
def attrs_gen_from_fspath(
    fspath: FsPath, h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (FsPath): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples

    """
    with File(str(fspath), mode="r") as f:
        yield from h5py_obj_attrs_gen(f, h5_path)
h5.core.datasets(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, Dataset)

Source code in libs/h5/h5/core.py 
432
433
434
435
436
437
438
439
def datasets(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Return a generator that yields tuples with: (HDF5-path, Dataset)"""
    if isinstance(h5_obj, (str, Path, PathLike)):
        yield from datasets_gen_from_fspath(str(h5_obj), h5_path=h5_path)
    else:
        yield from h5py_obj_dataset_gen(h5_obj, h5_path=h5_path)
h5.core.datasets_dict(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]] ⚓︎

Load an HDF5 file from a fspath into a dictionary

Given a fspath this method loads an HDF5 file into a dictionary where the key => value pairs are the HDF5-path => HDF5-dataset for the file. This method relies on the h5_dataset_gen that is in this very module to generate tuples of the form (HDF5-path, HDF5-dataset).

Parameters:

Name Type Description Default
h5_obj str

Filepath to an HDF5 format file

 required
h5_path str

Path to start from. Defaults to "".

 ''

Returns:

Type Description
Dict[str, Union[NDArray[Any], int8, float64]]

Dictionary with key => value paris of HDF5-path => HDF5-dataset
Source code in libs/h5/h5/core.py 
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
def datasets_dict(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]]:
    """Load an HDF5 file from a fspath into a dictionary

    Given a fspath this method loads an HDF5 file into a dictionary where the
    key => value pairs are the HDF5-path => HDF5-dataset for the file. This
    method relies on the h5_dataset_gen that is in this very module to
    generate tuples of the form (HDF5-path, HDF5-dataset).

    Args:
        h5_obj (str): Filepath to an HDF5 format file
        h5_path (str, optional): Path to start from. Defaults to "".

    Returns:
        Dictionary with key => value paris of HDF5-path => HDF5-dataset

    """
    if isinstance(h5_obj, (Path, str)):
        with File(h5_obj, mode="r") as h5file:
            datasets_dict = {
                h5_path: h5_dataset[()]
                for h5_path, h5_dataset in h5py_obj_dataset_gen(h5file, h5_path)
            }
            return datasets_dict
    else:
        return {
            h5_path: h5_dataset[()]
            for h5_path, h5_dataset in h5py_obj_dataset_gen(h5_obj, h5_path)
        }
h5.core.datasets_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, Dataset)

Source code in libs/h5/h5/core.py 
442
443
444
445
446
def datasets_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Return a generator that yields tuples with: (HDF5-path, Dataset)"""
    yield from datasets(h5_obj, h5_path)
h5.core.datasets_gen_from_fspath(fspath: str, h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath str

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, Dataset]]

Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples
Source code in libs/h5/h5/core.py 
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
def datasets_gen_from_fspath(
    fspath: str, h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (str): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples

    """
    with File(fspath, mode="r") as h5_obj:
        yield from h5py_obj_dataset_gen(h5_obj, h5_path=h5_path)
h5.core.datasets_keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str] ⚓︎

Return a list of all keys/paths for an h5py object.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group, FsPath]

h5-h5py group object

 required

Returns:

Type Description
List[str]

Generator that yields tuples; (h5-path, h5py.Dataset)
Source code in libs/h5/h5/core.py 
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
def datasets_keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str]:
    """Return a list of all keys/paths for an h5py object.

    Args:
        h5py_obj: h5-h5py group object

    Returns:
        Generator that yields tuples; (h5-path, h5py.Dataset)

    """
    if is_group(h5py_obj):
        keys = []

        def _fn(key: str, value: Union[File, Group, Dataset]) -> None:
            if is_h5py_dataset(value):
                keys.append(_leading_slash(key))

        h5py_obj.visititems(_fn)
        return keys
    with File(str(h5py_obj), mode="r") as f:
        return datasets_keys_list(f)
h5.core.fmt_h5_path(head: str, tail: str) -> str cached ⚓︎

Format function for HDF5-path-strings

Example

fmt_h5_path("foo", "bar") '/foo/bar'

Source code in libs/h5/h5/core.py 
134
135
136
137
138
139
140
141
142
143
@lru_cache(maxsize=128)
def fmt_h5_path(head: str, tail: str) -> str:
    """Format function for HDF5-path-strings

    Example:
        >>> fmt_h5_path("foo", "bar")
        '/foo/bar'

    """
    return _ensure_single_leading_slash(f"{head}/{tail}")
h5.core.groups(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, h5py.Group)

Source code in libs/h5/h5/core.py 
466
467
468
469
470
471
472
473
def groups(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, h5py.Group)"""
    if isinstance(h5_obj, (Path, str)):
        yield from groups_gen_from_fspath(str(h5_obj), h5_path=h5_path)
    else:
        yield from h5py_obj_groups_gen(h5_obj, h5_path=h5_path)
h5.core.groups_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Return a generator that yields tuples with: (HDF5-path, h5py.Group)

Source code in libs/h5/h5/core.py 
476
477
478
479
480
def groups_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Return a generator that yields tuples with: (HDF5-path, h5py.Group)"""
    yield from groups(h5_obj=h5_obj, h5_path=h5_path)
h5.core.groups_gen_from_fspath(fspath: FsPath, h5_path: str = '') -> Iterable[Tuple[str, Group]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath FsPath

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, Group]]

Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples
Source code in libs/h5/h5/core.py 
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
def groups_gen_from_fspath(
    fspath: FsPath, h5_path: str = ""
) -> Iterable[Tuple[str, Group]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (FsPath): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples

    """
    with File(str(fspath), mode="r") as f:
        yield from h5py_obj_groups_gen(f, h5_path)
h5.core.h5py_obj_attrs_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Recursive h5py.AttributeManager generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[Tuple[str, AttributeManager]]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
def h5py_obj_attrs_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[Tuple[str, AttributeManager]]:
    """Recursive h5py.AttributeManager generator.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield (h5py_obj.name, h5py_obj.attrs)

    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            (fmt_h5_path(h5_path, key), item.attrs)
            for key, item in h5py_obj.items()
            if isinstance(item, (Dataset, Group))
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_attrs_gen(item, fmt_h5_path(h5_path, key), root=False)
            for key, item in h5py_obj.items()
            if isinstance(item, Group)
        ),
    )
h5.core.h5py_obj_dataset_gen(h5py_obj: Union[File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Recursive h5 dataset generator.

Given an h5 group, which is what one gets after loading an h5 file via h5py, this function yields tuples containing (1.) a path (h5_path) to a dataset in the group, and (2.) the dataset itself as a numpy array.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''

Returns:

Type Description
Iterable[Tuple[str, Dataset]]

Generator that yields tuples; (h5-path, h5py.Dataset)
Source code in libs/h5/h5/core.py 
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
def h5py_obj_dataset_gen(
    h5py_obj: Union[File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Recursive h5 dataset generator.

    Given an h5 group, which is what one gets after loading an h5 file
    via h5py, this function yields tuples containing (1.) a path (h5_path) to
    a dataset in the group, and (2.) the dataset itself as a numpy array.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")

    Returns:
        Generator that yields tuples; (h5-path, h5py.Dataset)

    """
    return chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset
            (item.name, item) for item in h5py_obj.values() if isinstance(item, Dataset)
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_dataset_gen(item, h5_path or item.name)
            for item in h5py_obj.values()
            if isinstance(item, Group)
        ),
    )
h5.core.h5py_obj_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[Tuple[str, Union[Dataset, Group, File]]] ⚓︎

Recursive h5 dataset/group generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[Tuple[str, Union[Dataset, Group, File]]]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
def h5py_obj_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[Tuple[str, Union[Dataset, Group, File]]]:
    """Recursive h5 dataset/group generator.


    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield (h5py_obj.name, h5py_obj)
    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            (fmt_h5_path(h5_path, key), item) for key, item in h5py_obj.items()
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_gen(item, item.name, root=False)
            for key, item in h5py_obj.items()
            if isinstance(item, Group)
        ),
    )
h5.core.h5py_obj_groups_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[Tuple[str, Group]] ⚓︎

Recursive h5 groups generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[Tuple[str, Group]]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
def h5py_obj_groups_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[Tuple[str, Group]]:
    """Recursive h5 groups generator.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield h5py_obj.name, h5py_obj
    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            (fmt_h5_path(h5_path, key), item)
            for key, item in h5py_obj.items()
            if isinstance(item, (Group, File))
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_groups_gen(item, fmt_h5_path(h5_path, key), root=False)
            for key, item in h5py_obj.items()
            if isinstance(item, (Group, File))
        ),
    )
h5.core.h5py_obj_keys_gen(h5py_obj: Union[File, Group], h5_path: str = '', root: bool = True) -> Iterable[str] ⚓︎

Recursive h5 dataset generator.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group]

h5-h5py group object

 required
h5_path str

path so far (Default value = "")

 ''
root bool

if True, yield the root path (Default value = True)

 True

Returns:

Type Description
Iterable[str]

Generator that yields tuples; (h5-path, h5py.AttributeManager)
Source code in libs/h5/h5/core.py 
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def h5py_obj_keys_gen(
    h5py_obj: Union[File, Group], h5_path: str = "", root: bool = True
) -> Iterable[str]:
    """Recursive h5 dataset generator.

    Args:
        h5py_obj: h5-h5py group object
        h5_path: path so far (Default value = "")
        root: if True, yield the root path (Default value = True)

    Returns:
        Generator that yields tuples; (h5-path, h5py.AttributeManager)

    """
    if root:
        yield h5py_obj.name
    yield from chain(  # Chain of generators into one generator
        (  # Generator object if the current h5py object is a Dataset or Group
            item.name for item in h5py_obj.values()
        ),
        *(  # Unpack a generator that generates generators recursively
            h5py_obj_keys_gen(item, h5_path or item.name, root=False)
            # fmt_h5_path(h5_path, key))
            for item in h5py_obj.values()
            if isinstance(item, Group)
        ),
    )
h5.core.is_dataset(obj: Any) -> TypeGuard[Dataset] ⚓︎

h5py.Dataset type guard

Source code in libs/h5/h5/core.py 
118
119
120
def is_dataset(obj: Any) -> TypeGuard[Dataset]:
    """h5py.Dataset type guard"""
    return is_h5py_dataset(obj)
h5.core.is_file(obj: Any) -> TypeGuard[File] ⚓︎

h5py.File type guard

Source code in libs/h5/h5/core.py 
113
114
115
def is_file(obj: Any) -> TypeGuard[File]:
    """h5py.File type guard"""
    return is_h5py_file(obj)
h5.core.is_group(obj: Any) -> TypeGuard[Group] ⚓︎

h5py.Group type guard

Source code in libs/h5/h5/core.py 
104
105
106
def is_group(obj: Any) -> TypeGuard[Group]:
    """h5py.Group type guard"""
    return is_h5py_group(obj)
h5.core.is_h5py_dataset(obj: Any) -> TypeGuard[Dataset] ⚓︎

h5py.Dataset type guard

Source code in libs/h5/h5/core.py 
 99
100
101
def is_h5py_dataset(obj: Any) -> TypeGuard[Dataset]:
    """h5py.Dataset type guard"""
    return isinstance(obj, Dataset)
h5.core.is_h5py_file(obj: Any) -> TypeGuard[File] ⚓︎

h5py.File type guard

Source code in libs/h5/h5/core.py 
94
95
96
def is_h5py_file(obj: Any) -> TypeGuard[File]:
    """h5py.File type guard"""
    return isinstance(obj, File)
h5.core.is_h5py_group(obj: Any) -> TypeGuard[Group] ⚓︎

h5py.Group type guard

Source code in libs/h5/h5/core.py 
89
90
91
def is_h5py_group(obj: Any) -> TypeGuard[Group]:
    """h5py.Group type guard"""
    return isinstance(obj, Group)
h5.core.is_hdf5(path: FsPath) -> bool ⚓︎

Check if a file is an HDF5 file

Source code in libs/h5/h5/core.py 
84
85
86
def is_hdf5(path: FsPath) -> bool:
    """Check if a file is an HDF5 file"""
    return bool(_is_hdf5(str(path)))
h5.core.keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str] ⚓︎

Return a list of all keys/paths for an h5py object.

Parameters:

Name Type Description Default
h5py_obj Union[File, Group, FsPath]

h5-h5py group object

 required

Returns:

Type Description
List[str]

Generator that yields tuples; (h5-path, h5py.Dataset)
Source code in libs/h5/h5/core.py 
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
def keys_list(h5py_obj: Union[File, Group, FsPath]) -> List[str]:
    """Return a list of all keys/paths for an h5py object.

    Args:
        h5py_obj: h5-h5py group object

    Returns:
        Generator that yields tuples; (h5-path, h5py.Dataset)

    """
    if is_group(h5py_obj):
        keys = [h5py_obj.name]
        h5py_obj.visit(lambda key: keys.append(_leading_slash(key)))
        return keys
    with File(str(h5py_obj), mode="r") as f:
        return keys_list(f)

h5.dev ⚓︎

h5.dev ~ Under construction!

h5.legacy ⚓︎

Legacy; to be deprecated

Functions⚓︎
h5.legacy.h5_attrs_dict(fspath: str, h5_path: str = '') -> Dict[str, H5pyAttributesDict] ⚓︎

Alias for h5.attrs_dict

Source code in libs/h5/h5/legacy.py 
54
55
56
57
@_h5_deprecated
def h5_attrs_dict(fspath: str, h5_path: str = "") -> Dict[str, H5pyAttributesDict]:
    """Alias for h5.attrs_dict"""
    return attrs_dict(h5_obj=fspath, h5_path=h5_path)
h5.legacy.h5_attrs_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Alias for h5.datasets_gen

Source code in libs/h5/h5/legacy.py 
68
69
70
71
72
73
@_h5_deprecated
def h5_attrs_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Alias for h5.datasets_gen"""
    return attrs_gen(h5_obj=h5_obj, h5_path=h5_path)
h5.legacy.h5_attrs_gen_from_fspath(fspath: FsPath, h5_path: str = '') -> Iterable[Tuple[str, AttributeManager]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath FsPath

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, AttributeManager]]

Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples
Source code in libs/h5/h5/legacy.py 
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
@_h5_deprecated
def h5_attrs_gen_from_fspath(
    fspath: FsPath, h5_path: str = ""
) -> Iterable[Tuple[str, AttributeManager]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (FsPath): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.AttributeManager) tuples

    """
    return attrs_gen_from_fspath(fspath, h5_path)
h5.legacy.h5_datasets_dict(fspath: str, h5_path: str = '') -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]] ⚓︎

Alias for h5.datasets_dict

Source code in libs/h5/h5/legacy.py 
60
61
62
63
64
65
@_h5_deprecated
def h5_datasets_dict(
    fspath: str, h5_path: str = ""
) -> Dict[str, Union[npt.NDArray[Any], np.int8, np.float64]]:
    """Alias for h5.datasets_dict"""
    return datasets_dict(h5_obj=fspath, h5_path=h5_path)
h5.legacy.h5_datasets_gen(h5_obj: Union[FsPath, File, Group], h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Alias for h5.datasets_gen

Source code in libs/h5/h5/legacy.py 
110
111
112
113
114
115
@_h5_deprecated
def h5_datasets_gen(
    h5_obj: Union[FsPath, File, Group], h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Alias for h5.datasets_gen"""
    return datasets_gen(h5_obj=h5_obj, h5_path=h5_path)
h5.legacy.h5_datasets_gen_from_fspath(fspath: str, h5_path: str = '') -> Iterable[Tuple[str, Dataset]] ⚓︎

Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

Parameters:

Name Type Description Default
fspath str

fspath to h5 format file

 required
h5_path str

h5 path to start from. Defaults to "".

 ''

Returns:

Type Description
Iterable[Tuple[str, Dataset]]

Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples
Source code in libs/h5/h5/legacy.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
@_h5_deprecated
def h5_datasets_gen_from_fspath(
    fspath: str, h5_path: str = ""
) -> Iterable[Tuple[str, Dataset]]:
    """Given a fspath to an h5, yield (h5-path, h5py.Dataset) tuples

    Args:
        fspath (str): fspath to h5 format file
        h5_path (str, optional): h5 path to start from. Defaults to "".

    Returns:
        Generator that yields tuples of the form (h5-path, h5py.Dataset) tuples

    """
    yield from datasets_gen_from_fspath(fspath, h5_path=h5_path)

h5.testing ⚓︎

Functions⚓︎
h5.testing.make_test_hdf5_file(filepath: Union[str, Path]) -> str ⚓︎

Make test hdf5 file and return filepath

Source code in libs/h5/h5/testing.py 
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def make_test_hdf5_file(filepath: Union[str, Path]) -> str:
    """Make test hdf5 file and return filepath"""

    root_data = np.arange(10, dtype="i8").reshape(2, 5)
    a_data = np.arange(10, dtype="i8").reshape(2, 5)
    b_data = np.arange(10, dtype="i8").reshape(2, 5)
    aa_subsubgrp_data = np.arange(10, dtype="i8").reshape(2, 5)
    dset_vanilla_data = np.arange(100, dtype="i8")
    dset_chunked_data = np.arange(100, dtype="i8") * 2
    dset_filter_gzip_data = np.arange(100, dtype="i8") * 3
    dset_filter_lzf_data = np.arange(100, dtype="i8") * 4

    with h5.File(str(filepath), mode="w") as f:
        # set root group attributes
        f.attrs["root_attr_str"] = "root_attr_value"

        root_dataset = f.create_dataset("root_dataset", data=root_data)
        root_dataset.attrs["root_attr_str"] = "root_attr_value"
        root_dataset.attrs["root_attr_int"] = 123
        root_dataset.attrs["root_attr_float"] = 123.456
        root_dataset.attrs["root_attr_list"] = [1, 2, 3]
        root_dataset.attrs["root_attr_np_array"] = root_data

        a_subgrp = f.create_group("a_subgroup")
        a_subgrp.attrs["a_attr"] = "a_attr_value"
        a_dataset = a_subgrp.create_dataset("a_dataset", data=a_data)
        a_dataset.attrs["a_dataset_attr"] = "a_dataset_attr_value"

        b_subgrp = f.create_group("b_subgroup")
        b_subgrp.attrs["b_attr"] = "b_attr_value"
        b_dataset = b_subgrp.create_dataset("b_dataset", data=b_data)
        b_dataset.attrs["b_dataset_attr"] = "b_dataset_attr_value"

        aa_subsubgrp = a_subgrp.create_group("aa_subsubgroup")
        aa_subsubgrp.attrs["aa_subsubgroup_attr"] = "aa_subsubgroup_attr_value"
        aa_subsubgrp_dataset = aa_subsubgrp.create_dataset(
            "aa_subsubgroup_dataset", data=aa_subsubgrp_data
        )
        aa_subsubgrp_dataset.attrs["aa_subsubgroup_dataset_attr"] = (
            "aa_subsubgroup_dataset_attr_value"
        )

        dset_vanilla = f.create_dataset("vanilla", data=dset_vanilla_data)
        dset_vanilla.attrs["desc"] = "vanilla-dataset"
        dset_chunked = f.create_dataset("chunked", data=dset_chunked_data, chunks=(10,))
        dset_chunked.attrs["desc"] = "chunked-dataset"
        dset_filter_gzip = f.create_dataset(
            "filter-gzip",
            data=dset_filter_gzip_data,
            chunks=(10,),
            compression="gzip",
            compression_opts=5,
        )
        dset_filter_gzip.attrs["desc"] = "filter-gzip-dataset"
        dset_filter_lzf = f.create_dataset(
            "filter-lzf", data=dset_filter_lzf_data, chunks=(10,), compression="lzf"
        )
        dset_filter_lzf.attrs["desc"] = "filter-lzf-dataset"

    return str(filepath)
Modules⚓︎