Skip to content

Utils

read_dataset(dataset, index1=None, index2=None, max_retries=3)

Enhanced version of read_dataset with retry mechanism while maintaining the original functionality

Parameters:

Name Type Description Default
dataset
required
index1

(Default value = None)

 None
index2

(Default value = None)

 None
max_retries

(Default value = 3)

 3
Source code in june/hdf5_savers/utils.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
def read_dataset(dataset, index1=None, index2=None, max_retries=3):
    """Enhanced version of read_dataset with retry mechanism
    while maintaining the original functionality

    Args:
        dataset: 
        index1: (Default value = None)
        index2: (Default value = None)
        max_retries: (Default value = 3)

    """
    if index1 is None:
        index1 = 0
    if index2 is None:
        index2 = dataset.len()

    dataset_shape = dataset.shape

    # Determine load shape
    if len(dataset_shape) > 1:
        load_shape = [index2 - index1] + list(dataset_shape[1:])
    else:
        load_shape = index2 - index1

    # Create empty array for data
    ret = np.empty(load_shape, dtype=dataset.dtype)

    # Add retry mechanism
    for attempt in range(max_retries):
        try:
            dataset.read_direct(ret, np.s_[index1:index2], np.s_[0:index2-index1])
            return ret
        except (OSError, IOError) as e:
            if attempt == max_retries - 1:
                raise
            time.sleep(1)  # Wait before retry
            logger.warning(f"Retry {attempt + 1} of {max_retries} for dataset read")

write_dataset(group, dataset_name, data, index1=None, index2=None)

Parameters:

Name Type Description Default
group
required
dataset_name
required
data
required
index1

(Default value = None)

 None
index2

(Default value = None)

 None
Source code in june/hdf5_savers/utils.py 
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def write_dataset(group, dataset_name, data, index1=None, index2=None):
    """

    Args:
        group: 
        dataset_name: 
        data: 
        index1: (Default value = None)
        index2: (Default value = None)

    """
    if dataset_name not in group:
        if len(data.shape) > 1:
            maxshape = (None, *data.shape[1:])
        else:
            maxshape = (None,)
        group.create_dataset(dataset_name, data=data, maxshape=maxshape)
    else:
        if len(data.shape) > 1:
            newshape = (group[dataset_name].shape[0] + data.shape[0], *data.shape[1:])
        else:
            newshape = (group[dataset_name].shape[0] + data.shape[0],)
        group[dataset_name].resize(newshape)
        group[dataset_name][index1:index2] = data