Boarding school saver

load_boarding_schools_from_hdf5(file_path, chunk_size=50000, domain_super_areas=None, config_filename=None)

Loads boarding schools from an hdf5 file located at file_path. Note that this object will not be ready to use, as the links to object instances of other classes need to be restored first. This function should be rarely be called oustide world.py

Parameters:

Name	Type	Description	Default
file_path	str		required
chunk_size		(Default value = 50000)	50000
domain_super_areas		(Default value = None)	None
config_filename		(Default value = None)	None

Source code in june/hdf5_savers/boarding_school_saver.py

def load_boarding_schools_from_hdf5(
    file_path: str, chunk_size=50000, domain_super_areas=None, config_filename=None
):
    """Loads boarding schools from an hdf5 file located at ``file_path``.
    Note that this object will not be ready to use, as the links to
    object instances of other classes need to be restored first.
    This function should be rarely be called oustide world.py

    Args:
        file_path (str): 
        chunk_size: (Default value = 50000)
        domain_super_areas: (Default value = None)
        config_filename: (Default value = None)

    """
    BoardingSchool_Class = BoardingSchool
    disease_config = GlobalContext.get_disease_config()
    BoardingSchool_Class.subgroup_params = SubgroupParams.from_disease_config(disease_config)

    with h5py.File(file_path, "r", libver="latest", swmr=True) as f:
        boarding_schools = f["boarding_schools"]
        boarding_schools_list = []
        n_boarding_schools = boarding_schools.attrs["n_boarding_schools"]
        n_chunks = int(np.ceil(n_boarding_schools / chunk_size))

        # Check if registered members data exists
        has_subgroup_data = "registered_members_subgroups" in boarding_schools
        subgroup_counts = {}
        subgroup_members = {}
        subgroup_cumulative_counts = {}

        if has_subgroup_data:
            # Get all subgroups
            subgroups = read_dataset(boarding_schools["registered_members_subgroups"], 0, boarding_schools["registered_members_subgroups"].shape[0])

            # For each subgroup, prepare the count and flattened arrays
            for subgroup_id in subgroups:
                sg_key = f"registered_members_count_sg{subgroup_id}"
                ids_key = f"registered_members_ids_sg{subgroup_id}"

                if sg_key in boarding_schools and ids_key in boarding_schools:
                    # Read counts for this subgroup
                    counts = read_dataset(boarding_schools[sg_key], 0, n_boarding_schools)
                    subgroup_counts[subgroup_id] = counts

                    # Calculate cumulative counts for this subgroup
                    cumulative = np.concatenate(([0], np.cumsum(counts)))
                    subgroup_cumulative_counts[subgroup_id] = cumulative

                    # Read flattened member IDs for this subgroup
                    if boarding_schools[ids_key].shape[0] > 0:
                        member_ids = read_dataset(boarding_schools[ids_key], 0, boarding_schools[ids_key].shape[0])
                        subgroup_members[subgroup_id] = member_ids
                    else:
                        subgroup_members[subgroup_id] = np.array([])

        for chunk in range(n_chunks):
            idx1 = chunk * chunk_size
            idx2 = min((chunk + 1) * chunk_size, n_boarding_schools)
            ids = read_dataset(boarding_schools["id"], idx1, idx2)
            super_areas = read_dataset(boarding_schools["super_area"], idx1, idx2)

            # Read age proportions
            age_prop_0_15 = read_dataset(boarding_schools["age_prop_0_15"], idx1, idx2)
            age_prop_16_24 = read_dataset(boarding_schools["age_prop_16_24"], idx1, idx2)
            age_prop_25_34 = read_dataset(boarding_schools["age_prop_25_34"], idx1, idx2)
            age_prop_35_49 = read_dataset(boarding_schools["age_prop_35_49"], idx1, idx2)
            age_prop_50_64 = read_dataset(boarding_schools["age_prop_50_64"], idx1, idx2)
            age_prop_65_99 = read_dataset(boarding_schools["age_prop_65_99"], idx1, idx2)

            # Read target allocations
            target_n_total = read_dataset(boarding_schools["target_n_total"], idx1, idx2)
            target_n_0_15 = read_dataset(boarding_schools["target_n_0_15"], idx1, idx2)
            target_n_16_24 = read_dataset(boarding_schools["target_n_16_24"], idx1, idx2)
            target_n_25_34 = read_dataset(boarding_schools["target_n_25_34"], idx1, idx2)
            target_n_35_49 = read_dataset(boarding_schools["target_n_35_49"], idx1, idx2)
            target_n_50_64 = read_dataset(boarding_schools["target_n_50_64"], idx1, idx2)
            target_n_65_99 = read_dataset(boarding_schools["target_n_65_99"], idx1, idx2)

            for k in range(idx2 - idx1):
                if domain_super_areas is not None:
                    super_area = super_areas[k]
                    if super_area == nan_integer:
                        raise ValueError(
                            "if ``domain_super_areas`` is True, I expect not Nones super areas."
                        )
                    if super_area not in domain_super_areas:
                        continue

                # Get registered_members_ids for this boarding school if available
                registered_members_dict = {}

                if has_subgroup_data and subgroups.size > 0:
                    boarding_school_index = idx1 + k

                    # Get members for each subgroup
                    for subgroup_id in subgroups:
                        if subgroup_id in subgroup_counts and subgroup_id in subgroup_members:
                            n_members = subgroup_counts[subgroup_id][boarding_school_index]

                            if n_members > 0 and len(subgroup_members[subgroup_id]) > 0:
                                start_idx = subgroup_cumulative_counts[subgroup_id][boarding_school_index]
                                end_idx = subgroup_cumulative_counts[subgroup_id][boarding_school_index + 1]

                                if start_idx < len(subgroup_members[subgroup_id]):
                                    registered_members_dict[int(subgroup_id)] = subgroup_members[subgroup_id][start_idx:end_idx].tolist()

                # Construct age proportions dict
                age_proportions = {
                    'prop_0_15': float(age_prop_0_15[k]),
                    'prop_16_24': float(age_prop_16_24[k]),
                    'prop_25_34': float(age_prop_25_34[k]),
                    'prop_35_49': float(age_prop_35_49[k]),
                    'prop_50_64': float(age_prop_50_64[k]),
                    'prop_65_99': float(age_prop_65_99[k])
                }

                # Construct target allocations dict
                target_allocations = {
                    'n_total': int(target_n_total[k]),
                    'n_0_15': int(target_n_0_15[k]),
                    'n_16_24': int(target_n_16_24[k]),
                    'n_25_34': int(target_n_25_34[k]),
                    'n_35_49': int(target_n_35_49[k]),
                    'n_50_64': int(target_n_50_64[k]),
                    'n_65_99': int(target_n_65_99[k])
                }

                boarding_school = BoardingSchool_Class(
                    area=None,
                    registered_members_ids=registered_members_dict,
                    age_proportions=age_proportions,
                    target_allocations=target_allocations
                )
                boarding_school.id = ids[k]
                boarding_schools_list.append(boarding_school)
    return BoardingSchools(boarding_schools_list)

restore_boarding_schools_properties_from_hdf5(world, file_path, chunk_size=50000, domain_super_areas=None)

Loads boarding schools from an hdf5 file located at file_path. Note that this object will not be ready to use, as the links to object instances of other classes need to be restored first. This function should be rarely be called oustide world.py

Parameters:

Name	Type	Description	Default
world	World		required
file_path	str		required
chunk_size		(Default value = 50000)	50000
domain_super_areas		(Default value = None)	None

Source code in june/hdf5_savers/boarding_school_saver.py

def restore_boarding_schools_properties_from_hdf5(
    world: World, file_path: str, chunk_size=50000, domain_super_areas=None
):
    """Loads boarding schools from an hdf5 file located at ``file_path``.
    Note that this object will not be ready to use, as the links to
    object instances of other classes need to be restored first.
    This function should be rarely be called oustide world.py

    Args:
        world (World): 
        file_path (str): 
        chunk_size: (Default value = 50000)
        domain_super_areas: (Default value = None)

    """
    with h5py.File(file_path, "r", libver="latest", swmr=True) as f:
        boarding_schools = f["boarding_schools"]
        n_boarding_schools = boarding_schools.attrs["n_boarding_schools"]
        n_chunks = int(np.ceil(n_boarding_schools / chunk_size))

        # Check if registered members data exists
        has_subgroup_data = "registered_members_subgroups" in boarding_schools
        subgroup_counts = {}
        subgroup_members = {}
        subgroup_cumulative_counts = {}

        if has_subgroup_data:
            # Get all subgroups
            subgroups = read_dataset(boarding_schools["registered_members_subgroups"], 0, boarding_schools["registered_members_subgroups"].shape[0])

            # For each subgroup, prepare the count and flattened arrays
            for subgroup_id in subgroups:
                sg_key = f"registered_members_count_sg{subgroup_id}"
                ids_key = f"registered_members_ids_sg{subgroup_id}"

                if sg_key in boarding_schools and ids_key in boarding_schools:
                    # Read counts for this subgroup
                    counts = read_dataset(boarding_schools[sg_key], 0, n_boarding_schools)
                    subgroup_counts[subgroup_id] = counts

                    # Calculate cumulative counts for this subgroup
                    cumulative = np.concatenate(([0], np.cumsum(counts)))
                    subgroup_cumulative_counts[subgroup_id] = cumulative

                    # Read flattened member IDs for this subgroup
                    if boarding_schools[ids_key].shape[0] > 0:
                        member_ids = read_dataset(boarding_schools[ids_key], 0, boarding_schools[ids_key].shape[0])
                        subgroup_members[subgroup_id] = member_ids
                    else:
                        subgroup_members[subgroup_id] = np.array([])

        for chunk in range(n_chunks):
            idx1 = chunk * chunk_size
            idx2 = min((chunk + 1) * chunk_size, n_boarding_schools)
            ids = boarding_schools["id"][idx1:idx2]
            areas = boarding_schools["area"][idx1:idx2]
            super_areas = boarding_schools["super_area"][idx1:idx2]
            for k in range(idx2 - idx1):
                if domain_super_areas is not None:
                    super_area = super_areas[k]
                    if super_area == nan_integer:
                        raise ValueError(
                            "if ``domain_super_areas`` is True, I expect not Nones super areas."
                        )
                    if super_area not in domain_super_areas:
                        continue
                boarding_school = world.boarding_schools.get_from_id(ids[k])
                if areas[k] == nan_integer:
                    area = None
                else:
                    area = world.areas.get_from_id(areas[k])
                boarding_school.area = area
                if area is not None:
                    area.boarding_schools.append(boarding_school)

                # Restore registered_members_ids if available
                if has_subgroup_data and subgroups.size > 0:
                    boarding_school_index = idx1 + k
                    registered_members_dict = {}

                    # Process each subgroup
                    for subgroup_id in subgroups:
                        if subgroup_id in subgroup_counts and subgroup_id in subgroup_members:
                            n_members = subgroup_counts[subgroup_id][boarding_school_index]

                            if n_members > 0 and len(subgroup_members[subgroup_id]) > 0:
                                start_idx = subgroup_cumulative_counts[subgroup_id][boarding_school_index]
                                end_idx = subgroup_cumulative_counts[subgroup_id][boarding_school_index + 1]

                                if start_idx < len(subgroup_members[subgroup_id]):
                                    registered_members_dict[int(subgroup_id)] = subgroup_members[subgroup_id][start_idx:end_idx].tolist()

                    # Update the registered_members_ids with the data loaded from file
                    boarding_school.registered_members_ids = registered_members_dict

save_boarding_schools_to_hdf5(boarding_schools, file_path, chunk_size=50000)

Saves the boarding_schools object to hdf5 format file file_path. Currently for each boarding school, the following values are stored: - id, area, super_area, registered_members_ids, age_proportions, target_allocations

Parameters:

Name	Type	Description	Default
boarding_schools	BoardingSchools	BoardingSchools object containing boarding school instances	required
file_path	str	path of the saved hdf5 file	required
chunk_size	int	number of boarding schools to save at a time. Note that they have to be copied to be saved,	50000

so keep the number below 1e6. (Default value = 50000)

Source code in june/hdf5_savers/boarding_school_saver.py

def save_boarding_schools_to_hdf5(
    boarding_schools: BoardingSchools, file_path: str, chunk_size: int = 50000
):
    """Saves the boarding_schools object to hdf5 format file ``file_path``. Currently for each boarding school,
    the following values are stored:
    - id, area, super_area, registered_members_ids, age_proportions, target_allocations

    Args:
        boarding_schools (BoardingSchools): BoardingSchools object containing boarding school instances
        file_path (str): path of the saved hdf5 file
        chunk_size (int, optional): number of boarding schools to save at a time. Note that they have to be copied to be saved,
    so keep the number below 1e6. (Default value = 50000)

    """
    n_boarding_schools = len(boarding_schools)
    n_chunks = int(np.ceil(n_boarding_schools / chunk_size))
    with h5py.File(file_path, "a") as f:
        boarding_schools_dset = f.create_group("boarding_schools")
        for chunk in range(n_chunks):
            idx1 = chunk * chunk_size
            idx2 = min((chunk + 1) * chunk_size, n_boarding_schools)
            ids = []
            areas = []
            super_areas = []
            registered_members_ids = []  # Add for storing registered_members_ids

            # Age proportions
            age_prop_0_15 = []
            age_prop_16_24 = []
            age_prop_25_34 = []
            age_prop_35_49 = []
            age_prop_50_64 = []
            age_prop_65_99 = []

            # Target allocations
            target_n_total = []
            target_n_0_15 = []
            target_n_16_24 = []
            target_n_25_34 = []
            target_n_35_49 = []
            target_n_50_64 = []
            target_n_65_99 = []

            for boarding_school in boarding_schools[idx1:idx2]:
                ids.append(boarding_school.id)
                if boarding_school.area is None:
                    areas.append(nan_integer)
                    super_areas.append(nan_integer)
                else:
                    areas.append(boarding_school.area.id)
                    super_areas.append(boarding_school.super_area.id)

                # Process registered_members_ids
                boarding_school_registered_members = {}
                if hasattr(boarding_school, 'registered_members_ids') and boarding_school.registered_members_ids is not None:
                    for subgroup_id, members in boarding_school.registered_members_ids.items():
                        boarding_school_registered_members[subgroup_id] = np.array(members, dtype=np.int64)
                registered_members_ids.append(boarding_school_registered_members)

                # Store age proportions
                age_proportions = boarding_school.age_proportions
                age_prop_0_15.append(age_proportions.get('prop_0_15', 0.0))
                age_prop_16_24.append(age_proportions.get('prop_16_24', 0.0))
                age_prop_25_34.append(age_proportions.get('prop_25_34', 0.0))
                age_prop_35_49.append(age_proportions.get('prop_35_49', 0.0))
                age_prop_50_64.append(age_proportions.get('prop_50_64', 0.0))
                age_prop_65_99.append(age_proportions.get('prop_65_99', 0.0))

                # Store target allocations
                target_allocations = boarding_school.target_allocations
                target_n_total.append(target_allocations.get('n_total', 0))
                target_n_0_15.append(target_allocations.get('n_0_15', 0))
                target_n_16_24.append(target_allocations.get('n_16_24', 0))
                target_n_25_34.append(target_allocations.get('n_25_34', 0))
                target_n_35_49.append(target_allocations.get('n_35_49', 0))
                target_n_50_64.append(target_allocations.get('n_50_64', 0))
                target_n_65_99.append(target_allocations.get('n_65_99', 0))

            ids = np.array(ids, dtype=np.int64)
            areas = np.array(areas, dtype=np.int64)
            super_areas = np.array(super_areas, dtype=np.int64)

            # Convert age proportions to numpy arrays
            age_prop_0_15 = np.array(age_prop_0_15, dtype=np.float64)
            age_prop_16_24 = np.array(age_prop_16_24, dtype=np.float64)
            age_prop_25_34 = np.array(age_prop_25_34, dtype=np.float64)
            age_prop_35_49 = np.array(age_prop_35_49, dtype=np.float64)
            age_prop_50_64 = np.array(age_prop_50_64, dtype=np.float64)
            age_prop_65_99 = np.array(age_prop_65_99, dtype=np.float64)

            # Convert target allocations to numpy arrays
            target_n_total = np.array(target_n_total, dtype=np.int64)
            target_n_0_15 = np.array(target_n_0_15, dtype=np.int64)
            target_n_16_24 = np.array(target_n_16_24, dtype=np.int64)
            target_n_25_34 = np.array(target_n_25_34, dtype=np.int64)
            target_n_35_49 = np.array(target_n_35_49, dtype=np.int64)
            target_n_50_64 = np.array(target_n_50_64, dtype=np.int64)
            target_n_65_99 = np.array(target_n_65_99, dtype=np.int64)

            # Create datasets to store registered_members_ids
            # First, determine what subgroups exist across all boarding schools
            all_subgroups = set()
            for bs_subgroups in registered_members_ids:
                all_subgroups.update(bs_subgroups.keys())
            all_subgroups = sorted(list(all_subgroups))  # Ensure consistent ordering

            # Store subgroup IDs as integers
            subgroup_ids = np.array(all_subgroups, dtype=np.int64) if all_subgroups else np.array([], dtype=np.int64)

            # Create counts and flattened arrays for each subgroup
            subgroup_counts = {}
            flattened_subgroup_members = {}

            for subgroup_id in all_subgroups:
                # Count of members in this subgroup for each boarding school
                counts = np.array([len(bs_dict.get(subgroup_id, [])) for bs_dict in registered_members_ids], dtype=np.int64)
                subgroup_counts[subgroup_id] = counts

                # Flatten all member IDs for this subgroup
                member_arrays = [bs_dict.get(subgroup_id, np.array([], dtype=np.int64)) for bs_dict in registered_members_ids]
                flattened = np.concatenate(member_arrays) if any(len(arr) > 0 for arr in member_arrays) else np.array([], dtype=np.int64)
                flattened_subgroup_members[subgroup_id] = flattened

            if chunk == 0:
                boarding_schools_dset.attrs["n_boarding_schools"] = n_boarding_schools
                boarding_schools_dset.create_dataset("id", data=ids, maxshape=(None,))
                boarding_schools_dset.create_dataset("area", data=areas, maxshape=(None,))
                boarding_schools_dset.create_dataset(
                    "super_area", data=super_areas, maxshape=(None,)
                )

                # Create age proportions datasets
                boarding_schools_dset.create_dataset("age_prop_0_15", data=age_prop_0_15, maxshape=(None,))
                boarding_schools_dset.create_dataset("age_prop_16_24", data=age_prop_16_24, maxshape=(None,))
                boarding_schools_dset.create_dataset("age_prop_25_34", data=age_prop_25_34, maxshape=(None,))
                boarding_schools_dset.create_dataset("age_prop_35_49", data=age_prop_35_49, maxshape=(None,))
                boarding_schools_dset.create_dataset("age_prop_50_64", data=age_prop_50_64, maxshape=(None,))
                boarding_schools_dset.create_dataset("age_prop_65_99", data=age_prop_65_99, maxshape=(None,))

                # Create target allocations datasets
                boarding_schools_dset.create_dataset("target_n_total", data=target_n_total, maxshape=(None,))
                boarding_schools_dset.create_dataset("target_n_0_15", data=target_n_0_15, maxshape=(None,))
                boarding_schools_dset.create_dataset("target_n_16_24", data=target_n_16_24, maxshape=(None,))
                boarding_schools_dset.create_dataset("target_n_25_34", data=target_n_25_34, maxshape=(None,))
                boarding_schools_dset.create_dataset("target_n_35_49", data=target_n_35_49, maxshape=(None,))
                boarding_schools_dset.create_dataset("target_n_50_64", data=target_n_50_64, maxshape=(None,))
                boarding_schools_dset.create_dataset("target_n_65_99", data=target_n_65_99, maxshape=(None,))

                # Store registered_members_ids subgroups information
                if all_subgroups:  # Only create these datasets if we have subgroups
                    boarding_schools_dset.create_dataset("registered_members_subgroups", data=subgroup_ids, maxshape=(None,))

                    # Create datasets for each subgroup
                    for subgroup_id in all_subgroups:
                        # Store counts for this subgroup
                        boarding_schools_dset.create_dataset(
                            f"registered_members_count_sg{subgroup_id}", 
                            data=subgroup_counts[subgroup_id], 
                            maxshape=(None,)
                        )

                        # Store flattened members for this subgroup
                        flattened_members = flattened_subgroup_members[subgroup_id]
                        boarding_schools_dset.create_dataset(
                            f"registered_members_ids_sg{subgroup_id}", 
                            data=flattened_members, 
                            maxshape=(None,)
                        )
            else:
                newshape = (boarding_schools_dset["id"].shape[0] + ids.shape[0],)
                boarding_schools_dset["id"].resize(newshape)
                boarding_schools_dset["id"][idx1:idx2] = ids
                boarding_schools_dset["area"].resize(newshape)
                boarding_schools_dset["area"][idx1:idx2] = areas
                boarding_schools_dset["super_area"].resize(newshape)
                boarding_schools_dset["super_area"][idx1:idx2] = super_areas

                # Update age proportions datasets
                boarding_schools_dset["age_prop_0_15"].resize(newshape)
                boarding_schools_dset["age_prop_0_15"][idx1:idx2] = age_prop_0_15
                boarding_schools_dset["age_prop_16_24"].resize(newshape)
                boarding_schools_dset["age_prop_16_24"][idx1:idx2] = age_prop_16_24
                boarding_schools_dset["age_prop_25_34"].resize(newshape)
                boarding_schools_dset["age_prop_25_34"][idx1:idx2] = age_prop_25_34
                boarding_schools_dset["age_prop_35_49"].resize(newshape)
                boarding_schools_dset["age_prop_35_49"][idx1:idx2] = age_prop_35_49
                boarding_schools_dset["age_prop_50_64"].resize(newshape)
                boarding_schools_dset["age_prop_50_64"][idx1:idx2] = age_prop_50_64
                boarding_schools_dset["age_prop_65_99"].resize(newshape)
                boarding_schools_dset["age_prop_65_99"][idx1:idx2] = age_prop_65_99

                # Update target allocations datasets
                boarding_schools_dset["target_n_total"].resize(newshape)
                boarding_schools_dset["target_n_total"][idx1:idx2] = target_n_total
                boarding_schools_dset["target_n_0_15"].resize(newshape)
                boarding_schools_dset["target_n_0_15"][idx1:idx2] = target_n_0_15
                boarding_schools_dset["target_n_16_24"].resize(newshape)
                boarding_schools_dset["target_n_16_24"][idx1:idx2] = target_n_16_24
                boarding_schools_dset["target_n_25_34"].resize(newshape)
                boarding_schools_dset["target_n_25_34"][idx1:idx2] = target_n_25_34
                boarding_schools_dset["target_n_35_49"].resize(newshape)
                boarding_schools_dset["target_n_35_49"][idx1:idx2] = target_n_35_49
                boarding_schools_dset["target_n_50_64"].resize(newshape)
                boarding_schools_dset["target_n_50_64"][idx1:idx2] = target_n_50_64
                boarding_schools_dset["target_n_65_99"].resize(newshape)
                boarding_schools_dset["target_n_65_99"][idx1:idx2] = target_n_65_99

                # Update registered members for subgroups
                if all_subgroups:  # Only update these datasets if we have subgroups
                    for subgroup_id in all_subgroups:
                        # Update counts for this subgroup
                        count_dataset = boarding_schools_dset[f"registered_members_count_sg{subgroup_id}"]
                        count_dataset.resize(newshape)
                        count_dataset[idx1:idx2] = subgroup_counts[subgroup_id]

                        # Update flattened IDs for this subgroup (variable length)
                        ids_dataset = boarding_schools_dset[f"registered_members_ids_sg{subgroup_id}"]
                        flattened_members = flattened_subgroup_members[subgroup_id]

                        if flattened_members.shape[0] > 0:
                            current_length = ids_dataset.shape[0]
                            new_length = current_length + flattened_members.shape[0]
                            ids_dataset.resize((new_length,))
                            ids_dataset[current_length:new_length] = flattened_members