From dd5f684d9f05090cc9a719987e2f78abf12ff850 Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Thu, 16 Jan 2025 18:31:46 -0600 Subject: [PATCH 01/12] Add integration of shared chunk cache (H5SC) layer with top layer of library. --- src/H5CX.c | 69 +++++++++++- src/H5CXprivate.h | 2 + src/H5Dchunk.c | 95 +++------------- src/H5Dint.c | 23 +++- src/H5Dio.c | 219 ++++++++++++++++++++++-------------- src/H5Dpkg.h | 18 ++- src/H5Dprivate.h | 7 -- src/H5Fint.c | 13 +++ src/H5Fpkg.h | 3 + src/H5Fprivate.h | 3 + src/H5Fquery.c | 20 ++++ src/H5Oprivate.h | 7 +- src/H5Pdcpl.c | 12 +- src/H5Ppublic.h | 6 +- src/H5SC.c | 24 ++-- src/H5SCprivate.h | 115 +++++++++++-------- src/H5VLnative_dataset.c | 94 ++++++++-------- test/struct_chunk_storage.c | 6 + 18 files changed, 448 insertions(+), 288 deletions(-) diff --git a/src/H5CX.c b/src/H5CX.c index c6a43801d0b..50b90e3ff0f 100644 --- a/src/H5CX.c +++ b/src/H5CX.c @@ -2900,6 +2900,40 @@ H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t mpio_actual_chun FUNC_LEAVE_NOAPI_VOID } /* end H5CX_set_mpio_actual_chunk_opt() */ +/*------------------------------------------------------------------------- + * Function: H5CX_or_mpio_actual_chunk_opt + * + * Purpose: Performs a bitwise "or" operation on the the actual chunk + * optimization used for parallel I/O for the current API + * call context. + * + * Return: + * + *------------------------------------------------------------------------- + */ +void +H5CX_or_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t mpio_actual_chunk_opt) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + assert(head && *head); + assert(!((*head)->ctx.dxpl_id == H5P_DEFAULT || (*head)->ctx.dxpl_id == H5P_DATASET_XFER_DEFAULT)); + + /* If the value is already set, "or" it with the requested value, othewise simply set it */ + if ((*head)->ctx.mpio_actual_chunk_opt_set) + (*head)->ctx.mpio_actual_chunk_opt |= mpio_actual_chunk_opt; + else { + (*head)->ctx.mpio_actual_chunk_opt = mpio_actual_chunk_opt; + (*head)->ctx.mpio_actual_chunk_opt_set = true; + } + + FUNC_LEAVE_NOAPI_VOID +} /* end H5CX_or_mpio_actual_chunk_opt() */ + /*------------------------------------------------------------------------- * Function: H5CX_set_mpio_actual_io_mode * @@ -2926,7 +2960,40 @@ H5CX_set_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t mpio_actual_io_mode) (*head)->ctx.mpio_actual_io_mode_set = true; FUNC_LEAVE_NOAPI_VOID -} /* end H5CX_set_mpio_actual_chunk_opt() */ +} /* end H5CX_set_mpio_actual_io_mode() */ + +/*------------------------------------------------------------------------- + * Function: H5CX_or_mpio_actual_io_mode + * + * Purpose: Performs a bitwise "or" operation on the actual I/O mode + * used for parallel I/O for the current API call context. + * + * Return: + * + *------------------------------------------------------------------------- + */ +void +H5CX_or_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t mpio_actual_io_mode) +{ + H5CX_node_t **head = NULL; /* Pointer to head of API context list */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + assert(head && *head); + assert(!((*head)->ctx.dxpl_id == H5P_DEFAULT || (*head)->ctx.dxpl_id == H5P_DATASET_XFER_DEFAULT)); + + /* If the value is already set, "or" it with the requested value, othewise simply set it */ + if ((*head)->ctx.mpio_actual_io_mode_set) + (*head)->ctx.mpio_actual_io_mode |= mpio_actual_io_mode; + else { + (*head)->ctx.mpio_actual_io_mode = mpio_actual_io_mode; + (*head)->ctx.mpio_actual_io_mode_set = true; + } + + FUNC_LEAVE_NOAPI_VOID +} /* end H5CX_or_mpio_actual_io_mode() */ /*------------------------------------------------------------------------- * Function: H5CX_set_mpio_local_no_coll_cause diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index ad61a1a430e..63897dec2f3 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -395,7 +395,9 @@ H5_DLL void H5CX_set_actual_selection_io_mode(uint32_t actual_selection_io_mode) #ifdef H5_HAVE_PARALLEL H5_DLL void H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chunk_opt); +H5_DLL void H5CX_or_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chunk_opt); H5_DLL void H5CX_set_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t actual_io_mode); +H5_DLL void H5CX_or_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t actual_io_mode); H5_DLL void H5CX_set_mpio_local_no_coll_cause(uint32_t mpio_local_no_coll_cause); H5_DLL void H5CX_set_mpio_global_no_coll_cause(uint32_t mpio_global_no_coll_cause); #ifdef H5_HAVE_INSTRUMENTED_LIBRARY diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index a9b4817d4db..111f2e4c537 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -317,6 +317,7 @@ static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_ const void *fill_buf, const void *partial_chunk_fill_buf); static int H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2); #endif /* H5_HAVE_PARALLEL */ +static herr_t H5D__chunk_verify_offset(const H5D_t *dset, const hsize_t *offset); /* Debugging helper routine callback */ static int H5D__chunk_dump_index_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); @@ -378,7 +379,7 @@ H5FL_EXTERN(H5S_sel_iter_t); *------------------------------------------------------------------------- */ herr_t -H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, uint32_t data_size, const void *buf) +H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, const hsize_t *offset, uint32_t data_size, const void *buf) { const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ H5D_chunk_ud_t udata; /* User data for querying chunk info */ @@ -393,6 +394,11 @@ H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, uint32_t /* Sanity checks */ assert(layout->type == H5D_CHUNKED); + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). */ + if (H5D__chunk_verify_offset(dset, offset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + /* Allocate dataspace and initialize it if it hasn't been. */ if (!H5D__chunk_is_space_alloc(&layout->storage)) if (H5D__alloc_storage(dset, H5D_ALLOC_WRITE, false, NULL) < 0) @@ -483,7 +489,7 @@ H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, uint32_t *------------------------------------------------------------------------- */ herr_t -H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t *filters, void *buf) +H5D__chunk_direct_read(const H5D_t *dset, const hsize_t *offset, uint32_t *filters, void *buf) { const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */ @@ -501,6 +507,11 @@ H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t *filters, vo *filters = 0; + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). */ + if (H5D__chunk_verify_offset(dset, offset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + /* Allocate dataspace and initialize it if it hasn't been. */ if (!H5D__chunk_is_space_alloc(&layout->storage) && !H5D__chunk_is_data_cached(dset->shared)) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "storage is not initialized"); @@ -8222,20 +8233,16 @@ H5D__chunk_iter(H5D_t *dset, H5D_chunk_iter_op_t op, void *op_data) } /* end H5D__chunk_iter() */ /*------------------------------------------------------------------------- - * Function: H5D__chunk_get_offset_copy + * Function: H5D__chunk_verify_offset * - * Purpose: Copies an offset buffer and performs bounds checks on the - * values. - * - * This helper function ensures that the offset buffer given - * by the user is suitable for use with the rest of the library. + * Purpose: Performs bounds checks on the provided chunk offset values. * * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ -herr_t -H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *offset_copy) +static herr_t +H5D__chunk_verify_offset(const H5D_t *dset, const hsize_t *offset) { unsigned u; herr_t ret_value = SUCCEED; /* Return value */ @@ -8244,13 +8251,6 @@ H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *of assert(dset); assert(offset); - assert(offset_copy); - - /* The library's chunking code requires the offset to terminate with a zero. - * So transfer the offset array to an internal offset array that we - * can properly terminate (handled via the memset call). - */ - memset(offset_copy, 0, H5O_LAYOUT_NDIMS * sizeof(hsize_t)); for (u = 0; u < dset->shared->ndims; u++) { /* Make sure the offset doesn't exceed the dataset's dimensions */ @@ -8260,70 +8260,11 @@ H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *of /* Make sure the offset fall right on a chunk's boundary */ if (offset[u] % dset->shared->layout.u.chunk.dim[u]) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "offset doesn't fall on chunks's boundary"); - - offset_copy[u] = offset[u]; } done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__chunk_get_offset_copy() */ - -/*------------------------------------------------------------------------- - * Function: H5D__write_struct_chunk_direct - * - * Purpose: Internal routine to write a structured chunk directly into the file. - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -herr_t -H5D__write_struct_chunk_direct(H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, - H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE_NOERR - - /* Sanity checks */ - /* TBD: check for H5D_SPARSE_CHUNK */ - /* assert(layout->type == H5D_SPARSE_CHUNK); */ - - /* TBD: set up and call routine to write the structured chunk */ - /* FOR NOW: just return success */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__write_struct_chunk_direct() */ - -/*------------------------------------------------------------------------- - * Function: H5D__read_struct_chunk_direct - * - * Purpose: Internal routine to read a structured chunk directly from the file. - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -herr_t -H5D__read_struct_chunk_direct(const H5D_t H5_ATTR_UNUSED *dset, hsize_t H5_ATTR_UNUSED *offset, - H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, void H5_ATTR_UNUSED *buf[]) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE_NOERR - - /* Check args */ - /* TBD: check for H5D_SPARSE_CHUNK */ - /* assert(dset && H5D_SPARSE_CHUNK == layout->type); */ - assert(offset); - assert(chunk_info); - assert(buf); - - /* TBD: set up and call routine to read the structured chunk */ - /* FOR NOW: just return success */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__read_struct_chunk_direct() */ +} /* end H5D__chunk_verify_offset() */ /*------------------------------------------------------------------------- * Function: H5D__get_struct_chunk_info diff --git a/src/H5Dint.c b/src/H5Dint.c index c064e2b8d5a..862e6e8522d 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -29,6 +29,7 @@ #include "H5Iprivate.h" /* IDs */ #include "H5Lprivate.h" /* Links */ #include "H5MMprivate.h" /* Memory management */ +#include "H5SCprivate.h" /* Shared chunk cache */ #include "H5SLprivate.h" /* Skip Lists */ #include "H5VLprivate.h" /* Virtual Object Layer */ #include "H5VMprivate.h" /* Vector Functions */ @@ -2034,6 +2035,10 @@ H5D_close(H5D_t *dataset) #endif } /* end switch */ + /* Evict the dataset's entries in the shared chunk cache */ + if (dataset->shared->layout.sc_ops && H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, true) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to evict dataset's entries in shared chunk cache"); + /* Destroy any cached layout information for the dataset */ if (dataset->shared->layout.ops->dest && (dataset->shared->layout.ops->dest)(dataset) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to destroy layout info"); @@ -3096,7 +3101,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) expand = true; /* Chunked storage specific checks */ - if (H5D_CHUNKED == dset->shared->layout.type && dset->shared->ndims > 1) { + if (!dset->shared->layout.sc_ops && H5D_CHUNKED == dset->shared->layout.type && dset->shared->ndims > 1) { hsize_t scaled; /* Scaled value */ /* Compute the scaled dimension size value */ @@ -3146,7 +3151,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) *------------------------------------------------------------------------- */ /* Update the index values for the cached chunks for this dataset */ - if (H5D_CHUNKED == dset->shared->layout.type) { + if (!dset->shared->layout.sc_ops && H5D_CHUNKED == dset->shared->layout.type) { /* Set the cached chunk info */ if (H5D__chunk_set_info(dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to update # of chunks"); @@ -3197,7 +3202,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) * and if the chunks are written *------------------------------------------------------------------------- */ - if (H5D_CHUNKED == dset->shared->layout.type) { + if (!dset->shared->layout.sc_ops && H5D_CHUNKED == dset->shared->layout.type) { if (shrink && ((*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage) || (dset->shared->layout.ops->is_data_cached && (*dset->shared->layout.ops->is_data_cached)(dset->shared)))) @@ -3214,6 +3219,11 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to do update old edge chunks"); } /* end if */ + /* Notify the shared chunk cache that the extent has changed */ + if (dset->shared->layout.sc_ops) + if (H5SC_set_extent_notify(H5F_get_shared_cache(dset->oloc.file), dset, curr_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to notify shared chunk cache of extent change"); + /* Mark the dataspace as dirty, for later writing to the file */ if (H5D__mark(dset, H5D_MARK_SPACE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to mark dataspace as dirty"); @@ -3282,11 +3292,16 @@ H5D__flush_real(H5D_t *dataset) assert(dataset->shared); /* Avoid flushing the dataset (again) if it's closing */ - if (!dataset->shared->closing) + if (!dataset->shared->closing) { /* Flush cached raw data for each kind of dataset layout */ if (dataset->shared->layout.ops->flush && (dataset->shared->layout.ops->flush)(dataset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush raw data"); + /* Flush the dataset's entries in the shared chunk cache */ + if (dataset->shared->layout.sc_ops && H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, false) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush shared chunk cache"); + } + done: FUNC_LEAVE_NOAPI_TAG(ret_value) } /* end H5D__flush_real() */ diff --git a/src/H5Dio.c b/src/H5Dio.c index 436e9c95650..cb3646ace47 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -26,6 +26,7 @@ #include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5Sprivate.h" /* Dataspace */ +#include "H5SCprivate.h" /* Shared chunk cache */ /****************/ /* Local Macros */ @@ -96,6 +97,8 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) /* freed. */ H5D_storage_t store_local; /* Local buffer for store */ H5D_storage_t *store = &store_local; /* Union of EFL and chunk pointer in file space */ + bool any_scc = false; /* Whether any datasets support the shared chunk cache */ + bool any_nonscc = false; /* Whether any datasets do not support the shared chunk cache */ size_t io_op_init = 0; /* Number I/O ops that have been initialized */ size_t io_skipped = 0; /* Number I/O ops that have been skipped (due to the dataset not being allocated) */ @@ -286,10 +289,20 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) dset_info[i].skip_io = false; } - /* Call storage method's I/O initialization routine */ - if (dset_info[i].layout_ops.io_init && - (dset_info[i].layout_ops.io_init)(&io_info, &(dset_info[i])) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info"); + /* Check for shared chunk cache support */ + if (dset_info[i].dset->shared->layout.sc_ops) + /* Note that there is at least one dataset that supports shared chunk cache */ + any_scc = true; + else { + /* Call storage method's I/O initialization routine */ + if (dset_info[i].layout_ops.io_init && + (dset_info[i].layout_ops.io_init)(&io_info, &(dset_info[i])) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info"); + + /* Note that there is at least one dataset that does not support the shared chunk cache */ + any_nonscc = true; + } + io_op_init++; /* Reset metadata tagging */ @@ -302,22 +315,25 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) if (io_skipped == count) HGOTO_DONE(SUCCEED); - /* Perform second phase of type info initialization */ - if (H5D__typeinfo_init_phase2(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)"); + /* Initialize type info differently depending on if weŕe using the shared chunk cache or not */ + if (any_nonscc) { + /* Perform second phase of type info initialization */ + if (H5D__typeinfo_init_phase2(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)"); #ifdef H5_HAVE_PARALLEL - /* Adjust I/O info for any parallel or selection I/O */ - if (H5D__ioinfo_adjust(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to adjust I/O info for parallel or selection I/O"); + /* Adjust I/O info for any parallel or selection I/O */ + if (H5D__ioinfo_adjust(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to adjust I/O info for parallel or selection I/O"); #endif /* H5_HAVE_PARALLEL */ - /* Perform third phase of type info initialization */ - if (H5D__typeinfo_init_phase3(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)"); + /* Perform third phase of type info initialization */ + if (H5D__typeinfo_init_phase3(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)"); - H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); + H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); + } /* If multi dataset I/O callback is not provided, perform read IO via * single-dset path with looping */ @@ -386,26 +402,28 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) } /* Loop with serial & single-dset read IO path */ - for (i = 0; i < count; i++) { - /* Check for skipped I/O */ - if (dset_info[i].skip_io) - continue; + for (i = 0; i < count; i++) + /* Only call the legacy I/O code if weŕe not using the shared chunk cache */ + if (!dset_info[i].dset->shared->layout.sc_ops) { + /* Check for skipped I/O */ + if (dset_info[i].skip_io) + continue; + + /* Set metadata tagging with dset object header addr */ + H5AC_tag(dset_info[i].dset->oloc.addr, &prev_tag); - /* Set metadata tagging with dset object header addr */ - H5AC_tag(dset_info[i].dset->oloc.addr, &prev_tag); + /* Invoke correct "high level" I/O routine */ + if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0) { + /* Reset metadata tagging */ + H5AC_tag(prev_tag, NULL); + + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data"); + } - /* Invoke correct "high level" I/O routine */ - if ((*dset_info[i].io_ops.multi_read)(&io_info, &dset_info[i]) < 0) { /* Reset metadata tagging */ H5AC_tag(prev_tag, NULL); - - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data"); } - /* Reset metadata tagging */ - H5AC_tag(prev_tag, NULL); - } - /* Make final selection I/O call if the multi_read callbacks did not perform the actual I/O * (if using selection I/O and either multi dataset or type conversion) */ if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) { @@ -436,21 +454,28 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) /* Only report the collective I/O mode if we're actually performing collective I/O */ if (xfer_mode == H5FD_MPIO_COLLECTIVE) { - H5CX_set_mpio_actual_io_mode(io_info.actual_io_mode); + H5CX_or_mpio_actual_io_mode(io_info.actual_io_mode); /* If we did selection I/O, report that we used "link chunk" mode, since that's the most * analogous to what selection I/O does */ if (io_info.use_select_io == H5D_SELECTION_IO_MODE_ON) - H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); + H5CX_or_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); } } #endif /* H5_HAVE_PARALLEL */ } + /* Make shared chunk cache read call if appropriate */ + if (any_scc) { + assert(H5F_get_shared_cache(dset_info[0].dset->oloc.file)); + if (H5SC_read(H5F_get_shared_cache(dset_info[0].dset->oloc.file), count, dset_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "read through shared chunk cache failed"); + } + done: /* Shut down the I/O op information */ for (i = 0; i < io_op_init; i++) - if (dset_info[i].layout_ops.io_term && + if (!dset_info[i].dset->shared->layout.sc_ops && dset_info[i].layout_ops.io_term && (*dset_info[i].layout_ops.io_term)(&io_info, &(dset_info[i])) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down I/O op info"); @@ -525,6 +550,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) /* freed. */ H5D_storage_t store_local; /* Local buffer for store */ H5D_storage_t *store = &store_local; /* Union of EFL and chunk pointer in file space */ + bool any_scc = false; /* Whether any datasets support the shared chunk cache */ + bool any_nonscc = false; /* Whether any datasets do not support the shared chunk cache */ size_t io_op_init = 0; /* Number I/O ops that have been initialized */ size_t i; /* Local index variable */ char fake_char; /* Temporary variable for NULL buffer pointers */ @@ -722,12 +749,24 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize storage"); } /* end if */ - /* Call storage method's I/O initialization routine */ - /* Init io_info.dset_info[] and generate piece_info in skip list */ - if (dset_info[i].layout_ops.io_init && - (*dset_info[i].layout_ops.io_init)(&io_info, &(dset_info[i])) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info"); + /* We don't skip I/O for write operations */ dset_info[i].skip_io = false; + + /* Check for shared chunk cache support */ + if (dset_info[i].dset->shared->layout.sc_ops) + /* Note that there is at least one dataset that supports shared chunk cache */ + any_scc = true; + else { + /* Call storage method's I/O initialization routine */ + /* Init io_info.dset_info[] and generate piece_info in skip list */ + if (dset_info[i].layout_ops.io_init && + (*dset_info[i].layout_ops.io_init)(&io_info, &(dset_info[i])) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info"); + + /* Note that there is at least one dataset that does not support the shared chunk cache */ + any_nonscc = true; + } + io_op_init++; /* Reset metadata tagging */ @@ -736,22 +775,25 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) assert(io_op_init == count); - /* Perform second phase of type info initialization */ - if (H5D__typeinfo_init_phase2(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)"); + /* Initialize type info differently depending on if weŕe using the shared chunk cache or not */ + if (any_nonscc) { + /* Perform second phase of type info initialization */ + if (H5D__typeinfo_init_phase2(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (second phase)"); #ifdef H5_HAVE_PARALLEL - /* Adjust I/O info for any parallel or selection I/O */ - if (H5D__ioinfo_adjust(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to adjust I/O info for parallel or selection I/O"); + /* Adjust I/O info for any parallel or selection I/O */ + if (H5D__ioinfo_adjust(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to adjust I/O info for parallel or selection I/O"); #endif /* H5_HAVE_PARALLEL */ - /* Perform third phase of type info initialization */ - if (H5D__typeinfo_init_phase3(&io_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)"); + /* Perform third phase of type info initialization */ + if (H5D__typeinfo_init_phase3(&io_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up type info (third phase)"); - H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); + H5CX_set_no_selection_io_cause(io_info.no_selection_io_cause); + } /* If multi dataset I/O callback is not provided, perform write IO via * single-dset path with looping */ @@ -818,15 +860,18 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) for (i = 0; i < count; i++) { assert(!dset_info[i].skip_io); - /* Set metadata tagging with dset oheader addr */ - H5AC_tag(dset_info->dset->oloc.addr, &prev_tag); + /* Only call the legacy I/O code if weŕe not using the shared chunk cache */ + if (!dset_info[i].dset->shared->layout.sc_ops) { + /* Set metadata tagging with dset oheader addr */ + H5AC_tag(dset_info->dset->oloc.addr, &prev_tag); - /* Invoke correct "high level" I/O routine */ - if ((*dset_info[i].io_ops.multi_write)(&io_info, &dset_info[i]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data"); + /* Invoke correct "high level" I/O routine */ + if ((*dset_info[i].io_ops.multi_write)(&io_info, &dset_info[i]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data"); - /* Reset metadata tagging */ - H5AC_tag(prev_tag, NULL); + /* Reset metadata tagging */ + H5AC_tag(prev_tag, NULL); + } } /* Make final selection I/O call if the multi_write callbacks did not perform the actual I/O @@ -859,22 +904,29 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) /* Only report the collective I/O mode if we're actually performing collective I/O */ if (xfer_mode == H5FD_MPIO_COLLECTIVE) { - H5CX_set_mpio_actual_io_mode(io_info.actual_io_mode); + H5CX_or_mpio_actual_io_mode(io_info.actual_io_mode); /* If we did selection I/O, report that we used "link chunk" mode, since that's the most * analogous to what selection I/O does */ if (io_info.use_select_io == H5D_SELECTION_IO_MODE_ON) - H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); + H5CX_or_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); } } #endif /* H5_HAVE_PARALLEL */ } + /* Make shared chunk cache write call if appropriate */ + if (any_scc) { + assert(H5F_get_shared_cache(dset_info[0].dset->oloc.file)); + if (H5SC_write(H5F_get_shared_cache(dset_info[0].dset->oloc.file), count, dset_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "write through shared chunk cache failed"); + } + done: /* Shut down the I/O op information */ for (i = 0; i < io_op_init; i++) { assert(!dset_info[i].skip_io); - if (dset_info[i].layout_ops.io_term && + if (!dset_info[i].dset->shared->layout.sc_ops && dset_info[i].layout_ops.io_term && (*dset_info[i].layout_ops.io_term)(&io_info, &(dset_info[i])) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to shut down I/O op info"); } @@ -1003,30 +1055,33 @@ H5D__dset_ioinfo_init(H5D_t *dset, H5D_dset_io_info_t *dset_info, H5D_storage_t dset_info->dset = dset; dset_info->store = store; - /* Set I/O operations to initial values */ - dset_info->layout_ops = *dset->shared->layout.ops; - - /* Set the "high-level" I/O operations for the dataset */ - dset_info->io_ops.multi_read = dset->shared->layout.ops->ser_read; - dset_info->io_ops.multi_write = dset->shared->layout.ops->ser_write; - - /* Set the I/O operations for reading/writing single blocks on disk */ - if (dset_info->type_info.is_xform_noop && dset_info->type_info.is_conv_noop) { - /* - * If there is no data transform or type conversion then read directly - * into the application's buffer. - * This saves at least one mem-to-mem copy. - */ - dset_info->io_ops.single_read = H5D__select_read; - dset_info->io_ops.single_write = H5D__select_write; - } /* end if */ - else { - /* - * This is the general case (type conversion, usually). - */ - dset_info->io_ops.single_read = H5D__scatgath_read; - dset_info->io_ops.single_write = H5D__scatgath_write; - } /* end else */ + /* Only set layout ops if weŕe not using the shared chunk cache */ + if (!dset->shared->layout.sc_ops) { + /* Set I/O operations to initial values */ + dset_info->layout_ops = *dset->shared->layout.ops; + + /* Set the "high-level" I/O operations for the dataset */ + dset_info->io_ops.multi_read = dset->shared->layout.ops->ser_read; + dset_info->io_ops.multi_write = dset->shared->layout.ops->ser_write; + + /* Set the I/O operations for reading/writing single blocks on disk */ + if (dset_info->type_info.is_xform_noop && dset_info->type_info.is_conv_noop) { + /* + * If there is no data transform or type conversion then read directly + * into the application's buffer. + * This saves at least one mem-to-mem copy. + */ + dset_info->io_ops.single_read = H5D__select_read; + dset_info->io_ops.single_write = H5D__select_write; + } /* end if */ + else { + /* + * This is the general case (type conversion, usually). + */ + dset_info->io_ops.single_read = H5D__scatgath_read; + dset_info->io_ops.single_write = H5D__scatgath_write; + } /* end else */ + } FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__dset_ioinfo_init() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 9d707dba695..8b63f197a80 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -115,6 +115,13 @@ /* Package Private Typedefs */ /****************************/ +/* Typedef for cached dataset creation property list information */ +typedef struct H5D_dcpl_cache_t { + H5O_fill_t fill; /* Fill value info (H5D_CRT_FILL_VALUE_NAME) */ + H5O_pline_t pline; /* I/O pipeline info (H5O_CRT_PIPELINE_NAME) */ + H5O_efl_t efl; /* External file list info (H5D_CRT_EXT_FILE_LIST_NAME) */ +} H5D_dcpl_cache_t; + /* Typedef for datatype information for a single dataset in a raw data I/O operation */ typedef struct H5D_type_info_t { /* Initial values */ @@ -488,6 +495,12 @@ typedef struct H5D_chunk_cached_t { unsigned filter_mask; /*excluded filters */ } H5D_chunk_cached_t; +/* Chunk specific information passed to direct chunk read */ +typedef struct H5D_chunk_scc_udata_t { + uint32_t filters; + uint64_t size; +} H5D_chunk_scc_udata_t; + /****************************/ /* Virtual dataset typedefs */ /****************************/ @@ -761,10 +774,9 @@ H5_DLL herr_t H5D__chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src, H5 H5_DLL herr_t H5D__chunk_bh_info(const H5O_loc_t *loc, H5O_t *oh, H5O_layout_t *layout, hsize_t *btree_size); H5_DLL herr_t H5D__chunk_dump_index(H5D_t *dset, FILE *stream); H5_DLL herr_t H5D__chunk_delete(H5F_t *f, H5O_t *oh, H5O_storage_t *store); -H5_DLL herr_t H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *offset_copy); -H5_DLL herr_t H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, uint32_t data_size, +H5_DLL herr_t H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, const hsize_t *offset, uint32_t data_size, const void *buf); -H5_DLL herr_t H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t *filters, void *buf); +H5_DLL herr_t H5D__chunk_direct_read(const H5D_t *dset, const hsize_t *offset, uint32_t *filters, void *buf); #ifdef H5D_CHUNK_DEBUG H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers); #endif /* H5D_CHUNK_DEBUG */ diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index a3257c99192..71d00a1a63a 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -142,13 +142,6 @@ typedef struct H5D_obj_create_t H5D_obj_create_t; typedef struct H5D_io_type_info_t H5D_io_type_info_t; typedef struct H5D_dset_io_info_t H5D_dset_io_info_t; -/* Typedef for cached dataset creation property list information */ -typedef struct H5D_dcpl_cache_t { - H5O_fill_t fill; /* Fill value info (H5D_CRT_FILL_VALUE_NAME) */ - H5O_pline_t pline; /* I/O pipeline info (H5O_CRT_PIPELINE_NAME) */ - H5O_efl_t efl; /* External file list info (H5D_CRT_EXT_FILE_LIST_NAME) */ -} H5D_dcpl_cache_t; - /* Callback information for copying datasets */ typedef struct H5D_copy_file_ud_t { H5O_copy_file_ud_common_t common; /* Shared information (must be first) */ diff --git a/src/H5Fint.c b/src/H5Fint.c index 9f2b41f572f..46e1e81e782 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -33,6 +33,7 @@ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5SCprivate.h" /* Shared chunk cache */ #include "H5SMprivate.h" /* Shared Object Header Messages */ #include "H5Tprivate.h" /* Datatypes */ #include "H5VLprivate.h" /* Virtual Object Layer */ @@ -1348,6 +1349,10 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F if (H5AC_create(f, &(f->shared->mdc_initCacheCfg), &(f->shared->mdc_initCacheImageCfg)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create metadata cache"); + /* Create a shaerd chunk cache */ + if (NULL == (f->shared->shared_cache = H5SC_create(f, plist))) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create shared chunk cache"); + /* Create the file's "open object" information */ if (H5FO_create(f) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create open object data structure"); @@ -1557,6 +1562,10 @@ H5F__dest(H5F_t *f, bool flush, bool free_on_failure) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file"); + /* Destroy shared chunk cache */ + if (H5SC_destroy(f->shared->shared_cache) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to destroy shared chunk cache"); + /* Shutdown the metadata cache */ /* (Flushes any remaining dirty entries, which should only be the * superblock and / or driver info at this point) @@ -2300,6 +2309,10 @@ H5F__flush_phase1(H5F_t *f) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush dataset cache"); + /* Flush the shared chunk cache */ + if (H5SC_flush(f->shared->shared_cache) < 0) + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush shared chunk cache"); + /* Release any space allocated to space aggregators, so that the eoa value * corresponds to the end of the space written to in the file. */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 3fd7cb624d8..2259b533b99 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -361,6 +361,9 @@ struct H5F_shared_t { char *extpath; /* Path for searching target external link file */ + /* Shared chunk cache info */ + struct H5SC_t *shared_cache; /* Shared chunk/raw data cache */ + #ifdef H5_HAVE_PARALLEL H5P_coll_md_read_flag_t coll_md_read; /* Do all metadata reads collectively */ bool coll_md_write; /* Do all metadata writes collectively */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 1894a49cd1b..665ec1f812f 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -101,6 +101,7 @@ typedef struct H5F_t H5F_t; #define H5F_VOL_OBJ(F) ((F)->vol_obj) #define H5F_USE_FILE_LOCKING(F) ((F)->shared->use_file_locking) #define H5F_RFIC_FLAGS(F) ((F)->shared->rfic_flags) +#define H5F_SHARED_CACHE(F) ((F)->shared->shared_cache) #else /* H5F_MODULE */ #define H5F_LOW_BOUND(F) (H5F_get_low_bound(F)) #define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F)) @@ -165,6 +166,7 @@ typedef struct H5F_t H5F_t; #define H5F_VOL_OBJ(F) (H5F_get_vol_obj(F)) #define H5F_USE_FILE_LOCKING(F) (H5F_get_use_file_locking(F)) #define H5F_RFIC_FLAGS(F) (H5F_get_rfic_flags(F)) +#define H5F_SHARED_CACHE(F) (H5F_get_shared_cache(F)) #endif /* H5F_MODULE */ /* Macros to encode/decode offset/length's for storing in the file */ @@ -528,6 +530,7 @@ H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, bool minimize); H5_DLL H5VL_object_t *H5F_get_vol_obj(const H5F_t *f); H5_DLL bool H5F_get_use_file_locking(const H5F_t *f); H5_DLL uint64_t H5F_get_rfic_flags(const H5F_t *f); +H5_DLL struct H5SC_t *H5F_get_shared_cache(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 8d02a9dc9ab..8859162f5bb 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1399,3 +1399,23 @@ H5F_get_rfic_flags(const H5F_t *f) FUNC_LEAVE_NOAPI(f->shared->rfic_flags) } /* end H5F_get_rfic_flags */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_shared_cache + * + * Purpose: Get a pointer to the shared chunk cache for the file + * + * Return: Pointer to shared chunk cache, or NULL if there is none + * + *------------------------------------------------------------------------- + */ +struct H5SC_t * +H5F_get_shared_cache(const H5F_t *f) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + assert(f); + assert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->shared_cache) +} /* end H5F_get_shared_cache */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index d8e4a69431b..c0961ff8a25 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -677,9 +677,10 @@ typedef struct H5O_layout_struct_chunk_t { } H5O_layout_struct_chunk_t; typedef struct H5O_layout_t { - H5D_layout_t type; /* Type of layout */ - unsigned version; /* Version of message */ - const struct H5D_layout_ops_t *ops; /* Pointer to data layout I/O operations */ + H5D_layout_t type; /* Type of layout */ + unsigned version; /* Version of message */ + const struct H5D_layout_ops_t *ops; /* Pointer to data layout I/O operations */ + const struct H5SC_layout_ops_t *sc_ops; /* Pointer to shared chunk cache callback operations */ union { H5O_layout_chunk_t chunk; /* Information for chunked layout */ H5O_layout_struct_chunk_t struct_chunk; /* Information for structured chunk layout */ diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 689b17ace14..d2154e0c192 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -144,36 +144,36 @@ } #define H5D_DEF_LAYOUT_COMPACT \ { \ - H5D_COMPACT, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_COMPACT, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ + H5D_COMPACT, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_COMPACT, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ H5D_DEF_STORAGE_COMPACT \ } #define H5D_DEF_LAYOUT_CONTIG \ { \ - H5D_CONTIGUOUS, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CONTIG, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ + H5D_CONTIGUOUS, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CONTIG, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ H5D_DEF_STORAGE_CONTIG \ } #define H5D_DEF_LAYOUT_CHUNK \ { \ - H5D_CHUNKED, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CHUNK, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ + H5D_CHUNKED, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CHUNK, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ H5D_DEF_STORAGE_CHUNK \ } /* TBD: FOR NOW set to CHUNKED */ #ifdef TBD #define H5D_DEF_LAYOUT_STRUCT_CHUNK \ { \ - H5D_STRUCT_CHUNK, H5O_LAYOUT_VERSION_5, H5D_LOPS_CHUNK, \ + H5D_STRUCT_CHUNK, H5O_LAYOUT_VERSION_5, H5D_LOPS_CHUNK, NULL, \ {.struct_chunk = H5D_DEF_LAYOUT_STRUCT_CHUNK_INIT}, H5D_DEF_STORAGE_CHUNK \ } #endif /* TBD: same as H5D_CHUNKED */ #define H5D_DEF_LAYOUT_STRUCT_CHUNK \ { \ - H5D_CHUNKED, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CHUNK, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ + H5D_CHUNKED, H5O_LAYOUT_VERSION_DEFAULT, H5D_LOPS_CHUNK, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ H5D_DEF_STORAGE_CHUNK \ } #define H5D_DEF_LAYOUT_VIRTUAL \ { \ - H5D_VIRTUAL, H5O_LAYOUT_VERSION_4, H5D_LOPS_VIRTUAL, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ + H5D_VIRTUAL, H5O_LAYOUT_VERSION_4, H5D_LOPS_VIRTUAL, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, \ H5D_DEF_STORAGE_VIRTUAL \ } diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index d6a626bc235..879538d4bad 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -419,12 +419,12 @@ typedef herr_t (*H5P_iterate_t)(hid_t id, const char *name, void *iter_data); * non-collective I/O and contiguous collective I/O. */ typedef enum H5D_mpio_actual_chunk_opt_mode_t { - H5D_MPIO_NO_CHUNK_OPTIMIZATION = 0, + H5D_MPIO_NO_CHUNK_OPTIMIZATION = 0x0, /**< No chunk optimization was performed. Either no collective I/O was attempted or the dataset wasn't chunked. */ - H5D_MPIO_LINK_CHUNK, + H5D_MPIO_LINK_CHUNK = 0x1, /**< Collective I/O is performed on all chunks simultaneously. */ - H5D_MPIO_MULTI_CHUNK + H5D_MPIO_MULTI_CHUNK = 0x2 /**< Each chunk was individually assigned collective or independent I/O based on what fraction of processes access the chunk. If the fraction is greater than the multi chunk ratio threshold, collective I/O is performed on that diff --git a/src/H5SC.c b/src/H5SC.c index 95e637cc7ad..eb0c3bcfc94 100644 --- a/src/H5SC.c +++ b/src/H5SC.c @@ -63,7 +63,7 @@ bool H5_PKG_INIT_VAR = false; *------------------------------------------------------------------------- */ H5SC_t * -H5SC_create(H5F_t *file, hid_t fapl_id) +H5SC_create(H5F_t *file, H5P_genplist_t *fa_plist) { H5SC_t *cache = NULL; H5SC_t *ret_value = NULL; @@ -71,6 +71,7 @@ H5SC_create(H5F_t *file, hid_t fapl_id) FUNC_ENTER_NOAPI(NULL) assert(file); + assert(fa_plist); /* Allocated cache struct */ if (NULL == (cache = H5MM_malloc(sizeof(H5SC_t)))) @@ -155,13 +156,17 @@ H5SC_flush_dset(H5SC_t *cache, H5D_t *dset, bool evict) /*------------------------------------------------------------------------- * Function: H5SC_read * - * Purpose: Reads raw data through a shared chunk cache. + * Purpose: Reads raw data through a shared chunk cache. There may be + * datasets in the dset_info array that do not support the shared chunk + * cache. These datasets must be ignored by the shared chunk cache. There + * may also be datasets that have skip_io set. These datasets must also be + * skipped. * * Return: SUCCEED on success, FAIL on failure *------------------------------------------------------------------------- */ herr_t -H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info) +H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info) { herr_t ret_value = SUCCEED; @@ -169,7 +174,6 @@ H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_typ assert(cache); assert(count == 0 || dset_info); - assert(io_type_info); done: FUNC_LEAVE_NOAPI(ret_value) @@ -178,13 +182,15 @@ H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_typ /*------------------------------------------------------------------------- * Function: H5SC_write * - * Purpose: Writes raw data through a shared chunk cache. + * Purpose: Writes raw data through a shared chunk cache. There may be + * datasets in the dset_info array that do not support the shared chunk + * cache. These datasets must be ignored by the shared chunk cache. * * Return: SUCCEED on success, FAIL on failure *------------------------------------------------------------------------- */ herr_t -H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info) +H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info) { herr_t ret_value = SUCCEED; @@ -192,7 +198,6 @@ H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_ty assert(cache); assert(count == 0 || dset_info); - assert(io_type_info); done: FUNC_LEAVE_NOAPI(ret_value) @@ -210,7 +215,7 @@ H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_ty *------------------------------------------------------------------------- */ herr_t -H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, hsize_t *offset, void *buf) +H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, void *buf, size_t *buf_size) { herr_t ret_value = SUCCEED; @@ -220,6 +225,7 @@ H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, hsize_t *offset, void *buf) assert(dset); assert(offset); assert(buf); + assert(buf_size); done: FUNC_LEAVE_NOAPI(ret_value) @@ -237,7 +243,7 @@ H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, hsize_t *offset, void *buf) *------------------------------------------------------------------------- */ herr_t -H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, hsize_t *offset, const void *buf) +H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, const void *buf) { herr_t ret_value = SUCCEED; diff --git a/src/H5SCprivate.h b/src/H5SCprivate.h index 1ce1a4116bd..d75116c8179 100644 --- a/src/H5SCprivate.h +++ b/src/H5SCprivate.h @@ -22,6 +22,9 @@ #ifndef H5SCprivate_H #define H5SCprivate_H +/* Forward declarations that may be needed by the below headers */ +typedef struct H5SC_layout_ops_t H5SC_layout_ops_t; + /* Private headers needed by this file */ #include "H5private.h" /* Generic Functions */ #include "H5Dprivate.h" /* Datasets */ @@ -49,7 +52,7 @@ typedef struct H5SC_t H5SC_t; * *nbytes_used set to 0. *udata can be set to anything and will be passed through to H5SC_chunk_decode_t * and/or the selection or vector I/O routines, then freed with free() (we will create an H5SC_free_udata_t * callback if necessary). */ -typedef herr_t (*H5SC_chunk_lookup_t)(H5D_t *dset, size_t count, hsize_t *scaled[] /*in*/, +typedef herr_t (*H5SC_chunk_lookup_t)(struct H5D_t *dset, size_t count, hsize_t *scaled[] /*in*/, haddr_t *addr[] /*out*/, hsize_t *size[] /*out*/, hsize_t *defined_values_size[] /*out*/, size_t *size_hint[] /*out*/, size_t *defined_values_size_hint[] /*out*/, void **udata[] /*out*/); @@ -58,50 +61,55 @@ typedef herr_t (*H5SC_chunk_lookup_t)(H5D_t *dset, size_t count, hsize_t *scaled * if necessary. On entry, nbytes is the number of bytes used in the chunk buffer. On exit, it shall be set to * the total number of bytes used (not allocated) across all buffers for this chunk. On entry, alloc_size is * the size of the chunk buffer. On exit, it shall be set to the total number of bytes allocated across all - * buffers for this chunk. Optional, if not present, chunk is the same in cache as on disk. */ + * buffers for this chunk. Optional, if not present, chunk is the same in cache as on disk. partial_bound is + * true if the chunk was encoded with partial_bound set to true. If the dataset reported + * partial_bound_chunks_different_encoding as false, the setting of partial_bound is undefined. */ typedef herr_t (*H5SC_chunk_decode_t)(H5D_t *dset, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, - void **chunk /*in,out*/, void *udata); + bool partial_bound, void **chunk /*in,out*/, void *udata); /* The same as H5SC_chunk_decode_t but only decodes the defined values. Optional, if not present, the entire * chunk must always be decoded. */ typedef herr_t (*H5SC_chunk_decode_defined_values_t)(H5D_t *dset, size_t *nbytes /*in,out*/, - size_t *alloc_size /*in,out*/, void **chunk /*in,out*/); + size_t *alloc_size /*in,out*/, bool partial_bound, + void **chunk /*in,out*/, void *udata); /* Creates a new empty chunk. Does not insert into on disk chunk index. If fill is true, writes the fill value * to the chunk (unless this is a sparse chunk). The number of bytes used is returned in *nbytes and the size * of the chunk buffer is returned in *buf_size. */ typedef herr_t (*H5SC_new_chunk_t)(H5D_t *dset, bool fill, size_t *nbytes /*out*/, size_t *buf_size /*out*/, - void **chunk /*chunk*/); + void **chunk /*chunk*/, void **udata[] /*out*/); /* Reallocates buffers as necessary so the total allocated size of buffers for the chunk (alloc_size) is equal * to the total number of bytes used (nbytes). Optional, if not present the chunk cache will be more likely to * evict chunks if there is wasted space in the buffers. */ -typedef herr_t (*H5SC_chunk_condense_t)(H5D_t *dset, size_t *nbytes /*in, out*/, void **chunk /*in, out*/); +typedef herr_t (*H5SC_chunk_condense_t)(H5D_t *dset, size_t *nbytes /*in, out*/, void **chunk /*in, out*/, + void *udata); /* Compresses/encodes the chunk as necessary. If chunk is the same as cache_buf, leaves *write_buf as NULL. * This function leaves chunk alone and allocates write_buf if necessary to hold compressed data, sets * *write_size to the size of the data in write_buf, and sets *write_size_alloc to the size of write_buf, if - * it was allocated. */ + * it was allocated. partial_bound is true if the chunk is partially outside the bounds of the dataset. If the + * dataset reported partial_bound_chunks_different_encoding as false, the setting of partial_bound is + * undefined. */ typedef herr_t (*H5SC_chunk_encode_t)(H5D_t *dset, hsize_t *write_size /*out*/, - hsize_t *write_buf_alloc /*out*/, const void *chunk, - void **write_buf /*out*/); - -/* Frees chunk and all memory referenced by it. Optional, if not present free() is simply used. */ -typedef herr_t (*H5SC_chunk_evict_t)(H5D_t *dset, void *chunk); + hsize_t *write_buf_alloc /*out*/, bool partial_bound, const void *chunk, + void *udata, void **write_buf /*out*/); /* The same as H5SC_chunk_encode_t but does not preserve chunk buffer, encoding is performed in-place. Must * free all other data used. */ -typedef herr_t (*H5SC_chunk_encode_in_place_t)(H5D_t *dset, size_t *write_size /*out*/, - void **chunk /*in,out*/); +typedef herr_t (*H5SC_chunk_encode_in_place_t)(H5D_t *dset, size_t *write_size /*out*/, bool partial_bound, + void **chunk /*in,out*/, void *udata); + +/* Frees chunk and all memory referenced by it. Optional, if not present free() is simply used. */ +typedef herr_t (*H5SC_chunk_evict_t)(H5D_t *dset, void *chunk, void *udata); /* Inserts (or reinserts) count chunks into the chunk index if necessary. Old address and size (if any) of the * chunks on disk are passed as addr and old_disk_size, the new size is passed in as new_disk_size. This * function resizes and reallocates on disk if necessary, returning the address of the chunks on disk in - * *addr. If an element in chunk is passed as NULL then this function shall insert a chunk large enough and - * with properties set to (initially) hold only fill values. */ + * *addr. */ typedef herr_t (*H5SC_chunk_insert_t)(H5D_t *dset, size_t count, hsize_t *scaled[] /*in*/, haddr_t *addr[] /*in,out*/, hsize_t old_disk_size[], - hsize_t new_disk_size[], void *chunk[] /*in*/); + hsize_t new_disk_size[], void *chunk[] /*in*/, void *udata); /* Called when the chunk cache wants to read data directly from the disk to the user buffer via selection I/O. * If not possible due to compression, etc, returns select_possible=false. Otherwise transforms the file space @@ -109,10 +117,12 @@ typedef herr_t (*H5SC_chunk_insert_t)(H5D_t *dset, size_t count, hsize_t *scaled * If no transformation is necessary, leaves *file_space_out as NULL. chunk may be passed as NULL, and may * also be an in-cache chunk that only contains information on selected elements. Optional, if not present, * chunk I/O is only performed on entire chunks or with vector I/O. The H5SC code checks for type conversion - * before calling this. */ -typedef herr_t (*H5SC_chunk_selection_read_t)(H5D_t *dset, H5S_t *file_space_in, void *chunk /*in*/, - H5S_t **file_space_out /*out*/, bool *select_possible /*out*/, - void *udata); + * before calling this. partial_bound is true if the on-disk chunk was encoded with partial_bound set to true. + * If the dataset reported partial_bound_chunks_different_encoding as false, the setting of partial_bound is + * undefined. */ +typedef herr_t (*H5SC_chunk_selection_read_t)(H5D_t *dset, H5S_t *file_space_in, bool partial_bound, + void *chunk /*in*/, H5S_t **file_space_out /*out*/, + bool *select_possible /*out*/, void *udata); /* Called when the chunk cache wants to read data directly from the disk to the user buffer via vector I/O. If * not possible due to compression, etc, returns vector_possible=false. Otherwise returns the vector of @@ -120,9 +130,10 @@ typedef herr_t (*H5SC_chunk_selection_read_t)(H5D_t *dset, H5S_t *file_space_in, * with the number of vectors returned in vec_count. chunk may be passed as NULL, and may also be an in-cache * chunk that only contains information on selected elements. Optional, if not present, chunk I/O is only * performed on entire chunks or with selection I/O. The H5SC code checks for type conversion before calling - * this. */ + * this. partial_bound is true if the on-disk chunk was encoded with partial_bound set to true. If the dataset + * reported partial_bound_chunks_different_encoding as false, the setting of partial_bound is undefined. */ typedef herr_t (*H5SC_chunk_vector_read_t)(H5D_t *dset, haddr_t addr, H5S_t *file_space_in, - void *chunk /*in*/, size_t *vec_count /*out*/, + bool partial_bound, void *chunk /*in*/, size_t *vec_count /*out*/, haddr_t **offsets /*out*/, size_t **sizes /*out*/, bool *vector_possible /*out*/, void *udata); @@ -132,10 +143,12 @@ typedef herr_t (*H5SC_chunk_vector_read_t)(H5D_t *dset, haddr_t addr, H5S_t *fil * file_space_out). If no transformation is necessary, leaves *file_space_out as NULL. chunk may be passed as * NULL, and may also be an in-cache chunk that only contains information on selected elements. Optional, if * not present, chunk I/O is only performed on entire chunks or with vector I/O. The H5SC code checks for type - * conversion before calling this. */ -typedef herr_t (*H5SC_chunk_selection_write_t)(H5D_t *dset, H5S_t *file_space_in, void *chunk /*in*/, - H5S_t *file_space_out /*out*/, bool *select_possible /*out*/, - void *udata); + * conversion before calling this. partial_bound is true if the on-disk chunk was encoded with partial_bound + * set to true. If the dataset reported partial_bound_chunks_different_encoding as false, the setting of + * partial_bound is undefined. */ +typedef herr_t (*H5SC_chunk_selection_write_t)(H5D_t *dset, H5S_t *file_space_in, bool partial_bound, + void *chunk /*in*/, H5S_t *file_space_out /*out*/, + bool *select_possible /*out*/, void *udata); /* Called when the chunk cache wants to write data directly from the user buffer to the cache via vector I/O. * If not possible due to compression, etc, returns vector_possible=false. Otherwise returns the vector of @@ -143,9 +156,10 @@ typedef herr_t (*H5SC_chunk_selection_write_t)(H5D_t *dset, H5S_t *file_space_in * with the number of vectors returned in vec_count. chunk may be passed as NULL, and may also be an in-cache * chunk that only contains information on selected elements. Optional, if not present, chunk I/O is only * performed on entire chunks or with selection I/O. The H5SC code checks for type conversion before calling - * this. */ + * this. partial_bound is true if the on-disk chunk was encoded with partial_bound set to true. If the dataset + * reported partial_bound_chunks_different_encoding as false, the setting of partial_bound is undefined. */ typedef herr_t (*H5SC_chunk_vector_write_t)(H5D_t *dset, haddr_t addr, H5S_t *file_space_in, - void *chunk /*in*/, size_t *vec_count /*out*/, + bool partial_bound, void *chunk /*in*/, size_t *vec_count /*out*/, haddr_t **offsets /*out*/, size_t **sizes /*out*/, bool *vector_possible /*out*/, void *udata); @@ -155,8 +169,9 @@ typedef herr_t (*H5SC_chunk_vector_write_t)(H5D_t *dset, haddr_t addr, H5S_t *fi * within the chunk, offset appropriately within the full extent. Optional, if not present, chunk is the same * in memory as it is in cache, with the exception of type conversion (which will be handled by the H5SC * layer). If the layout stores variable length data within the chunk this callback must be defined. */ -typedef herr_t (*H5SC_chunk_scatter_mem_t)(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, - H5S_t *mem_space, H5S_t *file_space, const void *chunk); +typedef herr_t (*H5SC_chunk_scatter_mem_t)(H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info, + H5S_t *mem_space, H5S_t *file_space, const void *chunk, + void *udata); /* Gathers data from the memory buffer (in dset_info) into the chunk buffer, performing type conversion if * necessary. file_space's extent matches the chunk dimensions and the selection is within the chunk. @@ -165,24 +180,24 @@ typedef herr_t (*H5SC_chunk_scatter_mem_t)(H5D_io_info_t *io_info, H5D_dset_io_i * Optional, if not present, chunk is the same in memory as it is in cache, with the exception of type * conversion (which will be handled by H5SC layer). If the layout stores variable length data within the * chunk this callback must be defined. */ -typedef herr_t (*H5SC_chunk_gather_mem_t)(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, +typedef herr_t (*H5SC_chunk_gather_mem_t)(H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info, H5S_t *mem_space, H5S_t *file_space, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, size_t *buf_size_total /*in,out*/, - void *chunk); + void *chunk, void *udata); /* Propagates the fill value into the selected elements of the chunk buffer, performing type conversion if * necessary. space's extent matches the chunk dimensions and the selection is within the chunk. Optional, if * not present, chunk is the same in memory as it is in cache, with the exception of type conversion (which * will be handled by H5SC layer). If the layout stores variable length data within the chunk this callback * must be defined. */ -typedef herr_t (*H5SC_chunk_fill_t)(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, H5S_t *space, - size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, - size_t *buf_size_total /*in,out*/, void *chunk); +typedef herr_t (*H5SC_chunk_fill_t)(H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info, + H5S_t *space, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, + size_t *buf_size_total /*in,out*/, void *chunk, void *udata); /* Queries the defined elements in the chunk. selection may be passed as H5S_ALL. These selections are within * the logical chunk. Optional, if not present, all values are defined. */ typedef herr_t (*H5SC_chunk_defined_values_t)(H5D_t *dset, H5S_t *selection, void *chunk, - H5S_t **defined_values /*out*/); + H5S_t **defined_values /*out*/, void *udata); /* Erases the selected elements in the chunk, causing them to no longer be defined. If all values in the chunk * are erased and the chunk should be deleted, sets *delete_chunk to true, causing the cache to delete the @@ -191,28 +206,31 @@ typedef herr_t (*H5SC_chunk_defined_values_t)(H5D_t *dset, H5S_t *selection, voi * value will be written to the selection using H5SC_chunk_fill_t. */ typedef herr_t (*H5SC_chunk_erase_values_t)(H5D_t *dset, H5S_t *selection, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, void *chunk, - bool *delete_chunk /*out*/); + bool *delete_chunk /*out*/, void *udata); /* Frees the data values in the cached chunk and memory used by them (but does not reallocate - see * H5SC_chunk_condense_t), but leaves the defined values intact. Optional, if not present the entire chunk * will be evicted. */ typedef herr_t (*H5SC_chunk_evict_values_t)(H5D_t *dset, size_t *nbytes /*in,out*/, - size_t *alloc_size /*in,out*/, void *chunk); + size_t *alloc_size /*in,out*/, void *chunk, void *udata); + +/* Queries whether chunks that are partially outside the bounds of the dataset are encoded differently (for example, they may not have filters applied). If so, then chunks whose partial bound state changes will be re-encoded and re-inserted as necessary after the dataset extent changes to ensure they are encoded appropriately. */ +typedef herr_t (*H5SC_layout_opt_query)(H5D_t *dset, bool *partial_bound_chunks_different_encoding); /* Removes the chunk from the index and deletes it on disk. Only called if a chunk goes out of scope due to * H5Dset_extent() or if H5SC_chunk_erase_values_t returns *delete_chunk == true. */ typedef herr_t (*H5SC_delete_chunk_t)(H5D_t *dset, hsize_t *scaled /*in*/, haddr_t addr, hsize_t disk_size); /* Operations that are implemented by shared chunk cache clients */ -typedef struct H5SC_layout_ops_t { +struct H5SC_layout_ops_t { H5SC_chunk_lookup_t lookup; H5SC_chunk_decode_t decode; H5SC_chunk_decode_defined_values_t decode_defined_values; H5SC_new_chunk_t new_chunk; H5SC_chunk_condense_t condense; H5SC_chunk_encode_t encode; - H5SC_chunk_evict_t evict; H5SC_chunk_encode_in_place_t encode_in_place; + H5SC_chunk_evict_t evict; H5SC_chunk_insert_t insert; H5SC_chunk_selection_read_t selection_read; H5SC_chunk_vector_read_t vector_read; @@ -224,8 +242,9 @@ typedef struct H5SC_layout_ops_t { H5SC_chunk_defined_values_t defined_values; H5SC_chunk_erase_values_t erase_values; H5SC_chunk_evict_values_t evict_values; + H5SC_layout_opt_query layout_opt_query; H5SC_delete_chunk_t delete_chunk; -} H5SC_layout_ops_t; +}; /*****************************/ /* Library-private Variables */ @@ -236,7 +255,7 @@ typedef struct H5SC_layout_ops_t { /***************************************/ /* Functions that operate on a shared chunk cache */ -H5_DLL H5SC_t *H5SC_create(H5F_t *file, hid_t fapl_id); +H5_DLL H5SC_t *H5SC_create(H5F_t *file, H5P_genplist_t *fa_plist); H5_DLL herr_t H5SC_destroy(H5SC_t *cache); /* Flush functions */ @@ -244,12 +263,10 @@ H5_DLL herr_t H5SC_flush(H5SC_t *cache); H5_DLL herr_t H5SC_flush_dset(H5SC_t *cache, H5D_t *dset, bool evict); /* I/O functions */ -H5_DLL herr_t H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, - H5D_io_type_info_t *io_type_info); -H5_DLL herr_t H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, - H5D_io_type_info_t *io_type_info); -H5_DLL herr_t H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, hsize_t *offset, void *buf); -H5_DLL herr_t H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, hsize_t *offset, const void *buf); +H5_DLL herr_t H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info); +H5_DLL herr_t H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info); +H5_DLL herr_t H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, void *buf, size_t *buf_size); +H5_DLL herr_t H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, const void *buf); /* Other functions */ H5_DLL herr_t H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, hsize_t *old_dims); diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index 11ac2b48216..39765ec46d0 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -34,6 +34,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ #include "H5Sprivate.h" /* Dataspaces */ +#include "H5SCprivate.h" /* Shared chunk cache */ #include "H5VLprivate.h" /* Virtual Object Layer */ #include "H5VLnative_private.h" /* Native VOL connector */ @@ -755,27 +756,37 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* H5Dread_chunk */ case H5VL_NATIVE_DATASET_CHUNK_READ: { H5VL_native_dataset_chunk_read_t *chunk_read_args = &opt_args->chunk_read; - hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ /* Check arguments */ if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); - /* Make sure the dataset is chunked or sparse chunk */ - /* TBD: H5Dread_chunk() is supposed to work for sparse chunk */ - if (H5D_CHUNKED != dset->shared->layout.type) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); + /* Check for shared chunk cache support */ + if (dset->shared->layout.sc_ops) { + H5D_chunk_scc_udata_t udata; + size_t buf_size = SIZE_MAX; assert(0 && "set up the buf size from API when available. Either point directly into args struct in call to H5SC_direct_chunk_read() or fill in args struct afterwards"); - /* Copy the user's offset array so we can be sure it's terminated properly. - * (we don't want to mess with the user's buffer). - */ - if (H5D__chunk_get_offset_copy(dset, chunk_read_args->offset, offset_copy) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + /* Set up udata */ + udata.filters = 0; + udata.size = 0; - /* Read the raw chunk */ - if (H5D__chunk_direct_read(dset, offset_copy, &chunk_read_args->filters, chunk_read_args->buf) < - 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); + /* Dispatch call to H5SC layer */ + if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, chunk_read_args->offset, &udata, chunk_read_args->buf, &buf_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); + + /* Return info to caller */ + chunk_read_args->filters = udata.filters; + } + else { + /* Make sure the dataset is chunked */ + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); + + /* Read the raw chunk */ + if (H5D__chunk_direct_read(dset, chunk_read_args->offset, &chunk_read_args->filters, chunk_read_args->buf) < + 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); + } break; } @@ -783,25 +794,31 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* H5Dwrite_chunk */ case H5VL_NATIVE_DATASET_CHUNK_WRITE: { H5VL_native_dataset_chunk_write_t *chunk_write_args = &opt_args->chunk_write; - hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ /* Check arguments */ if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); - if (H5D_CHUNKED != dset->shared->layout.type) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); + /* Check for shared chunk cache support */ + if (dset->shared->layout.sc_ops) { + H5D_chunk_scc_udata_t udata; - /* Copy the user's offset array so we can be sure it's terminated properly. - * (we don't want to mess with the user's buffer). - */ - if (H5D__chunk_get_offset_copy(dset, chunk_write_args->offset, offset_copy) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + /* Set up direct chunk info struct */ + udata.filters = chunk_write_args->filters; + udata.size = (uint64_t)chunk_write_args->size; + + /* Dispatch call to H5SC layer */ + if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, chunk_write_args->offset, &udata, chunk_write_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); + } + else { + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); - /* Write chunk */ - if (H5D__chunk_direct_write(dset, chunk_write_args->filters, offset_copy, chunk_write_args->size, - chunk_write_args->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); + /* Write chunk */ + if (H5D__chunk_direct_write(dset, chunk_write_args->filters, chunk_write_args->offset, chunk_write_args->size, chunk_write_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); + } break; } @@ -843,7 +860,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* H5Dread_struct_chunk */ case H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK: { H5VL_native_dataset_read_struct_chunk_t *read_struct_chunk_args = &opt_args->read_struct_chunk; - hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + size_t buf_size = SIZE_MAX; assert(0 && "set up the buf size from API when available. Either point directly into args struct in call to H5SC_direct_chunk_read() or fill in args struct afterwards"); /* Check arguments */ if (NULL == dset->oloc.file) @@ -852,16 +869,11 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); - /* Copy the user's offset array so we can be sure it's terminated properly. - * (we don't want to mess with the user's buffer). - */ - if (H5D__chunk_get_offset_copy(dset, read_struct_chunk_args->offset, offset_copy) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + assert(dset->shared->layout.sc_ops); /* Read the structured chunk */ - if (H5D__read_struct_chunk_direct(dset, offset_copy, read_struct_chunk_args->chunk_info, - read_struct_chunk_args->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); + if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, read_struct_chunk_args->offset, &read_struct_chunk_args->chunk_info, read_struct_chunk_args->buf, &buf_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); break; } @@ -869,7 +881,6 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* H5Dwrite_struct_chunk */ case H5VL_NATIVE_DATASET_WRITE_STRUCT_CHUNK: { H5VL_native_dataset_write_struct_chunk_t *write_struct_chunk_args = &opt_args->write_struct_chunk; - hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ /* Check arguments */ if (NULL == dset->oloc.file) @@ -878,16 +889,11 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ if (H5D_CHUNKED != dset->shared->layout.type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); - /* Copy the user's offset array so we can be sure it's terminated properly. - * (we don't want to mess with the user's buffer). - */ - if (H5D__chunk_get_offset_copy(dset, write_struct_chunk_args->offset, offset_copy) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "failure to copy offset array"); + assert(dset->shared->layout.sc_ops); /* Write the structured chunk */ - if (H5D__write_struct_chunk_direct(dset, offset_copy, write_struct_chunk_args->chunk_info, - write_struct_chunk_args->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write structured chunk data"); + if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, write_struct_chunk_args->offset, &write_struct_chunk_args->chunk_info, write_struct_chunk_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write structured chunk data"); break; } diff --git a/test/struct_chunk_storage.c b/test/struct_chunk_storage.c index 52ca2badad1..8c6cf8625e6 100644 --- a/test/struct_chunk_storage.c +++ b/test/struct_chunk_storage.c @@ -235,6 +235,9 @@ test_sparse_direct_chunk(hid_t fapl) TESTING("APIs for direct chunk I/O on structured chunks"); + SKIPPED(); + return 0; + /* Create a file */ h5_fixname(FILENAME[2], fapl, filename, sizeof filename); if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) @@ -556,6 +559,9 @@ test_sparse_direct_chunk_query(hid_t fapl) TESTING("APIs for direct chunk I/O query on structured chunk"); + SKIPPED(); + return 0; + /* Create the file */ h5_fixname(FILENAME[3], fapl, filename, sizeof filename); From 876eb2df579af4813d799036911f7a72d516d77c Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Jan 2025 00:34:10 +0000 Subject: [PATCH 02/12] Committing clang-format changes --- src/H5Dchunk.c | 3 ++- src/H5Dint.c | 15 ++++++++++----- src/H5Dio.c | 4 ++-- src/H5Dpkg.h | 4 ++-- src/H5SC.c | 3 ++- src/H5SCprivate.h | 11 ++++++++--- src/H5VLnative_dataset.c | 40 ++++++++++++++++++++++++++-------------- 7 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 111f2e4c537..9eb4b2315b9 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -379,7 +379,8 @@ H5FL_EXTERN(H5S_sel_iter_t); *------------------------------------------------------------------------- */ herr_t -H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, const hsize_t *offset, uint32_t data_size, const void *buf) +H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, const hsize_t *offset, uint32_t data_size, + const void *buf) { const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ H5D_chunk_ud_t udata; /* User data for querying chunk info */ diff --git a/src/H5Dint.c b/src/H5Dint.c index 862e6e8522d..5d4b7adf939 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -2036,8 +2036,10 @@ H5D_close(H5D_t *dataset) } /* end switch */ /* Evict the dataset's entries in the shared chunk cache */ - if (dataset->shared->layout.sc_ops && H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, true) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to evict dataset's entries in shared chunk cache"); + if (dataset->shared->layout.sc_ops && + H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, true) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, + "unable to evict dataset's entries in shared chunk cache"); /* Destroy any cached layout information for the dataset */ if (dataset->shared->layout.ops->dest && (dataset->shared->layout.ops->dest)(dataset) < 0) @@ -3101,7 +3103,8 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) expand = true; /* Chunked storage specific checks */ - if (!dset->shared->layout.sc_ops && H5D_CHUNKED == dset->shared->layout.type && dset->shared->ndims > 1) { + if (!dset->shared->layout.sc_ops && H5D_CHUNKED == dset->shared->layout.type && + dset->shared->ndims > 1) { hsize_t scaled; /* Scaled value */ /* Compute the scaled dimension size value */ @@ -3222,7 +3225,8 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) /* Notify the shared chunk cache that the extent has changed */ if (dset->shared->layout.sc_ops) if (H5SC_set_extent_notify(H5F_get_shared_cache(dset->oloc.file), dset, curr_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to notify shared chunk cache of extent change"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to notify shared chunk cache of extent change"); /* Mark the dataspace as dirty, for later writing to the file */ if (H5D__mark(dset, H5D_MARK_SPACE) < 0) @@ -3298,7 +3302,8 @@ H5D__flush_real(H5D_t *dataset) HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush raw data"); /* Flush the dataset's entries in the shared chunk cache */ - if (dataset->shared->layout.sc_ops && H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, false) < 0) + if (dataset->shared->layout.sc_ops && + H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, false) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush shared chunk cache"); } diff --git a/src/H5Dio.c b/src/H5Dio.c index cb3646ace47..3075fe7554e 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -97,7 +97,7 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) /* freed. */ H5D_storage_t store_local; /* Local buffer for store */ H5D_storage_t *store = &store_local; /* Union of EFL and chunk pointer in file space */ - bool any_scc = false; /* Whether any datasets support the shared chunk cache */ + bool any_scc = false; /* Whether any datasets support the shared chunk cache */ bool any_nonscc = false; /* Whether any datasets do not support the shared chunk cache */ size_t io_op_init = 0; /* Number I/O ops that have been initialized */ size_t io_skipped = @@ -550,7 +550,7 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) /* freed. */ H5D_storage_t store_local; /* Local buffer for store */ H5D_storage_t *store = &store_local; /* Union of EFL and chunk pointer in file space */ - bool any_scc = false; /* Whether any datasets support the shared chunk cache */ + bool any_scc = false; /* Whether any datasets support the shared chunk cache */ bool any_nonscc = false; /* Whether any datasets do not support the shared chunk cache */ size_t io_op_init = 0; /* Number I/O ops that have been initialized */ size_t i; /* Local index variable */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 8b63f197a80..336e6b742da 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -774,8 +774,8 @@ H5_DLL herr_t H5D__chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src, H5 H5_DLL herr_t H5D__chunk_bh_info(const H5O_loc_t *loc, H5O_t *oh, H5O_layout_t *layout, hsize_t *btree_size); H5_DLL herr_t H5D__chunk_dump_index(H5D_t *dset, FILE *stream); H5_DLL herr_t H5D__chunk_delete(H5F_t *f, H5O_t *oh, H5O_storage_t *store); -H5_DLL herr_t H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, const hsize_t *offset, uint32_t data_size, - const void *buf); +H5_DLL herr_t H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, const hsize_t *offset, + uint32_t data_size, const void *buf); H5_DLL herr_t H5D__chunk_direct_read(const H5D_t *dset, const hsize_t *offset, uint32_t *filters, void *buf); #ifdef H5D_CHUNK_DEBUG H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers); diff --git a/src/H5SC.c b/src/H5SC.c index eb0c3bcfc94..42371257f7f 100644 --- a/src/H5SC.c +++ b/src/H5SC.c @@ -215,7 +215,8 @@ H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info) *------------------------------------------------------------------------- */ herr_t -H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, void *buf, size_t *buf_size) +H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, void *buf, + size_t *buf_size) { herr_t ret_value = SUCCEED; diff --git a/src/H5SCprivate.h b/src/H5SCprivate.h index d75116c8179..5b05d8379c5 100644 --- a/src/H5SCprivate.h +++ b/src/H5SCprivate.h @@ -214,7 +214,10 @@ typedef herr_t (*H5SC_chunk_erase_values_t)(H5D_t *dset, H5S_t *selection, size_ typedef herr_t (*H5SC_chunk_evict_values_t)(H5D_t *dset, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, void *chunk, void *udata); -/* Queries whether chunks that are partially outside the bounds of the dataset are encoded differently (for example, they may not have filters applied). If so, then chunks whose partial bound state changes will be re-encoded and re-inserted as necessary after the dataset extent changes to ensure they are encoded appropriately. */ +/* Queries whether chunks that are partially outside the bounds of the dataset are encoded differently (for + * example, they may not have filters applied). If so, then chunks whose partial bound state changes will be + * re-encoded and re-inserted as necessary after the dataset extent changes to ensure they are encoded + * appropriately. */ typedef herr_t (*H5SC_layout_opt_query)(H5D_t *dset, bool *partial_bound_chunks_different_encoding); /* Removes the chunk from the index and deletes it on disk. Only called if a chunk goes out of scope due to @@ -265,8 +268,10 @@ H5_DLL herr_t H5SC_flush_dset(H5SC_t *cache, H5D_t *dset, bool evict); /* I/O functions */ H5_DLL herr_t H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info); H5_DLL herr_t H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info); -H5_DLL herr_t H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, void *buf, size_t *buf_size); -H5_DLL herr_t H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, const void *buf); +H5_DLL herr_t H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, + void *buf, size_t *buf_size); +H5_DLL herr_t H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, + const void *buf); /* Other functions */ H5_DLL herr_t H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, hsize_t *old_dims); diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index 39765ec46d0..b8208793d00 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -764,14 +764,18 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* Check for shared chunk cache support */ if (dset->shared->layout.sc_ops) { H5D_chunk_scc_udata_t udata; - size_t buf_size = SIZE_MAX; assert(0 && "set up the buf size from API when available. Either point directly into args struct in call to H5SC_direct_chunk_read() or fill in args struct afterwards"); + size_t buf_size = SIZE_MAX; + assert(0 && "set up the buf size from API when available. Either point directly into args " + "struct in call to H5SC_direct_chunk_read() or fill in args struct afterwards"); /* Set up udata */ udata.filters = 0; - udata.size = 0; + udata.size = 0; /* Dispatch call to H5SC layer */ - if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, chunk_read_args->offset, &udata, chunk_read_args->buf, &buf_size) < 0) + if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, + chunk_read_args->offset, &udata, chunk_read_args->buf, + &buf_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); /* Return info to caller */ @@ -783,8 +787,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); /* Read the raw chunk */ - if (H5D__chunk_direct_read(dset, chunk_read_args->offset, &chunk_read_args->filters, chunk_read_args->buf) < - 0) + if (H5D__chunk_direct_read(dset, chunk_read_args->offset, &chunk_read_args->filters, + chunk_read_args->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); } @@ -799,16 +803,17 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file"); - /* Check for shared chunk cache support */ + /* Check for shared chunk cache support */ if (dset->shared->layout.sc_ops) { H5D_chunk_scc_udata_t udata; /* Set up direct chunk info struct */ udata.filters = chunk_write_args->filters; - udata.size = (uint64_t)chunk_write_args->size; + udata.size = (uint64_t)chunk_write_args->size; /* Dispatch call to H5SC layer */ - if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, chunk_write_args->offset, &udata, chunk_write_args->buf) < 0) + if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, + chunk_write_args->offset, &udata, chunk_write_args->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); } else { @@ -816,7 +821,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset"); /* Write chunk */ - if (H5D__chunk_direct_write(dset, chunk_write_args->filters, chunk_write_args->offset, chunk_write_args->size, chunk_write_args->buf) < 0) + if (H5D__chunk_direct_write(dset, chunk_write_args->filters, chunk_write_args->offset, + chunk_write_args->size, chunk_write_args->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); } @@ -860,7 +866,9 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ /* H5Dread_struct_chunk */ case H5VL_NATIVE_DATASET_READ_STRUCT_CHUNK: { H5VL_native_dataset_read_struct_chunk_t *read_struct_chunk_args = &opt_args->read_struct_chunk; - size_t buf_size = SIZE_MAX; assert(0 && "set up the buf size from API when available. Either point directly into args struct in call to H5SC_direct_chunk_read() or fill in args struct afterwards"); + size_t buf_size = SIZE_MAX; + assert(0 && "set up the buf size from API when available. Either point directly into args struct " + "in call to H5SC_direct_chunk_read() or fill in args struct afterwards"); /* Check arguments */ if (NULL == dset->oloc.file) @@ -872,8 +880,10 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ assert(dset->shared->layout.sc_ops); /* Read the structured chunk */ - if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, read_struct_chunk_args->offset, &read_struct_chunk_args->chunk_info, read_struct_chunk_args->buf, &buf_size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); + if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, + read_struct_chunk_args->offset, &read_struct_chunk_args->chunk_info, + read_struct_chunk_args->buf, &buf_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); break; } @@ -892,8 +902,10 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ assert(dset->shared->layout.sc_ops); /* Write the structured chunk */ - if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, write_struct_chunk_args->offset, &write_struct_chunk_args->chunk_info, write_struct_chunk_args->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write structured chunk data"); + if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, + write_struct_chunk_args->offset, &write_struct_chunk_args->chunk_info, + write_struct_chunk_args->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write structured chunk data"); break; } From 8703f44b20b067ccebd5e35592acd47050d6b76d Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Thu, 16 Jan 2025 19:44:48 -0600 Subject: [PATCH 03/12] Fix spelling and cmake error (hopefully) --- src/H5CX.c | 4 ++-- test/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/H5CX.c b/src/H5CX.c index 50b90e3ff0f..cf912c8d5f2 100644 --- a/src/H5CX.c +++ b/src/H5CX.c @@ -2923,7 +2923,7 @@ H5CX_or_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t mpio_actual_chunk assert(head && *head); assert(!((*head)->ctx.dxpl_id == H5P_DEFAULT || (*head)->ctx.dxpl_id == H5P_DATASET_XFER_DEFAULT)); - /* If the value is already set, "or" it with the requested value, othewise simply set it */ + /* If the value is already set, "or" it with the requested value, otherwise simply set it */ if ((*head)->ctx.mpio_actual_chunk_opt_set) (*head)->ctx.mpio_actual_chunk_opt |= mpio_actual_chunk_opt; else { @@ -2984,7 +2984,7 @@ H5CX_or_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t mpio_actual_io_mode) assert(head && *head); assert(!((*head)->ctx.dxpl_id == H5P_DEFAULT || (*head)->ctx.dxpl_id == H5P_DATASET_XFER_DEFAULT)); - /* If the value is already set, "or" it with the requested value, othewise simply set it */ + /* If the value is already set, "or" it with the requested value, otherwise simply set it */ if ((*head)->ctx.mpio_actual_io_mode_set) (*head)->ctx.mpio_actual_io_mode |= mpio_actual_io_mode; else { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a0f7519deb9..1d305a1570a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -386,7 +386,7 @@ set (H5_TESTS page_buffer dtypes dsets - sparse_storage + struct_chunk_storage select_io_dset chunk_info # compression lib link cmpd_dset From 14d0e9f947748ee6f26e41aa18e51d33d3ba0c9b Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Fri, 17 Jan 2025 10:22:56 -0600 Subject: [PATCH 04/12] Clean up shared chunk cache on failure in H5F_new() --- src/H5Fint.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/H5Fint.c b/src/H5Fint.c index 46e1e81e782..14fb0575052 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -1383,6 +1383,9 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F if (f->shared->fcpl_id > 0) if (H5I_dec_ref(f->shared->fcpl_id) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTDEC, NULL, "can't close property list"); + if (f->shared->shared_cache) + if (H5SC_destroy(f->shared->shared_cache) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, NULL, "unable to destroy shared chunk cache"); f->shared = H5FL_FREE(H5F_shared_t, f->shared); } From fc94e6d29ea7f3a09fd52cf7539e823d6aacbc9e Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Fri, 17 Jan 2025 11:24:23 -0600 Subject: [PATCH 05/12] Hopefully fix cmake --- src/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9655962e65c..458d9b77e27 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -829,6 +829,7 @@ set (common_SRCS ${H5UC_SOURCES} ${H5RS_SOURCES} ${H5S_SOURCES} + ${H5SC_SOURCES} ${H5SL_SOURCES} ${H5SM_SOURCES} ${H5T_SOURCES} @@ -869,6 +870,7 @@ set (H5_PUBLIC_HEADERS ${H5PL_HDRS} ${H5R_HDRS} ${H5S_HDRS} + ${H5SC_HDRS} ${H5SM_HDRS} ${H5T_HDRS} ${H5TS_HDRS} From 7164ff614f457015d60544284c5c0e67c157a730 Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Thu, 23 Jan 2025 10:38:50 -0600 Subject: [PATCH 06/12] Fix layout callback signatures and add chunk_dims parameter to layout_query --- src/H5SCprivate.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/H5SCprivate.h b/src/H5SCprivate.h index 5b05d8379c5..135585d2b21 100644 --- a/src/H5SCprivate.h +++ b/src/H5SCprivate.h @@ -77,7 +77,7 @@ typedef herr_t (*H5SC_chunk_decode_defined_values_t)(H5D_t *dset, size_t *nbytes * to the chunk (unless this is a sparse chunk). The number of bytes used is returned in *nbytes and the size * of the chunk buffer is returned in *buf_size. */ typedef herr_t (*H5SC_new_chunk_t)(H5D_t *dset, bool fill, size_t *nbytes /*out*/, size_t *buf_size /*out*/, - void **chunk /*chunk*/, void **udata[] /*out*/); + void **chunk /*chunk*/, void **udata /*out*/); /* Reallocates buffers as necessary so the total allocated size of buffers for the chunk (alloc_size) is equal * to the total number of bytes used (nbytes). Optional, if not present the chunk cache will be more likely to @@ -109,7 +109,7 @@ typedef herr_t (*H5SC_chunk_evict_t)(H5D_t *dset, void *chunk, void *udata); * *addr. */ typedef herr_t (*H5SC_chunk_insert_t)(H5D_t *dset, size_t count, hsize_t *scaled[] /*in*/, haddr_t *addr[] /*in,out*/, hsize_t old_disk_size[], - hsize_t new_disk_size[], void *chunk[] /*in*/, void *udata); + hsize_t new_disk_size[], void *chunk[] /*in*/, void *udata[]); /* Called when the chunk cache wants to read data directly from the disk to the user buffer via selection I/O. * If not possible due to compression, etc, returns select_possible=false. Otherwise transforms the file space @@ -214,11 +214,13 @@ typedef herr_t (*H5SC_chunk_erase_values_t)(H5D_t *dset, H5S_t *selection, size_ typedef herr_t (*H5SC_chunk_evict_values_t)(H5D_t *dset, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, void *chunk, void *udata); -/* Queries whether chunks that are partially outside the bounds of the dataset are encoded differently (for - * example, they may not have filters applied). If so, then chunks whose partial bound state changes will be - * re-encoded and re-inserted as necessary after the dataset extent changes to ensure they are encoded - * appropriately. */ -typedef herr_t (*H5SC_layout_opt_query)(H5D_t *dset, bool *partial_bound_chunks_different_encoding); +/* Queries data about the dataset from the layout client. The callbak shall set the chunk dimensions in the + * chunk_dims array (the number of dimensions is the same as the rank of the dataset), and shall set whether + * chunks that are partially outside the bounds of the dataset are encoded differently (for example, they may + * not have filters applied). If *partial_bound_chunks_different_encoding is set to true, then chunks whose + * partial bound state changes will be re-encoded and re-inserted as necessary after the dataset extent + * changes to ensure they are encoded appropriately. */ +typedef herr_t (*H5SC_layout_query)(H5D_t *dset, hsize_t *chunk_dims, bool *partial_bound_chunks_different_encoding); /* Removes the chunk from the index and deletes it on disk. Only called if a chunk goes out of scope due to * H5Dset_extent() or if H5SC_chunk_erase_values_t returns *delete_chunk == true. */ @@ -245,7 +247,7 @@ struct H5SC_layout_ops_t { H5SC_chunk_defined_values_t defined_values; H5SC_chunk_erase_values_t erase_values; H5SC_chunk_evict_values_t evict_values; - H5SC_layout_opt_query layout_opt_query; + H5SC_layout_query layout_query; H5SC_delete_chunk_t delete_chunk; }; From 5128774d41b6e0f2706facc7a379630a15cf42be Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 16:40:56 +0000 Subject: [PATCH 07/12] Committing clang-format changes --- src/H5SCprivate.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/H5SCprivate.h b/src/H5SCprivate.h index 135585d2b21..c6c67852b80 100644 --- a/src/H5SCprivate.h +++ b/src/H5SCprivate.h @@ -220,7 +220,8 @@ typedef herr_t (*H5SC_chunk_evict_values_t)(H5D_t *dset, size_t *nbytes /*in,out * not have filters applied). If *partial_bound_chunks_different_encoding is set to true, then chunks whose * partial bound state changes will be re-encoded and re-inserted as necessary after the dataset extent * changes to ensure they are encoded appropriately. */ -typedef herr_t (*H5SC_layout_query)(H5D_t *dset, hsize_t *chunk_dims, bool *partial_bound_chunks_different_encoding); +typedef herr_t (*H5SC_layout_query)(H5D_t *dset, hsize_t *chunk_dims, + bool *partial_bound_chunks_different_encoding); /* Removes the chunk from the index and deletes it on disk. Only called if a chunk goes out of scope due to * H5Dset_extent() or if H5SC_chunk_erase_values_t returns *delete_chunk == true. */ From 891769178327783b95111385c81e56dbbbbd583d Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Fri, 31 Jan 2025 15:00:59 -0600 Subject: [PATCH 08/12] Integrate shared chunk cache with get_defined and erase operations. Various other fixes/enhancements. --- src/H5Dchunk.c | 30 +++++++++++++---- src/H5Dint.c | 51 +++++++++++++++------------- src/H5Dio.c | 8 ++--- src/H5Dpkg.h | 8 ++--- src/H5SC.c | 73 +++++++++++++++++++++++++++++++++++++++- src/H5SCprivate.h | 42 ++++++++++++----------- src/H5VLnative_dataset.c | 8 ++--- 7 files changed, 158 insertions(+), 62 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 9eb4b2315b9..b343d82aa26 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -56,6 +56,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5MFprivate.h" /* File memory management */ #include "H5PBprivate.h" /* Page Buffer */ +#include "H5SCprivate.h" /* Shared chunk cache */ #include "H5SLprivate.h" /* Skip Lists */ #include "H5VMprivate.h" /* Vector and array functions */ @@ -8281,22 +8282,28 @@ H5D__chunk_verify_offset(const H5D_t *dset, const hsize_t *offset) *------------------------------------------------------------------------- */ herr_t -H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *space, +H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset, H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, hsize_t H5_ATTR_UNUSED *chunk_size) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE_NOERR + FUNC_ENTER_PACKAGE assert(dset); assert(dset->shared); assert(space); + /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */ + /* It should be possible to optimize this in the future by only flushing the target chunk, and later directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or index query completely if the shared chunk cache has all the needed information needed. For now, just mirror the previous algorithm for legacy chunks. */ + if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset"); + /* TBD: go get structured chunk information using chunk index */ /* FOR NOW: just return success */ +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__get_struct_chunk_info() */ @@ -8312,22 +8319,28 @@ H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR *------------------------------------------------------------------------- */ herr_t -H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsize_t H5_ATTR_UNUSED *offset, +H5D__get_struct_chunk_info_by_coord(H5D_t *dset, const hsize_t H5_ATTR_UNUSED *offset, H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, hsize_t H5_ATTR_UNUSED *chunk_size) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE_NOERR + FUNC_ENTER_PACKAGE /* Check args */ assert(dset); assert(dset->shared); assert(offset); + /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */ + /* It should be possible to optimize this in the future by only flushing the target chunk, and later directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or index query completely if the shared chunk cache has all the needed information needed. For now, just mirror the previous algorithm for legacy chunks. */ + if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset"); + /* TBD: go get structured chunk information using chunk coordinates */ /* FOR NOW: just return success */ +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__get_struct_chunk_info_by_coord() */ @@ -8341,19 +8354,24 @@ H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsiz *------------------------------------------------------------------------- */ herr_t -H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op, +H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op, void H5_ATTR_UNUSED *op_data) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE_NOERR + FUNC_ENTER_PACKAGE /* Check args */ assert(dset); assert(dset->shared); + /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */ + if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset"); + /* TBD: iterate over all the structured chunks in the dataset */ /* FOR NOW: just return success */ +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_iter() */ diff --git a/src/H5Dint.c b/src/H5Dint.c index 5d4b7adf939..1eef0c64c3a 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -2037,7 +2037,7 @@ H5D_close(H5D_t *dataset) /* Evict the dataset's entries in the shared chunk cache */ if (dataset->shared->layout.sc_ops && - H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, true) < 0) + H5SC_flush_dset(H5F_SHARED_CACHE(dataset->oloc.file), dataset, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to evict dataset's entries in shared chunk cache"); @@ -3224,7 +3224,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) /* Notify the shared chunk cache that the extent has changed */ if (dset->shared->layout.sc_ops) - if (H5SC_set_extent_notify(H5F_get_shared_cache(dset->oloc.file), dset, curr_dims) < 0) + if (H5SC_set_extent_notify(H5F_SHARED_CACHE(dset->oloc.file), dset, curr_dims) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to notify shared chunk cache of extent change"); @@ -3303,7 +3303,7 @@ H5D__flush_real(H5D_t *dataset) /* Flush the dataset's entries in the shared chunk cache */ if (dataset->shared->layout.sc_ops && - H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, false) < 0) + H5SC_flush_dset(H5F_SHARED_CACHE(dataset->oloc.file), dataset, false) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush shared chunk cache"); } @@ -4090,24 +4090,24 @@ H5D_get_dcpl_id(const H5D_obj_create_t *d) *------------------------------------------------------------------------- */ hid_t -H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace) +H5D__get_defined(H5D_t *dset, const H5S_t *fspace) { H5S_t *space = NULL; hid_t ret_value = H5I_INVALID_HID; FUNC_ENTER_PACKAGE - /* TBD: - if (dset->shared->layout.type == H5D_SPARSE_CHUNK) - call routine to get defined elements - else - if (NULL == (space = H5S_copy(fspace, false, true))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace"); - */ - - /* FOR NOW: return copy of fspace */ - if (NULL == (space = H5S_copy(fspace, false, true))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace"); + /* Check for shared chunk cache */ + if (dset->shared->layout.sc_ops) { + /* Forward to SCC layer */ + if (NULL == (space = H5SC_get_defined(H5F_SHARED_CACHE(dset->oloc.file), dset, fspace))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get defined values"); + } + else + /* No SCC support, so no support for defined values, entire dataset is defined */ + /* Clip to dataset extent? -NAF */ + if (NULL == (space = H5S_copy(fspace, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace"); /* Create an ID */ if ((ret_value = H5I_register(H5I_DATASPACE, space, true)) < 0) @@ -4134,19 +4134,22 @@ H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace) *------------------------------------------------------------------------- */ herr_t -H5D__erase(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *fspace) +H5D__erase(H5D_t *dset, const H5S_t *fspace) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE_NOERR + FUNC_ENTER_PACKAGE - /* TBD: - if (dset->shared->layout.type == H5D_SPARSE_CHUNK) - call routine to delete elements - else - return error - */ - /* FOR NOW: just return success */ + /* Check for shared chunk cache */ + if (dset->shared->layout.sc_ops) { + /* Forward to SCC layer */ + if (H5SC_erase(H5F_SHARED_CACHE(dset->oloc.file), dset, fspace) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to get defined values"); + } + else + /* No SCC support, so no support for defined values and hence cannot erase */ + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "dataset does not support erasing values"); +done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D__erase() */ diff --git a/src/H5Dio.c b/src/H5Dio.c index 3075fe7554e..d369e2387da 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -467,8 +467,8 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info) /* Make shared chunk cache read call if appropriate */ if (any_scc) { - assert(H5F_get_shared_cache(dset_info[0].dset->oloc.file)); - if (H5SC_read(H5F_get_shared_cache(dset_info[0].dset->oloc.file), count, dset_info) < 0) + assert(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file)); + if (H5SC_read(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file), count, dset_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "read through shared chunk cache failed"); } @@ -917,8 +917,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info) /* Make shared chunk cache write call if appropriate */ if (any_scc) { - assert(H5F_get_shared_cache(dset_info[0].dset->oloc.file)); - if (H5SC_write(H5F_get_shared_cache(dset_info[0].dset->oloc.file), count, dset_info) < 0) + assert(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file)); + if (H5SC_write(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file), count, dset_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "write through shared chunk cache failed"); } diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 336e6b742da..bcb2af47bb0 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -782,17 +782,17 @@ H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers); #endif /* H5D_CHUNK_DEBUG */ /* Functions that operate on H5D_SPARSE_CHUNK storage */ -H5_DLL hid_t H5D__get_defined(const H5D_t *dset, const H5S_t *fspace); -H5_DLL herr_t H5D__erase(const H5D_t *dset, const H5S_t *fspace); +H5_DLL hid_t H5D__get_defined(H5D_t *dset, const H5S_t *fspace); +H5_DLL herr_t H5D__erase(H5D_t *dset, const H5S_t *fspace); H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); -H5_DLL herr_t H5D__get_struct_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, +H5_DLL herr_t H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t chunk_idx, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); -H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, +H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(H5D_t *dset, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); H5_DLL herr_t H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t cb, void *op_data); diff --git a/src/H5SC.c b/src/H5SC.c index 42371257f7f..64e8ef08d73 100644 --- a/src/H5SC.c +++ b/src/H5SC.c @@ -148,6 +148,7 @@ H5SC_flush_dset(H5SC_t *cache, H5D_t *dset, bool evict) assert(cache); assert(dset); + assert(dset->shared->layout.sc_ops); done: FUNC_LEAVE_NOAPI(ret_value) @@ -224,6 +225,7 @@ H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void * assert(cache); assert(dset); + assert(dset->shared->layout.sc_ops); assert(offset); assert(buf); assert(buf_size); @@ -252,6 +254,7 @@ H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void assert(cache); assert(dset); + assert(dset->shared->layout.sc_ops); assert(offset); assert(buf); @@ -259,6 +262,73 @@ H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void FUNC_LEAVE_NOAPI(ret_value) } /* end H5SC_direct_chunk_write() */ +/*------------------------------------------------------------------------- + * Function: H5SC_get_defined + * + * Purpose: Returns a copy of file_space with only elements selected that are both selected in file_space and defined in dset. If file_space uses a point selection, the ordering of selected points will be preserved in the returned dataspace. + * + * Return: SUCCEED on success, FAIL on failure + *------------------------------------------------------------------------- + */ +H5S_t * +H5SC_get_defined(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space) +{ + H5S_t *defined = NULL; + H5S_t *ret_value = NULL; + + FUNC_ENTER_NOAPI(NULL) + + assert(cache); + assert(dset); + assert(dset->shared->layout.sc_ops); + assert(file_space); + + /* FOR NOW: just return copy of file_space */ + if (NULL == (defined = H5S_copy(file_space, false, true))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to copy dataspace"); + + /* Set return value */ + ret_value = defined; + defined = NULL; + +done: + if (defined) { + assert(!ret_value); + if (H5S_close(defined) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, NULL, "unable to release dataspace"); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SC_get_defined() */ + +/*------------------------------------------------------------------------- + * Function: H5SC_erase + * + * Purpose: Causes the elements selected in file_space to become undefined in dset. If dset does not support tracking defined elements, returns an error. + * + * Return: SUCCEED on success, FAIL on failure + *------------------------------------------------------------------------- + */ +herr_t +H5SC_erase(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + assert(cache); + assert(dset); + assert(dset->shared->layout.sc_ops); + assert(file_space); + + /* Check for support for erasing values */ + if (!dset->shared->layout.sc_ops->erase_values) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "dataset does not support erasing values"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SC_erase() */ + /*------------------------------------------------------------------------- * Function: H5SC_set_extent_notify * @@ -270,7 +340,7 @@ H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *------------------------------------------------------------------------- */ herr_t -H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, hsize_t *old_dims) +H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, const hsize_t *old_dims) { herr_t ret_value = SUCCEED; @@ -278,6 +348,7 @@ H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, hsize_t *old_dims) assert(cache); assert(dset); + assert(dset->shared->layout.sc_ops); assert(old_dims); done: diff --git a/src/H5SCprivate.h b/src/H5SCprivate.h index 135585d2b21..001cc444516 100644 --- a/src/H5SCprivate.h +++ b/src/H5SCprivate.h @@ -52,7 +52,7 @@ typedef struct H5SC_t H5SC_t; * *nbytes_used set to 0. *udata can be set to anything and will be passed through to H5SC_chunk_decode_t * and/or the selection or vector I/O routines, then freed with free() (we will create an H5SC_free_udata_t * callback if necessary). */ -typedef herr_t (*H5SC_chunk_lookup_t)(struct H5D_t *dset, size_t count, hsize_t *scaled[] /*in*/, +typedef herr_t (*H5SC_chunk_lookup_t)(struct H5D_t *dset, size_t count, const hsize_t *scaled[] /*in*/, haddr_t *addr[] /*out*/, hsize_t *size[] /*out*/, hsize_t *defined_values_size[] /*out*/, size_t *size_hint[] /*out*/, size_t *defined_values_size_hint[] /*out*/, void **udata[] /*out*/); @@ -107,7 +107,7 @@ typedef herr_t (*H5SC_chunk_evict_t)(H5D_t *dset, void *chunk, void *udata); * chunks on disk are passed as addr and old_disk_size, the new size is passed in as new_disk_size. This * function resizes and reallocates on disk if necessary, returning the address of the chunks on disk in * *addr. */ -typedef herr_t (*H5SC_chunk_insert_t)(H5D_t *dset, size_t count, hsize_t *scaled[] /*in*/, +typedef herr_t (*H5SC_chunk_insert_t)(H5D_t *dset, size_t count, const hsize_t *scaled[] /*in*/, haddr_t *addr[] /*in,out*/, hsize_t old_disk_size[], hsize_t new_disk_size[], void *chunk[] /*in*/, void *udata[]); @@ -120,7 +120,7 @@ typedef herr_t (*H5SC_chunk_insert_t)(H5D_t *dset, size_t count, hsize_t *scaled * before calling this. partial_bound is true if the on-disk chunk was encoded with partial_bound set to true. * If the dataset reported partial_bound_chunks_different_encoding as false, the setting of partial_bound is * undefined. */ -typedef herr_t (*H5SC_chunk_selection_read_t)(H5D_t *dset, H5S_t *file_space_in, bool partial_bound, +typedef herr_t (*H5SC_chunk_selection_read_t)(H5D_t *dset, const H5S_t *file_space_in, bool partial_bound, void *chunk /*in*/, H5S_t **file_space_out /*out*/, bool *select_possible /*out*/, void *udata); @@ -132,7 +132,7 @@ typedef herr_t (*H5SC_chunk_selection_read_t)(H5D_t *dset, H5S_t *file_space_in, * performed on entire chunks or with selection I/O. The H5SC code checks for type conversion before calling * this. partial_bound is true if the on-disk chunk was encoded with partial_bound set to true. If the dataset * reported partial_bound_chunks_different_encoding as false, the setting of partial_bound is undefined. */ -typedef herr_t (*H5SC_chunk_vector_read_t)(H5D_t *dset, haddr_t addr, H5S_t *file_space_in, +typedef herr_t (*H5SC_chunk_vector_read_t)(H5D_t *dset, haddr_t addr, const H5S_t *file_space_in, bool partial_bound, void *chunk /*in*/, size_t *vec_count /*out*/, haddr_t **offsets /*out*/, size_t **sizes /*out*/, bool *vector_possible /*out*/, void *udata); @@ -146,7 +146,7 @@ typedef herr_t (*H5SC_chunk_vector_read_t)(H5D_t *dset, haddr_t addr, H5S_t *fil * conversion before calling this. partial_bound is true if the on-disk chunk was encoded with partial_bound * set to true. If the dataset reported partial_bound_chunks_different_encoding as false, the setting of * partial_bound is undefined. */ -typedef herr_t (*H5SC_chunk_selection_write_t)(H5D_t *dset, H5S_t *file_space_in, bool partial_bound, +typedef herr_t (*H5SC_chunk_selection_write_t)(H5D_t *dset, const H5S_t *file_space_in, bool partial_bound, void *chunk /*in*/, H5S_t *file_space_out /*out*/, bool *select_possible /*out*/, void *udata); @@ -158,7 +158,7 @@ typedef herr_t (*H5SC_chunk_selection_write_t)(H5D_t *dset, H5S_t *file_space_in * performed on entire chunks or with selection I/O. The H5SC code checks for type conversion before calling * this. partial_bound is true if the on-disk chunk was encoded with partial_bound set to true. If the dataset * reported partial_bound_chunks_different_encoding as false, the setting of partial_bound is undefined. */ -typedef herr_t (*H5SC_chunk_vector_write_t)(H5D_t *dset, haddr_t addr, H5S_t *file_space_in, +typedef herr_t (*H5SC_chunk_vector_write_t)(H5D_t *dset, haddr_t addr, const H5S_t *file_space_in, bool partial_bound, void *chunk /*in*/, size_t *vec_count /*out*/, haddr_t **offsets /*out*/, size_t **sizes /*out*/, bool *vector_possible /*out*/, void *udata); @@ -170,7 +170,7 @@ typedef herr_t (*H5SC_chunk_vector_write_t)(H5D_t *dset, haddr_t addr, H5S_t *fi * in memory as it is in cache, with the exception of type conversion (which will be handled by the H5SC * layer). If the layout stores variable length data within the chunk this callback must be defined. */ typedef herr_t (*H5SC_chunk_scatter_mem_t)(H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info, - H5S_t *mem_space, H5S_t *file_space, const void *chunk, + const H5S_t *mem_space, const H5S_t *file_space, const void *chunk, void *udata); /* Gathers data from the memory buffer (in dset_info) into the chunk buffer, performing type conversion if @@ -181,7 +181,7 @@ typedef herr_t (*H5SC_chunk_scatter_mem_t)(H5D_dset_io_info_t *dset_info, H5D_io * conversion (which will be handled by H5SC layer). If the layout stores variable length data within the * chunk this callback must be defined. */ typedef herr_t (*H5SC_chunk_gather_mem_t)(H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info, - H5S_t *mem_space, H5S_t *file_space, size_t *nbytes /*in,out*/, + const H5S_t *mem_space, const H5S_t *file_space, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, size_t *buf_size_total /*in,out*/, void *chunk, void *udata); @@ -196,7 +196,7 @@ typedef herr_t (*H5SC_chunk_fill_t)(H5D_dset_io_info_t *dset_info, H5D_io_type_i /* Queries the defined elements in the chunk. selection may be passed as H5S_ALL. These selections are within * the logical chunk. Optional, if not present, all values are defined. */ -typedef herr_t (*H5SC_chunk_defined_values_t)(H5D_t *dset, H5S_t *selection, void *chunk, +typedef herr_t (*H5SC_chunk_defined_values_t)(H5D_t *dset, const H5S_t *selection, void *chunk, H5S_t **defined_values /*out*/, void *udata); /* Erases the selected elements in the chunk, causing them to no longer be defined. If all values in the chunk @@ -204,7 +204,7 @@ typedef herr_t (*H5SC_chunk_defined_values_t)(H5D_t *dset, H5S_t *selection, voi * chunk from cache, free it in memory using H5SC_chunk_evict_t, and delete it on disk using * H5SC_chunk_delete_t. These selections are within the logical chunk. Optional, if not present, the fill * value will be written to the selection using H5SC_chunk_fill_t. */ -typedef herr_t (*H5SC_chunk_erase_values_t)(H5D_t *dset, H5S_t *selection, size_t *nbytes /*in,out*/, +typedef herr_t (*H5SC_chunk_erase_values_t)(H5D_t *dset, const H5S_t *selection, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, void *chunk, bool *delete_chunk /*out*/, void *udata); @@ -214,17 +214,19 @@ typedef herr_t (*H5SC_chunk_erase_values_t)(H5D_t *dset, H5S_t *selection, size_ typedef herr_t (*H5SC_chunk_evict_values_t)(H5D_t *dset, size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, void *chunk, void *udata); -/* Queries data about the dataset from the layout client. The callbak shall set the chunk dimensions in the - * chunk_dims array (the number of dimensions is the same as the rank of the dataset), and shall set whether - * chunks that are partially outside the bounds of the dataset are encoded differently (for example, they may - * not have filters applied). If *partial_bound_chunks_different_encoding is set to true, then chunks whose - * partial bound state changes will be re-encoded and re-inserted as necessary after the dataset extent - * changes to ensure they are encoded appropriately. */ -typedef herr_t (*H5SC_layout_query)(H5D_t *dset, hsize_t *chunk_dims, bool *partial_bound_chunks_different_encoding); +/* Queries data about the dataset from the layout client. The callback shall set the chunk dimensions in the + * chunk_dims array (the number of dimensions is the same as the rank of the dataset), whether encoding and + * decoding is necessary for chunks between cache and disk, and shall set whether chunks that are partially + * outside the bounds of the dataset are encoded differently (for example, they may not have filters applied). + * If *partial_bound_chunks_different_encoding is set to true, then chunks whose partial bound state changes + * will be re-encoded and re-inserted as necessary after the dataset extent changes to ensure they are encoded + * appropriately. */ +typedef herr_t (*H5SC_layout_query)(H5D_t *dset, hsize_t *chunk_dims, bool *encode_decode_necessary, + bool *partial_bound_chunks_different_encoding); /* Removes the chunk from the index and deletes it on disk. Only called if a chunk goes out of scope due to * H5Dset_extent() or if H5SC_chunk_erase_values_t returns *delete_chunk == true. */ -typedef herr_t (*H5SC_delete_chunk_t)(H5D_t *dset, hsize_t *scaled /*in*/, haddr_t addr, hsize_t disk_size); +typedef herr_t (*H5SC_delete_chunk_t)(H5D_t *dset, const hsize_t *scaled /*in*/, haddr_t addr, hsize_t disk_size); /* Operations that are implemented by shared chunk cache clients */ struct H5SC_layout_ops_t { @@ -274,8 +276,10 @@ H5_DLL herr_t H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t * void *buf, size_t *buf_size); H5_DLL herr_t H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *udata, const void *buf); +H5_DLL H5S_t *H5SC_get_defined(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space); +H5_DLL herr_t H5SC_erase(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space); /* Other functions */ -H5_DLL herr_t H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, hsize_t *old_dims); +H5_DLL herr_t H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, const hsize_t *old_dims); #endif /* H5SCprivate_H */ diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index b8208793d00..bb428c1e3e4 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -773,7 +773,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ udata.size = 0; /* Dispatch call to H5SC layer */ - if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, + if (H5SC_direct_chunk_read(H5F_SHARED_CACHE(dset->oloc.file), dset, chunk_read_args->offset, &udata, chunk_read_args->buf, &buf_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); @@ -812,7 +812,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ udata.size = (uint64_t)chunk_write_args->size; /* Dispatch call to H5SC layer */ - if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, + if (H5SC_direct_chunk_write(H5F_SHARED_CACHE(dset->oloc.file), dset, chunk_write_args->offset, &udata, chunk_write_args->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); } @@ -880,7 +880,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ assert(dset->shared->layout.sc_ops); /* Read the structured chunk */ - if (H5SC_direct_chunk_read(H5F_get_shared_cache(dset->oloc.file), dset, + if (H5SC_direct_chunk_read(H5F_SHARED_CACHE(dset->oloc.file), dset, read_struct_chunk_args->offset, &read_struct_chunk_args->chunk_info, read_struct_chunk_args->buf, &buf_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read structured chunk data"); @@ -902,7 +902,7 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ assert(dset->shared->layout.sc_ops); /* Write the structured chunk */ - if (H5SC_direct_chunk_write(H5F_get_shared_cache(dset->oloc.file), dset, + if (H5SC_direct_chunk_write(H5F_SHARED_CACHE(dset->oloc.file), dset, write_struct_chunk_args->offset, &write_struct_chunk_args->chunk_info, write_struct_chunk_args->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write structured chunk data"); From 6d04fbff4def0e0afe7ca78d32a7826da0c6d6c3 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:05:51 +0000 Subject: [PATCH 09/12] Committing clang-format changes --- src/H5Dchunk.c | 17 +++++++++++------ src/H5Dpkg.h | 5 ++--- src/H5SC.c | 11 +++++++---- src/H5SCprivate.h | 9 +++++---- src/H5VLnative_dataset.c | 9 ++++----- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index b343d82aa26..c7016a68278 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -8282,10 +8282,9 @@ H5D__chunk_verify_offset(const H5D_t *dset, const hsize_t *offset) *------------------------------------------------------------------------- */ herr_t -H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, - hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset, - H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr, - hsize_t H5_ATTR_UNUSED *chunk_size) +H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t H5_ATTR_UNUSED chunk_idx, + hsize_t H5_ATTR_UNUSED *offset, H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, + haddr_t H5_ATTR_UNUSED *addr, hsize_t H5_ATTR_UNUSED *chunk_size) { herr_t ret_value = SUCCEED; /* Return value */ @@ -8296,7 +8295,10 @@ H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, assert(space); /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */ - /* It should be possible to optimize this in the future by only flushing the target chunk, and later directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or index query completely if the shared chunk cache has all the needed information needed. For now, just mirror the previous algorithm for legacy chunks. */ + /* It should be possible to optimize this in the future by only flushing the target chunk, and later + * directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or + * index query completely if the shared chunk cache has all the needed information needed. For now, just + * mirror the previous algorithm for legacy chunks. */ if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset"); @@ -8333,7 +8335,10 @@ H5D__get_struct_chunk_info_by_coord(H5D_t *dset, const hsize_t H5_ATTR_UNUSED *o assert(offset); /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */ - /* It should be possible to optimize this in the future by only flushing the target chunk, and later directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or index query completely if the shared chunk cache has all the needed information needed. For now, just mirror the previous algorithm for legacy chunks. */ + /* It should be possible to optimize this in the future by only flushing the target chunk, and later + * directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or + * index query completely if the shared chunk cache has all the needed information needed. For now, just + * mirror the previous algorithm for legacy chunks. */ if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset"); diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index bcb2af47bb0..e668a9bf9aa 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -788,9 +788,8 @@ H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, void *buf[]); -H5_DLL herr_t H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, - hsize_t chunk_idx, hsize_t *offset, - H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, +H5_DLL herr_t H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t chunk_idx, + hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, hsize_t *chunk_size); H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(H5D_t *dset, const hsize_t *offset, H5D_struct_chunk_info_t *chunk_info, haddr_t *addr, diff --git a/src/H5SC.c b/src/H5SC.c index 64e8ef08d73..731d1c98cf0 100644 --- a/src/H5SC.c +++ b/src/H5SC.c @@ -265,7 +265,9 @@ H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void /*------------------------------------------------------------------------- * Function: H5SC_get_defined * - * Purpose: Returns a copy of file_space with only elements selected that are both selected in file_space and defined in dset. If file_space uses a point selection, the ordering of selected points will be preserved in the returned dataspace. + * Purpose: Returns a copy of file_space with only elements selected that are both selected in file_space and + *defined in dset. If file_space uses a point selection, the ordering of selected points will be preserved in + *the returned dataspace. * * Return: SUCCEED on success, FAIL on failure *------------------------------------------------------------------------- @@ -273,7 +275,7 @@ H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void H5S_t * H5SC_get_defined(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space) { - H5S_t *defined = NULL; + H5S_t *defined = NULL; H5S_t *ret_value = NULL; FUNC_ENTER_NOAPI(NULL) @@ -289,7 +291,7 @@ H5SC_get_defined(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space) /* Set return value */ ret_value = defined; - defined = NULL; + defined = NULL; done: if (defined) { @@ -304,7 +306,8 @@ H5SC_get_defined(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space) /*------------------------------------------------------------------------- * Function: H5SC_erase * - * Purpose: Causes the elements selected in file_space to become undefined in dset. If dset does not support tracking defined elements, returns an error. + * Purpose: Causes the elements selected in file_space to become undefined in dset. If dset does not support + *tracking defined elements, returns an error. * * Return: SUCCEED on success, FAIL on failure *------------------------------------------------------------------------- diff --git a/src/H5SCprivate.h b/src/H5SCprivate.h index 001cc444516..05a8ec183f8 100644 --- a/src/H5SCprivate.h +++ b/src/H5SCprivate.h @@ -181,9 +181,9 @@ typedef herr_t (*H5SC_chunk_scatter_mem_t)(H5D_dset_io_info_t *dset_info, H5D_io * conversion (which will be handled by H5SC layer). If the layout stores variable length data within the * chunk this callback must be defined. */ typedef herr_t (*H5SC_chunk_gather_mem_t)(H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info, - const H5S_t *mem_space, const H5S_t *file_space, size_t *nbytes /*in,out*/, - size_t *alloc_size /*in,out*/, size_t *buf_size_total /*in,out*/, - void *chunk, void *udata); + const H5S_t *mem_space, const H5S_t *file_space, + size_t *nbytes /*in,out*/, size_t *alloc_size /*in,out*/, + size_t *buf_size_total /*in,out*/, void *chunk, void *udata); /* Propagates the fill value into the selected elements of the chunk buffer, performing type conversion if * necessary. space's extent matches the chunk dimensions and the selection is within the chunk. Optional, if @@ -226,7 +226,8 @@ typedef herr_t (*H5SC_layout_query)(H5D_t *dset, hsize_t *chunk_dims, bool *enco /* Removes the chunk from the index and deletes it on disk. Only called if a chunk goes out of scope due to * H5Dset_extent() or if H5SC_chunk_erase_values_t returns *delete_chunk == true. */ -typedef herr_t (*H5SC_delete_chunk_t)(H5D_t *dset, const hsize_t *scaled /*in*/, haddr_t addr, hsize_t disk_size); +typedef herr_t (*H5SC_delete_chunk_t)(H5D_t *dset, const hsize_t *scaled /*in*/, haddr_t addr, + hsize_t disk_size); /* Operations that are implemented by shared chunk cache clients */ struct H5SC_layout_ops_t { diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index bb428c1e3e4..c4474bad7b1 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -773,9 +773,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ udata.size = 0; /* Dispatch call to H5SC layer */ - if (H5SC_direct_chunk_read(H5F_SHARED_CACHE(dset->oloc.file), dset, - chunk_read_args->offset, &udata, chunk_read_args->buf, - &buf_size) < 0) + if (H5SC_direct_chunk_read(H5F_SHARED_CACHE(dset->oloc.file), dset, chunk_read_args->offset, + &udata, chunk_read_args->buf, &buf_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data"); /* Return info to caller */ @@ -812,8 +811,8 @@ H5VL__native_dataset_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_ udata.size = (uint64_t)chunk_write_args->size; /* Dispatch call to H5SC layer */ - if (H5SC_direct_chunk_write(H5F_SHARED_CACHE(dset->oloc.file), dset, - chunk_write_args->offset, &udata, chunk_write_args->buf) < 0) + if (H5SC_direct_chunk_write(H5F_SHARED_CACHE(dset->oloc.file), dset, chunk_write_args->offset, + &udata, chunk_write_args->buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data"); } else { From bd2553408abfaf474041237c88e2f2fe8d78bb8e Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Fri, 31 Jan 2025 15:35:39 -0600 Subject: [PATCH 10/12] Fix failing struct_chunk_storage test, since H5Derase() throws an error when it isn't supported. Also replace inappropriate usage of FAIL_STACK_ERROR with TEST_ERROR. --- test/struct_chunk_storage.c | 111 +++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/test/struct_chunk_storage.c b/test/struct_chunk_storage.c index 8c6cf8625e6..d73b893d8be 100644 --- a/test/struct_chunk_storage.c +++ b/test/struct_chunk_storage.c @@ -93,31 +93,32 @@ test_sparse_data(hid_t fapl) hsize_t dim[1] = {50}; /* 1-d dataspace */ hsize_t chunk_dim[1] = {5}; /* Chunk size */ int wbuf[50]; /* Write buffer */ + herr_t ret; TESTING("APIs for handling sparse data"); /* Create a file */ h5_fixname(FILENAME[1], fapl, filename, sizeof filename); if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Create dataspace */ if ((sid = H5Screate_simple(1, dim, NULL)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Create property list for compact dataset creation */ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: need to set to H5D_SPARSE_CHUNK */ if (H5Pset_layout(dcpl, H5D_STRUCT_CHUNK) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Pset_struct_chunk(dcpl, 1, chunk_dim, H5D_SPARSE_CHUNK) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Write sparse data to the dataset */ memset(wbuf, 0, sizeof(wbuf)); @@ -138,38 +139,44 @@ test_sparse_data(hid_t fapl) /* Get defined elements */ /* TBD: Verify that dataset with H5D_SPARSE_CHUNK layout will succeed; otherwise fail */ if ((sid1 = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: Verify defined elements in sid1 are as expected */ /* Erase all defined elements */ /* TBD: Verify that dataset with H5D_SPARSE_CHUNK layout will succeed; otherwise fail */ - if (H5Derase(did, sid1, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR; + /* Since it is not supported yet, it is expected to fail */ + H5E_BEGIN_TRY + { + ret = H5Derase(did, sid1, H5P_DEFAULT); + } + H5E_END_TRY + if (ret >= 0) + TEST_ERROR; /* Call H5Dget_defined() again after H5Derase() */ if ((sid2 = H5Dget_defined(did, H5S_ALL, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: Verify nothing is defined in sid2 */ if (H5Sclose(sid1) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Sclose(sid2) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Closing */ if (H5Sclose(sid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Pclose(dcpl) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Dclose(did) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Fclose(fid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; PASSED(); return SUCCEED; @@ -241,29 +248,29 @@ test_sparse_direct_chunk(hid_t fapl) /* Create a file */ h5_fixname(FILENAME[2], fapl, filename, sizeof filename); if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* * Create the data space with unlimited dimensions. */ if ((sid = H5Screate_simple(RANK, dims, maxdims)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: need to set to H5D_SPARSE_CHUNK */ if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* * Create a new dataset within the file using dcpl */ if ((did = H5Dcreate2(fid, SPARSE_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; start[0] = 3; start[1] = 2; @@ -272,11 +279,11 @@ test_sparse_direct_chunk(hid_t fapl) count[0] = count[1] = 1; /* Select the 2x3 block in chunk index 0 for writing */ if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Get the encoded size for the selection */ if (H5Sencode2(sid, NULL, &encode_size, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Set up section size for section 0 and section 1 */ wr_section_size[0] = encode_size; @@ -284,13 +291,13 @@ test_sparse_direct_chunk(hid_t fapl) /* Allocate buffers for section 0 (encoded selection) and section 1 (data) */ if ((wr_buf0 = (unsigned char *)calloc((size_t)1, encode_size)) == NULL) - FAIL_STACK_ERROR; + TEST_ERROR; if ((wr_buf1 = (int *)calloc((size_t)1, wr_section_size[1])) == NULL) - FAIL_STACK_ERROR; + TEST_ERROR; /* Encode selection into the buffer for section 0 */ if (H5Sencode2(sid, wr_buf0, &encode_size, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Set up data into the buffer for section 1 */ wr_buf1[0] = 32; @@ -312,27 +319,27 @@ test_sparse_direct_chunk(hid_t fapl) /* Write the structured chunk at offset [0,0]: chunk index 0 */ if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, wr_offset, &wr_chk_info, wr_buf) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Read the whole dataset */ if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: Verify buf read has data as in wr_buf1[] at location wr_buf0[] */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Sclose(sid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Pclose(dcpl) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if ((sid = H5Dget_space(did)) == H5I_INVALID_HID) - FAIL_STACK_ERROR; + TEST_ERROR; /* Select the 2x1 block in chunk index 3 for reading */ start[0] = 5; @@ -341,19 +348,19 @@ test_sparse_direct_chunk(hid_t fapl) block[1] = 1; count[0] = count[1] = 1; if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, NULL, count, block) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Sencode2(sid, NULL, &encode_size, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; rd_section_size[0] = encode_size; rd_section_size[1] = block[0] * block[1] * sizeof(int); /* Allocate buffers for section 0 (encoded selection) and section 1 (data) */ if ((rd_buf0 = (unsigned char *)calloc((size_t)1, encode_size)) == NULL) - FAIL_STACK_ERROR; + TEST_ERROR; if ((rd_buf1 = (int *)calloc((size_t)1, rd_section_size[1])) == NULL) - FAIL_STACK_ERROR; + TEST_ERROR; rd_buf[0] = rd_buf0; rd_buf[1] = rd_buf1; @@ -366,18 +373,18 @@ test_sparse_direct_chunk(hid_t fapl) /* Read the structured chunk at offset [5,5] */ if (H5Dread_struct_chunk(did, H5P_DEFAULT, rd_offset, &rd_chk_info, rd_buf) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Verify rd_chk_info and rd_buf are the same as wr_chk_info and wr_buf */ /* * Close/release resources. */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Sclose(sid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Fclose(fid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; PASSED(); return 0; @@ -579,7 +586,7 @@ test_sparse_direct_chunk_query(hid_t fapl) /* TBD: need to set to H5D_SPARSE_CHUNK */ if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) TEST_ERROR; @@ -600,7 +607,7 @@ test_sparse_direct_chunk_query(hid_t fapl) /* Write the structured chunk at offset */ if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, offset, &chk_info, write_buf) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Verify info of the first and only chunk via H5Dget_struct_chunk_info() */ if (verify_get_struct_chunk_info(did, H5S_ALL, 0, offset, &chk_info, CHK_SIZE) == FAIL) @@ -611,7 +618,7 @@ test_sparse_direct_chunk_query(hid_t fapl) /* Write the structured chunk at offset */ if (H5Dwrite_struct_chunk(did, H5P_DEFAULT, offset, &chk_info, write_buf) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Verify info of the chunk at offset [CHUNK_NX,CHUNK_NY] via H5Dget_struct_chunk_info_by_coord() */ if (verify_get_struct_chunk_info_by_coord(did, offset, &chk_info, CHK_SIZE) == FAIL) @@ -701,7 +708,7 @@ test_sparse_filter(hid_t fapl) /* TBD: need to set to H5D_SPARSE_CHUNK */ if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) TEST_ERROR; @@ -727,10 +734,10 @@ test_sparse_filter(hid_t fapl) TEST_ERROR; if ((did = H5Dopen2(fid, SPARSE_DSET, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if ((dcpl = H5Dget_create_plist(did)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: verify that filtn is H5Z_FILTER_DEFLATE and flags is H5Z_FLAG_MANDATORY */ filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, &flags, NULL, NULL, (size_t)0, NULL, NULL); @@ -742,16 +749,16 @@ test_sparse_filter(hid_t fapl) /* TBD: Modify the filter's flags to optional */ if (H5Pmodify_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 0, NULL) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: verify that flags is H5Z_FLAG_OPTIONAL */ if (H5Pget_filter_by_id3(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE, &flags, NULL, NULL, 0, NULL, NULL) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: Remove the filter */ if (H5Premove_filter2(dcpl, H5Z_FLAG_SPARSE_SELECTION, H5Z_FILTER_DEFLATE) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* TBD: verify that filtn is H5Z_FILTER_NONE */ filtn = H5Pget_filter3(dcpl, H5Z_FLAG_SPARSE_SELECTION, 0, NULL, NULL, NULL, (size_t)0, NULL, NULL); @@ -875,7 +882,7 @@ test_dense_chunk_api_on_sparse(hid_t fapl) /* TBD: need to set to H5D_SPARSE_CHUNK */ if (H5Pset_layout(dcpl, H5D_CHUNKED) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* The layout is set to H5D_CHUNKED as a side-effect */ if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) From 7398a04211b09e9d103805d0004e433b210e5e96 Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Tue, 18 Feb 2025 15:47:27 -0600 Subject: [PATCH 11/12] Adjust H5D_io_info_t struct and add more descriptions for the fields. --- src/H5Dpkg.h | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index e668a9bf9aa..db44a43d85a 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -141,18 +141,57 @@ typedef struct H5D_type_info_t { size_t request_nelmts; /* Requested strip mine */ } H5D_type_info_t; -/* Typedef for datatype information for all datasets in a raw data I/O operation */ +/**************************************************************************** + * + * structure H5D_io_type_info_t + * + * This structure stores some type conversion related info that is passed to + * shared chunk cache layout callbacks. + * + * The fields of this structure are discussed individually below: + * + * tconv_buf: The type conversion buffer. + * + * tconv_buf_size: The allocated size of tconv_buf in bytes. This is + * guaranteed to be at least large enough to convert a single element. + * + * bkg_buf: The background buffer for type conversion (if any). + * + * bkg_buf_size: The allocated size of bkg_buf in bytes. This is guaranteed to + * be at least large enough to hold a single element of the destination + * datatype, unless the background buffer is not needed for this + * dataset. + * + * vlen_buf_info: Structure containing a buffer used to store converted + * variable length data. Used when multiple variable length data arrays + * are to be stored concurrently in a single block. The callback must + * place all variable length data in this array for the operation, and + * must handle freeing of overwritten data and defragmentation + * appropriately (this may be handled by H5T_convert()). + * + * may_use_in_place_tconv: Boolean flag that is set to true when type + * conversion is not restricted by the public API or the cache from + * performing in-place type conversion. This is the case for read + * operations where the data in the file datatype does not need to be + * cached (or is about to be evicted), and for write operations when the + * user has used H5Pset_modify_write_buf() to indicate that the library + * may modify supplied write buffers. + * + * This being set to true does not necessarily indicate that in-place + * type conversion is possible. It is the client's responsibility to + * ensure that datatype sizes or a noncontiguous selection do no + * prohibit this (unless of course the algorithm is adjusted to support + * these cases). + * + ****************************************************************************/ struct H5D_io_type_info_t { uint8_t *tconv_buf; /* Datatype conv buffer */ - bool tconv_buf_allocated; /* Whether the type conversion buffer was allocated */ size_t tconv_buf_size; /* Size of type conversion buffer */ uint8_t *bkg_buf; /* Background buffer */ - bool bkg_buf_allocated; /* Whether the background buffer was allocated */ size_t bkg_buf_size; /* Size of background buffer */ H5T_vlen_buf_info_t vlen_buf_info; /* Vlen data buffer and info */ - bool must_fill_bkg; /* Whether any datasets need a background buffer filled with destination contents */ bool may_use_in_place_tconv; /* Whether datasets in this I/O could potentially use in-place type - conversion if the type sizes are compatible with it */ + conversion if the type sizes and situation are compatible with it */ }; /* Forward declaration of structs used below */ From d808f03626981cf55d2b878e848a6c428cae719b Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Feb 2025 21:49:44 +0000 Subject: [PATCH 12/12] Committing clang-format changes --- src/H5Dpkg.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index db44a43d85a..36d25c1740a 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -185,13 +185,13 @@ typedef struct H5D_type_info_t { * ****************************************************************************/ struct H5D_io_type_info_t { - uint8_t *tconv_buf; /* Datatype conv buffer */ - size_t tconv_buf_size; /* Size of type conversion buffer */ - uint8_t *bkg_buf; /* Background buffer */ - size_t bkg_buf_size; /* Size of background buffer */ - H5T_vlen_buf_info_t vlen_buf_info; /* Vlen data buffer and info */ - bool may_use_in_place_tconv; /* Whether datasets in this I/O could potentially use in-place type - conversion if the type sizes and situation are compatible with it */ + uint8_t *tconv_buf; /* Datatype conv buffer */ + size_t tconv_buf_size; /* Size of type conversion buffer */ + uint8_t *bkg_buf; /* Background buffer */ + size_t bkg_buf_size; /* Size of background buffer */ + H5T_vlen_buf_info_t vlen_buf_info; /* Vlen data buffer and info */ + bool may_use_in_place_tconv; /* Whether datasets in this I/O could potentially use in-place type + conversion if the type sizes and situation are compatible with it */ }; /* Forward declaration of structs used below */