Hi,
I suspect a overflow of a short in HDF5 when rewriting
variable-attributes more than 65533 times.
Our fortran model closes and re-opens netcdf files for each time-step it
processes (processing of a timestep is much, much longer than re-opening
of an netcdf-file). While writing the data, the model modifies also some
attributes, making sure that only the attribute-data is changed, but not
the attribute-name, or the attribute data-length. This works very well
and performant with netcdf3, but when using netcdf4, the model starts
crashing with a HDF5 error after a while.
I managed now to write a simple test-case, illustrating the problem.
After re-writing an attribute for 65533-times, we are no longer able to
write a data-attribute and get a HDF5 error.
In the model we write several attributes on several (hundred) variables,
but never more than a few thousand time-step. So it seems to be a
combination of re-writing attributes and re-opening files.
I tested the test-file against netcdf 4.3.0 / hdf 1.8.11 and netcdf
4.3.2 / hdf 1.8.13, with the same result. I hope you find time to have a
look at this bug:
...
*** SUCCESS writing example file simple_xy_nc4.nc: 64000
*** SUCCESS writing example file simple_xy_nc4.nc: 65000
65531
65532
Error: NetCDF: Can't open HDF5 attribute
Best regards,
Heiko
--
Dr. Heiko Klein Tel. + 47 22 96 32 58
Development Section / IT Department Fax. + 47 22 69 63 55
Norwegian Meteorological Institute http://www.met.no
P.O. Box 43 Blindern 0313 Oslo NORWAY
#include <stdlib.h>
#include <stdio.h>
#include <netcdf.h>
/* This is the name of the data file we will create. */
#define FILE_NAME "simple_xy_nc4.nc"
/* We are writing 2D data, a 6 x 12 grid. */
#define NDIMS 3
#define NX 6
#define NY 12
/* Handle errors by printing an error message and exiting with a
* non-zero status. */
#define ERRCODE 2
#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);}
int
main()
{
int ncid, x_dimid, y_dimid, t_dimid, varid;
int dimids[NDIMS];
size_t chunks[NDIMS], start[NDIMS], size[NDIMS];
int shuffle, deflate, deflate_level;
float data_out[NX][NY];
int x, y, retval;
/* Set chunking, shuffle, and deflate. */
shuffle = NC_SHUFFLE;
deflate = 1;
deflate_level = 1;
/* Create some pretend data. If this wasn't an example program, we
* would have some real data to write, for example, model output. */
for (x = 0; x < NX; x++)
for (y = 0; y < NY; y++)
data_out[x][y] = 0; rand() / RAND_MAX;
/* Create the file. The NC_NETCDF4 parameter tells netCDF to create
* a file in netCDF-4/HDF5 standard. */
if ((retval = nc_create(FILE_NAME, NC_NETCDF4, &ncid)))
ERR(retval);
/* Define the dimensions. */
if ((retval = nc_def_dim(ncid, "x", NX, &x_dimid)))
ERR(retval);
if ((retval = nc_def_dim(ncid, "y", NY, &y_dimid)))
ERR(retval);
if ((retval = nc_def_dim(ncid, "t", NC_UNLIMITED, &t_dimid)))
ERR(retval);
/* Set up variabe data. */
dimids[0] = t_dimid;
dimids[1] = y_dimid;
dimids[2] = x_dimid;
/* Define the variable. */
if ((retval = nc_def_var(ncid, "data", NC_FLOAT, NDIMS, dimids, &varid)))
ERR(retval);
/* putting attributes */
int i = -1;
if (retval = nc_put_att_int(ncid, varid, "record_size", NC_INT, 1, &i))
ERR(retval);
if (retval = nc_enddef(ncid))
ERR(retval);
/* Close the file. */
if ((retval = nc_close(ncid)))
ERR(retval);
for (i = 0; i < 2^31; ++i) {
/* reopen the file */
if ((retval = nc_open(FILE_NAME, NC_WRITE, &ncid)))
ERR(retval);
if (retval = nc_inq_varid(ncid, "data", &varid))
ERR(retval);
if (retval = nc_put_att_int(ncid, varid, "record_size", NC_INT, 1, &i))
ERR(retval);
start[0] = i;
start[1] = 0;
start[2] = 0;
size[0] = 1;
size[1] = NY;
size[2] = NX;
/* No need to explicitly end define mode for netCDF-4 files. Write
* the pretend data to the file. */
// if ((retval = nc_put_vara_float(ncid, varid, start, size,
&data_out[0][0])))
// ERR(retval);
/* Close the file. */
if ((retval = nc_close(ncid)))
ERR(retval);
if (i > 65530)
printf("%d\n", i);
if (i%1000 == 0)
printf("*** SUCCESS writing example file simple_xy_nc4.nc: %d\n", i);
}
return 0;
}
