Kari, im forwarding this to netcdfgroup@xxxxxxxxxxxxxxxx, which deals
with the C library.
-------- Original Message --------
Subject: Looks like a bug in the netCDF NC_64BIT_OFFSET variables
Date: Tue, 13 Apr 2010 17:48:33 -0500
From: Kari Hoijarvi <hoijarvi@xxxxxxxxxxxxxx>
To: support-netcdf@xxxxxxxxxxxxxxxx
CC: caron@xxxxxxxx, "Rudolf B. Husar (E-mail)" <rhusar@xxxxxxxxxxxx>,
"Stefan Falke (E-mail)" <stefan@xxxxxxxxxxxx>, Michael Decker
<m.decker@xxxxxxxxxxxxx>, Ben Domenico <Ben@xxxxxxxxxxxxxxxx>
Hello,
This looks like a bug in the NC_64BIT_OFFSET big variable handling.
I have a test here that creates 3000* 1000 * 2000 float variable, 24 GB
After nc_enddef, the file size is suspiciously 4 GB
After writing the data, the size is 16.0 GB (17,184,000,120 bytes),
instead of 24 GB.
Reading fails to produce expected results at strange offset 0, 516, 704
I attached my version netcdf/nc_test/nc_test.c, it has a new function
test_big_cube_without_unlimited_dim().
It should be easy to copy that over and run it.
Runs very slowly, since it creates 24 GB dummy data.
Kari
/*********************************************************************
* Copyright 1996-2005, UCAR/Unidata
* See COPYRIGHT file for copying and redistribution conditions.
* $Id: nc_test.c,v 1.44 2008/10/20 01:48:08 ed Exp $
*********************************************************************/
#include "tests.h"
/*
* Test driver for netCDF-3 interface. This program performs tests against
* the netCDF-3 specification for all user-level functions in an
* implementation of the netCDF library.
*
* Files:
* The read-only tests read files:
* test.nc (see below)
* tests.h (used merely as an example of a non-netCDF file)
*
* The write tests
* read test.nc (see below)
* write scratch.nc (deleted after each test)
*
* The file test.nc is created by running nc_test with the -c (create) option.
* It is described by the following global variables.
*/
/*
* global variables (defined by function init_gvars) describing file test.nc
*/
char dim_name[NDIMS][3];
size_t dim_len[NDIMS];
char var_name[NVARS][2+MAX_RANK];
nc_type var_type[NVARS];
size_t var_rank[NVARS];
int var_dimid[NVARS][MAX_RANK];
size_t var_shape[NVARS][MAX_RANK];
size_t var_nels[NVARS];
size_t var_natts[NVARS];
char att_name[NVARS][MAX_NATTS][2];
char gatt_name[NGATTS][3];
nc_type att_type[NVARS][NGATTS];
nc_type gatt_type[NGATTS];
size_t att_len[NVARS][MAX_NATTS];
size_t gatt_len[NGATTS];
/*
* command-line options
*/
int verbose; /* if 1, print details of tests */
int max_nmpt; /* max. number of messages per test */
/*
* Misc. global variables
*/
int nfails; /* number of failures in specific test */
char testfile[NC_MAX_NAME];
char scratch[] = "scratch.nc"; /* writable scratch file */
#define NC_TEST(func) \
print( "*** Testing " #func " ... ");\
nfails = 0;\
test_ ## func();\
nfailsTotal += nfails;\
if (verbose) \
print("\n"); \
if ( nfails == 0) \
print( "ok\n");\
else\
print( "\n\t### %d FAILURES TESTING %s! ###\n", nfails, #func)
#if 1 /* both CRAY MPP and OSF/1 Alpha systems need this */
#include <signal.h>
#endif /* T90 */
/* Test everything for classic and 64-bit offsetfiles. If netcdf-4 is
* included, that means another whole round of testing. */
#ifdef USE_NETCDF4
#define NUM_FORMATS (3)
#else
#define NUM_FORMATS (2)
#endif
void test_big_cube_without_unlimited_dim()
{
const int c_time = 3000;
const int c_lat = 1000;
const int c_lon = 2000;
int ncid, lat, lon, time, bbvar;
int dimids[3], start[3], count[3];
float *data = calloc(2000000, sizeof(*data));
int ec;
int itime, ilat, ilon;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = c_lat;
count[2] = c_lon;
// change to #if 0 if you already created the cube and want just to check it
#if 1
if (ec = nc_create("big_cube.nc", NC_CLOBBER | NC_64BIT_OFFSET, &ncid))
goto fail;
if (ec = nc_def_dim(ncid, "lat", c_lat, &lat))
goto fail;
if (ec = nc_def_dim(ncid, "lon", c_lon, &lon))
goto fail;
if (ec = nc_def_dim(ncid, "time", c_time, &time))
goto fail;
dimids[0] = time;
dimids[1] = lat;
dimids[2] = lon;
if (ec = nc_def_var(ncid, "bigbox", NC_FLOAT, 3, dimids, &bbvar))
goto fail;
if (ec = nc_enddef(ncid))
goto fail;
if (ec = nc_close(ncid))
goto fail;
if (ec = nc_open("big_cube.nc", NC_WRITE, &ncid))
goto fail;
for (itime = 0; itime < c_time; ++itime)
{
start[0] = itime;
for (ilat = 0; ilat < c_lat; ++ilat)
{
for (ilon = 0; ilon < c_lon; ++ilon)
{
data[ilon + c_lon*ilat] = (itime + ilat + ilon)
% 19;
}
}
if (ec = nc_put_vara_float(ncid, bbvar, start, count, data))
goto fail;
}
if (ec = nc_close(ncid))
goto fail;
#endif
if (ec = nc_open("big_cube.nc", NC_NOWRITE, &ncid))
goto fail;
if (ec = nc_inq_varid(ncid, "bigbox", &bbvar))
goto fail;
for (itime = 0; itime < c_time; ++itime)
{
start[0] = itime;
if (ec = nc_get_vara_float(ncid, bbvar, start, count, data))
goto fail;
for (ilat = 0; ilat < c_lat; ++ilat)
{
for (ilon = 0; ilon < c_lon; ++ilon)
{
if (data[ilon + c_lon*ilat] != (itime + ilat +
ilon) % 19)
{
printf("test_big_cube_without_unlimited_dim FAILS %d %d %d", itime, ilat, ilon);
}
}
}
}
if (0)
{
fail:
print("test_big_cube_without_unlimited_dim exited
unsuccessfully");
ec = nc_close(ncid);
}
free(data);
return;
}
int
main(int argc, char *argv[])
{
int i;
int nfailsTotal = 0; /* total number of failures */
/* Both CRAY MPP and OSF/1 Alpha systems need this. Some of the
* extreme test assignments in this program trigger floating point
* exceptions on CRAY T90
*/
(void) signal(SIGFPE, SIG_IGN);
test_big_cube_without_unlimited_dim();
verbose = 0;
max_nmpt = 8;
/* Initialize global variables defining test file */
init_gvars();
/* If you uncomment the nc_set_log_level line, you will get a lot
* of debugging info. If you set the number higher, you'll get
* more. 6 is max, 0 shows only errors. 3 is a good place to
* start. */
/*nc_set_log_level(3);*/
fprintf(stderr, "Testing %d different netCDF formats.\n", NUM_FORMATS);
/* Go thru formats and run all tests for each of two (for netCDF-3
* only builds), or 3 (for netCDF-4 builds) different formats. Do
* the netCDF-4 format last, however, because, as an additional
* test, the ../nc_test4/tst_nc_test_file program looks at the
* output of this program. */
for (i = 1; i <= NUM_FORMATS; i++)
{
switch (i)
{
case NC_FORMAT_CLASSIC:
nc_set_default_format(NC_FORMAT_CLASSIC, NULL);
fprintf(stderr, "\n\nSwitching to netCDF classic format.\n");
strcpy(testfile, "nc_test_classic.nc");
break;
case NC_FORMAT_64BIT:
nc_set_default_format(NC_FORMAT_64BIT, NULL);
fprintf(stderr, "\n\nSwitching to 64-bit offset format.\n");
strcpy(testfile, "nc_test_64bit.nc");
break;
#ifdef USE_NETCDF4
case NC_FORMAT_NETCDF4: /* actually it's _CLASSIC. */
nc_set_default_format(NC_FORMAT_NETCDF4_CLASSIC, NULL);
strcpy(testfile, "nc_test_netcdf4.nc");
fprintf(stderr, "\n\nSwitching to netCDF-4 format (with
NC_CLASSIC_MODEL).\n");
break;
#endif
default:
fprintf(stderr, "Unexpected format!\n");
return 2;
}
/* Write the test file, needed for the read-only tests below. */
write_file(testfile);
/* delete any existing scratch netCDF file */
(void) remove(scratch);
/* Test read-only functions, using pre-generated test-file */
NC_TEST(nc_strerror);
NC_TEST(nc_open);
NC_TEST(nc_close);
NC_TEST(nc_inq);
NC_TEST(nc_inq_dimid);
NC_TEST(nc_inq_dim);
NC_TEST(nc_inq_dimlen);
NC_TEST(nc_inq_dimname);
NC_TEST(nc_inq_varid);
NC_TEST(nc_inq_var);
NC_TEST(nc_inq_natts);
NC_TEST(nc_inq_ndims);
NC_TEST(nc_inq_nvars);
NC_TEST(nc_inq_unlimdim);
NC_TEST(nc_inq_vardimid);
NC_TEST(nc_inq_varname);
NC_TEST(nc_inq_varnatts);
NC_TEST(nc_inq_varndims);
NC_TEST(nc_inq_vartype);
NC_TEST(nc_get_var_text);
NC_TEST(nc_get_var_uchar);
NC_TEST(nc_get_var_schar);
NC_TEST(nc_get_var_short);
NC_TEST(nc_get_var_int);
NC_TEST(nc_get_var_long);
NC_TEST(nc_get_var_float);
NC_TEST(nc_get_var_double);
NC_TEST(nc_get_var1_text);
NC_TEST(nc_get_var1_uchar);
NC_TEST(nc_get_var1_schar);
NC_TEST(nc_get_var1_short);
NC_TEST(nc_get_var1_int);
NC_TEST(nc_get_var1_long);
NC_TEST(nc_get_var1_float);
NC_TEST(nc_get_var1_double);
NC_TEST(nc_get_var1);
NC_TEST(nc_get_vara_text);
NC_TEST(nc_get_vara_uchar);
NC_TEST(nc_get_vara_schar);
NC_TEST(nc_get_vara_short);
NC_TEST(nc_get_vara_int);
NC_TEST(nc_get_vara_long);
NC_TEST(nc_get_vara_float);
NC_TEST(nc_get_vara_double);
NC_TEST(nc_get_vara);
NC_TEST(nc_get_vars_text);
NC_TEST(nc_get_vars_uchar);
NC_TEST(nc_get_vars_schar);
NC_TEST(nc_get_vars_short);
NC_TEST(nc_get_vars_int);
NC_TEST(nc_get_vars_long);
NC_TEST(nc_get_vars_float);
NC_TEST(nc_get_vars_double);
NC_TEST(nc_get_vars);
NC_TEST(nc_get_varm_text);
NC_TEST(nc_get_varm_uchar);
NC_TEST(nc_get_varm_schar);
NC_TEST(nc_get_varm_short);
NC_TEST(nc_get_varm_int);
NC_TEST(nc_get_varm_long);
NC_TEST(nc_get_varm_float);
NC_TEST(nc_get_varm_double);
NC_TEST(nc_get_varm);
NC_TEST(nc_get_att_text);
NC_TEST(nc_get_att_uchar);
NC_TEST(nc_get_att_schar);
NC_TEST(nc_get_att_short);
NC_TEST(nc_get_att_int);
NC_TEST(nc_get_att_long);
NC_TEST(nc_get_att_float);
NC_TEST(nc_get_att_double);
NC_TEST(nc_get_att);
NC_TEST(nc_inq_att);
NC_TEST(nc_inq_attname);
NC_TEST(nc_inq_attid);
NC_TEST(nc_inq_attlen);
NC_TEST(nc_inq_atttype);
/* Test write functions */
NC_TEST(nc_create);
NC_TEST(nc_redef);
/* NC_TEST(nc_enddef); *//* redundant */
NC_TEST(nc_sync);
NC_TEST(nc_abort);
NC_TEST(nc_def_dim);
NC_TEST(nc_rename_dim);
NC_TEST(nc_def_var);
NC_TEST(nc_put_var_text);
NC_TEST(nc_put_var_uchar);
NC_TEST(nc_put_var_schar);
NC_TEST(nc_put_var_short);
NC_TEST(nc_put_var_int);
NC_TEST(nc_put_var_long);
NC_TEST(nc_put_var_float);
NC_TEST(nc_put_var_double);
NC_TEST(nc_put_var1_text);
NC_TEST(nc_put_var1_uchar);
NC_TEST(nc_put_var1_schar);
NC_TEST(nc_put_var1_short);
NC_TEST(nc_put_var1_int);
NC_TEST(nc_put_var1_long);
NC_TEST(nc_put_var1_float);
NC_TEST(nc_put_var1_double);
NC_TEST(nc_put_var1);
NC_TEST(nc_put_vara_text);
NC_TEST(nc_put_vara_uchar);
NC_TEST(nc_put_vara_schar);
NC_TEST(nc_put_vara_short);
NC_TEST(nc_put_vara_int);
NC_TEST(nc_put_vara_long);
NC_TEST(nc_put_vara_float);
NC_TEST(nc_put_vara_double);
NC_TEST(nc_put_vara);
NC_TEST(nc_put_vars_text);
NC_TEST(nc_put_vars_uchar);
NC_TEST(nc_put_vars_schar);
NC_TEST(nc_put_vars_short);
NC_TEST(nc_put_vars_int);
NC_TEST(nc_put_vars_long);
NC_TEST(nc_put_vars_float);
NC_TEST(nc_put_vars_double);
NC_TEST(nc_put_vars);
NC_TEST(nc_put_varm_text);
NC_TEST(nc_put_varm_uchar);
NC_TEST(nc_put_varm_schar);
NC_TEST(nc_put_varm_short);
NC_TEST(nc_put_varm_int);
NC_TEST(nc_put_varm_long);
NC_TEST(nc_put_varm_float);
NC_TEST(nc_put_varm_double);
NC_TEST(nc_put_varm);
NC_TEST(nc_rename_var);
NC_TEST(nc_put_att_text);
NC_TEST(nc_put_att_uchar);
NC_TEST(nc_put_att_schar);
NC_TEST(nc_put_att_short);
NC_TEST(nc_put_att_int);
NC_TEST(nc_put_att_long);
NC_TEST(nc_put_att_float);
NC_TEST(nc_put_att_double);
NC_TEST(nc_put_att);
NC_TEST(nc_copy_att);
NC_TEST(nc_rename_att);
NC_TEST(nc_del_att);
NC_TEST(nc_set_default_format);
}
fprintf(stderr, "\n*** Total number of failures: %d\n", nfailsTotal);
if (nfailsTotal)
{
fprintf(stderr, "*** nc_test FAILURE!!!\n");
return 2;
}
else
fprintf(stderr, "*** nc_test SUCCESS!!!\n");
return 0;
}
