Dear Philippe,
You have just remind me that I had modified the nccopy tool to extend
its functionality.
Like your version, it includes the deflating option and 2 more
capabilities: shuffling and remove unlimited dimensions
ncconvert [-k n] [-m n] [-d n] [-s] [-u] infile outfile
[-k n] kind of netCDF format for output file, default same as input
1 classic, 2 64-bit offset, 3 netCDF-4, 4 netCDF-4 classic
model
[-m n] size in bytes of copy buffer
[-d n] deflate vars (n: deflate level [0-9])
[-s] shuffle vars
[-u] remove unlimited dims
infile name of netCDF input file
outfile name for netCDF output file
I have modified the nccopy.c source file from the netcdf-4.1
distribution and rename as nconvert.c. I'm inlcuding also the nciter.c
and nciter.h (unmodified from the netcdf-4.1 source distribution) files
to compile the tool using your netcdf library.
Regards
--
Antonio S. Cofiño
Grupo de Meteorología de Santander
Dep. de Matemática Aplicada y
Ciencias de la Computación
Universidad de Cantabria
Escuela de Caminos
Avenida de los Castros, 44
39005 Santander, Spain
Tel: (+34) 942 20 1731
Fax: (+34) 942 20 1703
http://www.meteo.unican.es
mailto:antonio.cofino@xxxxxxxxx
El 21/07/2010 18:57, Poilbarbe Philippe escribió:
Hello,
Our main use of the NetCDF 4 is to compress files and have the hability
to uncompress 'on the fly' while reading variables.
nccopy is a good utility to convert files from one format to another
(any conversion can be done provided we use the 'classic' scheme).
But nccopy cannot compress variable from ones in a netCDF 3 file (since
it has no 'deflate' attribute) which is our main goal for this utility
So I have created nccompress from nccopy which can do this:
nccompress -d 1 -k 4 Infile.nc Outfile.nc
and all vars are compressed in a NetCDF4/HDF5 file (if -k is for a
classic file or a 64 bits file, -d is ignored).
I give you the patch to apply to nccopy.c if you are interested in this.
Maybe it is not the better way to do this (I have not filtered the no
dimension vars, for example) but it works.
Cheers,
Ph.P.
_______________________________________________
netcdfgroup mailing list
netcdfgroup@xxxxxxxxxxxxxxxx
For list information or to unsubscribe, visit:
http://www.unidata.ucar.edu/mailing_lists/
/*********************************************************************
* Copyright 2008, University Corporation for Atmospheric Research
* See netcdf/README file for copying and redistribution conditions.
* $ID$
*********************************************************************/
/*#include "config.h"*/ /* for USE_NETCDF4 macro */
#define USE_NETCDF4 1
#include <stdlib.h>
#include <stdio.h>
#ifndef _WIN32
#include <unistd.h>
#endif
#include <string.h>
#include <netcdf.h>
#include "nciter.h"
/* default bytes of memory we are willing to allocate for variable
* values during copy */
#define COPY_BUFFER_SIZE (5000000)
#define SAME_AS_INPUT (-1) /* default, if kind not specified */
#define CHECK(stat,f) if(stat != NC_NOERR) {check(stat,#f,__FILE__,__LINE__);}
else {}
#ifndef USE_NETCDF4
#define NC_CLASSIC_MODEL 0x0100 /* Enforce classic model when used with
NC_NETCDF4. */
#endif
/* These are in unistd.h; for use with getopt() */
extern int optind;
extern int opterr;
extern char *optarg;
static int DEFLATE_LEVEL=-1;
static int SHUFFLE_VARS=NC_NOSHUFFLE;
static int CONVERT_ULIM_DIMS=0;
static char *progname; /* for error messages */
static int nofill_flag = 1; /* default is not to fill, because fill
* values will be copied anyway */
static void
check(int err, const char* fcn, const char* file, const int line)
{
fprintf(stderr,"%s\n",nc_strerror(err));
fprintf(stderr,"Location: function %s; file %s; line %d\n",
fcn,file,line);
fflush(stderr); fflush(stdout);
exit(1);
}
/* Check error return from malloc, and allow malloc(0) with subsequent free */
static void *
emalloc (size_t size)
{
void *p;
p = (void *) malloc (size==0 ? 1 : size); /* don't malloc(0) */
if (p == 0) {
fprintf(stderr,"Out of memory\n");
}
return p;
}
/* Forward declaration, because copy_type, copy_vlen_type call each other */
static int copy_type(int igrp, nc_type typeid, int ogrp);
/* get group id in output corresponding to group igrp in input,
* given parent group id (or root group id) parid in output. */
static int
get_grpid(int igrp, int parid, int *ogrpp) {
int stat = NC_NOERR;
int ogid; /* like igrp but in output file */
#ifdef USE_NETCDF4
int inparid;
/* if not root group, get corresponding output groupid from group name */
stat = nc_inq_grp_parent(igrp, &inparid);
if(stat == NC_NOERR) { /* not root group */
char grpname[NC_MAX_NAME + 1];
stat = nc_inq_grpname(igrp, grpname);
CHECK(stat, nc_inq_grpname);
stat = nc_inq_grp_ncid(parid, grpname, &ogid);
CHECK(stat, nc_inq_grp_ncid);
} else if(stat == NC_ENOGRP) { /* root group */
ogid = parid;
stat = NC_NOERR;
} else {
CHECK(stat, nc_inq_grp_parent);
}
#else
ogid = parid;
#endif /* USE_NETCDF4 */
*ogrpp = ogid;
return stat;
}
#ifdef USE_NETCDF4
/* Return size of chunk in bytes for a variable varid in a group igrp, or 0 if
* layout is contiguous */
static int
inq_var_chunksize(int igrp, int varid, size_t* chunksizep) {
int stat = NC_NOERR;
int ndims;
size_t chunksizes[NC_MAX_DIMS];
int dim;
int contig = 1;
nc_type vartype;
size_t value_size;
size_t prod;
stat = nc_inq_vartype(igrp, varid, &vartype);
CHECK(stat, nc_inq_vartype);
/* from type, get size in memory needed for each value */
stat = nc_inq_type(igrp, vartype, NULL, &value_size);
CHECK(stat, nc_inq_type);
prod = value_size;
stat = nc_inq_varndims(igrp, varid, &ndims);
CHECK(stat, nc_inq_varndims);
if(ndims > 0) {
stat = nc_inq_var_chunking(igrp, varid, &contig, NULL);
CHECK(stat, nc_inq_var_chunking);
}
if(contig == 1) {
*chunksizep = 0;
return stat;
}
/* else chunked */
stat = nc_inq_var_chunking(igrp, varid, &contig, chunksizes);
CHECK(stat, nc_inq_var_chunking);
for(dim = 0; dim < ndims; dim++) {
prod *= chunksizes[dim];
}
*chunksizep = prod;
return stat;
}
/*
* copy a user-defined variable length type in the group igrp to the
* group ogrp
*/
static int
copy_vlen_type(int igrp, nc_type itype, int ogrp)
{
int stat = NC_NOERR;
nc_type ibasetype;
nc_type obasetype; /* base type in target group */
char name[NC_MAX_NAME];
size_t size;
char basename[NC_MAX_NAME];
size_t basesize;
nc_type vlen_type;
stat = nc_inq_vlen(igrp, itype, name, &size, &ibasetype);
CHECK(stat, nc_inq_vlen);
/* to get base type id in target group, use name of base type in
* source group */
stat = nc_inq_type(igrp, ibasetype, basename, &basesize);
CHECK(stat, nc_inq_type);
stat = nc_inq_typeid(ogrp, basename, &obasetype);
/* if no such type, create it now */
if(stat == NC_EBADTYPE) {
copy_type(igrp, ibasetype, ogrp);
CHECK(stat, copy_type);
stat = nc_inq_typeid(ogrp, basename, &obasetype);
}
CHECK(stat, nc_inq_typeid);
/* Now we know base type exists in output and we know its type id */
stat = nc_def_vlen(ogrp, name, obasetype, &vlen_type);
CHECK(stat, nc_copy_vlen_type);
return stat;
}
/*
* copy a user-defined opaque type in the group igrp to the group ogrp
*/
static int
copy_opaque_type(int igrp, nc_type itype, int ogrp)
{
int stat = NC_NOERR;
nc_type otype;
char name[NC_MAX_NAME];
size_t size;
stat = nc_inq_opaque(igrp, itype, name, &size);
CHECK(stat, nc_inq_opaque_type);
stat = nc_def_opaque(ogrp, size, name, &otype);
CHECK(stat, copy_opaque_type);
return stat;
}
/*
* copy a user-defined enum type in the group igrp to the group ogrp
*/
static int
copy_enum_type(int igrp, nc_type itype, int ogrp)
{
int stat = NC_NOERR;
nc_type otype;
nc_type basetype;
size_t basesize;
size_t nmembers;
char name[NC_MAX_NAME];
int i;
stat = nc_inq_enum(igrp, itype, name, &basetype, &basesize, &nmembers);
CHECK(stat, nc_inq_enum);
stat = nc_def_enum(ogrp, basetype, name, &otype);
CHECK(stat, nc_def_enum);
for(i = 0; i < nmembers; i++) { /* insert enum members */
char ename[NC_MAX_NAME];
long long val; /* large enough to hold any integer type */
stat = nc_inq_enum_member(igrp, itype, i, ename, &val);
CHECK(stat, nc_inq_enum_member);
stat = nc_insert_enum(ogrp, otype, ename, &val);
CHECK(stat, nc_insert_enum);
}
return stat;
}
/*
* copy a user-defined compound type in the group igrp to the group ogrp
*/
static int
copy_compound_type(int igrp, nc_type itype, int ogrp)
{
int stat = NC_NOERR;
char name[NC_MAX_NAME];
size_t size;
size_t nfields;
nc_type otype;
int fid;
stat = nc_inq_compound(igrp, itype, name, &size, &nfields);
CHECK(stat, nc_inq_compound);
stat = nc_def_compound(ogrp, size, name, &otype);
CHECK(stat, nc_def_compound);
for (fid = 0; fid < nfields; fid++) {
char fname[NC_MAX_NAME];
char ftypename[NC_MAX_NAME];
size_t foff;
nc_type iftype, oftype;
int fndims, fdimsizes[NC_MAX_DIMS];
stat = nc_inq_compound_field(igrp, itype, fid, fname, &foff, &iftype,
&fndims, fdimsizes);
CHECK(stat, nc_inq_compound_field);
/* type ids in source don't necessarily correspond to same
* typeids in destination, so look up destination typeid by using
* field type name */
stat = nc_inq_type(igrp, iftype, ftypename, NULL);
CHECK(stat, nc_inq_type);
stat = nc_inq_typeid(ogrp, ftypename, &oftype);
CHECK(stat, nc_inq_typeid);
if(fndims == 0) {
stat = nc_insert_compound(ogrp, otype, fname, foff, oftype);
CHECK(stat, nc_insert_compound);
} else { /* field is array type */
stat = nc_insert_array_compound(ogrp, otype, fname, foff,
oftype, fndims, fdimsizes);
CHECK(stat, nc_insert_array_compound);
}
}
return stat;
}
/*
* copy a user-defined type in the group igrp to the group ogrp
*/
static int
copy_type(int igrp, nc_type typeid, int ogrp)
{
int stat = NC_NOERR;
nc_type type_class;
stat = nc_inq_user_type(igrp, typeid, NULL, NULL, NULL, NULL, &type_class);
CHECK(stat, nc_inq_user_type);
switch(type_class) {
case NC_VLEN:
stat = copy_vlen_type(igrp, typeid, ogrp);
CHECK(stat, copy_vlen_type);
break;
case NC_OPAQUE:
stat = copy_opaque_type(igrp, typeid, ogrp);
CHECK(stat, copy_opaque_type);
break;
case NC_ENUM:
stat = copy_enum_type(igrp, typeid, ogrp);
CHECK(stat, copy_enum_type);
break;
case NC_COMPOUND:
stat = copy_compound_type(igrp, typeid, ogrp);
CHECK(stat, copy_compound_type);
break;
default:
stat = NC_EBADTYPE;
CHECK(stat, copy_type);
}
return stat;
}
/* Copy a group and all its subgroups, recursively, from group igrp in
* input to parent group ogrp in destination. This just creates all
* the groups in the destination, but doesn't copy anything that's in
* the groups. */
static int
copy_groups(int igrp, int ogrp)
{
int stat = NC_NOERR;
int inparid;
int ogid; /* like igrp but in output file */
int numgrps;
int *grpids;
int i;
/* if not root group, create corresponding new group in ogrp */
stat = nc_inq_grp_parent(igrp, &inparid);
if(stat == NC_NOERR) {
/* create new subgroup */
char grpname[NC_MAX_NAME + 1];
stat = nc_inq_grpname(igrp, grpname);
CHECK(stat, nc_inq_grpname);
stat = nc_def_grp(ogrp, grpname, &ogid);
CHECK(stat, nc_def_grp);
} else if(stat == NC_ENOGRP) {
ogid = ogrp;
stat = NC_NOERR;
} else {
CHECK(stat, nc_inq_grp_parent);
}
/* Copy any subgroups */
stat = nc_inq_grps(igrp, &numgrps, NULL);
grpids = (int *)emalloc(sizeof(int) * (numgrps + 1));
stat = nc_inq_grps(igrp, &numgrps, grpids);
CHECK(stat, nc_inq_grps);
for(i = 0; i < numgrps; i++) {
stat = copy_groups(grpids[i], ogid);
CHECK(stat, copy_group);
}
free(grpids);
return stat;
}
/*
* Copy the user-defined types in this group (igrp) and all its
* subgroups, recursively, to corresponding group in output (ogrp)
*/
static int
copy_types(int igrp, int ogrp)
{
int stat = NC_NOERR;
int ntypes;
nc_type *types = NULL;
int numgrps;
int *grpids = NULL;
int i;
stat = nc_inq_typeids(igrp, &ntypes, NULL);
CHECK(stat, nc_inq_typeids);
if(ntypes > 0) {
types = (nc_type *) emalloc(ntypes * sizeof(nc_type));
stat = nc_inq_typeids(igrp, &ntypes, types);
CHECK(stat, nc_inq_typeids);
for (i = 0; i < ntypes; i++) {
stat = copy_type(igrp, types[i], ogrp);
CHECK(stat, copy_type);
}
free(types);
}
/* Copy types from subgroups */
stat = nc_inq_grps(igrp, &numgrps, NULL);
CHECK(stat, nc_inq_grps);
if(numgrps > 0) {
grpids = (int *)emalloc(sizeof(int) * (numgrps + 1));
stat = nc_inq_grps(igrp, &numgrps, grpids);
CHECK(stat, nc_inq_grps);
for(i = 0; i < numgrps; i++) {
int ogid;
/* get groupid in output corresponding to grpids[i] in
* input, given parent group (or root group) ogrp in
* output */
stat = get_grpid(grpids[i], ogrp, &ogid);
CHECK(stat, get_grpid);
stat = copy_types(grpids[i], ogid);
CHECK(stat, copy_types);
}
free(grpids);
}
return stat;
}
/* Copy all netCDF-4 specific variable properties such as chunking,
* endianness, deflation, checksumming, fill, etc. */
static int
copy_var_specials(int igrp, int varid, int ogrp, int o_varid)
{
int stat = NC_NOERR;
{ /* handle chunking parameters */
int ndims;
stat = nc_inq_varndims(igrp, varid, &ndims);
CHECK(stat, nc_inq_varndims);
if (ndims > 0) { /* no chunking for scalar variables */
int contig = 0;
stat = nc_inq_var_chunking(igrp, varid, &contig, NULL);
CHECK(stat, nc_inq_var_chunking);
if(contig == 1) {
stat = nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL);
CHECK(stat, nc_def_var_chunking);
} else {
size_t chunkp[NC_MAX_DIMS];
stat = nc_inq_var_chunking(igrp, varid, NULL, chunkp);
CHECK(stat, nc_inq_var_chunking);
/* explicitly set chunking, even if default */
stat = nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, chunkp);
CHECK(stat, nc_def_var_chunking);
}
}
}
{ /* handle compression parameters */
int shuffle=NC_NOSHUFFLE, deflate=0, deflate_level=0;
stat = nc_inq_var_deflate(igrp, varid,
&shuffle, &deflate, &deflate_level);
CHECK(stat, nc_inq_var_deflate);
if(deflate != 0 || shuffle != 0) {
stat = nc_def_var_deflate(ogrp, o_varid,
shuffle, deflate, deflate_level);
CHECK(stat, nc_def_var_deflate);
}
}
{ /* handle checksum parameters */
int fletcher32 = 0;
stat = nc_inq_var_fletcher32(igrp, varid, &fletcher32);
CHECK(stat, nc_inq_var_fletcher32);
if(fletcher32 != 0) {
stat = nc_def_var_fletcher32(ogrp, o_varid, fletcher32);
CHECK(stat, nc_def_var_fletcher32);
}
}
{ /* handle endianness */
int endianness = 0;
stat = nc_inq_var_endian(igrp, varid, &endianness);
CHECK(stat, nc_inq_var_endian);
if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */
stat = nc_def_var_endian(ogrp, o_varid, endianness);
CHECK(stat, nc_def_var_endian);
}
}
return stat;
}
/* Release the variable chunk cache allocated for variable varid in
* group igrp with it. This is not necessary, but will save some
* memory if processing one variable at a time. */
static int
free_var_chunk_cache(int igrp, int varid)
{
int stat = NC_NOERR;
size_t chunk_cache_size = 1;
size_t cache_nelems = 1;
float cache_preemp = 0;
int inkind, outkind;
stat = nc_inq_format(igrp, &inkind);
CHECK(stat,nc_inq_format);
if(inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC) {
int contig = 1;
stat = nc_inq_var_chunking(igrp, varid, &contig, NULL);
CHECK(stat, nc_inq_var_chunking);
if(contig == 0) { /* chunked */
stat = nc_set_var_chunk_cache(igrp, varid, chunk_cache_size,
cache_nelems, cache_preemp);
CHECK(stat, nc_set_var_chunk_cache);
}
}
return stat;
}
#endif /* USE_NETCDF4 */
/* Copy dimensions from group igrp to group ogrp */
static int
copy_dims(int igrp, int ogrp)
{
int stat = NC_NOERR;
int ndims;
int nunlims;
int dgrp;
#ifdef USE_NETCDF4
int dimids[NC_MAX_DIMS];
int unlimids[NC_MAX_DIMS];
#else
int unlimid;
#endif /* USE_NETCDF4 */
stat = nc_inq_ndims(igrp, &ndims);
CHECK(stat, nc_inq_ndims);
#ifdef USE_NETCDF4
/* In netCDF-4 files, dimids may not be sequential because they
* may be defined in various groups, and we are only looking at one
* group at a time. */
/* Find the dimension ids in this group, don't include parents. */
stat = nc_inq_dimids(igrp, NULL, dimids, 0);
CHECK(stat, nc_inq_dimids);
/* Find the number of unlimited dimensions and get their IDs */
stat = nc_inq_unlimdims(igrp, &nunlims, unlimids);
CHECK(stat, nc_inq_unlimdims);
#else
stat = nc_inq_unlimdim(igrp, &unlimid);
CHECK(stat, nc_inq_unlimdim);
#endif /* USE_NETCDF4 */
/* Copy each dimension to output, including unlimited dimension(s) */
for (dgrp = 0; dgrp < ndims; dgrp++) {
char name[NC_MAX_NAME];
size_t length;
int is_unlim;
int uld;
int dimid;
is_unlim = 0;
#ifdef USE_NETCDF4
dimid = dimids[dgrp];
for (uld = 0; uld < nunlims; uld++) {
if(dimid == unlimids[uld]) {
is_unlim = 1;
break;
}
}
#else
dimid = dgrp;
if(unlimid != -1 && (dimid == unlimid)) {
is_unlim = 1;
}
#endif /* USE_NETCDF4 */
stat = nc_inq_dim(igrp, dimid, name, &length);
if (stat == NC_EDIMSIZE && sizeof(size_t) < 8) {
fprintf(stderr, "dimension \"%s\" requires 64-bit platform\n",
name);
}
CHECK(stat, nc_inq_dim);
if(is_unlim && !CONVERT_ULIM_DIMS) {
stat = nc_def_dim(ogrp, name, NC_UNLIMITED, NULL);
} else {
if(CONVERT_ULIM_DIMS && is_unlim) fprintf(stderr,"dim \"%s\" will
be converted from unlimited to limited dim\n",name);
stat = nc_def_dim(ogrp, name, length, NULL);
}
CHECK(stat, nc_def_dim);
}
return stat;
}
/* Copy the attributes for variable ivar in group igrp to variable
* ovar in group ogrp. Global (group) attributes are specified by
* using the varid NC_GLOBAL */
static int
copy_atts(int igrp, int ivar, int ogrp, int ovar)
{
int natts;
int iatt;
int stat = NC_NOERR;
stat = nc_inq_varnatts(igrp, ivar, &natts);
CHECK(stat, nc_inq_varnatts);
for(iatt = 0; iatt < natts; iatt++) {
char name[NC_MAX_NAME];
stat = nc_inq_attname(igrp, ivar, iatt, name);
CHECK(stat, nc_inq_attname);
stat = nc_copy_att(igrp, ivar, name, ogrp, ovar);
CHECK(stat, nc_copy_att);
}
return stat;
}
/* copy the schema for a single variable in group igrp to group ogrp */
static int
copy_var(int igrp, int varid, int ogrp)
{
int stat = NC_NOERR;
int ndims;
int idimids[NC_MAX_DIMS]; /* ids of dims for input variable */
int odimids[NC_MAX_DIMS]; /* ids of dims for output variable */
char name[NC_MAX_NAME];
nc_type typeid, o_typeid;
int natts;
int i;
int o_varid;
stat = nc_inq_varndims(igrp, varid, &ndims);
CHECK(stat, nc_inq_varndims);
stat = nc_inq_var(igrp, varid, name, &typeid, &ndims, idimids, &natts);
CHECK(stat, nc_inq_var);
o_typeid = typeid;
#ifdef USE_NETCDF4
if (typeid > NC_STRING) { /* user-defined type */
/* type ids in source don't necessarily correspond to same
* typeids in destination, so look up destination typeid by
* using type name */
char type_name[NC_MAX_NAME];
stat = nc_inq_type(igrp, typeid, type_name, NULL);
CHECK(stat, nc_inq_type);
stat = nc_inq_typeid(ogrp, type_name, &o_typeid);
CHECK(stat, nc_inq_typeid);
}
#endif /* USE_NETCDF4 */
/* get the corresponding dimids in the output file */
for(i = 0; i < ndims; i++) {
char dimname[NC_MAX_NAME];
stat = nc_inq_dimname(igrp, idimids[i], dimname);
CHECK(stat, nc_inq_dimname);
stat = nc_inq_dimid(ogrp, dimname, &odimids[i]);
CHECK(stat, nc_inq_dimid);
}
/* define the output variable */
stat = nc_def_var(ogrp, name, o_typeid, ndims, odimids, &o_varid);
CHECK(stat, nc_def_var);
/* attach the variable attributes to the output variable */
stat = copy_atts(igrp, varid, ogrp, o_varid);
CHECK(stat, copy_atts);
#ifdef USE_NETCDF4
{
int inkind;
int outkind;
stat = nc_inq_format(igrp, &inkind);
CHECK(stat,nc_inq_format);
stat = nc_inq_format(ogrp, &outkind);
CHECK(stat,nc_inq_format);
if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC)
&&
(outkind == NC_FORMAT_NETCDF4 || outkind ==
NC_FORMAT_NETCDF4_CLASSIC)) {
/* Copy all netCDF-4 specific variable properties such as
* chunking, endianness, deflation, checksumming, fill, etc. */
stat = copy_var_specials(igrp, varid, ogrp, o_varid);
CHECK(stat, copy_var_specials);
}
if(ndims>0 && DEFLATE_LEVEL>=0 && ((outkind == NC_FORMAT_NETCDF4 || outkind
== NC_FORMAT_NETCDF4_CLASSIC))) {
int shuffle=NC_NOSHUFFLE, deflate=0, deflate_level=0;
deflate=1;deflate_level=DEFLATE_LEVEL;
shuffle=SHUFFLE_VARS;
fprintf(stderr,"var \"%s\" will be deflated%s at level
%d\n",name,(SHUFFLE_VARS==NC_NOSHUFFLE)?"":" and shuffled",DEFLATE_LEVEL);
stat = nc_def_var_deflate(ogrp, o_varid,
shuffle, deflate, deflate_level);
CHECK(stat, nc_def_var_deflate);
}
}
#endif /* USE_NETCDF4 */
return stat;
}
/* copy the schema for all the variables in group igrp to group ogrp */
static int
copy_vars(int igrp, int ogrp)
{
int stat = NC_NOERR;
int nvars;
int varid;
stat = nc_inq_nvars(igrp, &nvars);
CHECK(stat, nc_inq_nvars);
for (varid = 0; varid < nvars; varid++) {
stat = copy_var(igrp, varid, ogrp);
CHECK(stat, copy_var);
}
return stat;
}
/* Copy the schema in a group and all its subgroups, recursively, from
* group igrp in input to parent group ogrp in destination. */
static int
copy_schema(int igrp, int ogrp)
{
int stat = NC_NOERR;
int ogid; /* like igrp but in output file */
int i;
/* get groupid in output corresponding to group igrp in input,
* given parent group (or root group) ogrp in output */
stat = get_grpid(igrp, ogrp, &ogid);
CHECK(stat, get_grpid);
stat = copy_dims(igrp, ogid);
CHECK(stat, copy_dims);
stat = copy_atts(igrp, NC_GLOBAL, ogid, NC_GLOBAL);
CHECK(stat, copy_atts);
stat = copy_vars(igrp, ogid);
CHECK(stat, copy_vars);
#ifdef USE_NETCDF4
{
int numgrps;
int *grpids;
/* Copy schema from subgroups */
stat = nc_inq_grps(igrp, &numgrps, NULL);
grpids = (int *)emalloc(sizeof(int) * (numgrps + 1));
stat = nc_inq_grps(igrp, &numgrps, grpids);
CHECK(stat, nc_inq_grps);
for(i = 0; i < numgrps; i++) {
stat = copy_schema(grpids[i], ogid);
CHECK(stat, copy_schema);
}
free(grpids);
}
#endif /* USE_NETCDF4 */
return stat;
}
/* Return number of values for a variable varid in a group igrp, as
* well as array of dimension sizes, assumed to be preallocated to
* hold one value for each dimension of variable */
static int
inq_nvals(int igrp, int varid,
size_t *dimsizes, long long *nvalsp) {
int stat = NC_NOERR;
int ndims;
int dimids[NC_MAX_DIMS];
int dim;
long long nvals = 1;
stat = nc_inq_varndims(igrp, varid, &ndims);
CHECK(stat, nc_inq_varndims);
stat = nc_inq_vardimid (igrp, varid, dimids);
CHECK(stat, nc_inq_vardimid);
for(dim = 0; dim < ndims; dim++) {
size_t len;
stat = nc_inq_dimlen(igrp, dimids[dim], &len);
CHECK(stat, nc_inq_dimlen);
nvals *= len;
if(dimsizes)
dimsizes[dim] = len;
}
if(nvalsp)
*nvalsp = nvals;
return stat;
}
/* Copy data from variable varid in group igrp to corresponding group
* ogrp. */
static int
copy_var_data(int igrp, int varid, int ogrp, size_t copybuf_size) {
int stat = NC_NOERR;
nc_type vartype;
long long nvalues; /* number of values for this variable */
size_t ntoget; /* number of values to access this iteration */
size_t value_size; /* size of a single value of this variable */
static void *buf = 0; /* buffer for the variable values */
char varname[NC_MAX_NAME];
int ovarid;
size_t start[NC_MAX_DIMS];
size_t count[NC_MAX_DIMS];
nciter_t iter; /* opaque structure for iteration status */
int do_realloc = 0;
size_t chunksize;
stat = inq_nvals(igrp, varid, count, &nvalues);
CHECK(stat, inq_nvals);
if(nvalues == 0)
return stat;
/* get corresponding output variable */
stat = nc_inq_varname(igrp, varid, varname);
CHECK(stat, nc_inq_varname);
stat = nc_inq_varid(ogrp, varname, &ovarid);
CHECK(stat, nc_inq_varid);
stat = nc_inq_vartype(igrp, varid, &vartype);
CHECK(stat, nc_inq_vartype);
/* from type, get size in memory needed for each value */
stat = nc_inq_type(igrp, vartype, NULL, &value_size);
CHECK(stat, nc_inq_type);
if(value_size > copybuf_size) {
copybuf_size = value_size;
do_realloc = 1;
}
#ifdef USE_NETCDF4
/* For chunked variables, copy_buf must also be at least as large as
* size of a chunk in input */
{
stat = inq_var_chunksize(igrp, varid, &chunksize);
CHECK(stat, inq_var_chunksize);
if(chunksize > copybuf_size) {
copybuf_size = chunksize;
do_realloc = 1;
}
}
#endif /* USE_NETCDF4 */
if(buf && do_realloc) {
free(buf);
buf = 0;
}
if(buf == 0) { /* first time or needs to grow */
buf = emalloc(copybuf_size);
memset((void*)buf,0,copybuf_size);
}
/* initialize variable iteration */
stat = nc_get_iter(igrp, varid, copybuf_size, &iter);
CHECK(stat, nc_get_iter);
/* Change start and count to iterate through whole variable. */
while((ntoget = nc_next_iter(&iter, start, count)) > 0) {
stat = nc_get_vara(igrp, varid, start, count, buf);
CHECK(stat, nc_get_vara);
stat = nc_put_vara(ogrp, ovarid, start, count, buf);
CHECK(stat, nc_put_vara);
#ifdef USE_NETCDF4
/* we have to explicitly free values for strings and vlens */
if(vartype == NC_STRING) {
stat = nc_free_string(ntoget, (char **)buf);
CHECK(stat, nc_free_string);
} else if(vartype > NC_STRING) { /* user-defined type */
nc_type vclass;
stat = nc_inq_user_type(igrp, vartype, NULL, NULL, NULL,
NULL, &vclass);
CHECK(stat, nc_inq_user_type);
if(vclass == NC_VLEN) {
stat = nc_free_vlens(ntoget, (nc_vlen_t *)buf);
CHECK(stat, nc_free_vlens);
}
}
/* We're all done with this input and output variable, so if
* either variable is chunked, we might as well free up its
* variable chunk cache */
stat = free_var_chunk_cache(igrp, varid);
CHECK(stat, free_var_chunk_cache);
stat = free_var_chunk_cache(ogrp, ovarid);
CHECK(stat, free_var_chunk_cache);
#endif /* USE_NETCDF4 */
} /* end main iteration loop */
return stat;
}
/* Copy data from variables in group igrp to variables in
* corresponding group with parent ogrp, and all subgroups
* recursively */
static int
copy_data(int igrp, int ogrp, size_t copybuf_size)
{
int stat = NC_NOERR;
int ogid;
int numgrps;
int *grpids;
int i;
int nvars;
int varid;
/* get groupid in output corresponding to group igrp in input,
* given parent group (or root group) ogrp in output */
stat = get_grpid(igrp, ogrp, &ogid);
CHECK(stat, get_grpid);
/* Copy data from this group */
stat = nc_inq_nvars(igrp, &nvars);
CHECK(stat, nc_inq_nvars);
for (varid = 0; varid < nvars; varid++) {
stat = copy_var_data(igrp, varid, ogid, copybuf_size);
CHECK(stat, copy_var_data);
}
#ifdef USE_NETCDF4
/* Copy data from subgroups */
stat = nc_inq_grps(igrp, &numgrps, NULL);
grpids = (int *)emalloc(sizeof(int) * (numgrps + 1));
stat = nc_inq_grps(igrp, &numgrps, grpids);
CHECK(stat, nc_inq_grps);
for(i = 0; i < numgrps; i++) {
stat = copy_data(grpids[i], ogid, copybuf_size);
CHECK(stat, copy_data);
}
free(grpids);
#endif /* USE_NETCDF4 */
return stat;
}
/* copy infile to outfile using netCDF API, kind specifies which
* netCDF format for output: 0 -> same as input, 1 -> classic, 2 ->
* 64-bit offset, 3 -> netCDF-4, 4 -> netCDF-4 classic model */
static int
copy(char* infile, char* outfile, int kind, size_t copybuf_size)
{
int stat = NC_NOERR;
int igrp, ogrp;
int inkind, outkind;
stat = nc_open(infile,NC_NOWRITE,&igrp);
CHECK(stat,nc_open);
stat = nc_inq_format(igrp, &inkind);
CHECK(stat,nc_inq_format);
outkind = (kind == SAME_AS_INPUT) ? inkind : kind;
switch(outkind) {
case NC_FORMAT_CLASSIC:
stat = nc_create(outfile,NC_CLOBBER,&ogrp);
break;
case NC_FORMAT_64BIT:
stat = nc_create(outfile,NC_CLOBBER|NC_64BIT_OFFSET,&ogrp);
break;
#ifdef USE_NETCDF4
case NC_FORMAT_NETCDF4:
stat = nc_create(outfile,NC_CLOBBER|NC_NETCDF4,&ogrp);
break;
case NC_FORMAT_NETCDF4_CLASSIC:
stat = nc_create(outfile,NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL,&ogrp);
break;
#else
case NC_FORMAT_NETCDF4:
case NC_FORMAT_NETCDF4_CLASSIC:
fprintf(stderr,
"%s built without ability to create netCDF-4 files\n",
progname);
exit(1);
#endif /* USE_NETCDF4 */
default:
fprintf(stderr,"%s: bad value (%d) for -k option\n", progname, kind);
exit(1);
}
CHECK(stat,nc_create);
if(nofill_flag) {
stat = nc_set_fill(ogrp,NC_NOFILL,NULL);
CHECK(stat,nc_set_fill);
}
#ifdef USE_NETCDF4
/* Because types in one group may depend on types in a different
* group, need to create all groups before defining types */
if(inkind == NC_FORMAT_NETCDF4) {
stat = copy_groups(igrp, ogrp);
CHECK(stat,copy_groups);
stat = copy_types(igrp, ogrp);
CHECK(stat,copy_types);
}
#endif /* USE_NETCDF4 */
stat = copy_schema(igrp, ogrp);
CHECK(stat,copy_schema);
stat = nc_enddef(ogrp);
CHECK(stat, nc_enddef);
stat = copy_data(igrp, ogrp, copybuf_size);
CHECK(stat,copy_data);
stat = nc_close(igrp);
CHECK(stat,nc_close);
stat = nc_close(ogrp);
CHECK(stat,nc_close);
return stat;
}
static void
usage(void)
{
#define USAGE "\
[-k n] kind of netCDF format for output file, default same as input\n\
1 classic, 2 64-bit offset, 3 netCDF-4, 4 netCDF-4 classic model\n\
[-m n] size in bytes of copy buffer\n\
[-d n] deflate vars (n: deflate level [0-9])\n\
[-s] shuffle vars\n\
[-u] remove unlimited dims\n\
infile name of netCDF input file\n\
outfile name for netCDF output file\n"
(void) fprintf(stderr,
"%s [-k n] [-m n] infile outfile\n%s",
progname,
USAGE);
}
int
main(int argc, char**argv)
{
char* inputfile = NULL;
char* outputfile = NULL;
int kind = SAME_AS_INPUT; /* default, output same format as input */
int c;
size_t copybuf_size = COPY_BUFFER_SIZE; /* default */
/* table of formats for legal -k values */
struct Kvalues {
char* name;
int kind;
} legalkinds[] = {
{"1", NC_FORMAT_CLASSIC},
{"classic", NC_FORMAT_CLASSIC},
/* The 64-bit offset kind (2) */
{"2", NC_FORMAT_64BIT},
{"64-bit-offset", NC_FORMAT_64BIT},
{"64-bit offset", NC_FORMAT_64BIT},
/* NetCDF-4 HDF5 format */
{"3", NC_FORMAT_NETCDF4},
{"hdf5", NC_FORMAT_NETCDF4},
{"netCDF-4", NC_FORMAT_NETCDF4},
{"netCDF4", NC_FORMAT_NETCDF4},
{"enhanced", NC_FORMAT_NETCDF4},
/* NetCDF-4 HDF5 format, but using only nc3 data model */
{"4", NC_FORMAT_NETCDF4_CLASSIC},
{"hdf5-nc3", NC_FORMAT_NETCDF4_CLASSIC},
{"netCDF-4 classic model", NC_FORMAT_NETCDF4_CLASSIC},
{"netCDF4_classic", NC_FORMAT_NETCDF4_CLASSIC},
{"enhanced-nc3", NC_FORMAT_NETCDF4_CLASSIC},
/* null terminate*/
{NULL,0}
};
opterr = 1;
progname = argv[0];
if (argc <= 1)
{
usage();
return 1;
}
while ((c = getopt(argc, argv, "k:m:d:us")) != EOF) {
switch(c) {
case 'k': /* for specifying variant of netCDF format to be generated
Possible values are:
1 (=> classic 32 bit)
2 (=> classic 64 bit offsets)
3 (=> netCDF-4/HDF5)
4 (=> classic, but stored in netCDF-4/HDF5 format)
Also allow string versions of above
"classic"
"64-bit-offset"
"64-bit offset"
"enhanced" | "hdf5" | "netCDF-4"
"enhanced-nc3" | "hdf5-nc3" | "netCDF-4 classic model"
*/
{
struct Kvalues* kvalue;
char *kind_name = (char *) emalloc(strlen(optarg)+1);
if (! kind_name) {
return 2;
}
(void)strcpy(kind_name, optarg);
for(kvalue=legalkinds;kvalue->name;kvalue++) {
if(strcmp(kind_name,kvalue->name) == 0) {
kind = kvalue->kind;
break;
}
}
if(kvalue->name == NULL) {
fprintf(stderr, "invalid format: %s\n", kind_name);
return 1;
}
}
break;
case 'm': /* non-default size of data copy buffer */
copybuf_size = strtoll(optarg, NULL, 10);
break;
case 'd': /* deflate vars */
DEFLATE_LEVEL = strtoll(optarg, NULL, 10);
break;
case 'u': /* deflate vars */
CONVERT_ULIM_DIMS = 1;
break;
case 's': /* deflate vars */
SHUFFLE_VARS = NC_SHUFFLE;
break;
default:
usage();
exit(1);
break;
}
}
argc -= optind;
argv += optind;
if (argc != 2) {
fprintf(stderr,"one input file and one output file required\n");
exit(1);
}
inputfile = argv[0];
outputfile = argv[1];
/* set nofill mode to speed up creation of output file, because we
* will copy fill values from input anyway */
nofill_flag = 1;
if(strcmp(inputfile, outputfile) == 0) {
fprintf(stderr,"output would overwrite input\n");
exit(1);
}
if(copy(inputfile, outputfile, kind, copybuf_size) != NC_NOERR)
exit(1);
return 0;
}
/*********************************************************************
* Copyright 2009, University Corporation for Atmospheric Research
* See netcdf/README file for copying and redistribution conditions.
* "$Header: /upc/share/CVS/netcdf-3/ncdump/nciter.c,v 1.7 2010/02/01
21:44:04 russ Exp $"
*********************************************************************/
/*#include "config.h"*/ /* for USE_NETCDF4 macro */
#define USE_NETCDF4 1
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "nciter.h"
#define CHECK(stat,f) if(stat != NC_NOERR) {check(stat,#f,__FILE__,__LINE__);}
else {}
/* forward declarations */
static int nc_blkio_init(size_t bufsize, size_t value_size, int rank,
const size_t *dims, int chunked, size_t *chunksizes,
nciter_t *iter);
static int up_start(int ndims, const size_t *dims, int incdim, size_t inc,
size_t* odom);
static int up_start_by_chunks(int ndims, const size_t *dims,
const size_t *chunks, size_t* odom);
static int inq_value_size(int igrp, nc_type vartype, size_t *value_sizep);
static void
check(int err, const char* fcn, const char* file, const int line)
{
fprintf(stderr,"%s\n",nc_strerror(err));
fprintf(stderr,"Location: function %s; file %s; line %d\n",
fcn,file,line);
fflush(stderr); fflush(stdout);
exit(1);
}
/* Initialize iteration for a variable. Just a wrapper for
* nc_blkio_init() that makes the netCDF calls needed to initialize
* lower-level iterator. */
int
nc_get_iter(int ncid,
int varid,
size_t bufsize, /* size in bytes of memory buffer */
nciter_t *iterp /* returned opaque iteration state */)
{
int stat = NC_NOERR;
nc_type vartype;
size_t value_size; /* size in bytes of each variable element */
int ndims; /* number of dimensions for variable */
int dimids[NC_MAX_DIMS];
size_t dimsizes[NC_MAX_VAR_DIMS]; /* variable dimension sizes */
size_t chunksizes[NC_MAX_VAR_DIMS]; /* corresponding chunk sizes */
long long nvalues = 1;
int dim;
int chunked = 0;
memset((void*)iterp,0,sizeof(nciter_t)); /* make sure it is initialized */
stat = nc_inq_varndims(ncid, varid, &ndims);
CHECK(stat, nc_inq_varndims);
stat = nc_inq_vardimid (ncid, varid, dimids);
CHECK(stat, nc_inq_vardimid);
for(dim = 0; dim < ndims; dim++) {
size_t len;
stat = nc_inq_dimlen(ncid, dimids[dim], &len);
CHECK(stat, nc_inq_dimlen);
nvalues *= len;
dimsizes[dim] = len;
}
stat = nc_inq_vartype(ncid, varid, &vartype);
CHECK(stat, nc_inq_vartype);
stat = inq_value_size(ncid, vartype, &value_size);
CHECK(stat, inq_value_size);
#ifdef USE_NETCDF4
{
int contig = 1;
if(ndims > 0) {
stat = nc_inq_var_chunking(ncid, varid, &contig, NULL);
CHECK(stat, nc_inq_var_chunking);
}
if(contig == 0) { /* chunked */
stat = nc_inq_var_chunking(ncid, varid, &contig, chunksizes);
CHECK(stat, nc_inq_var_chunking);
chunked = 1;
}
}
#endif /* USE_NETCDF4 */
stat = nc_blkio_init(bufsize, value_size, ndims, dimsizes,
chunked, chunksizes, iterp);
CHECK(stat, nc_blkio_init);
iterp->to_get = 0;
return stat;
}
/* Iterate on blocks for variables, by updating start and count vector
* for next vara call. Assumes nc_get_iter called first. Returns
* number of variable values to get, 0 if done, negative number if
* error, so use like this:
size_t to_get;
while((to_get = nc_next_iter(&iter, start, count)) > 0) {
... iteration ...
}
if(to_get < 0) { ... handle error ... }
*/
size_t
nc_next_iter(nciter_t *iter, /* returned opaque iteration state */
size_t *start, /* returned start vector for next vara call */
size_t *count /* returned count vector for next vara call */
) {
int i;
/* Note: special case for chunked variables is just an
* optimization, the contiguous code below is OK even
* for chunked variables, but in general will do more I/O ... */
if(iter->first) {
if(!iter->chunked) { /* contiguous storage */
for(i = 0; i < iter->right_dim; i++) {
start[i] = 0;
count[i] = 1;
}
start[iter->right_dim] = 0;
count[iter->right_dim] = iter->rows;
for(i = iter->right_dim + 1; i < iter->rank; i++) {
start[i] = 0;
count[i] = iter->dimsizes[i];
}
} else { /* chunked storage */
for(i = 0; i < iter->rank; i++) {
start[i] = 0;
count[i] = iter->chunksizes[i];
}
}
iter->first = 0;
} else {
if(!iter->chunked) { /* contiguous storage */
iter->more = up_start(iter->rank, iter->dimsizes, iter->right_dim,
iter->inc, start);
/* iterate on pieces of variable */
if(iter->cur < iter->numrows) {
iter->inc = iter->rows;
count[iter->right_dim] = iter->rows;
iter->cur++;
} else {
if(iter->leftover > 0) {
count[iter->right_dim] = iter->leftover;
iter->inc = iter->leftover;
iter->cur = 0;
}
}
} else { /* chunked storage */
iter->more = up_start_by_chunks(iter->rank, iter->dimsizes,
iter->chunksizes, start);
/* adjust count to stay in range of dimsizes */
for(i = 0; i < iter->rank; i++) {
int leftover = iter->dimsizes[i] - start[i];
count[i] = iter->chunksizes[i];
if(leftover < count[i])
count[i] = leftover;
}
}
}
iter->to_get = 1;
for(i = 0; i < iter->rank; i++) {
iter->to_get *= count[i];
}
return iter->more == 0 ? 0 : iter->to_get ;
}
/* Initialize block iteration for variables, including those that
* won't fit in the copy buffer all at once. Returns error if
* variable is chunked but size of chunks is too big to fit in bufsize
* bytes. */
static int
nc_blkio_init(size_t bufsize, /* size in bytes of in-memory copy buffer */
size_t value_size, /* size in bytes of each variable element */
int rank, /* number of dimensions for variable */
const size_t *dims, /* variable dimension sizes */
int chunked, /* 1 if variable is chunked, 0 otherwise */
size_t *chunksizes, /* if chunked, variable chunk sizes */
nciter_t *iter /* returned iteration state, don't mess with it */
) {
int stat = NC_NOERR;
int i;
long long prod;
iter->rank = rank;
iter->first = 1;
iter->more = 1;
iter->chunked = chunked;
for(i = 0; i < rank; i++)
iter->dimsizes[i] = dims[i];
prod = value_size;
if(iter->chunked == 0) { /* contiguous */
iter->right_dim = rank - 1;
for(i = rank; i > 0; i--) {
if(prod*dims[i-1] <= bufsize) {
prod *= dims[i-1];
iter->right_dim--;
} else {
break;
}
}
if (i > 0) { /* variable won't fit in bufsize bytes */
iter->rows = bufsize/prod;
iter->numrows = dims[iter->right_dim] / iter->rows;
iter->leftover = dims[iter->right_dim] - iter->numrows * iter->rows;
iter->cur = 1;
iter->inc = iter->rows;
return stat;
}
/* else, variable will fit in bufsize bytes of memory. */
iter->right_dim = 0;
iter->rows = dims[0];
iter->inc = 0;
return stat;
}
/* else, handle chunked case */
for(i = 0; i < rank; i++) {
iter->chunksizes[i] = chunksizes[i];
prod *= chunksizes[i];
}
if(prod > bufsize) {
stat = NC_ENOMEM;
fprintf(stderr, "chunksize (= %ld) > copy_buffer size (= %ld)\n", prod,
bufsize);
}
return stat;
}
/* From netCDF type in group igrp, get size in memory needed for each
* value. Wouldn't be needed if nc_inq_type() was a netCDF-3 function
* too. */
static int
inq_value_size(int igrp, nc_type vartype, size_t *value_sizep) {
int stat = NC_NOERR;
#ifdef USE_NETCDF4
stat = nc_inq_type(igrp, vartype, NULL, value_sizep);
CHECK(stat, nc_inq_type);
#else
switch(vartype) {
case NC_BYTE:
*value_sizep = sizeof(signed char);
break;
case NC_CHAR:
*value_sizep = sizeof(char);
break;
case NC_SHORT:
*value_sizep = sizeof(short);
break;
case NC_INT:
*value_sizep = sizeof(int);
break;
case NC_FLOAT:
*value_sizep = sizeof(float);
break;
case NC_DOUBLE:
*value_sizep = sizeof(double);
break;
default:
stat = NC_EBADTYPE;
CHECK(stat, inq_value_size);
break;
}
#endif /* USE_NETCDF4 */
return stat;
}
/*
* Updates a vector of size_t, odometer style. Returns 0 if odometer
* overflowed, else 1.
*/
static int
up_start(
int ndims, /* Number of dimensions */
const size_t *dims, /* The "odometer" limits for each dimension */
int incdim, /* the odmometer increment dimension */
size_t inc, /* the odometer increment for that dimension */
size_t* odom /* The "odometer" vector to be updated */
)
{
int id;
int ret = 1;
if(inc == 0) {
return 0;
}
odom[incdim] += inc;
for (id = incdim; id > 0; id--) {
if(odom[id] >= dims[id]) {
odom[id-1]++;
odom[id] -= dims[id];
}
}
if (odom[0] >= dims[0])
ret = 0;
return ret;
}
/*
* Updates a vector of size_t, odometer style, for chunk access.
* Returns 0 if odometer overflowed, else 1.
*/
static int
up_start_by_chunks(
int ndims, /* Number of dimensions */
const size_t *dims, /* The "odometer" limits for each dimension */
const size_t *chunks, /* the odometer increments for each dimension */
size_t* odom /* The "odometer" vector to be updated */
)
{
int incdim = ndims - 1;
int id;
int ret = 1;
odom[incdim] += chunks[incdim];
for (id = incdim; id > 0; id--) {
if(odom[id] >= dims[id]) {
odom[id-1] += chunks[id-1];
/* odom[id] -= dims[id]; */
odom[id] = 0;
}
}
if (odom[0] >= dims[0])
ret = 0;
return ret;
}
/*********************************************************************
* Copyright 2009, University Corporation for Atmospheric Research
* See netcdf/README file for copying and redistribution conditions.
* "$Id: nciter.h,v 1.4 2010/02/01 21:44:04 russ Exp $"
*********************************************************************/
#ifndef _NCITER_
#define _NCITER_
#include <netcdf.h>
#if defined(__cplusplus)
extern "C" {
#endif
/*
* The opaque structure to hold per-variable state of iteration
*/
typedef struct {
int first; /* false after associated next function invoked */
int right_dim; /* rightmost dimension for start of variable pieces */
size_t rows; /* how many subpiece rows in bufsiz */
size_t numrows; /* how many row pieces in right_dim dimension */
size_t cur; /* current "row" in loop over row pieces */
size_t leftover; /* how many rows left over after partitioning
* bufsiz into subpiece blocks */
int more; /* whether there is more data still to get */
size_t to_get; /* number of values to get on this access */
int rank; /* number of dimensions */
size_t inc; /* increment for right_dim element of start vector */
int chunked; /* 1 if chunked, 0 if contiguous */
size_t dimsizes[NC_MAX_DIMS];
size_t chunksizes[NC_MAX_DIMS]; /* ignored if not chunked */
} nciter_t;
/*
* The Interface
*/
/* Get iterator for a variable. */
extern int
nc_get_iter(int ncid, int varid, size_t bufsize, nciter_t *iterp);
/* Iterate over blocks of variable values, using start and count
* vectors. Returns number of values to access (product of counts),
* or 0 if done. */
extern size_t
nc_next_iter(nciter_t *iterp, size_t *start, size_t *count);
#if defined(__cplusplus)
}
#endif
#endif /* _NCITER_ */
Compiling as any program base on netCDF-4 library you need to know the compiler
an linker paramenters of you netCDF-4 installation. For example:
bash-3.2$ nc-config --all
This netCDF 4.1 has been built with the following features:
--cc -> cc
--cflags -> -I/usr/local/include
--libs -> -L/usr/local/lib -lnetcdf -L/usr/local/lib -lhdf5_hl -lhdf5
-lz -lm
--cxx -> g++
--has-c++ -> yes
--fc -> f77
--fflags -> -g -O2 /usr/local/include
--flibs -> -L/usr/local/lib -lnetcdf -L/usr/local/lib -lhdf5_hl -lhdf5
-lz -lm
--has-f77 -> yes
--has-f90 -> no
--has-dap -> no
--has-nc2 -> yes
--has-nc4 -> yes
--has-hdf5 -> yes
--has-hdf4 -> no
--has-szlib -> no
--prefix -> /usr/local
--includedir-> /usr/local/include
--version -> netCDF 4.1
Then you need the --cflags for compiling and --libs flags and compile
bash-3.2$ gcc -o ncconvert ncconvert.c nciter.c -I/usr/local/include
-L/usr/local/lib -lnetcdf -lhdf5_hl -lhdf5 -lz -lm
or
bash-3.2$ gcc -o ncconvert ncconvert.c nciter.c `nc-config --cflags` `nc-config
--libs`
for some help just execute:
bash-3.2$ ./ncconvert.exe
./ncconvert [-k n] [-m n] infile outfile
[-k n] kind of netCDF format for output file, default same as input
1 classic, 2 64-bit offset, 3 netCDF-4, 4 netCDF-4 classic model
[-m n] size in bytes of copy buffer
[-d n] deflate vars (n: deflate level [0-9])
[-s] shuffle vars
[-u] remove unlimited dims
infile name of netCDF input file
outfile name for netCDF output file
The the d,s and u options are only valid for netCDF format (-k 3 or -k 4)
Example of use
First we are getting an sample file (netCDF classic file) and dumping the
information
bash-3.2$ wget
http://www.unidata.ucar.edu/software/netcdf/examples/ECMWF_ERA-40_subset.nc
bash-3.2$ ncdump -c ECMWF_ERA-40_subset.nc
netcdf ECMWF_ERA-40_subset {
dimensions:
longitude = 144 ;
latitude = 73 ;
time = UNLIMITED ; // (62 currently)
variables:
float longitude(longitude) ;
longitude:units = "degrees_east" ;
longitude:long_name = "longitude" ;
float latitude(latitude) ;
latitude:units = "degrees_north" ;
latitude:long_name = "latitude" ;
int time(time) ;
time:units = "hours since 1900-01-01 00:00:0.0" ;
time:long_name = "time" ;
short tcw(time, latitude, longitude) ;
tcw:scale_factor = 0.0013500981745481 ;
tcw:add_offset = 44.3250482744756 ;
tcw:_FillValue = -32767s ;
tcw:missing_value = -32767s ;
tcw:units = "kg m**-2" ;
tcw:long_name = "Total column water" ;
short tcwv(time, latitude, longitude) ;
tcwv:scale_factor = 0.001327110772669 ;
tcwv:add_offset = 43.5704635546154 ;
tcwv:_FillValue = -32767s ;
tcwv:missing_value = -32767s ;
tcwv:units = "kg m**-2" ;
tcwv:long_name = "Total column water vapour" ;
short lsp(time, latitude, longitude) ;
lsp:scale_factor = 8.03329303850659e-07 ;
lsp:add_offset = 0.0263210846406669 ;
lsp:_FillValue = -32767s ;
lsp:missing_value = -32767s ;
lsp:units = "m" ;
lsp:long_name = "Stratiform precipitation (Large-scale
precipitation)" ;
short cp(time, latitude, longitude) ;
cp:scale_factor = 4.82483645945993e-07 ;
cp:add_offset = 0.0158085766594205 ;
cp:_FillValue = -32767s ;
cp:missing_value = -32767s ;
cp:units = "m" ;
cp:long_name = "Convective precipitation" ;
short msl(time, latitude, longitude) ;
msl:scale_factor = 0.1721754257462 ;
msl:add_offset = 99424.2653245743 ;
msl:_FillValue = -32767s ;
msl:missing_value = -32767s ;
msl:units = "Pa" ;
msl:long_name = "Mean sea level pressure" ;
short blh(time, latitude, longitude) ;
blh:scale_factor = 0.108739383344517 ;
blh:add_offset = 3570.14367055165 ;
blh:_FillValue = -32767s ;
blh:missing_value = -32767s ;
blh:units = "m" ;
blh:long_name = "Boundary layer height" ;
short tcc(time, latitude, longitude) ;
tcc:scale_factor = 1.52597204419215e-05 ;
tcc:add_offset = 0.499984740280558 ;
tcc:_FillValue = -32767s ;
tcc:missing_value = -32767s ;
tcc:units = "(0 - 1)" ;
tcc:long_name = "Total cloud cover" ;
short p10u(time, latitude, longitude) ;
p10u:scale_factor = 0.0007584155104299 ;
p10u:add_offset = -0.440509086897149 ;
p10u:_FillValue = -32767s ;
p10u:missing_value = -32767s ;
p10u:units = "m s**-1" ;
p10u:long_name = "10 metre U wind component" ;
short p10v(time, latitude, longitude) ;
p10v:scale_factor = 0.000664359461014752 ;
p10v:add_offset = -0.745888358484452 ;
p10v:_FillValue = -32767s ;
p10v:missing_value = -32767s ;
p10v:units = "m s**-1" ;
p10v:long_name = "10 metre V wind component" ;
short p2t(time, latitude, longitude) ;
p2t:scale_factor = 0.00183558351993706 ;
p2t:add_offset = 262.398478747535 ;
p2t:_FillValue = -32767s ;
p2t:missing_value = -32767s ;
p2t:units = "K" ;
p2t:long_name = "2 metre temperature" ;
short p2d(time, latitude, longitude) ;
p2d:scale_factor = 0.00161126451178551 ;
p2d:add_offset = 251.887106386855 ;
p2d:_FillValue = -32767s ;
p2d:missing_value = -32767s ;
p2d:units = "K" ;
p2d:long_name = "2 metre dewpoint temperature" ;
short e(time, latitude, longitude) ;
e:scale_factor = 1.16702451907916e-07 ;
e:add_offset = -0.00232199712964108 ;
e:_FillValue = -32767s ;
e:missing_value = -32767s ;
e:units = "m of water" ;
e:long_name = "Evaporation" ;
short lcc(time, latitude, longitude) ;
lcc:scale_factor = 1.52597204419215e-05 ;
lcc:add_offset = 0.499984740279558 ;
lcc:_FillValue = -32767s ;
lcc:missing_value = -32767s ;
lcc:units = "(0 - 1)" ;
lcc:long_name = "Low cloud cover" ;
short mcc(time, latitude, longitude) ;
mcc:scale_factor = 1.52597204419215e-05 ;
mcc:add_offset = 0.499984740279558 ;
mcc:_FillValue = -32767s ;
mcc:missing_value = -32767s ;
mcc:units = "(0 - 1)" ;
mcc:long_name = "Medium cloud cover" ;
short hcc(time, latitude, longitude) ;
hcc:scale_factor = 1.52597204419215e-05 ;
hcc:add_offset = 0.499984740280558 ;
hcc:_FillValue = -32767s ;
hcc:missing_value = -32767s ;
hcc:units = "(0 - 1)" ;
hcc:long_name = "High cloud cover" ;
short tco3(time, latitude, longitude) ;
tco3:scale_factor = 7.69770539069593e-08 ;
tco3:add_offset = 0.00736908367510674 ;
tco3:_FillValue = -32767s ;
tco3:missing_value = -32767s ;
tco3:units = "kg m**-2" ;
tco3:long_name = "Total column ozone" ;
short tp(time, latitude, longitude) ;
tp:scale_factor = 1.05226955985452e-06 ;
tp:add_offset = 0.0344776121286335 ;
tp:_FillValue = -32767s ;
tp:missing_value = -32767s ;
tp:units = "m" ;
tp:long_name = "Total precipitation" ;
// global attributes:
:Conventions = "CF-1.0" ;
:history = "2004-09-15 17:04:29 GMT by mars2netcdf-0.92" ;
data:
longitude = 0, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 27.5, 30,
32.5, 35, 37.5, 40, 42.5, 45, 47.5, 50, 52.5, 55, 57.5, 60, 62.5, 65,
67.5, 70, 72.5, 75, 77.5, 80, 82.5, 85, 87.5, 90, 92.5, 95, 97.5, 100,
102.5, 105, 107.5, 110, 112.5, 115, 117.5, 120, 122.5, 125, 127.5, 130,
132.5, 135, 137.5, 140, 142.5, 145, 147.5, 150, 152.5, 155, 157.5, 160,
162.5, 165, 167.5, 170, 172.5, 175, 177.5, 180, 182.5, 185, 187.5, 190,
192.5, 195, 197.5, 200, 202.5, 205, 207.5, 210, 212.5, 215, 217.5, 220,
222.5, 225, 227.5, 230, 232.5, 235, 237.5, 240, 242.5, 245, 247.5, 250,
252.5, 255, 257.5, 260, 262.5, 265, 267.5, 270, 272.5, 275, 277.5, 280,
282.5, 285, 287.5, 290, 292.5, 295, 297.5, 300, 302.5, 305, 307.5, 310,
312.5, 315, 317.5, 320, 322.5, 325, 327.5, 330, 332.5, 335, 337.5, 340,
342.5, 345, 347.5, 350, 352.5, 355, 357.5 ;
latitude = 90, 87.5, 85, 82.5, 80, 77.5, 75, 72.5, 70, 67.5, 65, 62.5, 60,
57.5, 55, 52.5, 50, 47.5, 45, 42.5, 40, 37.5, 35, 32.5, 30, 27.5, 25,
22.5, 20, 17.5, 15, 12.5, 10, 7.5, 5, 2.5, 0, -2.5, -5, -7.5, -10, -12.5,
-15, -17.5, -20, -22.5, -25, -27.5, -30, -32.5, -35, -37.5, -40, -42.5,
-45, -47.5, -50, -52.5, -55, -57.5, -60, -62.5, -65, -67.5, -70, -72.5,
-75, -77.5, -80, -82.5, -85, -87.5, -90 ;
time = 898476, 898482, 898500, 898506, 898524, 898530, 898548, 898554,
898572, 898578, 898596, 898602, 898620, 898626, 898644, 898650, 898668,
898674, 898692, 898698, 898716, 898722, 898740, 898746, 898764, 898770,
898788, 898794, 898812, 898818, 898836, 898842, 898860, 898866, 898884,
898890, 898908, 898914, 898932, 898938, 898956, 898962, 898980, 898986,
899004, 899010, 899028, 899034, 899052, 899058, 899076, 899082, 899100,
899106, 899124, 899130, 899148, 899154, 899172, 899178, 899196, 899202 ;
}
after we are converting the file from classic format to netCDF-4 classic mode
deflating all variables at level 9 (-d 9) and convert unlimited dims to limit
(-u)
bash-3.2$ ./ncconvert -d 9 -u -k 4 ECMWF_ERA-40_subset.nc
ECMWF_ERA-40_nc4classicmodel_subset_deflate9_noUlimDim.nc
dim "time" will be converted from unlimited to limited dim
var "longitude" will be deflated at level 9
var "latitude" will be deflated at level 9
var "time" will be deflated at level 9
var "tcw" will be deflated at level 9
var "tcwv" will be deflated at level 9
var "lsp" will be deflated at level 9
var "cp" will be deflated at level 9
var "msl" will be deflated at level 9
var "blh" will be deflated at level 9
var "tcc" will be deflated at level 9
var "p10u" will be deflated at level 9
var "p10v" will be deflated at level 9
var "p2t" will be deflated at level 9
var "p2d" will be deflated at level 9
var "e" will be deflated at level 9
var "lcc" will be deflated at level 9
var "mcc" will be deflated at level 9
var "hcc" will be deflated at level 9
var "tco3" will be deflated at level 9
var "tp" will be deflated at level 9
bash-3.2$ ncdump -c ECMWF_ERA-40_nc4classicmodel_subset_deflate9_noUlimDim.nc
netcdf ECMWF_ERA-40_nc4classicmodel_subset_deflate9_noUlimDim {
dimensions:
longitude = 144 ;
latitude = 73 ;
time = 62 ;
variables:
float longitude(longitude) ;
longitude:units = "degrees_east" ;
longitude:long_name = "longitude" ;
float latitude(latitude) ;
latitude:units = "degrees_north" ;
latitude:long_name = "latitude" ;
int time(time) ;
time:units = "hours since 1900-01-01 00:00:0.0" ;
time:long_name = "time" ;
short tcw(time, latitude, longitude) ;
tcw:scale_factor = 0.0013500981745481 ;
tcw:add_offset = 44.3250482744756 ;
tcw:_FillValue = -32767s ;
tcw:missing_value = -32767s ;
tcw:units = "kg m**-2" ;
tcw:long_name = "Total column water" ;
short tcwv(time, latitude, longitude) ;
tcwv:scale_factor = 0.001327110772669 ;
tcwv:add_offset = 43.5704635546154 ;
tcwv:_FillValue = -32767s ;
tcwv:missing_value = -32767s ;
tcwv:units = "kg m**-2" ;
tcwv:long_name = "Total column water vapour" ;
short lsp(time, latitude, longitude) ;
lsp:scale_factor = 8.03329303850659e-07 ;
lsp:add_offset = 0.0263210846406669 ;
lsp:_FillValue = -32767s ;
lsp:missing_value = -32767s ;
lsp:units = "m" ;
lsp:long_name = "Stratiform precipitation (Large-scale
precipitation)" ;
short cp(time, latitude, longitude) ;
cp:scale_factor = 4.82483645945993e-07 ;
cp:add_offset = 0.0158085766594205 ;
cp:_FillValue = -32767s ;
cp:missing_value = -32767s ;
cp:units = "m" ;
cp:long_name = "Convective precipitation" ;
short msl(time, latitude, longitude) ;
msl:scale_factor = 0.1721754257462 ;
msl:add_offset = 99424.2653245743 ;
msl:_FillValue = -32767s ;
msl:missing_value = -32767s ;
msl:units = "Pa" ;
msl:long_name = "Mean sea level pressure" ;
short blh(time, latitude, longitude) ;
blh:scale_factor = 0.108739383344517 ;
blh:add_offset = 3570.14367055165 ;
blh:_FillValue = -32767s ;
blh:missing_value = -32767s ;
blh:units = "m" ;
blh:long_name = "Boundary layer height" ;
short tcc(time, latitude, longitude) ;
tcc:scale_factor = 1.52597204419215e-05 ;
tcc:add_offset = 0.499984740280558 ;
tcc:_FillValue = -32767s ;
tcc:missing_value = -32767s ;
tcc:units = "(0 - 1)" ;
tcc:long_name = "Total cloud cover" ;
short p10u(time, latitude, longitude) ;
p10u:scale_factor = 0.0007584155104299 ;
p10u:add_offset = -0.440509086897149 ;
p10u:_FillValue = -32767s ;
p10u:missing_value = -32767s ;
p10u:units = "m s**-1" ;
p10u:long_name = "10 metre U wind component" ;
short p10v(time, latitude, longitude) ;
p10v:scale_factor = 0.000664359461014752 ;
p10v:add_offset = -0.745888358484452 ;
p10v:_FillValue = -32767s ;
p10v:missing_value = -32767s ;
p10v:units = "m s**-1" ;
p10v:long_name = "10 metre V wind component" ;
short p2t(time, latitude, longitude) ;
p2t:scale_factor = 0.00183558351993706 ;
p2t:add_offset = 262.398478747535 ;
p2t:_FillValue = -32767s ;
p2t:missing_value = -32767s ;
p2t:units = "K" ;
p2t:long_name = "2 metre temperature" ;
short p2d(time, latitude, longitude) ;
p2d:scale_factor = 0.00161126451178551 ;
p2d:add_offset = 251.887106386855 ;
p2d:_FillValue = -32767s ;
p2d:missing_value = -32767s ;
p2d:units = "K" ;
p2d:long_name = "2 metre dewpoint temperature" ;
short e(time, latitude, longitude) ;
e:scale_factor = 1.16702451907916e-07 ;
e:add_offset = -0.00232199712964108 ;
e:_FillValue = -32767s ;
e:missing_value = -32767s ;
e:units = "m of water" ;
e:long_name = "Evaporation" ;
short lcc(time, latitude, longitude) ;
lcc:scale_factor = 1.52597204419215e-05 ;
lcc:add_offset = 0.499984740279558 ;
lcc:_FillValue = -32767s ;
lcc:missing_value = -32767s ;
lcc:units = "(0 - 1)" ;
lcc:long_name = "Low cloud cover" ;
short mcc(time, latitude, longitude) ;
mcc:scale_factor = 1.52597204419215e-05 ;
mcc:add_offset = 0.499984740279558 ;
mcc:_FillValue = -32767s ;
mcc:missing_value = -32767s ;
mcc:units = "(0 - 1)" ;
mcc:long_name = "Medium cloud cover" ;
short hcc(time, latitude, longitude) ;
hcc:scale_factor = 1.52597204419215e-05 ;
hcc:add_offset = 0.499984740280558 ;
hcc:_FillValue = -32767s ;
hcc:missing_value = -32767s ;
hcc:units = "(0 - 1)" ;
hcc:long_name = "High cloud cover" ;
short tco3(time, latitude, longitude) ;
tco3:scale_factor = 7.69770539069593e-08 ;
tco3:add_offset = 0.00736908367510674 ;
tco3:_FillValue = -32767s ;
tco3:missing_value = -32767s ;
tco3:units = "kg m**-2" ;
tco3:long_name = "Total column ozone" ;
short tp(time, latitude, longitude) ;
tp:scale_factor = 1.05226955985452e-06 ;
tp:add_offset = 0.0344776121286335 ;
tp:_FillValue = -32767s ;
tp:missing_value = -32767s ;
tp:units = "m" ;
tp:long_name = "Total precipitation" ;
// global attributes:
:Conventions = "CF-1.0" ;
:history = "2004-09-15 17:04:29 GMT by mars2netcdf-0.92" ;
data:
longitude = 0, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 27.5, 30,
32.5, 35, 37.5, 40, 42.5, 45, 47.5, 50, 52.5, 55, 57.5, 60, 62.5, 65,
67.5, 70, 72.5, 75, 77.5, 80, 82.5, 85, 87.5, 90, 92.5, 95, 97.5, 100,
102.5, 105, 107.5, 110, 112.5, 115, 117.5, 120, 122.5, 125, 127.5, 130,
132.5, 135, 137.5, 140, 142.5, 145, 147.5, 150, 152.5, 155, 157.5, 160,
162.5, 165, 167.5, 170, 172.5, 175, 177.5, 180, 182.5, 185, 187.5, 190,
192.5, 195, 197.5, 200, 202.5, 205, 207.5, 210, 212.5, 215, 217.5, 220,
222.5, 225, 227.5, 230, 232.5, 235, 237.5, 240, 242.5, 245, 247.5, 250,
252.5, 255, 257.5, 260, 262.5, 265, 267.5, 270, 272.5, 275, 277.5, 280,
282.5, 285, 287.5, 290, 292.5, 295, 297.5, 300, 302.5, 305, 307.5, 310,
312.5, 315, 317.5, 320, 322.5, 325, 327.5, 330, 332.5, 335, 337.5, 340,
342.5, 345, 347.5, 350, 352.5, 355, 357.5 ;
latitude = 90, 87.5, 85, 82.5, 80, 77.5, 75, 72.5, 70, 67.5, 65, 62.5, 60,
57.5, 55, 52.5, 50, 47.5, 45, 42.5, 40, 37.5, 35, 32.5, 30, 27.5, 25,
22.5, 20, 17.5, 15, 12.5, 10, 7.5, 5, 2.5, 0, -2.5, -5, -7.5, -10, -12.5,
-15, -17.5, -20, -22.5, -25, -27.5, -30, -32.5, -35, -37.5, -40, -42.5,
-45, -47.5, -50, -52.5, -55, -57.5, -60, -62.5, -65, -67.5, -70, -72.5,
-75, -77.5, -80, -82.5, -85, -87.5, -90 ;
time = 898476, 898482, 898500, 898506, 898524, 898530, 898548, 898554,
898572, 898578, 898596, 898602, 898620, 898626, 898644, 898650, 898668,
898674, 898692, 898698, 898716, 898722, 898740, 898746, 898764, 898770,
898788, 898794, 898812, 898818, 898836, 898842, 898860, 898866, 898884,
898890, 898908, 898914, 898932, 898938, 898956, 898962, 898980, 898986,
899004, 899010, 899028, 899034, 899052, 899058, 899076, 899082, 899100,
899106, 899124, 899130, 899148, 899154, 899172, 899178, 899196, 899202 ;
}
