I've found a problem in which cache/memory and disk shape
information about variables will disagree with v2.2.22 of Java
Netcdf library.
When you add a new value to a variable, automatically increasing the
length of a dimension, subsequent reads can throw EOFException
because RandomAccessFile is instructed to read more values than the
file contains - the cached and actual shapes disagree.
I've created a runnable test case below to explain and demonstrate
success and failure conditions.
I am getting around this now by not interleaving read/write
operations on variables, but instead reading all variables' data to
memory, then performing any writes I need to after.
TestInsertRecord.java:
package com.metoceanengineers.datafeeds.netcdf.test;
import java.io.File;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import junit.framework.TestCase;
import ucar.ma2.Array;
import ucar.ma2.ArrayInt;
import ucar.ma2.DataType;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFileWriteable;
public class TestInsertRecord extends TestCase {
DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HHMM");
protected NetcdfFileWriteable createNc(String prefix) throws
IOException {
File mainline = File.createTempFile(prefix+"-", ".nc");
NetcdfFileWriteable mainlineNc =
NetcdfFileWriteable.createNew(mainline.getAbsolutePath(), false);
Dimension recordsDim =
mainlineNc.addUnlimitedDimension("records");
Dimension timeDims[] = {recordsDim};
Dimension var1Dims[] = {recordsDim}; // 1D
mainlineNc.addVariable("time", DataType.INT, timeDims);
mainlineNc.addVariable("var1", DataType.INT, var1Dims);
mainlineNc.create();
return mainlineNc;
}
protected String getNcInstance() throws Exception {
NetcdfFileWriteable mainlineNc = createNc("testfile");
int[] origin = {0};
ArrayInt.D1 timeArr = new ArrayInt.D1(2);
timeArr.set(0, (int)dateFormat.parse("20071130
0924").getTime());
timeArr.set(1, (int)dateFormat.parse("20071130
0926").getTime());
mainlineNc.write("time", origin, timeArr);
ArrayInt.D1 var1Arr = new ArrayInt.D1(2);
var1Arr.set(0, 10);
var1Arr.set(1, 12);
mainlineNc.write("var1", origin, var1Arr);
mainlineNc.close();
return mainlineNc.getLocation();
}
/**
* Append new data to end of existing variables.
*
* @throws Exception
*/
public void testAppendWorksOk() throws Exception {
String ncFilename = getNcInstance();
NetcdfFileWriteable ncFile =
NetcdfFileWriteable.openExisting(ncFilename, false);
/*
* Append value (20071130 0924, 11) into (time, var1)
*/
ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
newTimeValue.set(0, (int)dateFormat.parse("20071130
0925").getTime());
ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
newVarValue.set(0, 11);
int[] origin = {2};
/* The first write will expand the variables,
* but second write ok as we're just writing
* and not reading */
ncFile.write("time", origin, newTimeValue);
ncFile.write("var1", origin, newVarValue);
assertEquals(3,
ncFile.findDimension("records").getLength());
}
/**
* Test insertion of a record in between the 2 existing
* records by reading the existing tail, inserting new data
* and re-appending.
*
* Triggers EOFException through interleaved read/writes
*
* @throws Exception
*/
public void testInsertFails() throws Exception {
String ncFilename = getNcInstance();
NetcdfFileWriteable ncFile =
NetcdfFileWriteable.openExisting(ncFilename, false);
ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
newTimeValue.set(0, (int)dateFormat.parse("20071130
0925").getTime());
ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
newVarValue.set(0, 11);
/* Going to insert at 1, so read existing value,
* write down new one, and re-append old tail.
*/
int[] insertPointOrigin = {1};
int[] appendOrigin = {2};
int[] shape = {1};
Array tailTime =
ncFile.findVariable("time").read(insertPointOrigin, shape);
ncFile.write("time", insertPointOrigin, newTimeValue);
ncFile.write("time", appendOrigin, tailTime);
/* Next line excepts - why? Because the last write
above at
* records index 2 triggers an increase in the CACHED/MEMORY
* length of all variables to 3, but on disk it's still the
* original length 2.
*
* Therefore we get EOFException.
*/
Array tailVar1 =
ncFile.findVariable("var1").read(insertPointOrigin, shape);
ncFile.write("var1", insertPointOrigin, newVarValue);
ncFile.write("var1", appendOrigin, tailVar1);
assertEquals(3,
ncFile.findDimension("records").getLength());
}
}
