Sure. The rest is the same. I'll do binary search next.
First:
netcdf a {
dimensions:
depth = 2 ;
lat = 2001 ;
lon = 4500 ;
time = UNLIMITED ; // (1 currently)
variables:
double depth(depth) ;
depth:long_name = "Depth" ;
depth:standard_name = "depth" ;
depth:units = "m" ;
depth:positive = "down" ;
depth:axis = "Z" ;
depth:NAVO_code = 5 ;
dimensions:
depth = 2 ;
lat = 2001 ;
lon = 4500 ;
time = UNLIMITED ; // (1 currently)
variables:
double depth(depth) ;
depth:long_name = "Depth" ;
depth:standard_name = "depth" ;
depth:units = "m" ;
depth:positive = "down" ;
depth:axis = "Z" ;
depth:NAVO_code = 5 ;
depth:_Storage = "chunked" ;
depth:_ChunkSizes = 2 ;
depth:_DeflateLevel = 9 ;
double lat(lat) ;
lat:long_name = "Latitude" ;
lat:standard_name = "latitude" ;
lat:units = "degrees_north" ;
lat:point_spacing = "even" ;
lat:axis = "Y" ;
lat:NAVO_code = 1 ;
lat:_Storage = "chunked" ;
lat:_ChunkSizes = 2001 ;
lat:_DeflateLevel = 9 ;
double lon(lon) ;
lon:long_name = "Longitude" ;
lon:standard_name = "longitude" ;
lon:units = "degrees_east" ;
lon:modulo = "360 degrees" ;
lon:axis = "X" ;
lon:NAVO_code = 2 ;
lon:_Storage = "chunked" ;
lon:_ChunkSizes = 4500 ;
lon:_DeflateLevel = 9 ;
double time(time) ;
time:long_name = "Valid Time" ;
time:units = "hours since 2000-01-01 00:00:00" ;
time:time_origin = "2000-01-01 00:00:00" ;
time:calendar = "gregorian" ;
time:axis = "T" ;
time:NAVO_code = 13 ;
time:_Storage = "chunked" ;
time:_ChunkSizes = 1 ;
time:_DeflateLevel = 9 ;
short water_u(time, depth, lat, lon) ;
water_u:long_name = "Eastward Water Velocity" ;
water_u:standard_name = "eastward_sea_water_velocity" ;
water_u:units = "m/s" ;
water_u:_FillValue = -30000s ;
water_u:missing_value = -30000s ;
water_u:scale_factor = 0.001f ;
water_u:add_offset = 0.f ;
water_u:NAVO_code = 17 ;
water_u:_Storage = "chunked" ;
water_u:_ChunkSizes = 1, 1, 976, 2196 ;
water_u:_DeflateLevel = 9 ;
water_u:_Endianness = "little" ;
short water_v(time, depth, lat, lon) ;
water_v:long_name = "Northward Water Velocity" ;
water_v:standard_name = "northward_sea_water_velocity" ;
water_v:units = "m/s" ;
water_v:_FillValue = -30000s ;
water_v:missing_value = -30000s ;
water_v:scale_factor = 0.001f ;
water_v:add_offset = 0.f ;
water_v:NAVO_code = 18 ;
water_v:_Storage = "chunked" ;
water_v:_ChunkSizes = 1, 1, 976, 2196 ;
water_v:_DeflateLevel = 9 ;
water_v:_Endianness = "little" ;
// global attributes:
:classification_level = "UNCLASSIFIED" ;
:distribution_statement = "Approved for public release.
Distribution unlimited." ;
:downgrade_date = "not applicable" ;
:classification_authority = "not applicable" ;
:institution = "Naval Oceanographic Office" ;
:source = "HYCOM archive file" ;
:history = "Fri Nov 20 10:01:58 2015: ncks -4 --deflate 9
-v water_u,water_v -d depth,1,2,1 hycom_glb_911_2015110200_t000_uv3z.nc
hycom_glb_911_2015110200_t000_uv3z-test.nc\narchv2ncdf3z" ;
:field_type = "instantaneous" ;
:Conventions = "CF-1.0 NAVO_netcdf_v1.0" ;
:NCO = "4.4.2" ;
:_Format = "netCDF-4" ;
data:
depth = 2, 4 ;
lat = -80, -79.9199981689453, -79.8399963378906, -79.7600021362305,
-79.6800003051758, -79.5999984741211, -79.5199966430664,
-79.4400024414062, -79.3600006103516, -79.2799987792969,
-79.1999969482422, -79.120002746582, -79.0400009155273,
-78.9599990844727, -78.879997253418, -78.8000030517578,
-78.7200012207031, -78.6399993896484, -78.5599975585938,
-78.4800033569336, -78.4000015258789, -78.3199996948242,
-78.2399978637695, -78.1600036621094, -78.0800018310547, -78,
-77.9199981689453, -77.8399963378906, -77.7600021362305,
-77.6800003051758, -77.5999984741211, -77.5199966430664,
-77.4400024414062, -77.3600006103516, -77.2799987792969,
-77.1999969482422, -77.120002746582, -77.0400009155273,
-76.9599990844727, -76.879997253418, -76.8000030517578,
-76.7200012207031, -76.6399993896484, -76.5599975585938,
-76.4800033569336, -76.4000015258789, -76.3199996948242,
-76.2399978637695, -76.1600036621094, -76.0800018310547, -76,
-75.9199981689453, -75.8399963378906, -75.7600021362305,
-75.6800003051758, -75.5999984741211, -75.5199966430664,
-75.4400024414062, -75.3600006103516, -75.2799987792969,
-75.1999969482422, -75.120002746582, -75.0400009155273,
-74.9599990844727, -74.879997253418, -74.8000030517578,
-74.7200012207031, -74.6399993896484, -74.5599975585938,
simonf@shaman [~/downl/tax]
16:52:45 $ ncdump -d -s /tmp/a.nc
ncdump: unreasonable value for float significant digits: 0
simonf@shaman [~/downl/tax]
16:52:50 $ ncdump -s /tmp/a.nc |more
netcdf a {
dimensions:
depth = 2 ;
lat = 2001 ;
lon = 4500 ;
time = UNLIMITED ; // (1 currently)
variables:
double depth(depth) ;
depth:long_name = "Depth" ;
depth:standard_name = "depth" ;
depth:units = "m" ;
depth:positive = "down" ;
depth:axis = "Z" ;
depth:NAVO_code = 5 ;
depth:_Storage = "chunked" ;
depth:_ChunkSizes = 2 ;
depth:_DeflateLevel = 9 ;
double lat(lat) ;
lat:long_name = "Latitude" ;
lat:standard_name = "latitude" ;
lat:units = "degrees_north" ;
lat:point_spacing = "even" ;
lat:axis = "Y" ;
lat:NAVO_code = 1 ;
lat:_Storage = "chunked" ;
lat:_ChunkSizes = 2001 ;
lat:_DeflateLevel = 9 ;
double lon(lon) ;
lon:long_name = "Longitude" ;
lon:standard_name = "longitude" ;
lon:units = "degrees_east" ;
lon:modulo = "360 degrees" ;
lon:axis = "X" ;
lon:NAVO_code = 2 ;
lon:_Storage = "chunked" ;
lon:_ChunkSizes = 4500 ;
lon:_DeflateLevel = 9 ;
double time(time) ;
time:long_name = "Valid Time" ;
time:units = "hours since 2000-01-01 00:00:00" ;
time:time_origin = "2000-01-01 00:00:00" ;
time:calendar = "gregorian" ;
time:axis = "T" ;
time:NAVO_code = 13 ;
time:_Storage = "chunked" ;
time:_ChunkSizes = 1 ;
time:_DeflateLevel = 9 ;
short water_u(time, depth, lat, lon) ;
water_u:long_name = "Eastward Water Velocity" ;
water_u:standard_name = "eastward_sea_water_velocity" ;
water_u:units = "m/s" ;
water_u:_FillValue = -30000s ;
water_u:missing_value = -30000s ;
water_u:scale_factor = 0.001f ;
water_u:add_offset = 0.f ;
water_u:NAVO_code = 17 ;
water_u:_Storage = "chunked" ;
water_u:_ChunkSizes = 1, 1, 976, 2196 ;
water_u:_DeflateLevel = 9 ;
water_u:_Endianness = "little" ;
short water_v(time, depth, lat, lon) ;
water_v:long_name = "Northward Water Velocity" ;
water_v:standard_name = "northward_sea_water_velocity" ;
water_v:units = "m/s" ;
water_v:_FillValue = -30000s ;
water_v:missing_value = -30000s ;
water_v:scale_factor = 0.001f ;
water_v:add_offset = 0.f ;
water_v:NAVO_code = 18 ;
water_v:_Storage = "chunked" ;
water_v:_ChunkSizes = 1, 1, 976, 2196 ;
water_v:_DeflateLevel = 9 ;
water_v:_Endianness = "little" ;
// global attributes:
:classification_level = "UNCLASSIFIED" ;
:distribution_statement = "Approved for public release.
Distribution unlimited." ;
:downgrade_date = "not applicable" ;
:classification_authority = "not applicable" ;
:institution = "Naval Oceanographic Office" ;
:source = "HYCOM archive file" ;
:history = "Fri Nov 20 10:01:58 2015: ncks -4 --deflate 9
-v water_u,water_v -d depth,1,2,1 hycom_glb_911_2015110200_t000_uv3z.nc
hycom_glb_911_2015110200_t000_uv3z-test.nc\narchv2ncdf3z" ;
:field_type = "instantaneous" ;
:Conventions = "CF-1.0 NAVO_netcdf_v1.0" ;
:NCO = "4.4.2" ;
:_Format = "netCDF-4" ;
data:
Second:
netcdf Oa20_radiance {
dimensions:
columns = 4865 ;
rows = 3682 ;
variables:
ushort Oa20_radiance(rows, columns) ;
Oa20_radiance:_FillValue = -1US ;
Oa20_radiance:add_offset = 0.f ;
Oa20_radiance:ancillary_variables = "Oa20_radiance_err" ;
Oa20_radiance:coordinates = "time_stamp altitude latitude
longitude" ;
Oa20_radiance:long_name = "TOA radiance for OLCI
acquisition band Oa20" ;
Oa20_radiance:scale_factor = 0.00326378f ;
Oa20_radiance:standard_name =
"toa_upwelling_spectral_radiance" ;
Oa20_radiance:units = "mW.m-2.sr-1.nm-1" ;
Oa20_radiance:valid_max = -2US ;
Oa20_radiance:valid_min = 0US ;
Oa20_radiance:_Storage = "chunked" ;
Oa20_radiance:_ChunkSizes = 1260, 1665 ;
Oa20_radiance:_DeflateLevel = 2 ;
Oa20_radiance:_Shuffle = "true" ;
Oa20_radiance:_Endianness = "little" ;
// global attributes:
:absolute_orbit_number = 3517U ;
:ac_subsampling_factor = 64US ;
:al_subsampling_factor = 1US ;
:comment = " " ;
:contact = "EOHElp@xxxxxxx" ;
:creation_time = "2016-10-20T13:00:25Z" ;
:institution = "SVL" ;
:netCDF_version = "4.2 of Jul 5 2012 17:07:43 $" ;
:product_name =
"S3A_OL_1_EFR____20161020T102624_20161020T102906_20161020T130025_0162_010_079_4319_SVL_O_NR_002.SEN3"
;
:references = "S3IPF PDS 004 - i2r1 - Product Data Format
Specification - OLCI, S3IPF PDS 002 - i1r6 - Product Data Format
Specification - Product Structures, S3IPF DPM 002 - i2r1 - Detailed
Processing Model - OLCI Level 1" ;
:resolution = "[ 270 294 ]" ;
:source = "IPF-OL-1-EO 06.04" ;
:start_time = "2016-10-20T10:26:24.238891Z" ;
:stop_time = "2016-10-20T10:29:06.207560Z" ;
:title = "OLCI Level 1b Product, Radiance Oa20 Data Set" ;
:_Format = "netCDF-4" ;
data:
Oa20_radiance =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, 13132, 13132, 13132, 13132,
14559, 14559, 14559, 14050, 14050, 14050, 12835, 12835, 12835, 11065,
11065, 10654, 10654, 10654, 11167, 11167, 11167, 14414, 14414, 14414,
15897, 15897, 15897, 13592, 13592, 13592, 11068, 11068, 11322, 11322,
11322, 11955, 11955, 11955, 11647, 11647, 11647, 11324, 11324, 11324,
8303, 8303, 7950, 7950, 7950, 8410, 8410, 8410, 7343, 7343, 7343,
11477,
11477, 12327, 12327, 12327, 12137, 12137, 12137, 13351, 13351, 13351,
14739, 14739, 14739, 14464, 14464, 14455, 14455, 14455, 16642, 16642,
16642, 16603, 16603, 16603, 17591, 17591, 17591, 18271, 18271, 18175,
18175, 18175, 16549, 16549, 16549, 13523, 13523, 13523, 11172, 11172,
10405, 10405, 10405, 10272, 10272, 10272, 10776, 10776, 10776, 10999,
10999, 13601, 13601, 13601, 17339, 17339, 17339, 17690, 17690, 17690,
14465, 14465, 11978, 11978, 11978, 12253, 12253, 12253, 13162, 13162,
13185, 13185, 13185, 11546, 11546, 11546, 11859, 11859, 11859, 11160,
11160, 10914, 10914, 10914, 13108, 13108, 13108, 13960, 13960, 13960,
16132, 16132, 18585, 18585, 18585, 19838, 19838, 19838, 17879, 17879,
16333, 16333, 16333, 15546, 15546, 15546, 13680, 13680, 13060, 13060,
13060, 15861, 15861, 15861, 18300, 18300, 18472, 18472, 18472, 17279,
17279, 17279, 15611, 15611, 14673, 14673, 14673, 11660, 11660, 11660,
11020, 11020, 9921, 9921, 9921, 8624, 8624, 8624, 8671, 8671, 10214,
10214, 10214, 10864, 10864, 10864, 11475, 11475, 13118, 13118, 13118,
13033, 13033, 13033, 11779, 11779, 13134, 13134, 13134, 14151, 14151,
14151, 13842, 13842, 12836, 12836, 12836, 13101, 13101, 16060, 16060,
16060, 16074, 16074, 16074, 13860, 13860, 13847, 13847, 13847, 16115,
16115, 17292, 17292, 17292, 20331, 20331, 20331, 21380, 21380, 19915,
19915, 19915, 17696, 17696, 16059, 16059, 16059, 16525, 16525, 16525,
netcdf Oa20_radiance {
dimensions:
columns = 4865 ;
rows = 3682 ;
variables:
ushort Oa20_radiance(rows, columns) ;
Oa20_radiance:_FillValue = -1US ;
Oa20_radiance:add_offset = 0.f ;
Oa20_radiance:ancillary_variables = "Oa20_radiance_err" ;
Oa20_radiance:coordinates = "time_stamp altitude latitude
long
itude" ;
Oa20_radiance:long_name = "TOA radiance for OLCI
acquisition b
and Oa20" ;
Oa20_radiance:scale_factor = 0.00326378f ;
Oa20_radiance:standard_name =
"toa_upwelling_spectral_radiance
" ;
Oa20_radiance:units = "mW.m-2.sr-1.nm-1" ;
Oa20_radiance:valid_max = -2US ;
Oa20_radiance:valid_min = 0US ;
Oa20_radiance:_Storage = "chunked" ;
Oa20_radiance:_ChunkSizes = 1260, 1665 ;
Oa20_radiance:_DeflateLevel = 2 ;
Oa20_radiance:_Shuffle = "true" ;
Oa20_radiance:_Endianness = "little" ;
// global attributes:
:absolute_orbit_number = 3517U ;
:ac_subsampling_factor = 64US ;
:al_subsampling_factor = 1US ;
:comment = " " ;
:contact = "EOHElp@xxxxxxx" ;
:creation_time = "2016-10-20T13:00:25Z" ;
:institution = "SVL" ;
:netCDF_version = "4.2 of Jul 5 2012 17:07:43 $" ;
:product_name =
"S3A_OL_1_EFR____20161020T102624_20161020T1029
06_20161020T130025_0162_010_079_4319_SVL_O_NR_002.SEN3" ;
:references = "S3IPF PDS 004 - i2r1 - Product Data Format
Spec
ification - OLCI, S3IPF PDS 002 - i1r6 - Product Data Format Specification
- P
roduct Structures, S3IPF DPM 002 - i2r1 - Detailed Processing Model - OLCI
Level 1" ;
:resolution = "[ 270 294 ]" ;
:source = "IPF-OL-1-EO 06.04" ;
:start_time = "2016-10-20T10:26:24.238891Z" ;
:stop_time = "2016-10-20T10:29:06.207560Z" ;
:title = "OLCI Level 1b Product, Radiance Oa20 Data Set" ;
:_Format = "netCDF-4" ;
On Tue, Dec 13, 2016 at 10:04 AM, Ed Hartnett <edwardjameshartnett@xxxxxxxxx
> wrote:
> Can you do an ncdump with the -s option, so we can see the chunksizes?
>
> On Tue, Dec 13, 2016 at 10:54 AM, Chris Barker <chris.barker@xxxxxxxx>
> wrote:
>
>> if I understand the question, this is read times -- is this in fact with
>> exactly the same files? In which case the chunking is already set.
>>
>> less performant decompression??
>>
>> sounds like the binary search is needed.
>>
>> Is EVERYTHING else the same? disk system, OS, etc?
>>
>> -CHB
>>
>>
>> On Tue, Dec 13, 2016 at 9:21 AM, Charlie Zender <zender@xxxxxxx> wrote:
>>
>>> Hello Simon,
>>>
>>> Since both files are netCDF4 compressed that
>>> means they use chunking. My wild guess is that
>>> different chunking defaults cause the observed
>>> change in dumping time. You can see the
>>> chunk sizes employed with ncdump -s or ncks --hdn,
>>> and you can play with the chunk sizes/policy
>>> with either.
>>>
>>> Charlie
>>> --
>>> Charlie Zender, Earth System Sci. & Computer Sci.
>>> University of California, Irvine 949-891-2429 )'(
>>>
>>>
>>> _______________________________________________
>>> NOTE: All exchanges posted to Unidata maintained email lists are
>>> recorded in the Unidata inquiry tracking system and made publicly
>>> available through the web. Users who post to any of the lists we
>>> maintain are reminded to remove any personal information that they
>>> do not want to be made public.
>>>
>>>
>>> netcdfgroup mailing list
>>> netcdfgroup@xxxxxxxxxxxxxxxx
>>> For list information or to unsubscribe, visit:
>>> http://www.unidata.ucar.edu/mailing_lists/
>>>
>>
>>
>>
>> --
>>
>> Christopher Barker, Ph.D.
>> Oceanographer
>>
>> Emergency Response Division
>> NOAA/NOS/OR&R (206) 526-6959 voice
>> 7600 Sand Point Way NE (206) 526-6329 fax
>> Seattle, WA 98115 (206) 526-6317 main reception
>>
>> Chris.Barker@xxxxxxxx
>>
>> _______________________________________________
>> NOTE: All exchanges posted to Unidata maintained email lists are
>> recorded in the Unidata inquiry tracking system and made publicly
>> available through the web. Users who post to any of the lists we
>> maintain are reminded to remove any personal information that they
>> do not want to be made public.
>>
>>
>> netcdfgroup mailing list
>> netcdfgroup@xxxxxxxxxxxxxxxx
>> For list information or to unsubscribe, visit:
>> http://www.unidata.ucar.edu/mailing_lists/
>>
>
>
> _______________________________________________
> NOTE: All exchanges posted to Unidata maintained email lists are
> recorded in the Unidata inquiry tracking system and made publicly
> available through the web. Users who post to any of the lists we
> maintain are reminded to remove any personal information that they
> do not want to be made public.
>
>
> netcdfgroup mailing list
> netcdfgroup@xxxxxxxxxxxxxxxx
> For list information or to unsubscribe, visit:
> http://www.unidata.ucar.edu/mailing_lists/
>
