THREDDS Team: Did the XML file naming generator for the aggregation cache files (stored in cache/agg) change/flip from the dataset "ID" value to "urlPath" when going from v4.3.23 to v4.6.x? If so, why was this done, as it is preventing us from upgrading to the latest 4.6.3 due to the urlPath structure we currently use (which nicely mimics our FTP listing) and is important for keeping the same for obvious legacy reasons. e.g., we recently went in a changed all "/" to "-" in our dataset IDs (only) to fix this cache/agg file naming issue on our production v4.3.23 TDS server. What's odd is that there seems to have been a "collision detector" for creating these cache files, as some dirs had files with a "-" replacing the "/" when conflicts occurred - not so in v4.6.x. <dataset ID="GOMl0.04-expt_32.5" urlPath="GOMl0.04/expt_32.5">... <dataset ID="GOMl0.04-expt_32.5-2014" urlPath="GOMl0.04/expt_32.5/2014">... <dataset ID="GOMl0.04-expt_32.5-2014-hrly" urlPath="GOMl0.04/expt_32.5/2014/hrly">... v4.3.23 http://tds.hycom.org/thredds (agg cache works fine with no "/" in the dataset IDs - many flat files in the cache/agg with no directories) file naming structure (in v4.3.23) looks to be generated from the dataset "ID"s cache/agg/GOMl0.04-expt_32.5 cache/agg/GOMl0.04-expt_32.5-2014 cache/agg/GOMl0.04-expt_32.5-2014-hrly cache/agg/GOMl0.04-expt_32.5-2015 cache/agg/GOMl0.04-expt_32.5-2015-hrly ::now the problems occur: v4.6.3 http://beta.hycom.org/thredds (agg cache seems to be using the dataset "urlPath" for generating the XML filenames in cache/agg and there is no collision avoidance, as we have directories in our cache/agg, even though we changed all IDs to "/" to "-" in the catalogs) cache/agg/GOMl0.04/expt_32.5 cache/agg/GOMl0.04/expt_32.5/hrly cache/agg/GOMl0.04/expt_32.5/2014 cache/agg/GOMl0.04/expt_32.5/2015 cache/agg/GOMl0.04/expt_32.5/2016 so we get a "partial caching" of the datasets (i.e., the leaf datasets "GOMl0.04/expt_32.5/2015/hrly" are missing because the server cannot write a cache file due to there already being a *file* "GOMl0.04/expt_32.5/2015" in cache/agg. e.g., errors on our beta.hycom.org/thredds server running the latest v4.6.3 (identical catalogs as our v4.3.23 server) java.io.FileNotFoundException: /var/lib/tomcat/content/thredds/cache/agg/GOMl0.04/expt_32.5 (Is a directory) java.io.FileNotFoundException: /var/lib/tomcat/content/thredds/cache/agg/GOMl0.04/expt_32.5/2015/hrly (Not a directory) java.io.FileNotFoundException: /var/lib/tomcat/content/thredds/cache/agg/GOMl0.04/expt_32.5/2014/hrly (Not a directory) What's the fix for this? -- Michael McDonald Florida State University
Attachment:
threddsServlet.log
Description: Binary data
<?xml version="1.0" encoding="UTF-8"?>
<catalog name="THREDDS Catalog"
xmlns="http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0";
xmlns:xlink="http://www.w3.org/1999/xlink";>
<service name="all" serviceType="Compound" base="">
<service name="ncdods"
serviceType="OPeNDAP"
base="http://tds.hycom.org/thredds/dodsC/"/>
<service name="ncss"
serviceType="NetcdfSubset"
base="http://ncss.hycom.org/thredds/ncss/grid/"/>
<service name="wms"
serviceType="WMS"
base="http://wms.hycom.org/thredds/wms/"/>
<service name="wcs"
serviceType="WCS"
base="http://wcs.hycom.org/thredds/wcs/"/>
<service name="ftp"
serviceType="FTP"
base="ftp://ftp.hycom.org/datasets/"/>
</service>
<service name="all-ftp" serviceType="Compound" base="">
<service name="ncdods"
serviceType="OpenDAP"
base="http://tds.hycom.org/thredds/dodsC/"/>
<service name="ncss"
serviceType="NetcdfSubset"
base="http://ncss.hycom.org/thredds/ncss/grid/"/>
<service name="wms"
serviceType="WMS"
base="http://wms.hycom.org/thredds/wms/"/>
<service name="wcs"
serviceType="WCS"
base="http://wcs.hycom.org/thredds/wcs/"/>
</service>
<dataset name="HYCOM + NCODA Southeast United States 1/25 Degree Analysis">
<!-- Description -->
<!--
##########
## BASE ##
===============================================================================
##########
-->
<dataset name="GOMl0.04/expt_32.5">
<metadata inherited="true">
<authority>edu.ucar.unidata</authority>
<dataType>Grid</dataType>
<dataFormat>NetCDF</dataFormat>
<documentation type="rights"> Freely available </documentation>
<documentation
xlink:href="http://www.hycom.org/dataserver/goml0pt04/expt-32pt5";
xlink:title="GOMl0.04/expt_32.5
Documentation" />
<creator>
<name vocabulary="DIF">Naval Research Laboratory</name>
<contact url="http://www7320.nrlssc.navy.mil/hycomGOM/";
email="hycom_webmaster@xxxxxxxxxxxxxxx" />
</creator>
<publisher>
<name vocabulary="DIF">HYCOM.org</name>
<contact url="http://hycom.org/"; email="forum@xxxxxxxxx" />
</publisher>
</metadata>
<!--
##########
## TOP ##
===============================================================================
##########
-->
<dataset name="ALL 00Z Data: Apr-01-2014 to Present with Forecasts"
ID="GOMl0.04-expt_32.5" urlPath="GOMl0.04/expt_32.5">
<serviceName>all</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting" recheckEvery="120
min">
<netcdf
location="dods://tds.hycom.org/thredds/dodsC/GOMl0.04/expt_32.5/2014"/>
<netcdf
location="dods://tds.hycom.org/thredds/dodsC/GOMl0.04/expt_32.5/2015"/>
<netcdf
location="dods://tds.hycom.org/thredds/dodsC/GOMl0.04/expt_32.5/2016"/>
</aggregation>
</netcdf>
</dataset>
<dataset name="ALL Hourly Data: Apr-01-2014 to Present with Forecasts"
ID="GOMl0.04-expt_32.5-hrly" urlPath="GOMl0.04/expt_32.5/hrly">
<serviceName>all-ftp</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting" recheckEvery="120
min">
<netcdf
location="dods://tds.hycom.org/thredds/dodsC/GOMl0.04/expt_32.5/2014/hrly"/>
<netcdf
location="dods://tds.hycom.org/thredds/dodsC/GOMl0.04/expt_32.5/2015/hrly"/>
<netcdf
location="dods://tds.hycom.org/thredds/dodsC/GOMl0.04/expt_32.5/2016/hrly"/>
</aggregation>
</netcdf>
</dataset>
<!--
##########
## YEAR ##
===============================================================================
##########
-->
<dataset name="(2016 Data) Jan-01-2016 to Present with Forecasts">
<dataset name="Data at 00Z (3z)"
ID="GOMl0.04-expt_32.5-2016" urlPath="GOMl0.04/expt_32.5/2016">
<serviceName>all-ftp</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting" recheckEvery="120
min">
<scan
location="/hycom/ftp/datasets/GOMl0.04/expt_32.5/data/2016/"
suffix="???_00_3z.nc" subdirs="false" />
</aggregation>
</netcdf>
</dataset>
<dataset name="Hourly Data (3z)"
ID="GOMl0.04-expt_32.5-2016-hrly"
urlPath="GOMl0.04/expt_32.5/2016/hrly">
<serviceName>all-ftp</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting" recheckEvery="120
min">
<scan
location="/hycom/ftp/datasets/GOMl0.04/expt_32.5/data/2016/"
suffix="???_??_3z.nc" subdirs="false" />
</aggregation>
</netcdf>
</dataset>
</dataset> <!-- YEAR -->
<!--
##########
## YEAR ##
===============================================================================
##########
-->
<dataset name="(2015 Data) Jan-01-2015 to Present with Forecasts">
<dataset name="Data at 00Z (3z)"
ID="GOMl0.04-expt_32.5-2015" urlPath="GOMl0.04/expt_32.5/2015">
<serviceName>all-ftp</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting" recheckEvery="300
min">
<scan
location="/hycom/ftp/datasets/GOMl0.04/expt_32.5/data/2015/"
suffix="???_00_3z.nc" subdirs="false" />
</aggregation>
</netcdf>
</dataset>
<dataset name="Hourly Data (3z)"
ID="GOMl0.04-expt_32.5-2015-hrly"
urlPath="GOMl0.04/expt_32.5/2015/hrly">
<serviceName>all-ftp</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting" recheckEvery="300
min">
<scan
location="/hycom/ftp/datasets/GOMl0.04/expt_32.5/data/2015/"
suffix="???_??_3z.nc" subdirs="false" />
</aggregation>
</netcdf>
</dataset>
</dataset> <!-- YEAR -->
<!--
##########
## YEAR ##
===============================================================================
##########
-->
<dataset name="(2014 Data) Apr-01-2014 to Dec-31-2014">
<dataset name="Data at 00Z (3z)"
ID="GOMl0.04-expt_32.5-2014" urlPath="GOMl0.04/expt_32.5/2014">
<serviceName>all-ftp</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting">
<scan
location="/hycom/ftp/datasets/GOMl0.04/expt_32.5/data/2014/"
suffix="???_00_3z.nc" subdirs="false" />
</aggregation>
</netcdf>
</dataset>
<dataset name="Hourly Data (3z)"
ID="GOMl0.04-expt_32.5-2014-hrly"
urlPath="GOMl0.04/expt_32.5/2014/hrly">
<serviceName>all-ftp</serviceName>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2";>
<aggregation dimName="MT" type="joinExisting">
<scan
location="/hycom/ftp/datasets/GOMl0.04/expt_32.5/data/2014/"
suffix="???_??_3z.nc" subdirs="false" />
</aggregation>
</netcdf>
</dataset>
</dataset> <!-- YEAR -->
<!--
##########
## END ##
===============================================================================
##########
-->
</dataset> <!-- Model/Experiment -->
</dataset> <!-- Description -->
</catalog>
thredds archives: