You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2012/01/03 05:34:41 UTC
svn commit: r1226638 - in /oodt/trunk: ./
opendapps/src/main/java/org/apache/oodt/opendapps/
opendapps/src/main/java/org/apache/oodt/opendapps/util/
Author: mattmann
Date: Tue Jan 3 04:34:40 2012
New Revision: 1226638
URL: http://svn.apache.org/viewvc?rev=1226638&view=rev
Log:
- fix for OODT-366 Extension to opendapps module to extract ALL variables in DDS stream (contributed by Luca Cinquini)
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetCrawler.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/OpendapProfileHandler.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1226638&r1=1226637&r2=1226638&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Tue Jan 3 04:34:40 2012
@@ -4,6 +4,9 @@ Apache OODT Change Log
Release 0.4: Current Development
--------------------------------------------
+* OODT-366 Extension to opendapps module to extract ALL variables
+ in DDS stream (Luca Cinquini, mattmann)
+
* OODT-365 Main class to drive opendapps profile generation
(Luca Cinquini, mattmann)
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetCrawler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetCrawler.java?rev=1226638&r1=1226637&r2=1226638&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetCrawler.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetCrawler.java Tue Jan 3 04:34:40 2012
@@ -30,15 +30,20 @@ import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
-//APACHE imports
+//OODT imports
import org.apache.oodt.cas.metadata.Metadata;
+//Spring imports
+import org.springframework.util.StringUtils;
+
//OPeNDAP/THREDDS imports
import thredds.catalog.InvAccess;
import thredds.catalog.InvCatalogRef;
import thredds.catalog.InvDataset;
import thredds.catalog.InvDocumentation;
import thredds.catalog.InvProperty;
+import thredds.catalog.InvService;
+import thredds.catalog.ServiceType;
import thredds.catalog.ThreddsMetadata.Contributor;
import thredds.catalog.ThreddsMetadata.GeospatialCoverage;
import thredds.catalog.ThreddsMetadata.Range;
@@ -47,7 +52,6 @@ import thredds.catalog.ThreddsMetadata.V
import thredds.catalog.ThreddsMetadata.Variables;
import thredds.catalog.ThreddsMetadata.Vocab;
import thredds.catalog.crawl.CatalogCrawler;
-import thredds.catalog.InvService;
import ucar.nc2.units.DateType;
import ucar.unidata.geoloc.LatLonRect;
@@ -89,30 +93,26 @@ public class DatasetCrawler implements C
* .InvDataset, java.lang.Object)
*/
public void getDataset(InvDataset dd, Object context) {
- String url = this.datasetURL + dd.getCatalogUrl().split("#")[1];
- String id = dd.getID();
- this.datasetMet.put(url, this.extractDatasetMet(dd));
+ String url = this.datasetURL + dd.getCatalogUrl().split("#")[1];
+ String id = dd.getID();
LOG.log(Level.INFO, url + " is the computed access URL for this dataset");
+ // look for an OpenDAP access URL, only extract metadata if it is found
List<InvAccess> datasets = dd.getAccess();
if (dd.getAccess() != null && dd.getAccess().size() > 0) {
Iterator<InvAccess> sets = datasets.iterator();
while (sets.hasNext()) {
InvAccess single = sets.next();
InvService service = single.getService();
- if (service.getName().equals("odap")
- || service.getName().equals("rdbmDods")) // only get the opendap one
- {
- LOG.log(Level.INFO,
- "Found a service-specific dataset URL to over-ride the computed URL: "
- + single.getUrlPath());
- url = this.datasetURL + single.getUrlPath();
+ // note: select the OpenDAP access URL based on THREDDS service type
+ if (service.getServiceType()==ServiceType.OPENDAP) {
+ LOG.log(Level.INFO, "Found OpenDAP access URL: "+ single.getUrlPath());
+ String opendapurl = this.datasetURL + single.getUrlPath();
+ this.datasetMet.put(opendapurl, this.extractDatasetMet(dd));
+ this.urls.add(opendapurl);
break;
}
}
}
- if (url != null) {
- this.urls.add(url);
- }
}
/**
@@ -135,6 +135,9 @@ public class DatasetCrawler implements C
}
private Metadata extractDatasetMet(InvDataset dataset) {
+
+ LOG.log(Level.INFO, "Crawling catalog URL=" + dataset.getCatalogUrl()+" dataset ID="+dataset.getID());
+
Metadata met = new Metadata();
this.addIfNotNull(met, "Authority", dataset.getAuthority());
this.addIfNotNull(met, "CatalogUrl", dataset.getCatalogUrl());
@@ -281,7 +284,14 @@ public class DatasetCrawler implements C
this.addIfNotNull(met, "TimeCoverageResolution", dataset
.getTimeCoverage().getResolution().getText());
}
- this.addIfNotNull(met, "UniqueID", dataset.getUniqueID());
+ // dataset unique ID
+ if (StringUtils.hasText(dataset.getUniqueID()) && !dataset.getUniqueID().equalsIgnoreCase("null")) {
+ // note: globally unique ID, or string "null" if missing authority or ID
+ this.addIfNotNull(met, "UniqueID", dataset.getUniqueID());
+ } else {
+ // dataset ID is typically not null
+ this.addIfNotNull(met, "UniqueID", dataset.getID());
+ }
if (dataset.getVariables() != null) {
for (Variables vars : dataset.getVariables()) {
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java?rev=1226638&r1=1226637&r2=1226638&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java Tue Jan 3 04:34:40 2012
@@ -104,8 +104,8 @@ public class DatasetExtractor {
};
LOG.log(Level.INFO, "catalogURL: " + this.mainCatalogURL);
- CatalogCrawler crawler = new CatalogCrawler(CatalogCrawler.USE_ALL_DIRECT,
- false, listener);
+ // Note: look for all datasets, that have either a urlPath="" attribute, or a <access> subelement
+ CatalogCrawler crawler = new CatalogCrawler(CatalogCrawler.USE_ALL, false, listener);
crawler.crawl(this.mainCatalogURL, ignore, System.out, this);
this.allUrls = listener.getURLs();
this.datasetMet = listener.getDatasetMet();
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/OpendapProfileHandler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/OpendapProfileHandler.java?rev=1226638&r1=1226637&r2=1226638&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/OpendapProfileHandler.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/OpendapProfileHandler.java Tue Jan 3 04:34:40 2012
@@ -90,29 +90,43 @@ public class OpendapProfileHandler imple
List<DapRoot> roots = this.conf.getRoots();
for (DapRoot root : roots) {
+ LOG.log(Level.INFO,"Parsing DapRoot="+root.getDatasetUrl());
+
DatasetExtractor d = new DatasetExtractor(xmlQuery, root.getCatalogUrl()
.toExternalForm(), root.getDatasetUrl().toExternalForm());
if (d.getDapUrls() != null) {
for (String opendapUrl : d.getDapUrls()) {
- Profile profile = new Profile();
- DConnect dConn = null;
+
+ // wrap the profile generation in try-catch to avoid stopping the whole harvesting process in case an exception is thrown
try {
- dConn = new DConnect(opendapUrl, true);
- } catch (FileNotFoundException e) {
- LOG.log(Level.WARNING, "Opendap URL not found: [" + opendapUrl
- + "]: Message: " + e.getMessage());
- throw new ProfileException("Opendap URL not found: [" + opendapUrl
- + "]: Message: " + e.getMessage());
+
+ LOG.log(Level.INFO,"Connecting to opendapurl="+opendapUrl);
+
+ Profile profile = new Profile();
+ DConnect dConn = null;
+ try {
+ dConn = new DConnect(opendapUrl, true);
+ } catch (FileNotFoundException e) {
+ LOG.log(Level.WARNING, "Opendap URL not found: [" + opendapUrl
+ + "]: Message: " + e.getMessage());
+ throw new ProfileException("Opendap URL not found: [" + opendapUrl
+ + "]: Message: " + e.getMessage());
+ }
+
+ Metadata datasetMet = d.getDatasetMet(opendapUrl);
+ profile.setResourceAttributes(ProfileUtils.getResourceAttributes(
+ this.conf, opendapUrl, dConn, datasetMet));
+ profile.setProfileAttributes(ProfileUtils
+ .getProfileAttributes(this.conf, datasetMet));
+ profile.getProfileElements().putAll(
+ ProfileUtils.getProfileElements(this.conf, dConn, datasetMet, profile));
+ profiles.add(profile);
+ LOG.log(Level.INFO, "Added profile id="+profile.getProfileAttributes().getID());
+ } catch(Exception e) {
+ // in case of exception, don't harvest this dataset, but keep going
+ LOG.log(Level.WARNING,"Error while building profile for opendapurl="+opendapUrl);
}
- Metadata datasetMet = d.getDatasetMet(opendapUrl);
- profile.setResourceAttributes(ProfileUtils.getResourceAttributes(
- this.conf, opendapUrl, dConn, datasetMet));
- profile.setProfileAttributes(ProfileUtils
- .getProfileAttributes(this.conf, datasetMet));
- profile.getProfileElements().putAll(
- ProfileUtils.getProfileElements(this.conf, dConn, datasetMet, profile));
- profiles.add(profile);
}
}
}
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java?rev=1226638&r1=1226637&r2=1226638&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java Tue Jan 3 04:34:40 2012
@@ -19,6 +19,7 @@ package org.apache.oodt.opendapps.util;
//JDK imports
import java.util.Arrays;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -26,10 +27,19 @@ import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
-//OPeNDAP/THREDDS imports
+//OPeNDAP imports
+import opendap.dap.BaseType;
+import opendap.dap.DArray;
import opendap.dap.DConnect;
+import opendap.dap.DDS;
+import opendap.dap.DGrid;
-//APACHE imports
+//OODT imports
+import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.ENUM_ELEMENT_TYPE;
+import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.PROF_ATTR_SPEC_TYPE;
+import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.PROF_ELEM_SPEC_TYPE;
+import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.RANGED_ELEMENT_TYPE;
+import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.RES_ATTR_SPEC_TYPE;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.opendapps.OpendapProfileElementExtractor;
@@ -43,8 +53,6 @@ import org.apache.oodt.profile.ProfileAt
import org.apache.oodt.profile.ProfileElement;
import org.apache.oodt.profile.ResourceAttributes;
-import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.*;
-
/**
*
* Static methods for unraveling and generating {@link ProfileElement}s,
@@ -106,33 +114,49 @@ public class ProfileUtils {
}
public static Map<String, ProfileElement> getProfileElements(
- OpendapConfig conf, DConnect dConn, Metadata datasetMet, Profile profile) {
- // TODO: later, we should just read all attributes instead of just those
- // specified in the conf
+ OpendapConfig conf, DConnect dConn, Metadata datasetMet, Profile profile) throws Exception {
+
OpendapProfileElementExtractor pe = new OpendapProfileElementExtractor(conf);
Map<String, ProfileElement> profElements = new HashMap<String, ProfileElement>();
- for (RewriteSpec spec : conf.getRewriteSpecs()) {
- String peName = spec.getRename() != null && !spec.getRename().equals("") ? spec
- .getRename()
- : spec.getOrigName();
- try {
- if (spec.getElementType().equals(RANGED_ELEMENT_TYPE)) {
- profElements.put(peName, pe.extractRangedProfileElement(peName, spec
- .getOrigName(), profile, dConn.getDAS()));
- } else if (spec.getElementType().equals(ENUM_ELEMENT_TYPE)) {
- profElements.put(peName, pe.extractEnumeratedProfileElement(peName,
- spec.getOrigName(), profile, dConn.getDAS()));
- }
- } catch (Exception e) {
- e.printStackTrace();
- LOG
- .log(Level.WARNING, "Problem obtaining attribute: ["
- + spec.getOrigName() + "] from OPeNDAP: Message: "
- + e.getMessage());
- continue;
- }
-
+ // extracts all variables defined in DDS
+ try {
+
+ DDS dds = dConn.getDDS();
+
+ // loop over all variables found
+ Enumeration variables = dds.getVariables();
+ while (variables.hasMoreElements()) {
+
+ BaseType variable = (BaseType)variables.nextElement();
+ String varName = variable.getName();
+ if (variable instanceof DArray) {
+ LOG.log(Level.INFO, "Extracting Darray variable: "+varName);
+ } else if (variable instanceof DGrid) {
+ LOG.log(Level.INFO, "Extracting Dgrid variable: "+varName);
+ }
+
+ RewriteSpec spec = getProfileElementSpec(varName, conf);
+ if (spec!=null) {
+ // use configuration to set variable re-name and type
+ String peName = spec.getRename() != null && !spec.getRename().equals("") ? spec.getRename() : spec.getOrigName();
+ if (spec.getElementType().equals(RANGED_ELEMENT_TYPE)) {
+ profElements.put(peName, pe.extractRangedProfileElement(peName, spec.getOrigName(), profile, dConn.getDAS()));
+ } else if (spec.getElementType().equals(ENUM_ELEMENT_TYPE)) {
+ profElements.put(peName, pe.extractEnumeratedProfileElement(peName, spec.getOrigName(), profile, dConn.getDAS()));
+ }
+ } else {
+ // if not explicitly configured, assume variable if of RANGED_ELEMENT_TYPE
+ profElements.put(varName, pe.extractRangedProfileElement(varName, varName, profile, dConn.getDAS()));
+ }
+
+ }
+
+ } catch(Exception e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING, "Error extracting metadata from DDS ("+dConn.URL()+") :" +e.getMessage());
+ // rethrow the exception so that this dataset is not harvested
+ throw e;
}
if (datasetMet != null) {
@@ -243,5 +267,19 @@ public class ProfileUtils {
pe.setName(name);
return pe;
}
+
+ /**
+ * Utility method to discover the rewrite specification for a named variable, if available.
+ * @param name
+ * @param conf
+ */
+ private static RewriteSpec getProfileElementSpec(String origName, OpendapConfig conf) {
+ for (RewriteSpec spec : conf.getRewriteSpecs()) {
+ if (spec.getOrigName().equals(origName)) {
+ return spec;
+ }
+ }
+ return null;
+ }
}