You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by lu...@apache.org on 2012/03/20 23:44:09 UTC
svn commit: r1303182 - in
/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps: config/
extractors/ util/
Author: luca
Date: Tue Mar 20 22:44:09 2012
New Revision: 1303182
URL: http://svn.apache.org/viewvc?rev=1303182&view=rev
Log:
Implementing expanded syntax for environment variable replacement (OODT-405)
Modified:
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java Tue Mar 20 22:44:09 2012
@@ -69,5 +69,7 @@ public interface OpendapConfigMetKeys {
public static final String DATASET_MET_NAME_ATTR = "name";
public static final String DATASET_MET_VALUE_ATTR = "value";
+
+ public static final String RES_LOCATION_ATTR = "resLocation";
}
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java Tue Mar 20 22:44:09 2012
@@ -26,6 +26,8 @@ public interface OpendapProfileMetKeys {
public static final String VARIABLES = "Variables";
+ public static final String COORDINATES = "Coordinates";
+
public static final String VARIABLES_LONG_NAMES = "Variable Long Names";
public static final String CF_STANDARD_NAMES = "CF Standard Names";
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java Tue Mar 20 22:44:09 2012
@@ -96,19 +96,39 @@ public class DasMetadataExtractor implem
ProfileUtils.addIfNotExisting(metadata, key, att.getValues());
}
- // NetCDF variables
+ // NetCDF coordinates
} else {
- // store variable name
- ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES, attName);
- // store "standard_name", "long_name"
- while (e.hasMoreElements()) {
- String key = (String) e.nextElement();
- Attribute att = at.getAttribute(key);
- if (key.equalsIgnoreCase(STANDARD_NAME)) {
- ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES, att.getValueAt(0));
- } else if (key.equalsIgnoreCase(LONG_NAME)) {
- ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES, att.getValueAt(0));
- }
+
+ if ( attName.equalsIgnoreCase("lat") || attName.equalsIgnoreCase("latitude")
+ || attName.equalsIgnoreCase("lon") || attName.equalsIgnoreCase("longitude")
+ || attName.equalsIgnoreCase("time")
+ || attName.equalsIgnoreCase("alt") || attName.equalsIgnoreCase("altitude")
+ || attName.equalsIgnoreCase("depth")
+ ) {
+
+ // store coordinate name
+ ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.COORDINATES, attName);
+
+ } else if (attName.startsWith("time_")) {
+
+ // ignore for now - it's not a coordinate neither a variable you would want to search on
+
+ // NetCDF variables
+ } else {
+
+ // store variable name
+ ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES, attName);
+ // store "standard_name", "long_name"
+ while (e.hasMoreElements()) {
+ String key = (String) e.nextElement();
+ Attribute att = at.getAttribute(key);
+ if (key.equalsIgnoreCase(STANDARD_NAME)) {
+ ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES, att.getValueAt(0));
+ } else if (key.equalsIgnoreCase(LONG_NAME)) {
+ ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES, att.getValueAt(0));
+ }
+ }
+
}
}
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java Tue Mar 20 22:44:09 2012
@@ -273,6 +273,8 @@ public class ThreddsMetadataExtractor im
// add opendap access URL
if (type.equalsIgnoreCase(ServiceType.OPENDAP.toString())) {
+ // store opendap URL
+ ProfileUtils.addIfNotNull(met,"OpendapUrl",url+".dods");
// note: special processing of opendap endpoints since URL in thredds catalog is unusable without a suffix
ProfileUtils.addIfNotNull(met,"Access", this.encodeAccessTuple(url+".html", ProfileUtils.MIME_TYPE_OPENDAP_HTML, type));
}
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java?rev=1303182&r1=1303181&r2=1303182&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java Tue Mar 20 22:44:09 2012
@@ -82,6 +82,11 @@ public class ProfileChecker {
selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(), ProfileUtils.MIME_TYPE_HTML),
true, sb);
+ // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&Width=1280&BrowseImage=Aqua/MODIS MHchla|application/gis|NetViewer</resLocation>
+ ok = ok && checkResourceAttribute("Location of type "+ProfileUtils.MIME_TYPE_GIS,
+ selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(), ProfileUtils.MIME_TYPE_GIS),
+ true, sb);
+
// <elemName>mission_name</elemName>
// <elemName>sensor</elemName>
// <elemName>...</elemName>
Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java?rev=1303182&r1=1303181&r2=1303182&view=diff
==============================================================================
--- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java (original)
+++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java Tue Mar 20 22:44:09 2012
@@ -74,8 +74,9 @@ public class ProfileUtils {
// The delimiter must be a character that is not commonly used in the metadata values,
// and that it does not a special regular expression character.
// Cannot use '#' as it is used in URL anchors, such as THREDDS urls.
+ // Cannot user '?', '&' as they are used in URL query strings.
// Cannot use '|' as it is used as multi-part separators in encoding of metadata fields.
- PathUtils.DELIMITER = "&";
+ PathUtils.DELIMITER = "~";
}
// character separating multiple parts of the same metadata field,
@@ -97,6 +98,7 @@ public class ProfileUtils {
public final static String MIME_TYPE_OPENDAP_DDS = "application/opendap-dds";
public final static String MIME_TYPE_OPENDAP_HTML = "application/opendap-html";
public final static String MIME_TYPE_RSS = "application/rss+xml";
+ public final static String MIME_TYPE_GIS = "application/gis";
private static final Logger LOG = Logger.getLogger(ProfileUtils.class
@@ -108,10 +110,19 @@ public class ProfileUtils {
for (ConstantSpec spec : conf.getConstSpecs()) {
if (spec.getType().equals(RES_ATTR_SPEC_TYPE)) {
try {
- String value = PathUtils.replaceEnvVariables(spec.getValue(), datasetMet, true);
- if (StringUtils.hasText(value)) {
- setResourceAttributes(resAttr, spec.getName(), value);
+
+ // first process expanded '[@...]' instructions
+ List<String> values = multipleEnvVariablesReplacement(spec.getValue(), datasetMet);
+
+ // then process standard '[...]' instructions
+ for (String value : values) {
+ String _value = PathUtils.replaceEnvVariables(value, datasetMet, true);
+ if (StringUtils.hasText(_value)) {
+ setResourceAttributes(resAttr, spec.getName(), _value);
+ }
+
}
+
} catch (Exception e) {
e.printStackTrace();
LOG.log(Level.WARNING, "Error setting field: [" + spec.getName()
@@ -123,6 +134,44 @@ public class ProfileUtils {
return resAttr;
}
+
+ /**
+ * Utility method to process environment replacement instructions of the form '[@key]'
+ * resulting in as many output values as there are values for the environment variable 'key'.
+ * Note that currently only one such pattern '[@key']' can be processed.
+ *
+ * @param value
+ * @param metadata
+ * @return
+ */
+ private static List<String> multipleEnvVariablesReplacement(String value, Metadata metadata) {
+
+ List<String> newValues = new ArrayList<String>();
+
+ // regexp matching found > replace values
+ int start = value.indexOf("[@");
+ if (start>=0) {
+
+ int end = value.indexOf("]",start+2);
+ // remove '[@',']' to obtain environment variable key
+ String envKey = value.substring(start+2,end);
+ List<String> envValues = metadata.getAllMetadata(envKey);
+ if (envValues!=null) {
+ for (String envValue : envValues) {
+ // create new metadata value for this environment replacement
+ String newValue = value.replaceAll("\\[@"+envKey+"\\]", envValue);
+ newValues.add(newValue);
+ }
+ }
+
+ // regexp matching not found > return original value
+ } else {
+ newValues.add(value);
+ }
+
+ return newValues;
+
+ }
public static ProfileAttributes getProfileAttributes(OpendapConfig conf, Metadata datasetMet) {
ProfileAttributes profAttr = new ProfileAttributes();
Re: svn commit: r1303182 - in
/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps: config/
extractors/ util/
Posted by "Mattmann, Chris A (388J)" <ch...@jpl.nasa.gov>.
Thanks Luca!
Cheers,
Chris
On Mar 23, 2012, at 3:29 AM, Cinquini, Luca (3880) wrote:
> Hi Chris,
>
> On Mar 21, 2012, at 4:31 PM, Mattmann, Chris A (388J) wrote:
>
>> Hi Luca,
>>
>> On Mar 21, 2012, at 12:34 AM, Cinquini, Luca (3880) wrote:
>>
>>> Hey Chris,
>>> thanks for checking this out!
>>
>> Anytime!
>>
>>>
>>> Are you referring to the following line:
>>>
>>>> // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&Width=1280&BrowseImage=Aqua/MODIS MHchla|application/gis|NetViewer</resLocation>
>>
>> +1, yep that one.
>>
>>>
>>> That was meant to be an example of the kind of URLs that are checked by the ProfileChecker.... there are other examples of XML elements, all commented out, to help users understand what the checker does. I can remove them, or maybe better preface them with // example:..... ?
>>
>> I think we should have example stuff like this on the OODT wiki, rather than in the code. It's a personal preference, but
>> the less commented lines I have to look at, the happier I am :) I also think it helps developers that look at the software
>> and extend it to not see that type of documentation baked into comments, but provided easily as part of the wiki.
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Chris Mattmann, Ph.D.
Senior Computer Scientist
NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA
Office: 171-266B, Mailstop: 171-246
Email: chris.a.mattmann@nasa.gov
WWW: http://sunset.usc.edu/~mattmann/
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Adjunct Assistant Professor, Computer Science Department
University of Southern California, Los Angeles, CA 90089 USA
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Re: svn commit: r1303182 - in
/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps: config/
extractors/ util/
Posted by "Cinquini, Luca (3880)" <Lu...@jpl.nasa.gov>.
Hi Chris,
On Mar 21, 2012, at 4:31 PM, Mattmann, Chris A (388J) wrote:
> Hi Luca,
>
> On Mar 21, 2012, at 12:34 AM, Cinquini, Luca (3880) wrote:
>
>> Hey Chris,
>> thanks for checking this out!
>
> Anytime!
>
>>
>> Are you referring to the following line:
>>
>>> // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&Width=1280&BrowseImage=Aqua/MODIS MHchla|application/gis|NetViewer</resLocation>
>
> +1, yep that one.
>
>>
>> That was meant to be an example of the kind of URLs that are checked by the ProfileChecker.... there are other examples of XML elements, all commented out, to help users understand what the checker does. I can remove them, or maybe better preface them with // example:..... ?
>
> I think we should have example stuff like this on the OODT wiki, rather than in the code. It's a personal preference, but
> the less commented lines I have to look at, the happier I am :) I also think it helps developers that look at the software
> and extend it to not see that type of documentation baked into comments, but provided easily as part of the wiki.
>
> Would you be OK with me moving it there and starting a page?
Sure, although I wouldn't waste your time, I don't think it's that important... I removed those kind of comments from the latest version of the code and checked it in...
thanks again, Luca
>
> Cheers,
> Chris
>
> [1] https://cwiki.apache.org/confluence/display/OODT/Home
>
>
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> Chris Mattmann, Ph.D.
> Senior Computer Scientist
> NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA
> Office: 171-266B, Mailstop: 171-246
> Email: chris.a.mattmann@nasa.gov
> WWW: http://sunset.usc.edu/~mattmann/
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> Adjunct Assistant Professor, Computer Science Department
> University of Southern California, Los Angeles, CA 90089 USA
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>
Re: svn commit: r1303182 - in
/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps: config/
extractors/ util/
Posted by "Mattmann, Chris A (388J)" <ch...@jpl.nasa.gov>.
Hi Luca,
On Mar 21, 2012, at 12:34 AM, Cinquini, Luca (3880) wrote:
> Hey Chris,
> thanks for checking this out!
Anytime!
>
> Are you referring to the following line:
>
>> // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&Width=1280&BrowseImage=Aqua/MODIS MHchla|application/gis|NetViewer</resLocation>
+1, yep that one.
>
> That was meant to be an example of the kind of URLs that are checked by the ProfileChecker.... there are other examples of XML elements, all commented out, to help users understand what the checker does. I can remove them, or maybe better preface them with // example:..... ?
I think we should have example stuff like this on the OODT wiki, rather than in the code. It's a personal preference, but
the less commented lines I have to look at, the happier I am :) I also think it helps developers that look at the software
and extend it to not see that type of documentation baked into comments, but provided easily as part of the wiki.
Would you be OK with me moving it there and starting a page?
Cheers,
Chris
[1] https://cwiki.apache.org/confluence/display/OODT/Home
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Chris Mattmann, Ph.D.
Senior Computer Scientist
NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA
Office: 171-266B, Mailstop: 171-246
Email: chris.a.mattmann@nasa.gov
WWW: http://sunset.usc.edu/~mattmann/
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Adjunct Assistant Professor, Computer Science Department
University of Southern California, Los Angeles, CA 90089 USA
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Re: svn commit: r1303182 - in
/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps: config/
extractors/ util/
Posted by "Cinquini, Luca (3880)" <Lu...@jpl.nasa.gov>.
Hey Chris,
thanks for checking this out!
Are you referring to the following line:
> // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&Width=1280&BrowseImage=Aqua/MODIS MHchla|application/gis|NetViewer</resLocation>
That was meant to be an example of the kind of URLs that are checked by the ProfileChecker.... there are other examples of XML elements, all commented out, to help users understand what the checker does. I can remove them, or maybe better preface them with // example:..... ?
thanks, Luca
On Mar 20, 2012, at 5:14 PM, Mattmann, Chris A (388J) wrote:
> Hey Luca,
>
> One thing I caught here was that there was a commented out CMDS url -- I think we should
> remove that? If it's commented, looks extraneous...
>
> Keep up the good work!
>
> Cheers,
> Chris
>
> On Mar 20, 2012, at 11:44 PM, <lu...@apache.org> wrote:
>
>> Author: luca
>> Date: Tue Mar 20 22:44:09 2012
>> New Revision: 1303182
>>
>> URL: http://svn.apache.org/viewvc?rev=1303182&view=rev
>> Log:
>> Implementing expanded syntax for environment variable replacement (OODT-405)
>>
>> Modified:
>> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
>> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
>> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
>> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
>> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
>> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
>>
>> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
>> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
>> ==============================================================================
>> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java (original)
>> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java Tue Mar 20 22:44:09 2012
>> @@ -69,5 +69,7 @@ public interface OpendapConfigMetKeys {
>> public static final String DATASET_MET_NAME_ATTR = "name";
>>
>> public static final String DATASET_MET_VALUE_ATTR = "value";
>> +
>> + public static final String RES_LOCATION_ATTR = "resLocation";
>>
>> }
>>
>> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
>> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
>> ==============================================================================
>> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java (original)
>> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java Tue Mar 20 22:44:09 2012
>> @@ -26,6 +26,8 @@ public interface OpendapProfileMetKeys {
>>
>> public static final String VARIABLES = "Variables";
>>
>> + public static final String COORDINATES = "Coordinates";
>> +
>> public static final String VARIABLES_LONG_NAMES = "Variable Long Names";
>>
>> public static final String CF_STANDARD_NAMES = "CF Standard Names";
>>
>> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
>> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
>> ==============================================================================
>> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java (original)
>> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java Tue Mar 20 22:44:09 2012
>> @@ -96,19 +96,39 @@ public class DasMetadataExtractor implem
>> ProfileUtils.addIfNotExisting(metadata, key, att.getValues());
>> }
>>
>> - // NetCDF variables
>> + // NetCDF coordinates
>> } else {
>> - // store variable name
>> - ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES, attName);
>> - // store "standard_name", "long_name"
>> - while (e.hasMoreElements()) {
>> - String key = (String) e.nextElement();
>> - Attribute att = at.getAttribute(key);
>> - if (key.equalsIgnoreCase(STANDARD_NAME)) {
>> - ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES, att.getValueAt(0));
>> - } else if (key.equalsIgnoreCase(LONG_NAME)) {
>> - ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES, att.getValueAt(0));
>> - }
>> +
>> + if ( attName.equalsIgnoreCase("lat") || attName.equalsIgnoreCase("latitude")
>> + || attName.equalsIgnoreCase("lon") || attName.equalsIgnoreCase("longitude")
>> + || attName.equalsIgnoreCase("time")
>> + || attName.equalsIgnoreCase("alt") || attName.equalsIgnoreCase("altitude")
>> + || attName.equalsIgnoreCase("depth")
>> + ) {
>> +
>> + // store coordinate name
>> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.COORDINATES, attName);
>> +
>> + } else if (attName.startsWith("time_")) {
>> +
>> + // ignore for now - it's not a coordinate neither a variable you would want to search on
>> +
>> + // NetCDF variables
>> + } else {
>> +
>> + // store variable name
>> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES, attName);
>> + // store "standard_name", "long_name"
>> + while (e.hasMoreElements()) {
>> + String key = (String) e.nextElement();
>> + Attribute att = at.getAttribute(key);
>> + if (key.equalsIgnoreCase(STANDARD_NAME)) {
>> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES, att.getValueAt(0));
>> + } else if (key.equalsIgnoreCase(LONG_NAME)) {
>> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES, att.getValueAt(0));
>> + }
>> + }
>> +
>> }
>> }
>>
>>
>> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
>> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
>> ==============================================================================
>> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java (original)
>> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java Tue Mar 20 22:44:09 2012
>> @@ -273,6 +273,8 @@ public class ThreddsMetadataExtractor im
>>
>> // add opendap access URL
>> if (type.equalsIgnoreCase(ServiceType.OPENDAP.toString())) {
>> + // store opendap URL
>> + ProfileUtils.addIfNotNull(met,"OpendapUrl",url+".dods");
>> // note: special processing of opendap endpoints since URL in thredds catalog is unusable without a suffix
>> ProfileUtils.addIfNotNull(met,"Access", this.encodeAccessTuple(url+".html", ProfileUtils.MIME_TYPE_OPENDAP_HTML, type));
>> }
>>
>> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
>> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java?rev=1303182&r1=1303181&r2=1303182&view=diff
>> ==============================================================================
>> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java (original)
>> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java Tue Mar 20 22:44:09 2012
>> @@ -82,6 +82,11 @@ public class ProfileChecker {
>> selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(), ProfileUtils.MIME_TYPE_HTML),
>> true, sb);
>>
>> + // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&Width=1280&BrowseImage=Aqua/MODIS MHchla|application/gis|NetViewer</resLocation>
>> + ok = ok && checkResourceAttribute("Location of type "+ProfileUtils.MIME_TYPE_GIS,
>> + selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(), ProfileUtils.MIME_TYPE_GIS),
>> + true, sb);
>> +
>> // <elemName>mission_name</elemName>
>> // <elemName>sensor</elemName>
>> // <elemName>...</elemName>
>>
>> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
>> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java?rev=1303182&r1=1303181&r2=1303182&view=diff
>> ==============================================================================
>> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java (original)
>> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java Tue Mar 20 22:44:09 2012
>> @@ -74,8 +74,9 @@ public class ProfileUtils {
>> // The delimiter must be a character that is not commonly used in the metadata values,
>> // and that it does not a special regular expression character.
>> // Cannot use '#' as it is used in URL anchors, such as THREDDS urls.
>> + // Cannot user '?', '&' as they are used in URL query strings.
>> // Cannot use '|' as it is used as multi-part separators in encoding of metadata fields.
>> - PathUtils.DELIMITER = "&";
>> + PathUtils.DELIMITER = "~";
>> }
>>
>> // character separating multiple parts of the same metadata field,
>> @@ -97,6 +98,7 @@ public class ProfileUtils {
>> public final static String MIME_TYPE_OPENDAP_DDS = "application/opendap-dds";
>> public final static String MIME_TYPE_OPENDAP_HTML = "application/opendap-html";
>> public final static String MIME_TYPE_RSS = "application/rss+xml";
>> + public final static String MIME_TYPE_GIS = "application/gis";
>>
>>
>> private static final Logger LOG = Logger.getLogger(ProfileUtils.class
>> @@ -108,10 +110,19 @@ public class ProfileUtils {
>> for (ConstantSpec spec : conf.getConstSpecs()) {
>> if (spec.getType().equals(RES_ATTR_SPEC_TYPE)) {
>> try {
>> - String value = PathUtils.replaceEnvVariables(spec.getValue(), datasetMet, true);
>> - if (StringUtils.hasText(value)) {
>> - setResourceAttributes(resAttr, spec.getName(), value);
>> +
>> + // first process expanded '[@...]' instructions
>> + List<String> values = multipleEnvVariablesReplacement(spec.getValue(), datasetMet);
>> +
>> + // then process standard '[...]' instructions
>> + for (String value : values) {
>> + String _value = PathUtils.replaceEnvVariables(value, datasetMet, true);
>> + if (StringUtils.hasText(_value)) {
>> + setResourceAttributes(resAttr, spec.getName(), _value);
>> + }
>> +
>> }
>> +
>> } catch (Exception e) {
>> e.printStackTrace();
>> LOG.log(Level.WARNING, "Error setting field: [" + spec.getName()
>> @@ -123,6 +134,44 @@ public class ProfileUtils {
>>
>> return resAttr;
>> }
>> +
>> + /**
>> + * Utility method to process environment replacement instructions of the form '[@key]'
>> + * resulting in as many output values as there are values for the environment variable 'key'.
>> + * Note that currently only one such pattern '[@key']' can be processed.
>> + *
>> + * @param value
>> + * @param metadata
>> + * @return
>> + */
>> + private static List<String> multipleEnvVariablesReplacement(String value, Metadata metadata) {
>> +
>> + List<String> newValues = new ArrayList<String>();
>> +
>> + // regexp matching found > replace values
>> + int start = value.indexOf("[@");
>> + if (start>=0) {
>> +
>> + int end = value.indexOf("]",start+2);
>> + // remove '[@',']' to obtain environment variable key
>> + String envKey = value.substring(start+2,end);
>> + List<String> envValues = metadata.getAllMetadata(envKey);
>> + if (envValues!=null) {
>> + for (String envValue : envValues) {
>> + // create new metadata value for this environment replacement
>> + String newValue = value.replaceAll("\\[@"+envKey+"\\]", envValue);
>> + newValues.add(newValue);
>> + }
>> + }
>> +
>> + // regexp matching not found > return original value
>> + } else {
>> + newValues.add(value);
>> + }
>> +
>> + return newValues;
>> +
>> + }
>>
>> public static ProfileAttributes getProfileAttributes(OpendapConfig conf, Metadata datasetMet) {
>> ProfileAttributes profAttr = new ProfileAttributes();
>>
>>
>
>
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> Chris Mattmann, Ph.D.
> Senior Computer Scientist
> NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA
> Office: 171-266B, Mailstop: 171-246
> Email: chris.a.mattmann@nasa.gov
> WWW: http://sunset.usc.edu/~mattmann/
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> Adjunct Assistant Professor, Computer Science Department
> University of Southern California, Los Angeles, CA 90089 USA
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>
Re: svn commit: r1303182 - in
/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps: config/
extractors/ util/
Posted by "Mattmann, Chris A (388J)" <ch...@jpl.nasa.gov>.
Hey Luca,
One thing I caught here was that there was a commented out CMDS url -- I think we should
remove that? If it's commented, looks extraneous...
Keep up the good work!
Cheers,
Chris
On Mar 20, 2012, at 11:44 PM, <lu...@apache.org> wrote:
> Author: luca
> Date: Tue Mar 20 22:44:09 2012
> New Revision: 1303182
>
> URL: http://svn.apache.org/viewvc?rev=1303182&view=rev
> Log:
> Implementing expanded syntax for environment variable replacement (OODT-405)
>
> Modified:
> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
> oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java (original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java Tue Mar 20 22:44:09 2012
> @@ -69,5 +69,7 @@ public interface OpendapConfigMetKeys {
> public static final String DATASET_MET_NAME_ATTR = "name";
>
> public static final String DATASET_MET_VALUE_ATTR = "value";
> +
> + public static final String RES_LOCATION_ATTR = "resLocation";
>
> }
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java (original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java Tue Mar 20 22:44:09 2012
> @@ -26,6 +26,8 @@ public interface OpendapProfileMetKeys {
>
> public static final String VARIABLES = "Variables";
>
> + public static final String COORDINATES = "Coordinates";
> +
> public static final String VARIABLES_LONG_NAMES = "Variable Long Names";
>
> public static final String CF_STANDARD_NAMES = "CF Standard Names";
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java (original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java Tue Mar 20 22:44:09 2012
> @@ -96,19 +96,39 @@ public class DasMetadataExtractor implem
> ProfileUtils.addIfNotExisting(metadata, key, att.getValues());
> }
>
> - // NetCDF variables
> + // NetCDF coordinates
> } else {
> - // store variable name
> - ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES, attName);
> - // store "standard_name", "long_name"
> - while (e.hasMoreElements()) {
> - String key = (String) e.nextElement();
> - Attribute att = at.getAttribute(key);
> - if (key.equalsIgnoreCase(STANDARD_NAME)) {
> - ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES, att.getValueAt(0));
> - } else if (key.equalsIgnoreCase(LONG_NAME)) {
> - ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES, att.getValueAt(0));
> - }
> +
> + if ( attName.equalsIgnoreCase("lat") || attName.equalsIgnoreCase("latitude")
> + || attName.equalsIgnoreCase("lon") || attName.equalsIgnoreCase("longitude")
> + || attName.equalsIgnoreCase("time")
> + || attName.equalsIgnoreCase("alt") || attName.equalsIgnoreCase("altitude")
> + || attName.equalsIgnoreCase("depth")
> + ) {
> +
> + // store coordinate name
> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.COORDINATES, attName);
> +
> + } else if (attName.startsWith("time_")) {
> +
> + // ignore for now - it's not a coordinate neither a variable you would want to search on
> +
> + // NetCDF variables
> + } else {
> +
> + // store variable name
> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES, attName);
> + // store "standard_name", "long_name"
> + while (e.hasMoreElements()) {
> + String key = (String) e.nextElement();
> + Attribute att = at.getAttribute(key);
> + if (key.equalsIgnoreCase(STANDARD_NAME)) {
> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES, att.getValueAt(0));
> + } else if (key.equalsIgnoreCase(LONG_NAME)) {
> + ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES, att.getValueAt(0));
> + }
> + }
> +
> }
> }
>
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java (original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java Tue Mar 20 22:44:09 2012
> @@ -273,6 +273,8 @@ public class ThreddsMetadataExtractor im
>
> // add opendap access URL
> if (type.equalsIgnoreCase(ServiceType.OPENDAP.toString())) {
> + // store opendap URL
> + ProfileUtils.addIfNotNull(met,"OpendapUrl",url+".dods");
> // note: special processing of opendap endpoints since URL in thredds catalog is unusable without a suffix
> ProfileUtils.addIfNotNull(met,"Access", this.encodeAccessTuple(url+".html", ProfileUtils.MIME_TYPE_OPENDAP_HTML, type));
> }
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java (original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java Tue Mar 20 22:44:09 2012
> @@ -82,6 +82,11 @@ public class ProfileChecker {
> selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(), ProfileUtils.MIME_TYPE_HTML),
> true, sb);
>
> + // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&Width=1280&BrowseImage=Aqua/MODIS MHchla|application/gis|NetViewer</resLocation>
> + ok = ok && checkResourceAttribute("Location of type "+ProfileUtils.MIME_TYPE_GIS,
> + selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(), ProfileUtils.MIME_TYPE_GIS),
> + true, sb);
> +
> // <elemName>mission_name</elemName>
> // <elemName>sensor</elemName>
> // <elemName>...</elemName>
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java (original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java Tue Mar 20 22:44:09 2012
> @@ -74,8 +74,9 @@ public class ProfileUtils {
> // The delimiter must be a character that is not commonly used in the metadata values,
> // and that it does not a special regular expression character.
> // Cannot use '#' as it is used in URL anchors, such as THREDDS urls.
> + // Cannot user '?', '&' as they are used in URL query strings.
> // Cannot use '|' as it is used as multi-part separators in encoding of metadata fields.
> - PathUtils.DELIMITER = "&";
> + PathUtils.DELIMITER = "~";
> }
>
> // character separating multiple parts of the same metadata field,
> @@ -97,6 +98,7 @@ public class ProfileUtils {
> public final static String MIME_TYPE_OPENDAP_DDS = "application/opendap-dds";
> public final static String MIME_TYPE_OPENDAP_HTML = "application/opendap-html";
> public final static String MIME_TYPE_RSS = "application/rss+xml";
> + public final static String MIME_TYPE_GIS = "application/gis";
>
>
> private static final Logger LOG = Logger.getLogger(ProfileUtils.class
> @@ -108,10 +110,19 @@ public class ProfileUtils {
> for (ConstantSpec spec : conf.getConstSpecs()) {
> if (spec.getType().equals(RES_ATTR_SPEC_TYPE)) {
> try {
> - String value = PathUtils.replaceEnvVariables(spec.getValue(), datasetMet, true);
> - if (StringUtils.hasText(value)) {
> - setResourceAttributes(resAttr, spec.getName(), value);
> +
> + // first process expanded '[@...]' instructions
> + List<String> values = multipleEnvVariablesReplacement(spec.getValue(), datasetMet);
> +
> + // then process standard '[...]' instructions
> + for (String value : values) {
> + String _value = PathUtils.replaceEnvVariables(value, datasetMet, true);
> + if (StringUtils.hasText(_value)) {
> + setResourceAttributes(resAttr, spec.getName(), _value);
> + }
> +
> }
> +
> } catch (Exception e) {
> e.printStackTrace();
> LOG.log(Level.WARNING, "Error setting field: [" + spec.getName()
> @@ -123,6 +134,44 @@ public class ProfileUtils {
>
> return resAttr;
> }
> +
> + /**
> + * Utility method to process environment replacement instructions of the form '[@key]'
> + * resulting in as many output values as there are values for the environment variable 'key'.
> + * Note that currently only one such pattern '[@key']' can be processed.
> + *
> + * @param value
> + * @param metadata
> + * @return
> + */
> + private static List<String> multipleEnvVariablesReplacement(String value, Metadata metadata) {
> +
> + List<String> newValues = new ArrayList<String>();
> +
> + // regexp matching found > replace values
> + int start = value.indexOf("[@");
> + if (start>=0) {
> +
> + int end = value.indexOf("]",start+2);
> + // remove '[@',']' to obtain environment variable key
> + String envKey = value.substring(start+2,end);
> + List<String> envValues = metadata.getAllMetadata(envKey);
> + if (envValues!=null) {
> + for (String envValue : envValues) {
> + // create new metadata value for this environment replacement
> + String newValue = value.replaceAll("\\[@"+envKey+"\\]", envValue);
> + newValues.add(newValue);
> + }
> + }
> +
> + // regexp matching not found > return original value
> + } else {
> + newValues.add(value);
> + }
> +
> + return newValues;
> +
> + }
>
> public static ProfileAttributes getProfileAttributes(OpendapConfig conf, Metadata datasetMet) {
> ProfileAttributes profAttr = new ProfileAttributes();
>
>
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Chris Mattmann, Ph.D.
Senior Computer Scientist
NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA
Office: 171-266B, Mailstop: 171-246
Email: chris.a.mattmann@nasa.gov
WWW: http://sunset.usc.edu/~mattmann/
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Adjunct Assistant Professor, Computer Science Department
University of Southern California, Los Angeles, CA 90089 USA
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++