You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2011/03/25 00:46:42 UTC
svn commit: r1085203 - in /oodt/trunk: ./
pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/
pushpull/src/main/resources/policy/
Author: bfoster
Date: Thu Mar 24 23:46:41 2011
New Revision: 1085203
URL: http://svn.apache.org/viewvc?rev=1085203&view=rev
Log:
- added ability for puspull to dynamically generate ProductName for a given mime-type
----------------------
OODT-166
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFileMetKeys.java
oodt/trunk/pushpull/src/main/resources/policy/mimetypes.xml
Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1085203&r1=1085202&r2=1085203&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Thu Mar 24 23:46:41 2011
@@ -4,6 +4,8 @@ Apache OODT Change Log
Release 0.3-SNAPSHOT (in progress)
--------------------------------------------
+* OODT-166 Ability for puspull to dynamically generate ProductName for a given mime-type (bfoster)
+
* OODT-167 ProcessedPageInfo isLastPage fails for case PageNum = 1 and totalPages = 0 (bfoster)
* OODT-164 AcqusitionDate Versioner (mattmann)
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java?rev=1085203&r1=1085202&r2=1085203&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java Thu Mar 24 23:46:41 2011
@@ -408,16 +408,28 @@ public class FileRetrievalSystem {
+ "' because it is already on the download queue");
return false;
}
- String productType = this.mimeTypeDetection.getMimeType(file.getName());
- String superType = null;
- if (productType != null
- && !productType.equals("application/octet-stream")) {
- superType = this.mimeTypeDetection
- .getSuperTypeForMimeType(productType);
+
+ RemoteFile remoteFile = new RemoteFile(file);
+ remoteFile.addMetadata(RemoteFile.RENAMING_STRING, renamingString);
+ remoteFile.addMetadata(RemoteFile.DELETE_AFTER_DOWNLOAD,
+ deleteAfterDownload + "");
+
+ String mimeType = this.mimeTypeDetection.getMimeType(file.getName());
+ if (mimeType != null
+ && !mimeType.equals("application/octet-stream")) {
+ remoteFile.addMetadata(RemoteFile.MIME_TYPE, mimeType);
+ remoteFile.addMetadata(RemoteFile.SUPER_TYPE, this.mimeTypeDetection
+ .getSuperTypeForMimeType(mimeType));
String description = this.mimeTypeDetection
- .getDescriptionForMimeType(productType);
- if (description != null)
- productType = description;
+ .getDescriptionForMimeType(mimeType);
+ if (description != null) {
+ for (String field : description.split("\\&\\&")) {
+ String[] keyval = field.split("\\=");
+ remoteFile.addMetadata(keyval[0].trim(), keyval[1].trim());
+ }
+ if (remoteFile.getMetadata(RemoteFile.UNIQUE_ELEMENT) != null)
+ uniqueMetadataElement = remoteFile.getMetadata(RemoteFile.UNIQUE_ELEMENT);
+ }
} else if (config.onlyDownloadDefinedTypes()) {
throw new UndefinedTypeException("File '" + file
+ "' is not a defined type");
@@ -428,22 +440,15 @@ public class FileRetrievalSystem {
+ downloadToDir.getPath());
if (!this.isStagingAreaInitialized(downloadToDir))
this.initializeStagingArea(downloadToDir);
-
- RemoteFile remoteFile = new RemoteFile(file);
- remoteFile
- .addMetadata(
- RemoteFile.PRODUCT_NAME,
- remoteFile
- .getMetadata(uniqueMetadataElement == null ? RemoteFile.FILENAME
- : uniqueMetadataElement));
- remoteFile.addMetadata(RemoteFile.RENAMING_STRING, renamingString);
- remoteFile.addMetadata(RemoteFile.DOWNLOAD_TO_DIR, downloadToDir
- .getAbsolutePath());
- remoteFile.addMetadata(RemoteFile.PRODUCT_TYPE, productType);
- remoteFile.addMetadata(RemoteFile.SUPER_TYPE, superType);
- remoteFile.addMetadata(RemoteFile.DELETE_AFTER_DOWNLOAD,
- deleteAfterDownload + "");
-
+
+ remoteFile.addMetadata(RemoteFile.DOWNLOAD_TO_DIR, downloadToDir.getAbsolutePath());
+
+ if (remoteFile.getMetadata(RemoteFile.PRODUCT_NAME_GENERATOR) != null) {
+ remoteFile.addMetadata(RemoteFile.PRODUCT_NAME, RenamingConvention.rename(remoteFile.getProtocolFile(), remoteFile.getMetadata(RemoteFile.PRODUCT_NAME_GENERATOR)));
+ }else {
+ remoteFile.setUniqueMetadataElement(uniqueMetadataElement == null ? RemoteFile.FILENAME : uniqueMetadataElement);
+ }
+
if (!isAlreadyInDatabase(remoteFile)) {
// get download location
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFileMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFileMetKeys.java?rev=1085203&r1=1085202&r2=1085203&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFileMetKeys.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RemoteFileMetKeys.java Thu Mar 24 23:46:41 2011
@@ -44,8 +44,14 @@ public interface RemoteFileMetKeys {
public static final String PRODUCT_TYPE = "ProductType";
+ public static final String MIME_TYPE = "MimeType";
+
public static final String SUPER_TYPE = "SuperType";
public static final String DELETE_AFTER_DOWNLOAD = "DeleteAfterDownload";
+
+ public static final String PRODUCT_NAME_GENERATOR = "ProductNameGenerator";
+
+ public static final String UNIQUE_ELEMENT = "UniqueElement";
}
Modified: oodt/trunk/pushpull/src/main/resources/policy/mimetypes.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/resources/policy/mimetypes.xml?rev=1085203&r1=1085202&r2=1085203&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/resources/policy/mimetypes.xml (original)
+++ oodt/trunk/pushpull/src/main/resources/policy/mimetypes.xml Thu Mar 24 23:46:41 2011
@@ -18,6 +18,7 @@
<mime-info>
<mime-type type="product/pgp_signature">
+ <_comment><![CDATA[ProductNameGenerator=[GREP_RM('\.sig$','[FILENAME]')]]]></_comment>
<glob pattern="*.sig"/>
</mime-type>
@@ -32,25 +33,25 @@
<mime-type type="product/iasi_xxx_1c">
<sub-class-of type="metop_a/noaa"/>
- <_comment>MOA_IASI_L1C</_comment>
+ <_comment>ProductType=MOA_IASI_L1C</_comment>
<glob pattern="(?:L\d{1,}\.){0,1}IASI_xxx_1C_\w{3}_\w{15}_\w{15}_\w_\w_\w{15}__\w{14}" isregex="true"/>
</mime-type>
<mime-type type="product/mhsx">
<sub-class-of type="metop_a/noaa"/>
- <_comment>MOA_MHS_L1B</_comment>
+ <_comment>ProductType=MOA_MHS_L1B</_comment>
<glob pattern="(?:L\d{1,}\.){0,1}NSS\.MHSX\.\w{2}\.D\d{5}\.S\d{4}\.E\d{4}\.B\d{7}\.\w{2}" isregex="true"/>
</mime-type>
<mime-type type="product/amsu_a">
<sub-class-of type="metop_a/noaa"/>
- <_comment>MOA_AMSUA_L1B</_comment>
+ <_comment>ProductType=MOA_AMSUA_L1B</_comment>
<glob pattern="(?:L\d{1,}\.){0,1}NSS\.AMAX\.\w{2}\.D\d{5}\.S\d{4}\.E\d{4}\.B\d{7}\.\w{2}" isregex="true"/>
</mime-type>
<mime-type type="product/rtg_sst">
<sub-class-of type="ncep/model"/>
- <_comment>RTG_SST</_comment>
+ <_comment><![CDATA[ProductType=RTG_SST && UniqueElement=RetrievedFromLoc]]></_comment>
<glob pattern="rtg_sst_grb_0\.5\.\d{8}" isregex="true"/>
<glob pattern="sst\.\d{8}_rtgssthr_grb_0\.5\.grib2" isregex="true"/>
</mime-type>