You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2012/08/01 23:25:55 UTC
svn commit: r1368254 - in
/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull:
config/RemoteSpecs.java retrievalsystem/FileRetrievalSystem.java
Author: bfoster
Date: Wed Aug 1 21:25:54 2012
New Revision: 1368254
URL: http://svn.apache.org/viewvc?rev=1368254&view=rev
Log:
- Fixed: CAS-Pushpull uniqueMetadataElement is being loaded as empty String instead of null
-------------
OODT-481
Modified:
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java?rev=1368254&r1=1368253&r2=1368254&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/config/RemoteSpecs.java Wed Aug 1 21:25:54 2012
@@ -14,8 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.pushpull.config;
//OODT imports
@@ -33,18 +31,19 @@ import java.io.FileInputStream;
import java.net.URL;
import java.util.HashMap;
import java.util.LinkedList;
+
+//DOM imports
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
+//Google imports
+import com.google.common.base.Strings;
+
/**
- *
- * @author bfoster
- * @version $Revision$
- *
- * <p>
- * Describe your class here
- * </p>.
+ * Remote Site Crawling specifications.
+ *
+ * @author bfoster (Brian Foster)
*/
public class RemoteSpecs implements ConfigParserMetKeys {
@@ -218,10 +217,10 @@ public class RemoteSpecs implements Conf
if (afterUseList.getLength() > 0) {
Element afterUse = (Element) afterUseList.item(0);
File onSuccessDir = new File(PathUtils
- .replaceEnvVariables(((Element) afterUse)
+ .replaceEnvVariables(afterUse
.getAttribute(MOVEON_TO_SUCCESS_ATTR)));
File onFailDir = new File(PathUtils
- .replaceEnvVariables(((Element) afterUse)
+ .replaceEnvVariables(afterUse
.getAttribute(MOVEON_TO_FAIL_ATTR)));
pfi.setAfterUseEffects(onSuccessDir, onFailDir);
}
@@ -238,11 +237,18 @@ public class RemoteSpecs implements Conf
DataFilesInfo dfi = null;
if (dataInfoList.getLength() > 0) {
Node dataInfo = dataInfoList.item(0);
- String queryElement = PathUtils
- .replaceEnvVariables(((Element) dataInfo)
- .getAttribute(QUERY_ELEM_ATTR));
+ String queryElement = ((Element) dataInfo)
+ .getAttribute(QUERY_ELEM_ATTR);
+ if (Strings.isNullOrEmpty(queryElement)) {
+ queryElement = null;
+ } else {
+ queryElement = PathUtils.replaceEnvVariables(queryElement);
+ }
String renamingConv = ((Element) dataInfo)
.getAttribute(RENAMING_CONV_ATTR);
+ if (Strings.isNullOrEmpty(renamingConv)) {
+ renamingConv = null;
+ }
boolean allowAliasOverride = PathUtils.replaceEnvVariables(
((Element) dataInfo)
.getAttribute(ALLOW_ALIAS_OVERRIDE_ATTR))
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java?rev=1368254&r1=1368253&r2=1368254&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java (original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java Wed Aug 1 21:25:54 2012
@@ -55,6 +55,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
/**
* <pre>
@@ -113,7 +114,7 @@ import com.google.common.base.Preconditi
* and every encountered file will be downloaded.
* </pre>
*
- * @author bfoster
+ * @author bfoster (Brian Foster)
*/
public class FileRetrievalSystem {
@@ -419,31 +420,33 @@ public class FileRetrievalSystem {
remoteFile.addMetadata(RemoteFile.DELETE_AFTER_DOWNLOAD,
deleteAfterDownload + "");
- String mimeType = this.mimeTypeDetection.getMimeType(file.getName());
- if (mimeType != null
- && !mimeType.equals("application/octet-stream")) {
- remoteFile.addMetadata(RemoteFile.MIME_TYPE, mimeType);
- remoteFile.addMetadata(RemoteFile.SUPER_TYPE, this.mimeTypeDetection
- .getSuperTypeForMimeType(mimeType));
- String description = this.mimeTypeDetection
- .getDescriptionForMimeType(mimeType);
- if (description != null) {
- if(description.indexOf("&") != -1){
- for (String field : description.split("\\&\\&")) {
- String[] keyval = field.split("\\=");
- remoteFile.addMetadata(keyval[0].trim(), keyval[1].trim());
- }
- }
- else{
- // it's the ProductType
- remoteFile.addMetadata(RemoteFile.PRODUCT_TYPE, description);
- }
- if (remoteFile.getMetadata(RemoteFile.UNIQUE_ELEMENT) != null)
- uniqueMetadataElement = remoteFile.getMetadata(RemoteFile.UNIQUE_ELEMENT);
- }
- } else if (config.onlyDownloadDefinedTypes()) {
- throw new UndefinedTypeException("File '" + file
+ if (config.onlyDownloadDefinedTypes()) {
+ String mimeType = this.mimeTypeDetection.getMimeType(file.getName());
+ if (mimeType != null
+ && !mimeType.equals("application/octet-stream")) {
+ remoteFile.addMetadata(RemoteFile.MIME_TYPE, mimeType);
+ remoteFile.addMetadata(RemoteFile.SUPER_TYPE, this.mimeTypeDetection
+ .getSuperTypeForMimeType(mimeType));
+ String description = this.mimeTypeDetection
+ .getDescriptionForMimeType(mimeType);
+ if (!Strings.isNullOrEmpty(description)) {
+ if(description.indexOf("&") != -1){
+ for (String field : description.split("\\&\\&")) {
+ String[] keyval = field.split("\\=");
+ remoteFile.addMetadata(keyval[0].trim(), keyval[1].trim());
+ }
+ } else{
+ // it's the ProductType
+ remoteFile.addMetadata(RemoteFile.PRODUCT_TYPE, description);
+ }
+ if (remoteFile.getMetadata(RemoteFile.UNIQUE_ELEMENT) != null) {
+ uniqueMetadataElement = remoteFile.getMetadata(RemoteFile.UNIQUE_ELEMENT);
+ }
+ }
+ } else {
+ throw new UndefinedTypeException("File '" + file
+ "' is not a defined type");
+ }
}
downloadToDir = new File(downloadToDir.isAbsolute() ? downloadToDir