You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2018/02/27 05:22:48 UTC
[2/4] oodt git commit: This fixes [OODT-970] and [OODT-969] - Tests
for Avro File Manager and solve the NPE problem in NettyTransceiver.
This fixes [OODT-970] and [OODT-969] - Tests for Avro File Manager and solve the NPE problem in NettyTransceiver.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/36f320e3
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/36f320e3
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/36f320e3
Branch: refs/heads/development
Commit: 36f320e35155ff38eebdde60fb90a07074ad4148
Parents: 75d292c
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed Jan 10 16:29:41 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Wed Jan 10 16:29:41 2018 +0530
----------------------------------------------------------------------
core/pom.xml | 10 +
filemgr/pom.xml | 4 -
filemgr/src/main/assembly/assembly.xml | 6 +-
.../filemgr/system/AvroFileManagerClient.java | 335 ++-----------------
.../filemgr/system/AvroFileManagerServer.java | 2 +-
.../filemgr/util/RpcCommunicationFactory.java | 2 +
filemgr/src/main/resources/filemgr.properties | 8 +-
.../system/AbstractFileManagerServerTest.java | 172 ++++++++++
.../system/TestAvroFileManagerClient.java | 220 ++++++++++++
.../system/TestAvroFileManagerServer.java | 155 +++++++++
.../cas/filemgr/system/TestRpcFileManager.java | 1 +
.../system/TestRpcFileManagerClient.java | 1 +
.../filemgr/system/TestXmlRpcFileManager.java | 303 -----------------
.../system/TestXmlRpcFileManagerClient.java | 268 ++++-----------
.../system/TestXmlRpcFileManagerServer.java | 149 +++++++++
.../TestDistributedXmlRpcFileManager.java | 4 +-
16 files changed, 805 insertions(+), 835 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index a21c37c..a01df20 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -75,6 +75,16 @@ the License.
<version>1.7.4</version>
</dependency>
<dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-ipc</artifactId>
+ <version>1.8.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.8.2</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>19.0</version>
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index 9d3c424..9718319 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -54,13 +54,10 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
- <version>1.8.1</version>
</dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.avro/avro-ipc -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-ipc</artifactId>
- <version>1.8.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -291,7 +288,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <!--forkMode>pertest</forkMode-->
<forkMode>pertest</forkMode>
<!-- so we can DEBUG -->
<argLine>-enableassertions</argLine>
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/filemgr/src/main/assembly/assembly.xml b/filemgr/src/main/assembly/assembly.xml
index 07a8a9c..2e90515 100644
--- a/filemgr/src/main/assembly/assembly.xml
+++ b/filemgr/src/main/assembly/assembly.xml
@@ -45,9 +45,9 @@ $Id$
<fileSet>
<directory>${basedir}/src/main/resources</directory>
<outputDirectory>logs</outputDirectory>
- <includes>
- <include>REMOVE.log</include>
- </includes>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/resources</directory>
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
index 043330a..ca3839d 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
@@ -17,20 +17,11 @@
package org.apache.oodt.cas.filemgr.system;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import org.apache.avro.AvroRemoteException;
import org.apache.avro.ipc.NettyTransceiver;
import org.apache.avro.ipc.Transceiver;
import org.apache.avro.ipc.specific.SpecificRequestor;
import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer;
-import org.apache.oodt.cas.filemgr.exceptions.FileManagerException;
import org.apache.oodt.cas.filemgr.structs.Element;
import org.apache.oodt.cas.filemgr.structs.FileTransferStatus;
import org.apache.oodt.cas.filemgr.structs.Product;
@@ -58,6 +49,15 @@ import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.filemgr.versioning.Versioner;
import org.apache.oodt.cas.metadata.Metadata;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
/**
* @author radu
*
@@ -68,11 +68,11 @@ public class AvroFileManagerClient implements FileManagerClient {
private static Logger LOG = Logger.getLogger(AvroFileManagerClient.class
.getName());
- /* Avro-Rpc client */
- Transceiver client;
+ /** Avro-Rpc client */
+ private Transceiver client;
- /* proxy for the server */
- AvroFileManager proxy;
+ /** proxy for the server */
+ private AvroFileManager proxy;
/* URL where the fileManager is */
private URL fileManagerUrl;
@@ -89,9 +89,8 @@ public class AvroFileManagerClient implements FileManagerClient {
try {
this.fileManagerUrl = url;
InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(),this.fileManagerUrl.getPort());
- this.client = new NettyTransceiver(inetSocketAddress);
+ this.client = new NettyTransceiver(inetSocketAddress, 40000L);
proxy = (AvroFileManager) SpecificRequestor.getClient(AvroFileManager.class, client);
-
} catch (IOException e) {
e.printStackTrace();
LOG.log(Level.WARNING, "IOException when connecting to filemgr: ["
@@ -99,8 +98,7 @@ public class AvroFileManagerClient implements FileManagerClient {
}
if (testConnection && !isAlive()) {
- throw new ConnectionException("Exception connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ throw new ConnectionException("Exception connecting to filemgr: [" + this.fileManagerUrl + "]");
}
}
@@ -124,8 +122,9 @@ public class AvroFileManagerClient implements FileManagerClient {
boolean success;
try {
- if(proxy != null)
+ if (proxy != null) {
success = proxy.isAlive();
+ }
else return false;
} catch (AvroRemoteException e) {
LOG.log(Level.WARNING, "AvroRemoteException when connecting to filemgr: ["
@@ -144,12 +143,6 @@ public class AvroFileManagerClient implements FileManagerClient {
success = proxy.transferringProduct(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
return success;
@@ -162,13 +155,7 @@ public class AvroFileManagerClient implements FileManagerClient {
success = proxy.removeProductTransferStatus(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
- }
+ }
return success;
}
@@ -179,12 +166,6 @@ public class AvroFileManagerClient implements FileManagerClient {
success = this.proxy.isTransferComplete(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
return success;
}
@@ -196,12 +177,6 @@ public class AvroFileManagerClient implements FileManagerClient {
success = this.proxy.moveProduct(AvroTypeFactory.getAvroProduct(product), newPath);
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
return success;
}
@@ -213,12 +188,6 @@ public class AvroFileManagerClient implements FileManagerClient {
success = this.proxy.modifyProduct(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
return success;
}
@@ -230,12 +199,6 @@ public class AvroFileManagerClient implements FileManagerClient {
success = this.proxy.removeProduct(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
return success;
}
@@ -246,12 +209,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getFileTransferStatus(this.proxy.getCurrentFileTransfer());
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
}
@@ -264,12 +221,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
return fileTransferStatuses;
}
@@ -280,12 +231,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.getProductPctTransferred(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
}
@@ -295,12 +240,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.getRefPctTransferred(AvroTypeFactory.getAvroReference(reference));
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
}
@@ -315,12 +254,6 @@ public class AvroFileManagerClient implements FileManagerClient {
));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -330,12 +263,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getProductPage(this.proxy.getFirstPage(AvroTypeFactory.getAvroProductType(type)));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -345,12 +272,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getProductPage(this.proxy.getLastPage(AvroTypeFactory.getAvroProductType(type)));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -363,12 +284,6 @@ public class AvroFileManagerClient implements FileManagerClient {
));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -381,12 +296,6 @@ public class AvroFileManagerClient implements FileManagerClient {
));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -396,12 +305,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.addProductType(AvroTypeFactory.getAvroProductType(type));
} catch (AvroRemoteException e) {
throw new RepositoryManagerException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new RepositoryManagerException(e.getMessage());
- }
}
}
@@ -411,12 +314,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.hasProduct(productName);
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -426,12 +323,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.getNumProducts(AvroTypeFactory.getAvroProductType(type));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -444,12 +335,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
return products;
}
@@ -463,12 +348,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
return products;
}
@@ -479,12 +358,6 @@ public class AvroFileManagerClient implements FileManagerClient {
this.proxy.setProductTransferStatus(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -494,12 +367,6 @@ public class AvroFileManagerClient implements FileManagerClient {
this.proxy.addProductReferences(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -510,12 +377,6 @@ public class AvroFileManagerClient implements FileManagerClient {
AvroTypeFactory.getAvroMetadata(metadata));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -529,12 +390,6 @@ public class AvroFileManagerClient implements FileManagerClient {
);
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -544,12 +399,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.catalogProduct(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -559,12 +408,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getMetadata(this.proxy.getMetadata(AvroTypeFactory.getAvroProduct(product)));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -575,12 +418,6 @@ public class AvroFileManagerClient implements FileManagerClient {
this.proxy.getReducedMetadata(AvroTypeFactory.getAvroProduct(product), (List<String>) elements));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -590,12 +427,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.removeFile(filePath);
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
}
@@ -605,12 +436,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return this.proxy.retrieveFile(filePath, offset, numBytes).array();
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
}
@@ -620,14 +445,7 @@ public class AvroFileManagerClient implements FileManagerClient {
this.proxy.transferFile(filePath, ByteBuffer.wrap(fileData), offset, numBytes);
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new DataTransferException(e.getMessage());
- }
}
-
}
@Override
@@ -641,14 +459,7 @@ public class AvroFileManagerClient implements FileManagerClient {
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
-
}
@Override
@@ -660,12 +471,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new ValidationLayerException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new ValidationLayerException(e.getMessage());
- }
}
return products;
}
@@ -676,12 +481,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getElement(this.proxy.getElementById(elementId));
} catch (AvroRemoteException e) {
throw new ValidationLayerException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new ValidationLayerException(e.getMessage());
- }
}
}
@@ -691,12 +490,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getElement(this.proxy.getElementByName(elementName));
} catch (AvroRemoteException e) {
throw new ValidationLayerException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new ValidationLayerException(e.getMessage());
- }
}
}
@@ -710,12 +503,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
return queryResults;
}
@@ -729,12 +516,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
return products;
}
@@ -745,12 +526,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getProductType(this.proxy.getProductTypeByName(productTypeName));
} catch (AvroRemoteException e) {
throw new RepositoryManagerException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new RepositoryManagerException(e.getMessage());
- }
}
}
@@ -760,12 +535,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getProductType(this.proxy.getProductTypeById(productTypeId));
} catch (AvroRemoteException e) {
throw new RepositoryManagerException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new RepositoryManagerException(e.getMessage());
- }
}
}
@@ -778,12 +547,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new RepositoryManagerException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new RepositoryManagerException(e.getMessage());
- }
}
return productTypes;
}
@@ -797,12 +560,6 @@ public class AvroFileManagerClient implements FileManagerClient {
}
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
return references;
}
@@ -813,12 +570,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getProduct(this.proxy.getProductById(productId));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -828,12 +579,6 @@ public class AvroFileManagerClient implements FileManagerClient {
return AvroTypeFactory.getProduct(this.proxy.getProductByName(productName));
} catch (AvroRemoteException e) {
throw new CatalogException(e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new CatalogException(e.getMessage());
- }
}
}
@@ -848,9 +593,7 @@ public class AvroFileManagerClient implements FileManagerClient {
clientTransfer);
if (clientTransfer) {
- LOG.log(Level.FINEST,
- "File Manager Client: clientTransfer enabled: "
- + "transfering product ["
+ LOG.log(Level.FINEST, "File Manager Client: clientTransfer enabled: transfering product ["
+ product.getProductName() + "]");
// we need to transfer the product ourselves
@@ -948,60 +691,28 @@ public class AvroFileManagerClient implements FileManagerClient {
}
throw new Exception("Failed to ingest product [" + product + "] : "
+ e.getMessage());
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new FileManagerException(e.getMessage());
- }
}
-
}
@Override
public Metadata getCatalogValues(Metadata metadata, ProductType productType) throws Exception {
- try {
- return AvroTypeFactory.getMetadata(this.proxy.getCatalogValues(
+ return AvroTypeFactory.getMetadata(this.proxy.getCatalogValues(
AvroTypeFactory.getAvroMetadata(metadata),
AvroTypeFactory.getAvroProductType(productType)));
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new FileManagerException(e.getMessage());
- }
- }
}
@Override
public Metadata getOrigValues(Metadata metadata, ProductType productType) throws Exception {
- try {
- return AvroTypeFactory.getMetadata(this.proxy.getOrigValues(
+ return AvroTypeFactory.getMetadata(this.proxy.getOrigValues(
AvroTypeFactory.getAvroMetadata(metadata),
AvroTypeFactory.getAvroProductType(productType)));
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new FileManagerException(e.getMessage());
- }
- }
}
@Override
public Query getCatalogQuery(Query query, ProductType productType) throws Exception {
- try {
- return AvroTypeFactory.getQuery(this.proxy.getCatalogQuery(
+ return AvroTypeFactory.getQuery(this.proxy.getCatalogQuery(
AvroTypeFactory.getAvroQuery(query),
- AvroTypeFactory.getAvroProductType(productType)
- ));
- } finally {
- try {
- this.client.close();
- } catch (IOException e) {
- throw new FileManagerException(e.getMessage());
- }
- }
+ AvroTypeFactory.getAvroProductType(productType)));
}
@Override
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java
index 386a1c4..91258d9 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java
@@ -70,7 +70,7 @@ public class AvroFileManagerServer implements AvroFileManager, FileManagerServer
private Server server;
/* file manager tools */
- FileManager fileManager;
+ private FileManager fileManager;
public AvroFileManagerServer(int port){
this.port = port;
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java
index 861d374..0ea46b2 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java
@@ -124,6 +124,8 @@ public class RpcCommunicationFactory {
* @throws IOException
*/
public static FileManagerServer createServer(int port) throws IOException {
+ setPror();
+
String serverFactory = System.getProperty("filemgr.server",
"org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory");
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/main/resources/filemgr.properties
----------------------------------------------------------------------
diff --git a/filemgr/src/main/resources/filemgr.properties b/filemgr/src/main/resources/filemgr.properties
index ff12d63..2297555 100644
--- a/filemgr/src/main/resources/filemgr.properties
+++ b/filemgr/src/main/resources/filemgr.properties
@@ -16,10 +16,10 @@
# Configuration properties for the File Manager
# rpc configuration, uncomment the avro implementations to use AvroRPC
-filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory
-filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory
-#filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory
-#filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory
+#filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory
+#filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory
+filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory
+filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory
# repository factory
filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java
new file mode 100644
index 0000000..486532b
--- /dev/null
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/AbstractFileManagerServerTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.filemgr.system;
+
+import junit.framework.TestCase;
+import org.apache.oodt.cas.filemgr.ingest.StdIngester;
+import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.SerializableMetadata;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.net.URL;
+import java.util.Properties;
+import java.util.logging.Logger;
+
+public abstract class AbstractFileManagerServerTest extends TestCase {
+
+ private static final Logger LOG = Logger.getLogger(AbstractFileManagerServerTest.class.getName());
+
+ protected static final int FM_PORT = 50001;
+ protected static final String transferServiceFacClass =
+ "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory";
+
+ protected FileManagerServer fileManagerServer;
+ private String luceneCatLoc;
+ private Properties initialProperties = new Properties(System.getProperties());
+
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ startFileManagerServer();
+ ingestTestFile();
+ }
+
+ private void startFileManagerServer() {
+ Properties properties = new Properties(System.getProperties());
+
+ // first make sure to load properties for the file manager
+ // and make sure to load logging properties as well
+
+ // set the log levels
+ URL loggingPropertiesUrl = this.getClass().getResource("/test.logging.properties");
+ properties.setProperty("java.util.logging.config.file", new File(loggingPropertiesUrl.getFile())
+ .getAbsolutePath());
+
+ // first load the example configuration
+ try {
+ URL filemgrPropertiesUrl = this.getClass().getResource("/filemgr.properties");
+ properties.load(new FileInputStream(new File(filemgrPropertiesUrl.getFile())));
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+
+ // override the catalog to use: we'll use lucene
+ try {
+ URL ingestUrl = this.getClass().getResource("/ingest");
+ luceneCatLoc = new File(ingestUrl.getFile()).getCanonicalPath() + "/cat";
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+
+ properties.setProperty("filemgr.catalog.factory", "org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory");
+ properties.setProperty("org.apache.oodt.cas.filemgr.catalog.lucene.idxPath", luceneCatLoc);
+
+ // now override the repo mgr policy
+ URL fmpolicyUrl = this.getClass().getResource("/ingest/fmpolicy");
+ try {
+ properties.setProperty("org.apache.oodt.cas.filemgr.repositorymgr.dirs",
+ "file://" + new File(fmpolicyUrl.getFile()).getCanonicalPath());
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+
+ // now override the val layer ones
+ properties.setProperty("org.apache.oodt.cas.filemgr.validation.dirs",
+ "file://" + new File(fmpolicyUrl.getFile()).getAbsolutePath());
+
+ // set up mime repo path
+ URL mimeTypesUrl = this.getClass().getResource("/mime-types.xml");
+ properties.setProperty("org.apache.oodt.cas.filemgr.mime.type.repository",
+ new File(mimeTypesUrl.getFile()).getAbsolutePath());
+
+ // override expand product met
+ properties.setProperty("org.apache.oodt.cas.filemgr.metadata.expandProduct",
+ Boolean.toString(shouldExpandProduct()));
+
+ System.setProperties(properties);
+
+ setProperties();
+
+ try {
+ fileManagerServer = newFileManagerServer(FM_PORT);
+ fileManagerServer.startUp();
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ fileManagerServer.shutdown();
+ fileManagerServer = null;
+
+ // blow away lucene cat
+ deleteAllFiles(luceneCatLoc);
+
+ // blow away test file
+ deleteAllFiles("/tmp/test.txt");
+
+ // Reset the System properties to initial values.
+ System.setProperties(initialProperties);
+ super.tearDown();
+ }
+
+ protected void deleteAllFiles(String startDir) {
+ File startDirFile = new File(startDir);
+ File[] delFiles = startDirFile.listFiles();
+
+ if (delFiles != null && delFiles.length > 0) {
+ for (File delFile : delFiles) {
+ delFile.delete();
+ }
+ }
+
+ startDirFile.delete();
+ }
+
+ @SuppressWarnings("Duplicates")
+ private void ingestTestFile() {
+ Metadata prodMet;
+ StdIngester ingester = new StdIngester(transferServiceFacClass);
+
+ try {
+ URL ingestUrl = this.getClass().getResource("/ingest");
+ URL refUrl = this.getClass().getResource("/ingest/test.txt");
+ URL metUrl = this.getClass().getResource("/ingest/test.txt.met");
+
+ prodMet = new SerializableMetadata(new FileInputStream(new File(metUrl.getFile())));
+
+ // now add the right file location
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath());
+ prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
+ ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(refUrl.getFile()), prodMet);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ protected abstract void setProperties();
+
+ protected abstract FileManagerServer newFileManagerServer(int port) throws Exception;
+
+ protected abstract boolean shouldExpandProduct();
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java
new file mode 100644
index 0000000..30796ff
--- /dev/null
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerClient.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.filemgr.system;
+
+import org.apache.oodt.cas.filemgr.ingest.StdIngester;
+import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery;
+import org.apache.oodt.cas.filemgr.structs.query.QueryResult;
+import org.apache.oodt.cas.filemgr.util.SqlParser;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.SerializableMetadata;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Vector;
+
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
+public class TestAvroFileManagerClient extends AbstractFileManagerServerTest {
+
+ /**
+ * @since OODT-161
+ */
+ public void testGetReducedMetadata() {
+ List<String> arrayListElems = new ArrayList<>();
+ List<String> vectorElemList = new Vector<>();
+ List<String> linkedListElemList = new LinkedList<>();
+
+ arrayListElems.add(CoreMetKeys.FILENAME);
+ vectorElemList.add(CoreMetKeys.FILENAME);
+ linkedListElemList.add(CoreMetKeys.FILENAME);
+
+ try {
+ FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:" + FM_PORT));
+
+ Metadata reducedMet;
+ List pTypes = fmc.getProductTypes();
+ assertNotNull(pTypes);
+ assertTrue(pTypes.size() > 0);
+ ProductType genericFileType = fmc.getProductTypeByName("GenericFile");
+ assertNotNull(genericFileType);
+ List products = fmc.getProductsByProductType(genericFileType);
+ assertNotNull(products);
+ assertTrue(products.size() > 0);
+ Product product = (Product) products.get(0);
+ assertNotNull(product);
+
+ reducedMet = fmc.getReducedMetadata(product, arrayListElems);
+ assertNotNull(reducedMet);
+ assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME));
+ assertEquals(reducedMet.getMap().keySet().size(), 1);
+
+ reducedMet = fmc.getReducedMetadata(product, vectorElemList);
+ assertNotNull(reducedMet);
+ assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME));
+ assertEquals(reducedMet.getMap().keySet().size(), 1);
+
+ reducedMet = fmc.getReducedMetadata(product, linkedListElemList);
+ assertNotNull(reducedMet);
+ assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME));
+ assertEquals(reducedMet.getMap().keySet().size(), 1);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ public void testIngest() throws Exception {
+ URL ingestUrl = this.getClass().getResource("/ingest");
+ URL refUrl = this.getClass().getResource("/ingest/test.txt");
+
+ Metadata prodMet = new Metadata();
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(
+ ingestUrl.getFile()).getCanonicalPath());
+ prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
+
+ StdIngester ingester = new StdIngester(transferServiceFacClass);
+ String productId = ingester.ingest(
+ new URL("http://localhost:" + FM_PORT),
+ new File(refUrl.getFile()), prodMet);
+ FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:" + FM_PORT));
+ Metadata m = fmc.getMetadata(fmc.getProductById(productId));
+ assertEquals(m.getMetadata("Filename"), "test.txt");
+ deleteAllFiles("/tmp/test-type");
+ }
+
+ /**
+ * todo This is an incomplete test. No verification done after the file is deleted
+ *
+ * @throws Exception
+ */
+ public void testRemoveFile() throws Exception {
+ Path tmpFilePath = Paths.get("/tmp", "test-delete.txt");
+ URL ingestUrl = this.getClass().getResource("/ingest");
+ URL refUrl = this.getClass().getResource("/ingest/test-delete.txt");
+ Files.copy(Paths.get(refUrl.toURI()), tmpFilePath, REPLACE_EXISTING);
+
+ try {
+ Metadata prodMet = new Metadata();
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath());
+ prodMet.addMetadata(CoreMetKeys.FILENAME, "test-delete.txt");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
+
+ StdIngester ingester = new StdIngester(transferServiceFacClass);
+ String productId = ingester.ingest(
+ new URL("http://localhost:" + FM_PORT),
+ new File(refUrl.getFile()), prodMet);
+ FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:" + FM_PORT));
+ Metadata m = fmc.getMetadata(fmc.getProductById(productId));
+ assertEquals(m.getMetadata("Filename"), "test-delete.txt");
+ String loc = m.getMetadata("FileLocation");
+ fmc.removeFile(loc + "/" + m.getMetadata("Filename"));
+ fmc.getProductById(productId);
+ } finally {
+ Files.copy(tmpFilePath, Paths.get(refUrl.toURI()));
+ }
+ deleteAllFiles("/tmp/test-type");
+ }
+
+ /**
+ * @since OODT-404
+ */
+ public void testMetadataPersistence() throws Exception {
+ URL ingestUrl = this.getClass().getResource("/ingest");
+ URL refUrl = this.getClass().getResource("/ingest/test-file-3.txt");
+ URL metUrl = this.getClass().getResource("/ingest/test-file-3.txt.met");
+
+ Metadata prodMet;
+ StdIngester ingester = new StdIngester(transferServiceFacClass);
+ prodMet = new SerializableMetadata(new FileInputStream(metUrl.getFile()));
+ // now add the right file location
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath());
+ String productId = ingester.ingest(
+ new URL("http://localhost:" + FM_PORT),
+ new File(refUrl.getFile()), prodMet);
+ FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:" + FM_PORT));
+
+ Metadata m = fmc.getMetadata(fmc.getProductById(productId));
+ assertEquals(m.getAllMetadata("TestElement").size(), 4);
+ assertEquals(m.getMetadata("TestElement"), "fe");
+ }
+
+ public void testComplexQuery() throws Exception {
+ URL ingestUrl = this.getClass().getResource("/ingest");
+ URL refUrl1 = this.getClass().getResource("/ingest/test-file-1.txt");
+ URL refUrl2 = this.getClass().getResource("/ingest/test-file-2.txt");
+
+ StdIngester ingester = new StdIngester(transferServiceFacClass);
+
+ //ingest first file
+ Metadata prodMet = new Metadata();
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath());
+ prodMet.addMetadata(CoreMetKeys.FILENAME, "test-file-1.txt");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile1");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
+ ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(refUrl1.getFile()), prodMet);
+
+ //ingest second file
+ prodMet.replaceMetadata(CoreMetKeys.FILENAME, "test-file-2.txt");
+ prodMet.replaceMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile2");
+ ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(refUrl2.getFile()), prodMet);
+
+ //perform complex query
+ ComplexQuery complexQuery = new ComplexQuery();
+ List<String> reducedMetadata = new Vector<String>();
+ reducedMetadata.add(CoreMetKeys.FILENAME);
+ complexQuery.setReducedMetadata(reducedMetadata);
+ List<String> productTypeNames = new Vector<String>();
+ productTypeNames.add("GenericFile");
+ complexQuery.setReducedProductTypeNames(productTypeNames);
+ complexQuery.setSortByMetKey(CoreMetKeys.FILENAME);
+ complexQuery.setToStringResultFormat("$" + CoreMetKeys.FILENAME);
+ complexQuery.addCriterion(SqlParser.parseSqlWhereClause("Filename != 'test.txt'"));
+ FileManagerClient fmc = new AvroFileManagerClient(new URL("http://localhost:" + FM_PORT));
+ List<QueryResult> queryResults = fmc.complexQuery(complexQuery);
+ assertEquals("[test-file-1.txt, test-file-2.txt]", queryResults.toString());
+ }
+
+ @Override
+ protected void setProperties() {
+ System.setProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory");
+ System.setProperty("filemgr.client", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory");
+ }
+
+ @Override
+ protected FileManagerServer newFileManagerServer(int port) throws Exception {
+ return new AvroFileManagerServer(port);
+ }
+
+ @Override
+ protected boolean shouldExpandProduct() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java
new file mode 100644
index 0000000..9cbf617
--- /dev/null
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestAvroFileManagerServer.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.filemgr.system;
+
+import org.apache.oodt.cas.filemgr.metadata.ProductMetKeys;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.metadata.Metadata;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class TestAvroFileManagerServer extends AbstractFileManagerServerTest {
+
+ private static Logger LOG = Logger.getLogger(TestAvroFileManagerServer.class.getName());
+
+ public void testExpandProductMet() {
+ AvroFileManagerClient fmc = null;
+ try {
+ fmc = new AvroFileManagerClient(new URL("http://localhost:" + FM_PORT));
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+
+ Metadata met = null;
+ try {
+ met = fmc.getMetadata(fmc.getProductByName("test.txt"));
+ } catch (CatalogException e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ fail(e.getMessage());
+ }
+
+ assertNotNull(met);
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID));
+ assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME));
+ assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE));
+ assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
+ assertEquals(Product.STATUS_RECEIVED, met
+ .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
+
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES));
+
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size());
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)
+ .size());
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)
+ .size());
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)
+ .size());
+
+ URL refUrl = this.getClass().getResource("/ingest/test.txt");
+
+ String origPath = null;
+ try {
+ origPath = new File(refUrl.getFile()).getCanonicalPath();
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS));
+ assertEquals("/tmp/test.txt/test.txt", met
+ .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS));
+
+ assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES));
+ assertEquals("text/plain", met
+ .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES));
+
+ try {
+ met = fmc.getReducedMetadata(fmc.getProductByName("test.txt"),
+ Collections.EMPTY_LIST);
+ } catch (CatalogException e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ fail(e.getMessage());
+ }
+
+ assertNotNull(met);
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID));
+ assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME));
+ assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE));
+ assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
+ assertEquals(Product.STATUS_RECEIVED, met
+ .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
+
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES));
+ assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES));
+
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size());
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)
+ .size());
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)
+ .size());
+ assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)
+ .size());
+
+ origPath = null;
+ try {
+ origPath = new File(refUrl.getFile()).getCanonicalPath();
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS));
+ assertEquals("/tmp/test.txt/test.txt", met
+ .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS));
+
+ assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES));
+ assertEquals("text/plain", met
+ .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES));
+
+ }
+
+ @Override
+ protected void setProperties() {
+ System.setProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory");
+ System.setProperty("filemgr.client", "org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory");
+ }
+
+ @Override
+ protected FileManagerServer newFileManagerServer(int port) throws Exception {
+ return new AvroFileManagerServer(port);
+ }
+
+ @Override
+ protected boolean shouldExpandProduct() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java
index ee14ef7..256f019 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java
@@ -44,6 +44,7 @@ import junit.framework.TestCase;
* @since OODT-72
*
*/
+@Deprecated
public class TestRpcFileManager extends TestCase {
private static final int FM_PORT = 50002;
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java
index 18e3cc9..bf60141 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java
@@ -51,6 +51,7 @@ import junit.framework.TestCase;
* Test suite for the {@link XmlRpcFileManagerClient}
* </p>.
*/
+@Deprecated
public class TestRpcFileManagerClient extends TestCase {
private static int FM_PORT = 50001;
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java
deleted file mode 100644
index 1e3db7e..0000000
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManager.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.cas.filemgr.system;
-
-import org.apache.oodt.cas.filemgr.ingest.StdIngester;
-import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
-import org.apache.oodt.cas.filemgr.metadata.ProductMetKeys;
-import org.apache.oodt.cas.filemgr.structs.Product;
-import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
-import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.metadata.SerializableMetadata;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.util.Collections;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import junit.framework.TestCase;
-
-
-/**
- *
- * Test harness for the XmlRpcFileManager.
- *
- * @since OODT-72
- *
- */
-public class TestXmlRpcFileManager extends TestCase {
- private static Logger LOG = Logger.getLogger(TestXmlRpcFileManager.class.getName());
- private static final int FM_PORT = 50001;
-
- private XmlRpcFileManager fm;
-
- private String luceneCatLoc;
-
- private static final String transferServiceFacClass =
- "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory";
-
- private Properties initialProperties = new Properties(
- System.getProperties());
-
- /**
- * @since OODT-72
- */
- public void testExpandProductMet() {
- XmlRpcFileManagerClient fmc = null;
- try {
- fmc = new XmlRpcFileManagerClient(new URL("http://localhost:" + FM_PORT));
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- Metadata met = null;
- try {
- met = fmc.getMetadata(fmc.getProductByName("test.txt"));
- } catch (CatalogException e) {
- LOG.log(Level.SEVERE, e.getMessage());
- fail(e.getMessage());
- }
-
- assertNotNull(met);
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID));
- assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME));
- assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE));
- assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
- assertEquals(Product.STATUS_RECEIVED, met
- .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
-
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES));
-
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size());
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)
- .size());
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)
- .size());
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)
- .size());
-
- URL refUrl = this.getClass().getResource("/ingest/test.txt");
-
- String origPath = null;
- try {
- origPath = new File(refUrl.getFile()).getCanonicalPath();
- } catch (IOException e) {
- fail(e.getMessage());
- }
- assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS));
- assertEquals("/tmp/test.txt/test.txt", met
- .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS));
-
- assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES));
- assertEquals("text/plain", met
- .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES));
-
- try {
- met = fmc.getReducedMetadata(fmc.getProductByName("test.txt"),
- Collections.EMPTY_LIST);
- } catch (CatalogException e) {
- LOG.log(Level.SEVERE, e.getMessage());
- fail(e.getMessage());
- }
-
- assertNotNull(met);
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ID));
- assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_ID));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_NAME));
- assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_NAME));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_STRUCTURE));
- assertEquals("Flat", met.getMetadata(ProductMetKeys.PRODUCT_STRUCTURE));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
- assertEquals(Product.STATUS_RECEIVED, met
- .getMetadata(ProductMetKeys.PRODUCT_TRANSFER_STATUS));
-
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_ORIG_REFS));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_DATASTORE_REFS));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_FILE_SIZES));
- assertTrue(met.containsKey(ProductMetKeys.PRODUCT_MIME_TYPES));
-
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_ORIG_REFS).size());
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS)
- .size());
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_FILE_SIZES)
- .size());
- assertEquals(1, met.getAllMetadata(ProductMetKeys.PRODUCT_MIME_TYPES)
- .size());
-
- origPath = null;
- try {
- origPath = new File(refUrl.getFile()).getCanonicalPath();
- } catch (IOException e) {
- fail(e.getMessage());
- }
- assertEquals(origPath, met.getMetadata(ProductMetKeys.PRODUCT_ORIG_REFS));
- assertEquals("/tmp/test.txt/test.txt", met
- .getMetadata(ProductMetKeys.PRODUCT_DATASTORE_REFS));
-
- assertNotNull(met.getMetadata(ProductMetKeys.PRODUCT_FILE_SIZES));
- assertEquals("text/plain", met
- .getMetadata(ProductMetKeys.PRODUCT_MIME_TYPES));
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see junit.framework.TestCase#setUp()
- */
- @Override
- public void setUp() throws Exception {
- startXmlRpcFileManager();
- ingestTestFile();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see junit.framework.TestCase#tearDown()
- */
- @Override
- public void tearDown() throws Exception {
- fm.shutdown();
- fm = null;
-
- // blow away lucene cat
- deleteAllFiles(luceneCatLoc);
-
- // blow away test file
- deleteAllFiles("/tmp/test.txt");
-
- // Reset the System properties to initial values.
- System.setProperties(initialProperties);
- }
-
- private void deleteAllFiles(String startDir) {
- File startDirFile = new File(startDir);
- File[] delFiles = startDirFile.listFiles();
-
- if (delFiles != null && delFiles.length > 0) {
- for (File delFile : delFiles) {
- delFile.delete();
- }
- }
-
- startDirFile.delete();
-
- }
-
- private void ingestTestFile() {
- Metadata prodMet;
- StdIngester ingester = new StdIngester(transferServiceFacClass);
-
- try {
- URL ingestUrl = this.getClass().getResource("/ingest");
- URL refUrl = this.getClass().getResource("/ingest/test.txt");
- URL metUrl = this.getClass().getResource("/ingest/test.txt.met");
-
- prodMet = new SerializableMetadata(new FileInputStream(
- new File(metUrl.getFile())));
-
- // now add the right file location
- prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(
- ingestUrl.getFile()).getCanonicalPath());
- prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt");
- prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
- ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(
- refUrl.getFile()), prodMet);
- } catch (Exception e) {
- fail(e.getMessage());
- }
- }
-
- private void startXmlRpcFileManager() {
-
- Properties properties = new Properties(System.getProperties());
-
- // first make sure to load properties for the file manager
- // and make sure to load logging properties as well
-
- // set the log levels
- URL loggingPropertiesUrl = this.getClass().getResource(
- "/test.logging.properties");
- properties.setProperty("java.util.logging.config.file", new File(
- loggingPropertiesUrl.getFile()).getAbsolutePath());
-
- // first load the example configuration
- try {
- URL filemgrPropertiesUrl = this.getClass().getResource(
- "/filemgr.properties");
- properties.load(
- new FileInputStream(new File(filemgrPropertiesUrl.getFile())));
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- // override the catalog to use: we'll use lucene
- try {
- URL ingestUrl = this.getClass().getResource("/ingest");
- luceneCatLoc = new File(ingestUrl.getFile()).getCanonicalPath() + "/cat";
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- properties.setProperty("filemgr.catalog.factory",
- "org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory");
- properties.setProperty("org.apache.oodt.cas.filemgr.catalog.lucene.idxPath",
- luceneCatLoc);
-
- // now override the repo mgr policy
- URL fmpolicyUrl = this.getClass().getResource("/ingest/fmpolicy");
- try {
- properties.setProperty("org.apache.oodt.cas.filemgr.repositorymgr.dirs",
- "file://" + new File(fmpolicyUrl.getFile()).getCanonicalPath());
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- // now override the val layer ones
- properties.setProperty("org.apache.oodt.cas.filemgr.validation.dirs",
- "file://" + new File(fmpolicyUrl.getFile()).getAbsolutePath());
-
- // set up mime repo path
- URL mimeTypesUrl = this.getClass().getResource("/mime-types.xml");
- properties.setProperty("org.apache.oodt.cas.filemgr.mime.type.repository",
- new File(mimeTypesUrl.getFile()).getAbsolutePath());
-
- // override expand product met
- properties.setProperty("org.apache.oodt.cas.filemgr.metadata.expandProduct",
- "true");
-
- System.setProperties(properties);
-
- try {
- fm = new XmlRpcFileManager(FM_PORT);
- } catch (Exception e) {
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/oodt/blob/36f320e3/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java
index 8ad4c70..c368d48 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestXmlRpcFileManagerClient.java
@@ -19,6 +19,7 @@
package org.apache.oodt.cas.filemgr.system;
//OODT imports
+
import org.apache.oodt.cas.filemgr.ingest.StdIngester;
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
@@ -29,62 +30,47 @@ import org.apache.oodt.cas.filemgr.util.SqlParser;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
-//JDK imports
import java.io.File;
import java.io.FileInputStream;
import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
-import java.util.Properties;
import java.util.Vector;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
+//JDK imports
//Junit imports
-import junit.framework.TestCase;
/**
* @author mattmann
* @version $Revision$
- *
+ * <p>
* <p>
* Test suite for the {@link XmlRpcFileManagerClient}
* </p>.
*/
-public class TestXmlRpcFileManagerClient extends TestCase {
-
- private static final int FM_PORT = 50001;
-
- private XmlRpcFileManager fm;
-
- private String luceneCatLoc;
-
- private static final String transferServiceFacClass = "org.apache.oodt.cas."
- + "filemgr.datatransfer.LocalDataTransferFactory";
-
- private Properties initialProperties = new Properties(
- System.getProperties());
-
- public TestXmlRpcFileManagerClient() {
- }
+public class TestXmlRpcFileManagerClient extends AbstractFileManagerServerTest {
-
/**
* @since OODT-161
- *
*/
public void testGetReducedMetadata() {
- List arrayListElems = new ArrayList();
- List vectorElemList = new Vector();
- List linkedListElemList = new LinkedList();
+ List<String> arrayListElems = new ArrayList<>();
+ List<String> vectorElemList = new Vector<>();
+ List<String> linkedListElemList = new LinkedList<>();
arrayListElems.add(CoreMetKeys.FILENAME);
vectorElemList.add(CoreMetKeys.FILENAME);
linkedListElemList.add(CoreMetKeys.FILENAME);
try {
- XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL(
- "http://localhost:" + FM_PORT));
-
+ FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:" + FM_PORT));
+
Metadata reducedMet;
List pTypes = fmc.getProductTypes();
assertNotNull(pTypes);
@@ -111,66 +97,63 @@ public class TestXmlRpcFileManagerClient extends TestCase {
assertNotNull(reducedMet);
assertTrue(reducedMet.containsKey(CoreMetKeys.FILENAME));
assertEquals(reducedMet.getMap().keySet().size(), 1);
-
} catch (Exception e) {
fail(e.getMessage());
- }finally {
- fm.shutdown();
}
-
}
-
+
public void testIngest() throws Exception {
URL ingestUrl = this.getClass().getResource("/ingest");
URL refUrl = this.getClass().getResource("/ingest/test.txt");
Metadata prodMet = new Metadata();
prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(
- ingestUrl.getFile()).getCanonicalPath());
+ ingestUrl.getFile()).getCanonicalPath());
prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt");
prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile");
prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
StdIngester ingester = new StdIngester(transferServiceFacClass);
String productId = ingester.ingest(
- new URL("http://localhost:" + FM_PORT),
- new File(refUrl.getFile()), prodMet);
- XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL(
- "http://localhost:" + FM_PORT));
+ new URL("http://localhost:" + FM_PORT),
+ new File(refUrl.getFile()), prodMet);
+ FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:" + FM_PORT));
Metadata m = fmc.getMetadata(fmc.getProductById(productId));
assertEquals(m.getMetadata("Filename"), "test.txt");
deleteAllFiles("/tmp/test-type");
}
public void testRemoveFile() throws Exception {
+ Path tmpFilePath = Paths.get("/tmp", "test-delete.txt");
URL ingestUrl = this.getClass().getResource("/ingest");
URL refUrl = this.getClass().getResource("/ingest/test-delete.txt");
+ Files.copy(Paths.get(refUrl.toURI()), tmpFilePath, REPLACE_EXISTING);
- Metadata prodMet = new Metadata();
- prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(
- ingestUrl.getFile()).getCanonicalPath());
- prodMet.addMetadata(CoreMetKeys.FILENAME, "test-delete.txt");
- prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile");
- prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
-
- StdIngester ingester = new StdIngester(transferServiceFacClass);
- String productId = ingester.ingest(
- new URL("http://localhost:" + FM_PORT),
- new File(refUrl.getFile()), prodMet);
- XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL(
- "http://localhost:" + FM_PORT));
- Metadata m = fmc.getMetadata(fmc.getProductById(productId));
- assertEquals(m.getMetadata("Filename"), "test-delete.txt");
- String loc = m.getMetadata("FileLocation");
- fmc.removeFile(loc+"/"+m.getMetadata("Filename"));
+ try {
+ Metadata prodMet = new Metadata();
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath());
+ prodMet.addMetadata(CoreMetKeys.FILENAME, "test-delete.txt");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile");
+ prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
- fmc.getProductById(productId);
+ StdIngester ingester = new StdIngester(transferServiceFacClass);
+ String productId = ingester.ingest(
+ new URL("http://localhost:" + FM_PORT),
+ new File(refUrl.getFile()), prodMet);
+ FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:" + FM_PORT));
+ Metadata m = fmc.getMetadata(fmc.getProductById(productId));
+ assertEquals(m.getMetadata("Filename"), "test-delete.txt");
+ String loc = m.getMetadata("FileLocation");
+ fmc.removeFile(loc + "/" + m.getMetadata("Filename"));
+ fmc.getProductById(productId);
+ } finally {
+ Files.copy(tmpFilePath, Paths.get(refUrl.toURI()));
+ }
deleteAllFiles("/tmp/test-type");
}
/**
* @since OODT-404
- *
*/
public void testMetadataPersistence() throws Exception {
URL ingestUrl = this.getClass().getResource("/ingest");
@@ -179,23 +162,19 @@ public class TestXmlRpcFileManagerClient extends TestCase {
Metadata prodMet;
StdIngester ingester = new StdIngester(transferServiceFacClass);
- prodMet = new SerializableMetadata(new FileInputStream(
- metUrl.getFile()));
+ prodMet = new SerializableMetadata(new FileInputStream(metUrl.getFile()));
// now add the right file location
- prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(
- ingestUrl.getFile()).getCanonicalPath());
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath());
String productId = ingester.ingest(
- new URL("http://localhost:" + FM_PORT),
- new File(refUrl.getFile()), prodMet);
- XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL(
- "http://localhost:" + FM_PORT));
-
+ new URL("http://localhost:" + FM_PORT),
+ new File(refUrl.getFile()), prodMet);
+ FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:" + FM_PORT));
+
Metadata m = fmc.getMetadata(fmc.getProductById(productId));
assertEquals(m.getAllMetadata("TestElement").size(), 4);
assertEquals(m.getMetadata("TestElement"), "fe");
}
-
public void testComplexQuery() throws Exception {
URL ingestUrl = this.getClass().getResource("/ingest");
URL refUrl1 = this.getClass().getResource("/ingest/test-file-1.txt");
@@ -205,20 +184,17 @@ public class TestXmlRpcFileManagerClient extends TestCase {
//ingest first file
Metadata prodMet = new Metadata();
- prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(
- ingestUrl.getFile()).getCanonicalPath());
+ prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(ingestUrl.getFile()).getCanonicalPath());
prodMet.addMetadata(CoreMetKeys.FILENAME, "test-file-1.txt");
prodMet.addMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile1");
prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
- ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(
- refUrl1.getFile()), prodMet);
-
+ ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(refUrl1.getFile()), prodMet);
+
//ingest second file
prodMet.replaceMetadata(CoreMetKeys.FILENAME, "test-file-2.txt");
prodMet.replaceMetadata(CoreMetKeys.PRODUCT_NAME, "TestFile2");
- ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(
- refUrl2.getFile()), prodMet);
-
+ ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(refUrl2.getFile()), prodMet);
+
//perform complex query
ComplexQuery complexQuery = new ComplexQuery();
List<String> reducedMetadata = new Vector<String>();
@@ -230,144 +206,24 @@ public class TestXmlRpcFileManagerClient extends TestCase {
complexQuery.setSortByMetKey(CoreMetKeys.FILENAME);
complexQuery.setToStringResultFormat("$" + CoreMetKeys.FILENAME);
complexQuery.addCriterion(SqlParser.parseSqlWhereClause("Filename != 'test.txt'"));
- XmlRpcFileManagerClient fmc = new XmlRpcFileManagerClient(new URL(
- "http://localhost:" + FM_PORT));
+ FileManagerClient fmc = new XmlRpcFileManagerClient(new URL("http://localhost:" + FM_PORT));
List<QueryResult> queryResults = fmc.complexQuery(complexQuery);
assertEquals("[test-file-1.txt, test-file-2.txt]", queryResults.toString());
}
- /*
- * (non-Javadoc)
- *
- * @see junit.framework.TestCase#setUp()
- */
- protected void setUp() throws Exception {
- startXmlRpcFileManager();
- ingestTestFile();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see junit.framework.TestCase#tearDown()
- */
- protected void tearDown() throws Exception {
- fm.shutdown();
- fm = null;
-
- // blow away lucene cat
- deleteAllFiles(luceneCatLoc);
-
- // blow away test file
- deleteAllFiles("/tmp/test.txt");
-
- // Reset the System properties to initial values.
- System.setProperties(initialProperties);
- }
-
- private void deleteAllFiles(String startDir) {
- File startDirFile = new File(startDir);
- File[] delFiles = startDirFile.listFiles();
-
- if (delFiles != null && delFiles.length > 0) {
- for (File delFile : delFiles) {
- delFile.delete();
- }
- }
-
- startDirFile.delete();
-
+ @Override
+ protected void setProperties() {
+ System.setProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory");
+ System.setProperty("filemgr.client", "org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory");
}
- private void ingestTestFile() {
- Metadata prodMet;
- StdIngester ingester = new StdIngester(transferServiceFacClass);
-
- try {
- URL ingestUrl = this.getClass().getResource("/ingest");
- URL refUrl = this.getClass().getResource("/ingest/test.txt");
- URL metUrl = this.getClass().getResource("/ingest/test.txt.met");
-
- prodMet = new SerializableMetadata(new FileInputStream(
- new File(metUrl.getFile())));
-
- // now add the right file location
- prodMet.addMetadata(CoreMetKeys.FILE_LOCATION, new File(
- ingestUrl.getFile()).getCanonicalPath());
- prodMet.addMetadata(CoreMetKeys.FILENAME, "test.txt");
- prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
- ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(
- refUrl.getFile()), prodMet);
- } catch (Exception e) {
- fail(e.getMessage());
- }
+ @Override
+ protected FileManagerServer newFileManagerServer(int port) throws Exception {
+ return new XmlRpcFileManagerServer(port);
}
- private void startXmlRpcFileManager() {
-
- Properties properties = new Properties(System.getProperties());
-
- // first make sure to load properties for the file manager
- // and make sure to load logging properties as well
-
- // set the log levels
- URL loggingPropertiesUrl = this.getClass().getResource(
- "/test.logging.properties");
- properties.setProperty("java.util.logging.config.file", new File(
- loggingPropertiesUrl.getFile()).getAbsolutePath());
-
- // first load the example configuration
- try {
- URL filemgrPropertiesUrl = this.getClass().getResource(
- "/filemgr.properties");
- properties.load(
- new FileInputStream(new File(filemgrPropertiesUrl.getFile())));
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- // override the catalog to use: we'll use lucene
- try {
- URL ingestUrl = this.getClass().getResource("/ingest");
- luceneCatLoc = new File(ingestUrl.getFile()).getCanonicalPath()
- + "/cat";
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- properties.setProperty("filemgr.catalog.factory",
- "org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory");
- properties.setProperty(
- "org.apache.oodt.cas.filemgr.catalog.lucene.idxPath",
- luceneCatLoc);
-
- // now override the repo mgr policy
- URL fmpolicyUrl = this.getClass().getResource("/ingest/fmpolicy");
- try {
- properties.setProperty(
- "org.apache.oodt.cas.filemgr.repositorymgr.dirs",
- "file://" + new File(fmpolicyUrl.getFile()).getCanonicalPath());
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- // now override the val layer ones
- properties.setProperty("org.apache.oodt.cas.filemgr.validation.dirs",
- "file://" + new File(fmpolicyUrl.getFile()).getAbsolutePath());
-
- // set up mime repo path
- URL mimeTypesUrl = this.getClass().getResource("/mime-types.xml");
- properties.setProperty(
- "org.apache.oodt.cas.filemgr.mime.type.repository",
- new File(mimeTypesUrl.getFile()).getAbsolutePath());
-
- System.setProperties(properties);
-
- try {
- fm = new XmlRpcFileManager(FM_PORT);
- } catch (Exception e) {
- fail(e.getMessage());
- }
+ @Override
+ protected boolean shouldExpandProduct() {
+ return false;
}
-
}