You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2014/05/03 04:00:59 UTC
svn commit: r1592138 - in /oodt/trunk/filemgr/src:
main/java/org/apache/oodt/cas/filemgr/datatransfer/
main/java/org/apache/oodt/cas/filemgr/system/
test/org/apache/oodt/cas/filemgr/datatransfer/
Author: bfoster
Date: Sat May 3 02:00:58 2014
New Revision: 1592138
URL: http://svn.apache.org/r1592138
Log:
- Filemgr deletes files using DataTransferer now ... OODT-680
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java?rev=1592138&r1=1592137&r2=1592138&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java Sat May 3 02:00:58 2014
@@ -72,4 +72,6 @@ public interface DataTransfer {
*/
public void retrieveProduct(Product product, File directory) throws DataTransferException,
IOException;
+
+ public void deleteProduct(Product product) throws DataTransferException, IOException;
}
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java?rev=1592138&r1=1592137&r2=1592138&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java Sat May 3 02:00:58 2014
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.oodt.cas.filemgr.datatransfer;
//OODT imports
@@ -30,68 +29,44 @@ import java.util.logging.Logger;
import java.util.logging.Level;
import java.net.URL;
-/**
- * @author mattmann
- * @version $Revision$
- *
- * <p>
- * An implementation of the {@link DataTransfer} interface that leaves products
- * in the same place (i.e., "in place") and doesn't transfer them at all.
- * </p>
- *
+/**
+ * An implementation of the {@link DataTransfer} interface that leaves
+ * products in the same place (i.e., "in place") and doesn't transfer
+ * them at all.
+ *
+ * @author mattmann@apache.org (Chris Mattmann)
*/
public class InPlaceDataTransferer implements DataTransfer {
- /* our log stream */
- private static final Logger LOG = Logger.getLogger(InPlaceDataTransferer.class
- .getName());
-
- /* file manager client */
- private XmlRpcFileManagerClient client = null;
-
- /**
- * <p>
- * Default Constructor
- * </p>
- */
- public InPlaceDataTransferer() {
- }
+ private static final Logger LOG = Logger.getLogger(InPlaceDataTransferer.class.getName());
- /*
- * (non-Javadoc)
- *
- * @see org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl(java.net.URL)
- */
- public void setFileManagerUrl(URL url) {
- try {
- client = new XmlRpcFileManagerClient(url);
- LOG.log(Level.INFO, "In Place Data Transfer to: ["
- + client.getFileManagerUrl().toString() + "] enabled");
- } catch (ConnectionException e) {
- LOG.log(Level.WARNING, "Connection exception for filemgr: [" + url
- + "]");
- }
- }
+ private XmlRpcFileManagerClient client = null;
- /*
- * (non-Javadoc)
- *
- * @see org.apache.oodt.cas.datatransfer.DataTransfer#transferProduct(org.apache.oodt.cas.data.structs.Product)
- */
- public void transferProduct(Product product) throws DataTransferException,
- IOException {
- // do nothing
- }
+ public InPlaceDataTransferer() {}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#retrieveProduct(org.
- * apache.oodt.cas.filemgr.structs.Product, java.io.File)
- */
- public void retrieveProduct(Product product, File directory)
- throws DataTransferException, IOException {
- // do nothing
+ public void setFileManagerUrl(URL url) {
+ try {
+ client = new XmlRpcFileManagerClient(url);
+ LOG.log(Level.INFO, "In Place Data Transfer to: [" + client.getFileManagerUrl().toString()
+ + "] enabled");
+ } catch (ConnectionException e) {
+ LOG.log(Level.WARNING, "Connection exception for filemgr: [" + url + "]");
}
+ }
+
+ @Override
+ public void transferProduct(Product product) throws DataTransferException, IOException {
+ // do nothing
+ }
+
+ @Override
+ public void retrieveProduct(Product product, File directory) throws DataTransferException,
+ IOException {
+ // do nothing
+ }
+
+ @Override
+ public void deleteProduct(Product product) throws DataTransferException, IOException {
+ // do nothing
+ }
}
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java?rev=1592138&r1=1592137&r2=1592138&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java Sat May 3 02:00:58 2014
@@ -170,6 +170,17 @@ public class LocalDataTransferer impleme
}
}
+ @Override
+ public void deleteProduct(Product product) throws DataTransferException, IOException {
+ for (Reference ref : product.getProductReferences()) {
+ File dataFile = new File(URI.create(ref.getDataStoreReference()).toURL().getPath());
+ if (!dataFile.delete()) {
+ throw new IOException(String.format("Failed to delete file %s - delete returned false",
+ dataFile));
+ }
+ }
+ }
+
/**
* @param args
*/
@@ -427,5 +438,4 @@ public class LocalDataTransferer impleme
return;
}
}
-
}
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java?rev=1592138&r1=1592137&r2=1592138&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java Sat May 3 02:00:58 2014
@@ -24,6 +24,7 @@ import org.apache.oodt.cas.filemgr.struc
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+
//JDK imports
import java.io.File;
import java.io.FileInputStream;
@@ -195,6 +196,17 @@ public class RemoteDataTransferer implem
}
}
+ @Override
+ public void deleteProduct(Product product) throws DataTransferException, IOException {
+ for (Reference ref : product.getProductReferences()) {
+ File dataFile = new File(URI.create(ref.getDataStoreReference()).toURL().getPath());
+ if (!dataFile.delete()) {
+ throw new IOException(String.format("Failed to delete file %s - delete returned false",
+ dataFile));
+ }
+ }
+ }
+
private void remoteTransfer(Reference reference, Product product)
throws URISyntaxException {
// get the file path
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java?rev=1592138&r1=1592137&r2=1592138&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java Sat May 3 02:00:58 2014
@@ -32,6 +32,7 @@ import org.apache.tika.io.IOUtils;
import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
@@ -74,12 +75,27 @@ public class S3DataTransferer implements
public void retrieveProduct(Product product, File directory) throws DataTransferException,
IOException {
for (Reference ref : product.getProductReferences()) {
- GetObjectRequest request = new GetObjectRequest(bucketName, stripProtocol(ref.getDataStoreReference(), true));
+ GetObjectRequest request = new GetObjectRequest(bucketName, stripProtocol(
+ ref.getDataStoreReference(), true));
S3Object file = s3Client.getObject(request);
stageFile(file, ref, directory);
}
}
+ @Override
+ public void deleteProduct(Product product) throws DataTransferException, IOException {
+ for (Reference ref : product.getProductReferences()) {
+ DeleteObjectRequest request = new DeleteObjectRequest(bucketName, stripProtocol(
+ ref.getDataStoreReference(), true));
+ try {
+ s3Client.deleteObject(request);
+ } catch (AmazonClientException e) {
+ throw new DataTransferException(String.format(
+ "Failed to delete product reference %s from S3", ref.getDataStoreReference()), e);
+ }
+ }
+ }
+
private void stageFile(S3Object file, Reference ref, File directory) throws IOException {
S3ObjectInputStream inStream = null;
FileOutputStream outStream = null;
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java?rev=1592138&r1=1592137&r2=1592138&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java Sat May 3 02:00:58 2014
@@ -56,6 +56,9 @@ import org.apache.oodt.cas.filemgr.versi
import org.apache.oodt.cas.filemgr.versioning.VersioningUtils;
import org.apache.oodt.cas.filemgr.datatransfer.TransferStatusTracker;
+
+import com.google.common.collect.Lists;
+
//JDK imports
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -926,8 +929,14 @@ public class XmlRpcFileManager {
"Moving of heirarhical products not supported yet");
}
- public boolean removeFile(String filePath) {
- return new File(filePath).delete();
+ public boolean removeFile(String filePath) throws DataTransferException, IOException {
+ // TODO(bfoster): Clean this up so that it deletes by product not file.
+ Product product = new Product();
+ Reference r = new Reference();
+ r.setDataStoreReference(filePath);
+ product.setProductReferences(Lists.newArrayList(r));
+ dataTransfer.deleteProduct(product);
+ return true;
}
public boolean modifyProduct(Hashtable productHash) throws CatalogException {
Modified: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java?rev=1592138&r1=1592137&r2=1592138&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java (original)
+++ oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java Sat May 3 02:00:58 2014
@@ -39,6 +39,7 @@ import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
@@ -54,8 +55,10 @@ import com.google.common.collect.Lists;
public class TestS3DataTransferer {
private final static String S3_BUCKET_NAME = "TestBucket";
- private final static String ORGINAL_REF = "/path/to/file";
- private final static String DATA_STORE_REF = "/path/in/s3/storage/file";
+ private final static String ORGINAL_REF = "file:/path/to/file";
+ private final static String DATA_STORE_REF = "s3:/path/in/s3/storage/file";
+ private final static String EXPECTED_ORGINAL_REF = "/path/to/file";
+ private final static String EXPECTED_DATA_STORE_REF = "path/in/s3/storage/file";
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
@@ -93,8 +96,8 @@ public class TestS3DataTransferer {
PutObjectRequest request = argument.getValue();
assertThat(request.getBucketName(), is(S3_BUCKET_NAME));
- assertThat(request.getKey(), is(DATA_STORE_REF));
- assertThat(request.getFile().getAbsolutePath(), is(ORGINAL_REF));
+ assertThat(request.getKey(), is(EXPECTED_DATA_STORE_REF));
+ assertThat(request.getFile().getAbsolutePath(), is(EXPECTED_ORGINAL_REF));
}
@Test
@@ -106,6 +109,19 @@ public class TestS3DataTransferer {
GetObjectRequest request = argument.getValue();
assertThat(request.getBucketName(), is(S3_BUCKET_NAME));
- assertThat(request.getKey(), is(DATA_STORE_REF));
+ assertThat(request.getKey(), is(EXPECTED_DATA_STORE_REF));
+ }
+
+ @Test
+ public void testDeleteProduct() throws DataTransferException, IOException {
+ dataTransferer.deleteProduct(product);
+
+ ArgumentCaptor<DeleteObjectRequest> argument = ArgumentCaptor
+ .forClass(DeleteObjectRequest.class);
+ verify(s3Client).deleteObject(argument.capture());
+
+ DeleteObjectRequest request = argument.getValue();
+ assertThat(request.getBucketName(), is(S3_BUCKET_NAME));
+ assertThat(request.getKey(), is(EXPECTED_DATA_STORE_REF));
}
}