You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2012/03/09 08:57:07 UTC
svn commit: r1298728 - in /oodt/trunk/filemgr/src:
main/java/org/apache/oodt/cas/filemgr/cli/action/
main/java/org/apache/oodt/cas/filemgr/datatransfer/
main/java/org/apache/oodt/cas/filemgr/system/ main/resources/
test/org/apache/oodt/cas/filemgr/cli/...
Author: bfoster
Date: Fri Mar 9 07:57:06 2012
New Revision: 1298728
URL: http://svn.apache.org/viewvc?rev=1298728&view=rev
Log:
- Ability for File Manager to stage an ingested Product to one of its clients
-------------------
OODT-84
Added:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java (with props)
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
oodt/trunk/filemgr/src/main/resources/cmd-line-actions.xml
oodt/trunk/filemgr/src/main/resources/cmd-line-options.xml
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestLocalDataTransferer.java
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/system/MockXmlRpcFileManagerClient.java
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java?rev=1298728&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java Fri Mar 9 07:57:06 2012
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.cli.action;
+
+//JDK imports
+import java.io.File;
+
+//OODT imports
+import org.apache.oodt.cas.cli.exception.CmdLineActionException;
+import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer;
+import org.apache.oodt.cas.filemgr.datatransfer.DataTransferFactory;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+
+/**
+ * Retrieves files for a given {@link Product}.
+ *
+ * @author bfoster (Brian Foster)
+ */
+public class RetrieveFilesCliAction extends FileManagerCliAction {
+
+ private String productId;
+ private String productName;
+ private DataTransfer dt;
+ private File destination;
+
+ @Override
+ public void execute(ActionMessagePrinter printer)
+ throws CmdLineActionException {
+ try {
+ XmlRpcFileManagerClient fmClient = getClient();
+ dt.setFileManagerUrl(fmClient.getFileManagerUrl());
+ Product product = null;
+ if (productId != null) {
+ product = fmClient.getProductById(productId);
+ } else if (productName != null) {
+ product = fmClient.getProductByName(productName);
+ } else {
+ throw new Exception("Must specify either productId or productName");
+ }
+ if (product != null) {
+ dt.retrieveProduct(product, destination);
+ } else {
+ throw new Exception("Product was not found");
+ }
+ dt.retrieveProduct(product, destination);
+ } catch (Exception e) {
+ throw new CmdLineActionException("Failed to retrieve files for product : " + e.getMessage(), e);
+ }
+ }
+
+ public void setProductId(String productId) {
+ this.productId = productId;
+ }
+
+ public void setProductName(String productName) {
+ this.productName = productName;
+ }
+
+ public void setDataTransferFactory(DataTransferFactory dtFactory) {
+ dt = dtFactory.createDataTransfer();
+ }
+
+ public void setDestination(File destination) {
+ this.destination = destination;
+ }
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/DataTransfer.java Fri Mar 9 07:57:06 2012
@@ -22,6 +22,7 @@ import org.apache.oodt.cas.filemgr.struc
import org.apache.oodt.cas.filemgr.structs.Product;
//JDK imports
+import java.io.File;
import java.io.IOException;
import java.net.URL;
@@ -60,4 +61,15 @@ public interface DataTransfer {
public void transferProduct(Product product) throws DataTransferException,
IOException;
+ /**
+ * Requires that the data store reference be set, nothing else is used
+ * @param product The product whose data store reference will be copied
+ * @param directory The directory where the data store reference will be copied to
+ * @throws DataTransferException
+ * If a general error occurs during the transfer.
+ * @throws IOException
+ * If there is an IO eerror when performing the transfer.
+ */
+ public void retrieveProduct(Product product, File directory) throws DataTransferException,
+ IOException;
}
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java Fri Mar 9 07:57:06 2012
@@ -24,6 +24,7 @@ import org.apache.oodt.cas.filemgr.struc
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
//JDK imports
+import java.io.File;
import java.io.IOException;
import java.util.logging.Logger;
import java.util.logging.Level;
@@ -81,4 +82,16 @@ public class InPlaceDataTransferer imple
IOException {
// do nothing
}
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#retrieveProduct(org.
+ * apache.oodt.cas.filemgr.structs.Product, java.io.File)
+ */
+ public void retrieveProduct(Product product, File directory)
+ throws DataTransferException, IOException {
+ // do nothing
+ }
}
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java Fri Mar 9 07:57:06 2012
@@ -49,277 +49,376 @@ import java.net.URISyntaxException;
* @author bfoster
* @version $Revision$
*
- * <p>
- * An implementation of the {@link DataTransfer} interface that moves products
- * that are available via URIs on the same machine, through an NFS mounted disk,
- * or via the locally mounted file repository.
- * </p>
+ * <p>
+ * An implementation of the {@link DataTransfer} interface that moves
+ * products that are available via URIs on the same machine, through an
+ * NFS mounted disk, or via the locally mounted file repository.
+ * </p>
*
*/
public class LocalDataTransferer implements DataTransfer {
- /* our log stream */
- private static final Logger LOG = Logger.getLogger(LocalDataTransferer.class
- .getName());
-
- /* file manager client */
- private XmlRpcFileManagerClient client = null;
-
- /**
- * <p>
- * Default Constructor
- * </p>
- */
- public LocalDataTransferer() {
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl(java.net.URL)
- */
- public void setFileManagerUrl(URL url) {
- try {
- client = new XmlRpcFileManagerClient(url);
- LOG.log(Level.INFO, "Local Data Transfer to: ["
- + client.getFileManagerUrl().toString() + "] enabled");
- } catch (ConnectionException e) {
- e.printStackTrace();
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.oodt.cas.datatransfer.DataTransfer#transferProduct(org.apache.oodt.cas.data.structs.Product)
- */
- public void transferProduct(Product product) throws DataTransferException,
- IOException {
- // check whether or not it's a set of files, or it's actually a dir
- // structure
- if (product.getProductStructure()
- .equals(Product.STRUCTURE_HIERARCHICAL)) {
- try {
- moveDirToProductRepo(product);
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING, "URI Syntax Exception when moving dir "
+ /* our log stream */
+ private static final Logger LOG = Logger.getLogger(LocalDataTransferer.class
+ .getName());
+
+ /* file manager client */
+ private XmlRpcFileManagerClient client = null;
+
+ /**
+ * <p>
+ * Default Constructor
+ * </p>
+ */
+ public LocalDataTransferer() {
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl
+ * (java.net.URL)
+ */
+ public void setFileManagerUrl(URL url) {
+ try {
+ client = new XmlRpcFileManagerClient(url);
+ LOG.log(Level.INFO, "Local Data Transfer to: ["
+ + client.getFileManagerUrl().toString() + "] enabled");
+ } catch (ConnectionException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.datatransfer.DataTransfer#transferProduct(org.apache
+ * .oodt.cas.data.structs.Product)
+ */
+ public void transferProduct(Product product) throws DataTransferException,
+ IOException {
+ // check whether or not it's a set of files, or it's actually a dir
+ // structure
+ if (product.getProductStructure().equals(Product.STRUCTURE_HIERARCHICAL)) {
+ try {
+ moveDirToProductRepo(product);
+ } catch (URISyntaxException e) {
+ LOG.log(
+ Level.WARNING,
+ "URI Syntax Exception when moving dir "
+ ((Reference) product.getProductReferences().get(0))
- .getOrigReference() + ": Message: "
+ .getOrigReference() + ": Message: "
+ e.getMessage());
- throw new DataTransferException(e);
- }
- } else if (product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
- try {
- moveFilesToProductRepo(product);
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING,
- "URI Syntax Exception when moving files: Message: "
- + e.getMessage());
- throw new DataTransferException(e);
+ throw new DataTransferException(e);
+ }
+ } else if (product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
+ try {
+ moveFilesToProductRepo(product);
+ } catch (URISyntaxException e) {
+ LOG.log(
+ Level.WARNING,
+ "URI Syntax Exception when moving files: Message: "
+ + e.getMessage());
+ throw new DataTransferException(e);
+ }
+ } else {
+ throw new DataTransferException(
+ "Cannot transfer product on unknown ProductStructure: "
+ + product.getProductStructure());
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#retrieveProduct(org.
+ * apache.oodt.cas.filemgr.structs.Product, java.io.File)
+ */
+ public void retrieveProduct(Product product, File directory)
+ throws DataTransferException, IOException {
+ // check whether or not it's a set of files, or it's actually a dir
+ // structure
+ if (product.getProductStructure().equals(Product.STRUCTURE_HIERARCHICAL)) {
+ try {
+ copyDirToDir(product, directory);
+ } catch (URISyntaxException e) {
+ LOG.log(
+ Level.WARNING,
+ "URI Syntax Exception when moving dir "
+ + ((Reference) product.getProductReferences().get(0))
+ .getDataStoreReference() + ": Message: "
+ + e.getMessage());
+ throw new DataTransferException(e);
+ }
+ } else if (product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
+ try {
+ copyFilesToDir(product, directory);
+ } catch (URISyntaxException e) {
+ LOG.log(
+ Level.WARNING,
+ "URI Syntax Exception when moving files: Message: "
+ + e.getMessage());
+ throw new DataTransferException(e);
+ }
+ } else {
+ throw new DataTransferException(
+ "Cannot transfer product on unknown ProductStructure: "
+ + product.getProductStructure());
+ }
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) throws DataTransferException,
+ IOException, URISyntaxException {
+ String usage = "LocalFileTransfer --productName <name> --productRepo <repo> [--dir <dirRef>] [--files <origRef 1>...<origRef N>]\n";
+
+ MimeTypes mimeTypeRepo;
+ try {
+ mimeTypeRepo = MimeTypesFactory
+ .create(System
+ .getProperty("org.apache.oodt.cas.filemgr.mime.type.repository"));
+ } catch (MimeTypeException e) {
+ e.printStackTrace();
+ throw new IOException(e.getMessage());
+ }
+
+ String productName = null;
+ String productRepo = null;
+ String transferType = null;
+ Reference dirReference = null;
+
+ List<Reference> fileReferences = null;
+
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("--dir")) {
+ transferType = "dir";
+ dirReference = new Reference();
+ dirReference.setOrigReference(new File(new URI(args[++i])).toURI()
+ .toString());
+ LOG.log(Level.FINER,
+ "LocalFileTransfer.main: Generated orig reference: "
+ + dirReference.getOrigReference());
+ } else if (args[i].equals("--files")) {
+ transferType = "files";
+ fileReferences = new Vector<Reference>();
+ for (int j = i + 1; j < args.length; j++) {
+ LOG.log(Level.FINER, "LocalFileTransfer.main: Adding file ref: "
+ + args[j]);
+ fileReferences.add(new Reference(args[j], null,
+ new File(args[j]).length(), mimeTypeRepo
+ .getMimeType(args[j])));
}
- } else {
- throw new DataTransferException(
- "Cannot transfer product on unknown ProductStructure: "
- + product.getProductStructure());
- }
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) throws DataTransferException,
- IOException, URISyntaxException {
- String usage = "LocalFileTransfer --productName <name> --productRepo <repo> [--dir <dirRef>] [--files <origRef 1>...<origRef N>]\n";
-
- MimeTypes mimeTypeRepo;
- try {
- mimeTypeRepo = MimeTypesFactory.create(System
- .getProperty("org.apache.oodt.cas.filemgr.mime.type.repository"));
- } catch (MimeTypeException e) {
- e.printStackTrace();
- throw new IOException(e.getMessage());
- }
-
- String productName = null;
- String productRepo = null;
- String transferType = null;
- Reference dirReference = null;
-
- List<Reference> fileReferences = null;
-
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("--dir")) {
- transferType = "dir";
- dirReference = new Reference();
- dirReference.setOrigReference(new File(new URI(args[++i]))
- .toURI().toString());
- LOG.log(Level.FINER,
- "LocalFileTransfer.main: Generated orig reference: "
- + dirReference.getOrigReference());
- } else if (args[i].equals("--files")) {
- transferType = "files";
- fileReferences = new Vector<Reference>();
- for (int j = i + 1; j < args.length; j++) {
- LOG.log(Level.FINER,
- "LocalFileTransfer.main: Adding file ref: "
- + args[j]);
- fileReferences.add(new Reference(args[j], null, new File(
- args[j]).length(), mimeTypeRepo
- .getMimeType(args[j])));
- }
- } else if (args[i].equals("--productName")) {
- productName = args[++i];
- } else if (args[i].equals("--productRepo")) {
- productRepo = args[++i];
+ } else if (args[i].equals("--productName")) {
+ productName = args[++i];
+ } else if (args[i].equals("--productRepo")) {
+ productRepo = args[++i];
+ }
+ }
+
+ if (transferType == null
+ || (transferType != null && ((transferType.equals("dir") && dirReference == null)
+ || (transferType.equals("files") && fileReferences == null)
+ || (transferType != null && !(transferType.equals("dir") || transferType
+ .equals("files"))) || productName == null || productRepo == null))) {
+ System.err.println(usage);
+ System.exit(1);
+ }
+
+ // construct a new Product
+ Product p = new Product();
+ p.setProductName(productName);
+
+ if (transferType.equals("dir")) {
+ p.setProductStructure(Product.STRUCTURE_HIERARCHICAL);
+ dirReference.setDataStoreReference(new File(new URI(productRepo))
+ .toURL().toExternalForm()
+ + URLEncoder.encode(p.getProductName(), "UTF-8") + "/");
+ p.getProductReferences().add(dirReference);
+ /* we'll do a simple versioning scheme ourselves: no versioning! */
+ p.getProductReferences().addAll(
+ VersioningUtils.getReferencesFromDir(new File(new URI(
+ dirReference.getOrigReference()))));
+ VersioningUtils.createBasicDataStoreRefsHierarchical(p
+ .getProductReferences());
+ } else if (transferType.equals("files")) {
+ p.setProductStructure("Flat");
+ p.getProductReferences().addAll(fileReferences);
+ VersioningUtils.createBasicDataStoreRefsFlat(productName, productRepo,
+ p.getProductReferences());
+ }
+
+ DataTransfer transfer = new LocalDataTransferer();
+ transfer.transferProduct(p);
+
+ }
+
+ private void copyDirToDir(Product product, File directory)
+ throws IOException, URISyntaxException {
+ Reference dirRef = (Reference) product.getProductReferences().get(0);
+ LOG.log(
+ Level.INFO,
+ "LocalDataTransferer: Staging Directory: "
+ + dirRef.getDataStoreReference() + " into directory "
+ + directory.getAbsolutePath());
+
+ for (Iterator<Reference> i = product.getProductReferences().iterator(); i
+ .hasNext();) {
+ Reference r = i.next();
+ File fileRef = new File(new URI(r.getDataStoreReference()));
+
+ if (fileRef.isFile()) {
+ copyFile(r, directory);
+ } else if (fileRef.isDirectory()
+ && (fileRef.list() != null && fileRef.list().length == 0)) {
+ // if it's a directory and it doesn't exist yet, we should
+ // create it
+ // just in case there's no files in it
+ File dest = new File(directory, fileRef.getName());
+ if (!new File(new URI(dest.getAbsolutePath())).exists()) {
+ LOG.log(Level.FINER, "Directory: [" + dest.getAbsolutePath()
+ + "] doesn't exist: creating it");
+ if (!new File(new URI(dest.getAbsolutePath())).mkdirs()) {
+ LOG.log(
+ Level.WARNING,
+ "Unable to create directory: ["
+ + dest.getAbsolutePath()
+ + "] in local data transferer");
+ }
}
- }
-
- if (transferType == null
- || (transferType != null && ((transferType.equals("dir") && dirReference == null)
- || (transferType.equals("files") && fileReferences == null)
- || (transferType != null && !(transferType
- .equals("dir") || transferType.equals("files")))
- || productName == null || productRepo == null))) {
- System.err.println(usage);
- System.exit(1);
- }
-
- // construct a new Product
- Product p = new Product();
- p.setProductName(productName);
-
- if (transferType.equals("dir")) {
- p.setProductStructure(Product.STRUCTURE_HIERARCHICAL);
- dirReference.setDataStoreReference(new File(new URI(productRepo))
- .toURL().toExternalForm()
- + URLEncoder.encode(p.getProductName(), "UTF-8") + "/");
- p.getProductReferences().add(dirReference);
- /* we'll do a simple versioning scheme ourselves: no versioning! */
- p.getProductReferences().addAll(
- VersioningUtils.getReferencesFromDir(new File(new URI(
- dirReference.getOrigReference()))));
- VersioningUtils.createBasicDataStoreRefsHierarchical(p
- .getProductReferences());
- } else if (transferType.equals("files")) {
- p.setProductStructure("Flat");
- p.getProductReferences().addAll(fileReferences);
- VersioningUtils.createBasicDataStoreRefsFlat(productName,
- productRepo, p.getProductReferences());
- }
-
- DataTransfer transfer = new LocalDataTransferer();
- transfer.transferProduct(p);
-
- }
-
- private void moveDirToProductRepo(Product product) throws IOException,
- URISyntaxException {
- Reference dirRef = (Reference) product.getProductReferences().get(0);
- LOG.log(Level.INFO, "LocalDataTransferer: Moving Directory: "
- + dirRef.getOrigReference() + " to "
- + dirRef.getDataStoreReference());
-
- // notify the file manager that we started
- quietNotifyTransferProduct(product);
-
- for (Iterator<Reference> i = product.getProductReferences().iterator(); i
- .hasNext();) {
- Reference r = i.next();
- File fileRef = new File(new URI(r.getOrigReference()));
-
- if (fileRef.isFile()) {
- moveFile(r, false);
- } else if (fileRef.isDirectory()
- && (fileRef.list() != null && fileRef.list().length == 0)) {
- // if it's a directory and it doesn't exist yet, we should
- // create it
- // just in case there's no files in it
- if (!new File(new URI(r.getDataStoreReference())).exists()) {
- LOG.log(Level.FINER, "Directory: ["
- + r.getDataStoreReference()
- + "] doesn't exist: creating it");
- if (!new File(new URI(r.getDataStoreReference())).mkdirs()) {
- LOG.log(Level.WARNING, "Unable to create directory: ["
- + r.getDataStoreReference()
- + "] in local data transferer");
- }
- }
+ }
+ }
+ }
+
+ private void moveDirToProductRepo(Product product) throws IOException,
+ URISyntaxException {
+ Reference dirRef = (Reference) product.getProductReferences().get(0);
+ LOG.log(
+ Level.INFO,
+ "LocalDataTransferer: Moving Directory: "
+ + dirRef.getOrigReference() + " to "
+ + dirRef.getDataStoreReference());
+
+ // notify the file manager that we started
+ quietNotifyTransferProduct(product);
+
+ for (Iterator<Reference> i = product.getProductReferences().iterator(); i
+ .hasNext();) {
+ Reference r = i.next();
+ File fileRef = new File(new URI(r.getOrigReference()));
+
+ if (fileRef.isFile()) {
+ moveFile(r, false);
+ } else if (fileRef.isDirectory()
+ && (fileRef.list() != null && fileRef.list().length == 0)) {
+ // if it's a directory and it doesn't exist yet, we should
+ // create it
+ // just in case there's no files in it
+ if (!new File(new URI(r.getDataStoreReference())).exists()) {
+ LOG.log(Level.FINER, "Directory: [" + r.getDataStoreReference()
+ + "] doesn't exist: creating it");
+ if (!new File(new URI(r.getDataStoreReference())).mkdirs()) {
+ LOG.log(
+ Level.WARNING,
+ "Unable to create directory: ["
+ + r.getDataStoreReference()
+ + "] in local data transferer");
+ }
}
- }
+ }
+ }
- // notify the file manager that we're done
- quietNotifyProductTransferComplete(product);
+ // notify the file manager that we're done
+ quietNotifyProductTransferComplete(product);
- }
+ }
- private void moveFilesToProductRepo(Product product) throws IOException,
- URISyntaxException {
- List<Reference> refs = product.getProductReferences();
-
- // notify the file manager that we started
- quietNotifyTransferProduct(product);
-
- for (Iterator<Reference> i = refs.iterator(); i.hasNext();) {
- Reference r = (Reference) i.next();
- moveFile(r, true);
- }
-
- // notify the file manager that we're done
- quietNotifyProductTransferComplete(product);
- }
-
- private void moveFile(Reference r, boolean log) throws IOException,
- URISyntaxException {
- if (log) {
- LOG
- .log(Level.INFO, "LocalDataTransfer: Moving File: "
- + r.getOrigReference() + " to "
- + r.getDataStoreReference());
- }
- File srcFileRef = new File(new URI(r.getOrigReference()));
- File destFileRef = new File(new URI(r.getDataStoreReference()));
-
- FileUtils.copyFile(srcFileRef, destFileRef);
- }
-
- private void quietNotifyTransferProduct(Product p) {
- if (client == null) {
- LOG
- .log(Level.WARNING,
- "File Manager service not defined: this transfer will not be tracked");
- return;
- }
-
- try {
- client.transferringProduct(p);
- } catch (DataTransferException e) {
- e.printStackTrace();
- LOG.log(Level.WARNING,
- "Error notifying file manager of product transfer initiation for product: ["
- + p.getProductId() + "]: Message: "
- + e.getMessage());
- return;
- }
- }
-
- private void quietNotifyProductTransferComplete(Product p) {
- if (client == null) {
- LOG
- .log(Level.WARNING,
- "File Manager service not defined: this transfer will not be tracked");
- return;
- }
-
- try {
- client.removeProductTransferStatus(p);
- } catch (DataTransferException e) {
- e.printStackTrace();
- LOG.log(Level.WARNING,
- "Error notifying file manager of product transfer completion for product: ["
- + p.getProductId() + "]: Message: "
- + e.getMessage());
- return;
- }
- }
+ private void moveFilesToProductRepo(Product product) throws IOException,
+ URISyntaxException {
+ List<Reference> refs = product.getProductReferences();
+
+ // notify the file manager that we started
+ quietNotifyTransferProduct(product);
+
+ for (Iterator<Reference> i = refs.iterator(); i.hasNext();) {
+ Reference r = (Reference) i.next();
+ moveFile(r, true);
+ }
+
+ // notify the file manager that we're done
+ quietNotifyProductTransferComplete(product);
+ }
+
+ private void copyFilesToDir(Product product, File directory)
+ throws IOException, URISyntaxException {
+ List<Reference> refs = product.getProductReferences();
+ for (Iterator<Reference> i = refs.iterator(); i.hasNext();) {
+ Reference r = (Reference) i.next();
+ copyFile(r, directory);
+ }
+ }
+
+ private void moveFile(Reference r, boolean log) throws IOException,
+ URISyntaxException {
+ if (log) {
+ LOG.log(Level.INFO,
+ "LocalDataTransfer: Moving File: " + r.getOrigReference()
+ + " to " + r.getDataStoreReference());
+ }
+ File srcFileRef = new File(new URI(r.getOrigReference()));
+ File destFileRef = new File(new URI(r.getDataStoreReference()));
+
+ FileUtils.copyFile(srcFileRef, destFileRef);
+ }
+
+ private void copyFile(Reference r, File directory) throws IOException,
+ URISyntaxException {
+ File srcFileRef = new File(new URI(r.getDataStoreReference()));
+ LOG.log(Level.INFO,
+ "LocalDataTransfer: Copying File: " + r.getDataStoreReference()
+ + " to file:" + directory.getAbsolutePath() + "/"
+ + srcFileRef.getName());
+ FileUtils.copyFile(srcFileRef, new File(directory, srcFileRef.getName()));
+ }
+
+ private void quietNotifyTransferProduct(Product p) {
+ if (client == null) {
+ LOG.log(Level.WARNING,
+ "File Manager service not defined: this transfer will not be tracked");
+ return;
+ }
+
+ try {
+ client.transferringProduct(p);
+ } catch (DataTransferException e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "Error notifying file manager of product transfer initiation for product: ["
+ + p.getProductId() + "]: Message: " + e.getMessage());
+ return;
+ }
+ }
+
+ private void quietNotifyProductTransferComplete(Product p) {
+ if (client == null) {
+ LOG.log(Level.WARNING,
+ "File Manager service not defined: this transfer will not be tracked");
+ return;
+ }
+
+ try {
+ client.removeProductTransferStatus(p);
+ } catch (DataTransferException e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "Error notifying file manager of product transfer completion for product: ["
+ + p.getProductId() + "]: Message: " + e.getMessage());
+ return;
+ }
+ }
}
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java Fri Mar 9 07:57:06 2012
@@ -27,6 +27,7 @@ import org.apache.oodt.cas.filemgr.syste
//JDK imports
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -40,187 +41,236 @@ import java.util.logging.Logger;
* @author bfoster
* @version $Revision$
*
- * <p>
- * An implementation of the {@link DataTransfer} interface that transfers files
- * to a remote file manager over XML-RPC, using the File Manager Client.
- * </p>
+ * <p>
+ * An implementation of the {@link DataTransfer} interface that
+ * transfers files to a remote file manager over XML-RPC, using the
+ * File Manager Client.
+ * </p>
*
*/
public class RemoteDataTransferer implements DataTransfer {
- /*
- * the url pointer to the file manager that we'll remotely transfer the file
- * to
- */
- private URL fileManagerUrl = null;
-
- /*
- * the size of the chunks that files should be transferred over XML-RPC
- * using
- */
- private int chunkSize = 1024;
-
- /* our file manager client */
- private XmlRpcFileManagerClient client = null;
-
- /* our log stream */
- private static final Logger LOG = Logger.getLogger(RemoteDataTransferer.class
- .getName());
-
- /**
- *
- */
- public RemoteDataTransferer(int chunkSz) {
- this.chunkSize = chunkSz;
- }
+ /*
+ * the url pointer to the file manager that we'll remotely transfer the file
+ * to
+ */
+ private URL fileManagerUrl = null;
+
+ /*
+ * the size of the chunks that files should be transferred over XML-RPC using
+ */
+ private int chunkSize = 1024;
+
+ /* our file manager client */
+ private XmlRpcFileManagerClient client = null;
+
+ /* our log stream */
+ private static final Logger LOG = Logger
+ .getLogger(RemoteDataTransferer.class.getName());
- /*
- * (non-Javadoc)
+ /**
*
- * @see org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl(java.net.URL)
*/
- public void setFileManagerUrl(URL url) {
- try {
- client = new XmlRpcFileManagerClient(url);
- this.fileManagerUrl = url;
- LOG.log(Level.INFO, "Remote Data Transfer to: ["
- + client.getFileManagerUrl().toString() + "] enabled");
- } catch (ConnectionException e) {
- LOG.log(Level.WARNING, "Connection exception for filemgr: [" + url
- + "]");
- }
- }
+ public RemoteDataTransferer(int chunkSz) {
+ this.chunkSize = chunkSz;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl
+ * (java.net.URL)
+ */
+ public void setFileManagerUrl(URL url) {
+ try {
+ client = new XmlRpcFileManagerClient(url);
+ this.fileManagerUrl = url;
+ LOG.log(Level.INFO, "Remote Data Transfer to: ["
+ + client.getFileManagerUrl().toString() + "] enabled");
+ } catch (ConnectionException e) {
+ LOG.log(Level.WARNING, "Connection exception for filemgr: [" + url
+ + "]");
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#transferProduct(
+ * org.apache.oodt.cas.filemgr.structs.Product)
+ */
+ public void transferProduct(Product product) throws DataTransferException,
+ IOException {
+
+ if (fileManagerUrl == null) {
+ throw new DataTransferException(
+ "No file manager url specified for remote data transfer: cannot transfer product: ["
+ + product.getProductName() + "]!");
+ }
+
+ quietNotifyTransferProduct(product);
+
+ // for each file reference, transfer the file to the remote file manager
+ for (Iterator<Reference> i = product.getProductReferences().iterator(); i
+ .hasNext();) {
+ Reference r = i.next();
+ // test whether or not the reference is a directory or a file
+ File refFile = null;
+ try {
+ refFile = new File(new URI(r.getOrigReference()));
+ } catch (URISyntaxException e) {
+ LOG.log(Level.WARNING,
+ "Unable to test if reference: [" + r.getOrigReference()
+ + "] is a directory: skipping it");
+ continue;
+ }
- /*
- * (non-Javadoc)
- *
- * @see org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#transferProduct(org.apache.oodt.cas.filemgr.structs.Product)
- */
- public void transferProduct(Product product) throws DataTransferException,
- IOException {
+ if (!refFile.isDirectory()) {
+ LOG.log(Level.FINE, "Reference: [" + r.getOrigReference()
+ + "] is file: transferring it");
- if (fileManagerUrl == null) {
- throw new DataTransferException(
- "No file manager url specified for remote data transfer: cannot transfer product: ["
- + product.getProductName() + "]!");
- }
-
- quietNotifyTransferProduct(product);
-
- // for each file reference, transfer the file to the remote file manager
- for (Iterator<Reference> i = product.getProductReferences().iterator(); i
- .hasNext();) {
- Reference r = i.next();
- // test whether or not the reference is a directory or a file
- File refFile = null;
try {
- refFile = new File(new URI(r.getOrigReference()));
+ remoteTransfer(r, product);
} catch (URISyntaxException e) {
- LOG.log(Level.WARNING, "Unable to test if reference: ["
- + r.getOrigReference()
- + "] is a directory: skipping it");
- continue;
+ LOG.log(Level.WARNING,
+ "Error transferring file: [" + r.getOrigReference()
+ + "]: URISyntaxException: " + e.getMessage());
}
-
- if (!refFile.isDirectory()) {
- LOG.log(Level.FINE, "Reference: [" + r.getOrigReference()
- + "] is file: transferring it");
-
- try {
- remoteTransfer(r, product);
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING, "Error transferring file: ["
- + r.getOrigReference() + "]: URISyntaxException: "
- + e.getMessage());
- }
- } else {
- LOG.log(Level.FINE, "RemoteTransfer: skipping reference: ["
+ } else {
+ LOG.log(
+ Level.FINE,
+ "RemoteTransfer: skipping reference: ["
+ refFile.getAbsolutePath() + "] of product: ["
+ product.getProductName() + "]: ref is a directory");
- }
- }
-
- quietNotifyProductTransferComplete(product);
-
- }
-
- private void remoteTransfer(Reference reference, Product product)
- throws URISyntaxException {
- // get the file path
- File origFile = new File(new URI(reference.getOrigReference()));
- File destFile = new File(new URI(reference.getDataStoreReference()));
- String origFilePath = origFile.getAbsolutePath();
- String destFilePath = destFile.getAbsolutePath();
+ }
+ }
- // read the file in chunk by chunk
+ quietNotifyProductTransferComplete(product);
- byte[] buf = new byte[chunkSize];
+ }
- FileInputStream is = null;
-
- try {
- is = new FileInputStream(origFile);
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.oodt.cas.filemgr.datatransfer.DataTransfer#retrieveProduct(org.
+ * apache.oodt.cas.filemgr.structs.Product, java.io.File)
+ */
+ public void retrieveProduct(Product product, File directory)
+ throws DataTransferException, IOException {
+ for (Reference reference : product.getProductReferences()) {
+ FileOutputStream fOut = null;
+ try {
+ File dataStoreFile = new File(new URI(
+ reference.getDataStoreReference()));
+ File dest = new File(directory, dataStoreFile.getName());
+ fOut = new FileOutputStream(dest, false);
+ LOG.log(
+ Level.INFO,
+ "RemoteDataTransfer: Copying File: " + "fmp:"
+ + dataStoreFile.getAbsolutePath() + " to " + "file:"
+ + dest.getAbsolutePath());
+ byte[] fileData = null;
int offset = 0;
- int numBytes = 0;
-
- // remove the file if it already exists: this operation
- // is an overwrite
- if (!client.removeFile(destFilePath)) {
- LOG.log(Level.WARNING,
- "RemoteDataTransfer: attempt to perform overwrite of dest file: ["
- + destFilePath + "] failed");
+ while (true) {
+ fileData = (byte[]) client.retrieveFile(
+ dataStoreFile.getAbsolutePath(), offset, 1024);
+ if (fileData.length <= 0)
+ break;
+ fOut.write(fileData);
+ if (fileData.length < 1024)
+ break;
+ offset += 1024;
}
-
- while ((numBytes = is.read(buf, offset, chunkSize)) != -1) {
- client.transferFile(destFilePath, buf, offset, numBytes);
+ } catch (Exception e) {
+ throw new DataTransferException("", e);
+ } finally {
+ try {
+ fOut.close();
+ } catch (Exception e) {
}
- } catch (IOException e) {
- LOG.log(Level.WARNING,
- "Error opening input stream to read file to transfer: Message: "
- + e.getMessage());
- return;
- } catch (DataTransferException e) {
+ }
+ }
+ }
+
+ private void remoteTransfer(Reference reference, Product product)
+ throws URISyntaxException {
+ // get the file path
+ File origFile = new File(new URI(reference.getOrigReference()));
+ File destFile = new File(new URI(reference.getDataStoreReference()));
+ String origFilePath = origFile.getAbsolutePath();
+ String destFilePath = destFile.getAbsolutePath();
+
+ // read the file in chunk by chunk
+
+ byte[] buf = new byte[chunkSize];
+
+ FileInputStream is = null;
+
+ try {
+ is = new FileInputStream(origFile);
+ int offset = 0;
+ int numBytes = 0;
+
+ // remove the file if it already exists: this operation
+ // is an overwrite
+ if (!client.removeFile(destFilePath)) {
LOG.log(Level.WARNING,
- "DataTransferException when transfering file: ["
- + origFilePath + "] to [" + destFilePath
- + "]: Message: " + e.getMessage());
- return;
- } finally {
- if (is != null) {
- try {
- is.close();
- } catch (Exception ignore) {
- }
-
- is = null;
+ "RemoteDataTransfer: attempt to perform overwrite of dest file: ["
+ + destFilePath + "] failed");
+ }
+
+ while ((numBytes = is.read(buf, offset, chunkSize)) != -1) {
+ client.transferFile(destFilePath, buf, offset, numBytes);
+ }
+ } catch (IOException e) {
+ LOG.log(Level.WARNING,
+ "Error opening input stream to read file to transfer: Message: "
+ + e.getMessage());
+ return;
+ } catch (DataTransferException e) {
+ LOG.log(
+ Level.WARNING,
+ "DataTransferException when transfering file: [" + origFilePath
+ + "] to [" + destFilePath + "]: Message: "
+ + e.getMessage());
+ return;
+ } finally {
+ if (is != null) {
+ try {
+ is.close();
+ } catch (Exception ignore) {
}
- }
- }
- private void quietNotifyTransferProduct(Product p) {
- try {
- client.transferringProduct(p);
- } catch (DataTransferException e) {
- e.printStackTrace();
- LOG.log(Level.WARNING,
- "Error notifying file manager of product transfer initiation for product: ["
- + p.getProductId() + "]: Message: "
- + e.getMessage());
- return;
- }
- }
-
- private void quietNotifyProductTransferComplete(Product p) {
- try {
- client.removeProductTransferStatus(p);
- } catch (DataTransferException e) {
- e.printStackTrace();
- LOG.log(Level.WARNING,
- "Error notifying file manager of product transfer completion for product: ["
- + p.getProductId() + "]: Message: "
- + e.getMessage());
- return;
- }
- }
+ is = null;
+ }
+ }
+ }
+
+ private void quietNotifyTransferProduct(Product p) {
+ try {
+ client.transferringProduct(p);
+ } catch (DataTransferException e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "Error notifying file manager of product transfer initiation for product: ["
+ + p.getProductId() + "]: Message: " + e.getMessage());
+ return;
+ }
+ }
+
+ private void quietNotifyProductTransferComplete(Product p) {
+ try {
+ client.removeProductTransferStatus(p);
+ } catch (DataTransferException e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "Error notifying file manager of product transfer completion for product: ["
+ + p.getProductId() + "]: Message: " + e.getMessage());
+ return;
+ }
+ }
}
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java Fri Mar 9 07:57:06 2012
@@ -771,7 +771,32 @@ public class XmlRpcFileManager {
}
-
+ public byte[] retrieveFile(String filePath, int offset, int numBytes)
+ throws DataTransferException {
+ FileInputStream is = null;
+ try {
+ byte[] fileData = new byte[numBytes];
+ (is = new FileInputStream(filePath)).skip(offset);
+ int bytesRead = is.read(fileData);
+ if (bytesRead != -1) {
+ byte[] fileDataTruncated = new byte[bytesRead];
+ System.arraycopy(fileData, 0, fileDataTruncated, 0, bytesRead);
+ return fileDataTruncated;
+ } else {
+ return new byte[0];
+ }
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "Failed to read '" + numBytes
+ + "' bytes from file '" + filePath + "' at index '" + offset
+ + "' : " + e.getMessage(), e);
+ throw new DataTransferException("Failed to read '" + numBytes
+ + "' bytes from file '" + filePath + "' at index '" + offset
+ + "' : " + e.getMessage(), e);
+ } finally {
+ try { is.close(); } catch (Exception e) {}
+ }
+ }
+
public boolean transferFile(String filePath, byte[] fileData, int offset,
int numBytes) {
File outFile = new File(filePath);
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java Fri Mar 9 07:57:06 2012
@@ -826,6 +826,22 @@ public class XmlRpcFileManagerClient {
return success;
}
+ public byte[] retrieveFile(String filePath, int offset, int numBytes)
+ throws DataTransferException {
+ Vector<Object> argList = new Vector<Object>();
+ argList.add(filePath);
+ argList.add(new Integer(offset));
+ argList.add(new Integer(numBytes));
+
+ try {
+ return (byte[]) client.execute("filemgr.retrieveFile", argList);
+ } catch (XmlRpcException e) {
+ throw new DataTransferException(e.getMessage());
+ } catch (IOException e) {
+ throw new DataTransferException(e.getMessage());
+ }
+ }
+
public void transferFile(String filePath, byte[] fileData, int offset,
int numBytes) throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Modified: oodt/trunk/filemgr/src/main/resources/cmd-line-actions.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/resources/cmd-line-actions.xml?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/resources/cmd-line-actions.xml (original)
+++ oodt/trunk/filemgr/src/main/resources/cmd-line-actions.xml Fri Mar 9 07:57:06 2012
@@ -78,6 +78,12 @@
<bean id="luceneQuery" class="org.apache.oodt.cas.filemgr.cli.action.LuceneQueryCliAction">
<property name="description" value="Queries by parsing an Lucene-like query into a FileManager Query" />
</bean>
+ <bean id="retrieveFilesById" class="org.apache.oodt.cas.filemgr.cli.action.RetrieveFilesCliAction">
+ <property name="description" value="Retrieve a Product's files by Product ID" />
+ </bean>
+ <bean id="retrieveFilesByName" class="org.apache.oodt.cas.filemgr.cli.action.RetrieveFilesCliAction">
+ <property name="description" value="Retrieve a Product's files by Product name" />
+ </bean>
<bean id="sqlQuery" class="org.apache.oodt.cas.filemgr.cli.action.SqlQueryCliAction">
<property name="description" value="Queries by parsing an SQL-like query into a FileManager Query" />
<property name="detailedDescription">
Modified: oodt/trunk/filemgr/src/main/resources/cmd-line-options.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/resources/cmd-line-options.xml?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/resources/cmd-line-options.xml (original)
+++ oodt/trunk/filemgr/src/main/resources/cmd-line-options.xml Fri Mar 9 07:57:06 2012
@@ -89,6 +89,10 @@
p:option-ref="dumpMetadata" p:required="false" />
<bean class="org.apache.oodt.cas.cli.option.GroupSubOption"
p:option-ref="luceneQuery" p:required="false" />
+ <bean class="org.apache.oodt.cas.cli.option.GroupSubOption"
+ p:option-ref="retrieveFilesById" p:required="false" />
+ <bean class="org.apache.oodt.cas.cli.option.GroupSubOption"
+ p:option-ref="retrieveFilesByName" p:required="false" />
</list>
</property>
</bean>
@@ -772,6 +776,94 @@
</property>
</bean>
+ <!-- retrieveFiles Options -->
+ <bean id="retrieveFilesById" class="org.apache.oodt.cas.cli.option.ActionCmdLineOption"
+ p:isSubOption="true">
+ <property name="shortOption" value="rfbyid" />
+ <property name="longOption" value="retrieveFilesById" />
+ <property name="description" value="Triggers retrieveFilesById Action" />
+ <property name="hasArgs" value="false" />
+ <property name="staticArgs">
+ <list>
+ <value>retrieveFilesById</value>
+ </list>
+ </property>
+ <property name="requirementRules">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesById" p:relation="REQUIRED" />
+ </list>
+ </property>
+ </bean>
+
+ <bean id="retrieveFilesByName" class="org.apache.oodt.cas.cli.option.ActionCmdLineOption"
+ p:isSubOption="true">
+ <property name="shortOption" value="rfbyn" />
+ <property name="longOption" value="retrieveFilesByName" />
+ <property name="description" value="Triggers retrieveFilesByName Action" />
+ <property name="hasArgs" value="false" />
+ <property name="staticArgs">
+ <list>
+ <value>retrieveFilesByName</value>
+ </list>
+ </property>
+ <property name="requirementRules">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesByName" p:relation="REQUIRED" />
+ </list>
+ </property>
+ </bean>
+
+ <bean id="transferer" class="org.apache.oodt.cas.cli.option.AdvancedCmdLineOption">
+ <property name="shortOption" value="trans" />
+ <property name="longOption" value="transferer" />
+ <property name="description" value="Factory for creating DataTransfer which will perform the file transfer(s)" />
+ <property name="type" value="org.apache.oodt.cas.filemgr.datatransfer.DataTransferFactory" />
+ <property name="hasArgs" value="true" />
+ <property name="argsDescription" value="transfer factory class" />
+ <property name="requirementRules">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesById" p:relation="REQUIRED" />
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesByName" p:relation="REQUIRED" />
+ </list>
+ </property>
+ <property name="handler">
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToActionHandler">
+ <property name="applyToActions">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="retrieveFilesById" p:methodName="setDataTransferFactory" />
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="retrieveFilesByName" p:methodName="setDataTransferFactory" />
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+
+ <bean id="destination" class="org.apache.oodt.cas.cli.option.AdvancedCmdLineOption">
+ <property name="shortOption" value="dest" />
+ <property name="longOption" value="destination" />
+ <property name="description" value="Directory to transfer Product file to" />
+ <property name="type" value="java.io.File" />
+ <property name="hasArgs" value="true" />
+ <property name="argsDescription" value="directory" />
+ <property name="requirementRules">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesById" p:relation="REQUIRED" />
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesByName" p:relation="REQUIRED" />
+ </list>
+ </property>
+ <property name="handler">
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToActionHandler" />
+ </property>
+ </bean>
+
<!-- Options used for multiple Actions -->
<bean id="productId" class="org.apache.oodt.cas.cli.option.AdvancedCmdLineOption">
<property name="shortOption" value="pid" />
@@ -789,6 +881,8 @@
p:actionName="deleteProductById" p:relation="REQUIRED" />
<bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
p:actionName="dumpMetadata" p:relation="REQUIRED" />
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesById" p:relation="REQUIRED" />
</list>
</property>
<property name="handler">
@@ -812,6 +906,8 @@
p:actionName="getProductByName" p:relation="REQUIRED" />
<bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
p:actionName="deleteProductByName" p:relation="REQUIRED" />
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="retrieveFilesByName" p:relation="REQUIRED" />
</list>
</property>
<property name="handler">
Modified: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java (original)
+++ oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java Fri Mar 9 07:57:06 2012
@@ -365,4 +365,30 @@ public class TestFileManagerCli extends
assertEquals(Lists.newArrayList(reducedProductTypes.split(" ")), complexQuery.getReducedProductTypeNames());
assertEquals(sortBy, complexQuery.getSortByMetKey());
}
+
+ public void testRetrieveFilesById() {
+ String productId = "TestProductId";
+ String destination = "/tmp/toDir";
+ String transferer = "org.apache.oodt.cas.filemgr.datatransfer.InPlaceDataTransferFactory";
+ cmdLineUtility
+ .run(("--url http://localhost:9000 --operation --retrieveFilesById"
+ + " --productId " + productId + " --destination " + destination
+ + " --transferer " + transferer).split(" "));
+ MethodCallDetails methodCallDetails = client.getLastMethodCallDetails();
+ assertEquals("getProductById", methodCallDetails.getMethodName());
+ assertEquals(productId, methodCallDetails.getArgs().get(0));
+ }
+
+ public void testRetrieveFilesByName() {
+ String productName = "TestProductName";
+ String destination = "/tmp/toDir";
+ String transferer = "org.apache.oodt.cas.filemgr.datatransfer.InPlaceDataTransferFactory";
+ cmdLineUtility
+ .run(("--url http://localhost:9000 --operation --retrieveFilesByName"
+ + " --productName " + productName + " --destination " + destination
+ + " --transferer " + transferer).split(" "));
+ MethodCallDetails methodCallDetails = client.getLastMethodCallDetails();
+ assertEquals("getProductByName", methodCallDetails.getMethodName());
+ assertEquals(productName, methodCallDetails.getArgs().get(0));
+ }
}
Modified: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestLocalDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestLocalDataTransferer.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestLocalDataTransferer.java (original)
+++ oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestLocalDataTransferer.java Fri Mar 9 07:57:06 2012
@@ -14,72 +14,89 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
package org.apache.oodt.cas.filemgr.datatransfer;
+//Apache imports
+import org.apache.commons.io.FileUtils;
+
//OODT imports
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
//JDK imports
import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
//Junit imports
import junit.framework.TestCase;
/**
- * @author mattmann
- * @version $Revision$
- *
- * <p>
- * Describe your class here
- * </p>.
+ * Test class for {@link LocalDataTransferer}.
+ *
+ * @author mattmann (Chris Mattmann)
+ * @author bfoster (Brian Foster)
*/
public class TestLocalDataTransferer extends TestCase {
- private LocalDataTransferer transfer;
-
- private String productOrigLoc;
-
- private String productExpectedLoc;
-
- public TestLocalDataTransferer() {
- transfer = (LocalDataTransferer) new LocalDataTransferFactory()
- .createDataTransfer();
-
- try {
- File tempFileSrc = new File("./src/testdata/test.txt");
- productOrigLoc = tempFileSrc.getCanonicalPath();
- File tempFileDest = File.createTempFile("foo", ".txt");
- productExpectedLoc = tempFileDest.getAbsolutePath();
- // now delete the file so that it can be created
- assertTrue(tempFileDest.delete());
-
- } catch (Exception e) {
- fail(e.getMessage());
- }
- }
-
- public void testTransfer() {
- Product testProduct = Product.getDefaultFlatProduct("test",
- "urn:oodt:GenericFile");
- testProduct.getProductReferences().add(
- new Reference("file://" + productOrigLoc, "file://"
- + productExpectedLoc, new File(productOrigLoc).length()));
-
- System.out.println("Added ref: ["+((Reference)testProduct.getProductReferences().get(0))+"]");
-
- try {
- transfer.transferProduct(testProduct);
- } catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
-
- // assert that it didn't transfer the file anywhere
- assertTrue(new File(productExpectedLoc).exists());
- new File(productExpectedLoc).deleteOnExit();
- }
+ private LocalDataTransferer transfer;
+ private File origFile;
+ private File testDir;
+ private File repoDir;
+ private File repoFile;
+ private File destDir;
+ private File destFile;
+
+ public void setUp() throws Exception {
+ transfer = (LocalDataTransferer) new LocalDataTransferFactory()
+ .createDataTransfer();
+ origFile = new File("./src/testdata/test.txt");
+ File testFile = File.createTempFile("test", ".txt");
+ testDir = new File(testFile.getParentFile(), UUID.randomUUID().toString());
+ repoDir = new File(testDir, "repo");
+ if (!repoDir.mkdirs()) {
+ throw new Exception("Failed to create repo directory!");
+ }
+ repoFile = new File(repoDir, "test.txt");
+ destDir = new File(testDir, "dest");
+ if (!destDir.mkdirs()) {
+ throw new Exception("Failed to create destination directory!");
+ }
+ destFile = new File(destDir, "test.txt");
+ }
+
+ public void tearDown() throws Exception {
+ FileUtils.forceDelete(testDir);
+ }
+
+ public void testTransferAndRetrieve() throws DataTransferException, IOException {
+ Product testProduct = createDummyProduct();
+
+ // Test transfer.
+ transfer.transferProduct(testProduct);
+
+ // Check that file was successfully transfered.
+ assertTrue("Repo file does not exist", repoFile.exists());
+ assertTrue("Repo file does not have the same contents as orig file",
+ FileUtils.contentEquals(origFile, repoFile));
+
+ // Test retrieve
+ transfer.retrieveProduct(testProduct, destDir);
+
+ // Check that file was successfully transfered.
+ assertTrue("Destination file does not exist", destFile.exists());
+ assertTrue("Destination file does not have the same contents as orig file",
+ FileUtils.contentEquals(origFile, destFile));
+ }
+
+ private Product createDummyProduct() {
+ Product testProduct = Product.getDefaultFlatProduct("test",
+ "urn:oodt:GenericFile");
+ testProduct.getProductReferences().add(
+ new Reference(origFile.toURI().toString(), new File(repoDir,
+ "test.txt").toURI().toString(), origFile.length()));
+ return testProduct;
+ }
}
Modified: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/system/MockXmlRpcFileManagerClient.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/system/MockXmlRpcFileManagerClient.java?rev=1298728&r1=1298727&r2=1298728&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/system/MockXmlRpcFileManagerClient.java (original)
+++ oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/system/MockXmlRpcFileManagerClient.java Fri Mar 9 07:57:06 2012
@@ -30,6 +30,7 @@ import org.apache.oodt.cas.filemgr.struc
import org.apache.oodt.cas.filemgr.structs.Query;
import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery;
import org.apache.oodt.cas.filemgr.structs.query.QueryResult;
import org.apache.oodt.cas.metadata.Metadata;
@@ -118,6 +119,14 @@ public class MockXmlRpcFileManagerClient
}
@Override
+ public byte[] retrieveFile(String filePath, int offset, int numBytes)
+ throws DataTransferException {
+ lastMethodCallDetails = new MethodCallDetails("removeFile",
+ Lists.newArrayList((Object) filePath, offset, numBytes));
+ return new byte[0];
+ }
+
+ @Override
public FileTransferStatus getCurrentFileTransfer() {
lastMethodCallDetails = new MethodCallDetails("getCurrentFileTransfer",
Lists.newArrayList());