You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ra...@apache.org on 2011/08/15 19:24:09 UTC
svn commit: r1157916 - in /incubator/airavata/trunk/modules:
commons/gfac-schema/src/main/java/org/apache/airavata/core/gfac/type/
gfac-core/ gfac-core/src/main/java/org/apache/airavata/core/gfac/external/
gfac-core/src/main/java/org/apache/airavata/co...
Author: raminder
Date: Mon Aug 15 17:24:09 2011
New Revision: 1157916
URL: http://svn.apache.org/viewvc?rev=1157916&view=rev
Log:
applied patch for AIRAVATA-77 by Patnachai
Modified:
incubator/airavata/trunk/modules/commons/gfac-schema/src/main/java/org/apache/airavata/core/gfac/type/DataType.java
incubator/airavata/trunk/modules/gfac-core/pom.xml
incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/external/GridFtp.java
incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImpl.java
incubator/airavata/trunk/modules/gfac-core/src/main/resources/service.properties
incubator/airavata/trunk/modules/gfac-core/src/test/resources/service.properties
Modified: incubator/airavata/trunk/modules/commons/gfac-schema/src/main/java/org/apache/airavata/core/gfac/type/DataType.java
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/commons/gfac-schema/src/main/java/org/apache/airavata/core/gfac/type/DataType.java?rev=1157916&r1=1157915&r2=1157916&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/commons/gfac-schema/src/main/java/org/apache/airavata/core/gfac/type/DataType.java (original)
+++ incubator/airavata/trunk/modules/commons/gfac-schema/src/main/java/org/apache/airavata/core/gfac/type/DataType.java Mon Aug 15 17:24:09 2011
@@ -26,5 +26,6 @@ public enum DataType{
Double,
Integer,
Float,
- Boolean
+ Boolean,
+ File
}
\ No newline at end of file
Modified: incubator/airavata/trunk/modules/gfac-core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/pom.xml?rev=1157916&r1=1157915&r2=1157916&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/pom.xml (original)
+++ incubator/airavata/trunk/modules/gfac-core/pom.xml Mon Aug 15 17:24:09 2011
@@ -46,6 +46,11 @@
<type>jar</type>
<version>cog-4_1_6</version>
</dependency>
+ <dependency>
+ <groupId>cog-4_1_6_rc2</groupId>
+ <artifactId>cryptix32</artifactId>
+ <version>cog-4_1_6</version>
+ </dependency>
<dependency>
<groupId>cog-4_1_6_rc2</groupId>
<artifactId>cryptix-asn1</artifactId>
Modified: incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/external/GridFtp.java
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/external/GridFtp.java?rev=1157916&r1=1157915&r2=1157916&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/external/GridFtp.java (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/external/GridFtp.java Mon Aug 15 17:24:09 2011
@@ -25,18 +25,37 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import org.apache.airavata.core.gfac.exception.GfacException;
+import org.apache.airavata.core.gfac.type.host.GlobusHost;
import org.apache.airavata.core.gfac.utils.ContactInfo;
+import org.apache.airavata.core.gfac.utils.GfacUtils;
import org.globus.ftp.DataChannelAuthentication;
+import org.globus.ftp.DataSource;
+import org.globus.ftp.DataSourceStream;
import org.globus.ftp.GridFTPClient;
+import org.globus.ftp.HostPort;
+import org.globus.ftp.Marker;
+import org.globus.ftp.MarkerListener;
+import org.globus.ftp.Session;
import org.globus.ftp.exception.ClientException;
import org.globus.ftp.exception.ServerException;
import org.globus.gsi.gssapi.auth.HostAuthorization;
import org.ietf.jgss.GSSCredential;
public class GridFtp {
+
+ public static final String GSIFTP_SCHEME = "gsiftp";
+
+ /**
+ * Make directory at remote location
+ *
+ * @param destURI
+ * @param gssCred
+ * @throws GfacException
+ */
public void makeDir(URI destURI, GSSCredential gssCred) throws GfacException {
GridFTPClient destClient = null;
try {
@@ -81,7 +100,60 @@ public class GridFtp {
}
}
- public String readRemoteFile(URI destURI, GSSCredential gsCredential, File localFile) throws GfacException {
+ /**
+ * Upload file from stream
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param localFile
+ * @throws GfacException
+ */
+ public void updateFile(URI destURI, GSSCredential gsCredential, InputStream io) throws GfacException {
+ GridFTPClient ftpClient = null;
+
+ try {
+ ContactInfo contactInfo = new ContactInfo(destURI.getHost(), destURI.getPort());
+ String remoteFile = destURI.getPath();
+
+ ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
+ ftpClient.setAuthorization(new HostAuthorization("host"));
+ ftpClient.authenticate(gsCredential);
+ ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+
+ System.out.println("the remote file is " + remoteFile);
+
+ ftpClient.put(remoteFile, new DataSourceStream(io), new MarkerListener() {
+ @Override
+ public void markerArrived(Marker marker) {
+ }
+ });
+
+ } catch (ServerException e) {
+ e.printStackTrace();
+ } catch (ClientException e) {
+ e.printStackTrace();
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (ftpClient != null)
+ try {
+ ftpClient.close();
+ } catch (Exception e) {
+ // no op
+ }
+ }
+ }
+ /**
+ * Upload file to remote location
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param localFile
+ * @throws GfacException
+ */
+ public void updateFile(URI destURI, GSSCredential gsCredential, File localFile) throws GfacException {
GridFTPClient ftpClient = null;
try {
@@ -93,6 +165,81 @@ public class GridFtp {
ftpClient.authenticate(gsCredential);
ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+ System.out.println("the local temp file is " + localFile);
+ System.out.println("the remote file is " + remoteFile);
+
+ ftpClient.put(localFile, remoteFile, false);
+ } catch (ServerException e) {
+ e.printStackTrace();
+ } catch (ClientException e) {
+ e.printStackTrace();
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (ftpClient != null)
+ try {
+ ftpClient.close();
+ } catch (Exception e) {
+ // no op
+ }
+ }
+ }
+
+ /**
+ * Download File from remote location
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param localFile
+ * @throws GfacException
+ */
+ public void downloadFile(URI destURI, GSSCredential gsCredential, File localFile) throws GfacException {
+ GridFTPClient ftpClient = null;
+
+ try {
+ ContactInfo contactInfo = new ContactInfo(destURI.getHost(), destURI.getPort());
+ String remoteFile = destURI.getPath();
+
+ ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
+ ftpClient.setAuthorization(new HostAuthorization("host"));
+ ftpClient.authenticate(gsCredential);
+ ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+
+ System.out.println("the local temp file is " + localFile);
+ System.out.println("the remote file is " + remoteFile);
+
+ ftpClient.get(remoteFile, localFile);
+ } catch (ServerException e) {
+ e.printStackTrace();
+ } catch (ClientException e) {
+ e.printStackTrace();
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (ftpClient != null)
+ try {
+ ftpClient.close();
+ } catch (Exception e) {
+ // no op
+ }
+ }
+ }
+
+ /**
+ * Stream remote file
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param localFile
+ * @return
+ * @throws GfacException
+ */
+ public String readRemoteFile(URI destURI, GSSCredential gsCredential, File localFile) throws GfacException {
+ try {
File localTempfile;
if (localFile == null) {
localTempfile = File.createTempFile("stderr", "err");
@@ -100,10 +247,8 @@ public class GridFtp {
localTempfile = localFile;
}
- System.out.println("the local temp file is " + localTempfile);
- System.out.println("the remote file is " + remoteFile);
+ downloadFile(destURI, gsCredential, localTempfile);
- ftpClient.get(remoteFile, localTempfile);
FileInputStream instream = new FileInputStream(localTempfile);
int size = instream.available();
byte[] buf = new byte[size];
@@ -111,23 +256,81 @@ public class GridFtp {
instream.read(buf);
return new String(buf);
- } catch (ServerException e) {
- e.printStackTrace();
- } catch (ClientException e) {
- e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * Transfer data from one gridFTp endpoint to another gridFTP endpoint
+ *
+ * @param srchost
+ * @param desthost
+ * @param gssCred
+ * @param srcActive
+ * @throws ServerException
+ * @throws ClientException
+ * @throws IOException
+ */
+ public void transfer(URI srchost, URI desthost, GSSCredential gssCred, boolean srcActive) throws ServerException,
+ ClientException, IOException {
+ GridFTPClient destClient = null;
+ GridFTPClient srcClient = null;
+
+ try {
+ destClient = new GridFTPClient(desthost.getHost(), desthost.getPort());
+ destClient.setAuthorization(new HostAuthorization("host"));
+ destClient.authenticate(gssCred);
+ destClient.setType(Session.TYPE_IMAGE);
+
+ srcClient = new GridFTPClient(srchost.getHost(), srchost.getPort());
+ srcClient.setAuthorization(new HostAuthorization("host"));
+ srcClient.authenticate(gssCred);
+ srcClient.setType(Session.TYPE_IMAGE);
+
+ if (srcActive) {
+ HostPort hp = destClient.setPassive();
+ srcClient.setActive(hp);
+ } else {
+ HostPort hp = srcClient.setPassive();
+ destClient.setActive(hp);
+ }
+
+ /**
+ * Transfer a file. The transfer() function blocks until the
+ * transfer is complete.
+ */
+ srcClient.transfer(srchost.getPath(), destClient, desthost.getPath(), false, null);
+ if (srcClient.getSize(srchost.getPath()) == destClient.getSize(desthost.getPath())) {
+ System.out.println("CHECK SUM OK");
+ } else {
+ System.out.println("CHECK SUM FAIL");
+ }
+
+ } catch (ServerException e) {
+ throw e;
+ } catch (ClientException e) {
+ throw e;
+ } catch (IOException e) {
+ throw e;
} finally {
- if (ftpClient != null)
+ if (destClient != null) {
try {
- ftpClient.close();
+ destClient.close();
+ } catch (Exception e) {
+ // no op
+ }
+ }
+ if (srcClient != null) {
+ try {
+ srcClient.close();
} catch (Exception e) {
// no op
}
+ }
}
- return null;
}
-
}
Modified: incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImpl.java
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImpl.java?rev=1157916&r1=1157915&r2=1157916&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImpl.java (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImpl.java Mon Aug 15 17:24:09 2011
@@ -38,6 +38,8 @@ import org.apache.airavata.core.gfac.ext
import org.apache.airavata.core.gfac.extension.PostExecuteChain;
import org.apache.airavata.core.gfac.extension.PreExecuteChain;
import org.apache.airavata.core.gfac.scheduler.Scheduler;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
/**
* This generic service implementation will load Registry service and Data Catalog from property file. It selects
@@ -46,6 +48,8 @@ import org.apache.airavata.core.gfac.sch
*/
public class PropertiesBasedServiceImpl extends AbstractSimpleService {
+ private static Log log = LogFactory.getLog(PropertiesBasedServiceImpl.class);
+
private static final String FILENAME = "service.properties";
public static final String SCHEDULER_CLASS = "scheduler.class";
public static final String DATA_CHAIN_CLASS = "datachain.classes";
@@ -63,7 +67,7 @@ public class PropertiesBasedServiceImpl
private PostExecuteChain[] postChain;
private DataServiceChain[] dataChain;
- private Registry registryService;
+ private Registry registryService;
/*
* (non-Javadoc)
@@ -154,6 +158,8 @@ public class PropertiesBasedServiceImpl
}
this.scheduler = spiClass.newInstance();
+
+ log.info("Scheduler:" + className + " is loaded");
} catch (ClassNotFoundException ex) {
throw new GfacException("Scheduler " + className + " not found", ex);
@@ -245,8 +251,7 @@ public class PropertiesBasedServiceImpl
T[] chain = (T[]) Array.newInstance(type, classNames.length);
for (int i = 0; i < classNames.length; i++) {
- String className = classNames[i];
- System.out.println(className);
+ String className = classNames[i].trim();
try {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
@@ -259,6 +264,9 @@ public class PropertiesBasedServiceImpl
}
chain[i] = (T) spiClass.newInstance();
+
+ log.info(type.getName() + " : " + className + " is loaded");
+
} catch (ClassNotFoundException ex) {
ex.printStackTrace();
// TODO proper throw out
Modified: incubator/airavata/trunk/modules/gfac-core/src/main/resources/service.properties
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/main/resources/service.properties?rev=1157916&r1=1157915&r2=1157916&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/main/resources/service.properties (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/main/resources/service.properties Mon Aug 15 17:24:09 2011
@@ -25,5 +25,5 @@ jcr.class=org.apache.jackrabbit.core.Rep
scheduler.class= org.apache.airavata.core.gfac.scheduler.impl.SchedulerImpl
datachain.classes=org.apache.airavata.core.gfac.extension.data.RegistryDataService
-#prechain.classes=
-#postchain.classes
+prechain.classes=org.apache.airavata.core.gfac.extension.pre.GridFtpInputStaging, org.apache.airavata.core.gfac.extension.pre.HttpInputStaging
+postchain.classes=org.apache.airavata.core.gfac.extension.post.GridFtpOutputStaging
Modified: incubator/airavata/trunk/modules/gfac-core/src/test/resources/service.properties
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/test/resources/service.properties?rev=1157916&r1=1157915&r2=1157916&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/test/resources/service.properties (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/test/resources/service.properties Mon Aug 15 17:24:09 2011
@@ -19,12 +19,11 @@
#
#
-
jcr.class=org.apache.jackrabbit.core.RepositoryFactoryImpl
#jcr.class=org.apache.jackrabbit.rmi.repository.RmiRepositoryFactory
#org.apache.jackrabbit.repository.uri=http://localhost:8080/rmi
scheduler.class= org.apache.airavata.core.gfac.scheduler.impl.SchedulerImpl
datachain.classes=org.apache.airavata.core.gfac.extension.data.RegistryDataService
-#prechain.classes=
-#postchain.classes
+prechain.classes=org.apache.airavata.core.gfac.extension.pre.GridFtpInputStaging, org.apache.airavata.core.gfac.extension.pre.HttpInputStaging
+postchain.classes=org.apache.airavata.core.gfac.extension.post.GridFtpOutputStaging