You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/05/01 20:29:24 UTC
[6/9] Separating gfac-monitoring implementation
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
deleted file mode 100644
index 0c9dca8..0000000
--- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.job;
-
-import com.google.common.eventbus.EventBus;
-import com.google.common.eventbus.Subscribe;
-import org.apache.airavata.commons.gfac.type.HostDescription;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
-import org.apache.airavata.gfac.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.UserMonitorData;
-import org.apache.airavata.gfac.monitor.event.MonitorPublisher;
-import org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor;
-import org.apache.airavata.gfac.monitor.state.JobStatusChangeRequest;
-import org.apache.airavata.schemas.gfac.GsisshHostType;
-import org.junit.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-public class QstatMonitorTestWithMyProxyAuth {
- private String myProxyUserName;
- private String myProxyPassword;
- private String certificateLocation;
- private String pbsFilePath;
- private String workingDirectory;
- private HostDescription hostDescription;
- private MonitorPublisher monitorPublisher;
- private BlockingQueue<UserMonitorData> pullQueue;
- private Thread monitorThread;
-
- @org.testng.annotations.BeforeClass
- public void setUp() throws Exception {
- System.setProperty("myproxy.username", "ogce");
- System.setProperty("myproxy.password", "OpenGwy14");
- System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh");
- System.setProperty("gsi.working.directory", "/home/ogce");
- System.setProperty("trusted.cert.location", "/Users/lahirugunathilake/Downloads/certificates");
- myProxyUserName = System.getProperty("myproxy.username");
- myProxyPassword = System.getProperty("myproxy.password");
- workingDirectory = System.getProperty("gsi.working.directory");
- certificateLocation = System.getProperty("trusted.cert.location");
- if (myProxyUserName == null || myProxyPassword == null || workingDirectory == null) {
- System.out.println(">>>>>> Please run tests with my proxy user name and password. " +
- "E.g :- mvn clean install -Dmyproxy.username=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<");
- throw new Exception("Need my proxy user name password to run tests.");
- }
-
- monitorPublisher = new MonitorPublisher(new EventBus());
- class InnerClassQstat {
-
- @Subscribe
- private void getStatus(JobStatusChangeRequest status) {
- Assert.assertNotNull(status);
- System.out.println(status.getState().toString());
- monitorThread.interrupt();
- }
- }
- monitorPublisher.registerListener(this);
- pullQueue = new LinkedBlockingQueue<UserMonitorData>();
- QstatMonitor qstatMonitor = new
- QstatMonitor(pullQueue, monitorPublisher);
- try {
- monitorThread = (new Thread(qstatMonitor));
- monitorThread.start();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- hostDescription = new HostDescription(GsisshHostType.type);
- hostDescription.getType().setHostAddress("trestles.sdsc.edu");
- hostDescription.getType().setHostName("gsissh-gordon");
- ((GsisshHostType) hostDescription.getType()).setPort(22);
- ((GsisshHostType)hostDescription.getType()).setInstalledPath("/opt/torque/bin/");
- }
-
- @Test
- public void testQstatMonitor() throws SSHApiException {
- /* now have to submit a job to some machine and add that job to the queue */
- //Create authentication
- GSIAuthenticationInfo authenticationInfo
- = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
- 7512, 17280000, certificateLocation);
-
- // Server info
- ServerInfo serverInfo = new ServerInfo("ogce", hostDescription.getType().getHostAddress());
-
-
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
-
-
- // Execute command
- System.out.println("Target PBS file path: " + workingDirectory);
- // constructing the job object
- JobDescriptor jobDescriptor = new JobDescriptor();
- jobDescriptor.setWorkingDirectory(workingDirectory);
- jobDescriptor.setShellName("/bin/bash");
- jobDescriptor.setJobName("GSI_SSH_SLEEP_JOB");
- jobDescriptor.setExecutablePath("/bin/echo");
- jobDescriptor.setAllEnvExport(true);
- jobDescriptor.setMailOptions("n");
- jobDescriptor.setStandardOutFile(workingDirectory + File.separator + "application.out");
- jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err");
- jobDescriptor.setNodes(1);
- jobDescriptor.setProcessesPerNode(1);
- jobDescriptor.setQueueName("normal");
- jobDescriptor.setMaxWallTime("60");
- jobDescriptor.setAcountString("sds128");
- List<String> inputs = new ArrayList<String>();
- jobDescriptor.setOwner("ogce");
- inputs.add("Hello World");
- jobDescriptor.setInputValues(inputs);
- //finished construction of job object
- System.out.println(jobDescriptor.toXML());
- for (int i = 0; i < 1; i++) {
- String jobID = pbsCluster.submitBatchJob(jobDescriptor);
- System.out.println("Job submitted successfully, Job ID: " + jobID);
- MonitorID monitorID = new MonitorID(hostDescription, jobID,null,null,null, "ogce");
- monitorID.setAuthenticationInfo(authenticationInfo);
- try {
- org.apache.airavata.gfac.monitor.util.CommonUtils.addMonitortoQueue(pullQueue, monitorID);
- } catch (Exception e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- }
- try {
-
- monitorThread.join();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Subscribe
- public void testCaseShutDown(JobStatusChangeRequest status) {
- Assert.assertNotNull(status.getState());
- monitorThread.stop();
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestGlobalHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestGlobalHandler.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestGlobalHandler.java
new file mode 100644
index 0000000..c245140
--- /dev/null
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestGlobalHandler.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.job;
+
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+
+import java.util.Map;
+
+public class TestGlobalHandler extends AbstractHandler {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestInHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestInHandler.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestInHandler.java
new file mode 100644
index 0000000..1c41d8e
--- /dev/null
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestInHandler.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.job;
+
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+
+import java.util.Map;
+
+public class TestInHandler extends AbstractHandler {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestOutHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestOutHandler.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestOutHandler.java
new file mode 100644
index 0000000..2ebf3d8
--- /dev/null
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestOutHandler.java
@@ -0,0 +1,32 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.job;
+
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+
+import java.util.Map;
+
+public class TestOutHandler extends AbstractHandler {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestProvider.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestProvider.java
new file mode 100644
index 0000000..3de320a
--- /dev/null
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestProvider.java
@@ -0,0 +1,47 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.job;
+
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+
+import java.util.Map;
+
+public class TestProvider extends AbstractProvider {
+
+ public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestThreadedHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestThreadedHandler.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestThreadedHandler.java
new file mode 100644
index 0000000..08f9c33
--- /dev/null
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/TestThreadedHandler.java
@@ -0,0 +1,39 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.job;
+
+import junit.framework.Assert;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.handler.ThreadedHandler;
+
+import java.util.Map;
+
+public class TestThreadedHandler extends ThreadedHandler {
+
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+ }
+
+ public void run() {
+ System.out.println("Invoking TestThreadedHandler");
+ Assert.assertTrue(true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-core/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/resources/gfac-config.xml b/modules/gfac/gfac-core/src/test/resources/gfac-config.xml
index 7c7e704..da92462 100644
--- a/modules/gfac/gfac-core/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-core/src/test/resources/gfac-config.xml
@@ -9,82 +9,57 @@
WARRANTIES OR CONDITIONS OF ANY ~ KIND, either express or implied. See the
License for the ~ specific language governing permissions and limitations
~ under the License. -->
-
+
<GFac>
+ <DaemonHandlers>
+ <Handler class="org.apache.airavata.job.TestThreadedHandler">
+ </Handler>
+ </DaemonHandlers>
<GlobalHandlers>
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.AppDescriptorCheckHandler">
+ <Handler class="org.apache.airavata.job.TestGlobalHandler">
<property name="name" value="value"/>
</Handler>
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
- </InHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
- <property name="name" value="value"/>
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler">
- <property name="name" value="value"/>
- </Handler>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Provider>
-
- <Provider class="org.apache.airavata.gfac.ec2.EC2Provider" host="org.apache.airavata.schemas.gfac.impl.Ec2HostTypeImpl">
- <InHandlers/>
- <OutHandlers/>
- </Provider>
-
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
- </InHandlers>
- <OutHandlers/>
- </Provider>
- <Application name="UltraScan">
+ <Application name="UltraScan" class="org.apache.airavata.job.TestProvider">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.job.TestInHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.job.TestOutHandler"/>
</OutHandlers>
</Application>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHOutputHandler"/>
- </OutHandlers>
- </Provider>
+
+ <Provider class="org.apache.airavata.job.TestProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl" executionMode="async">
+ <InHandlers>
+ <Handler class="org.apache.airavata.job.TestInHandler">
+ <property name="name1" value="value1"/>
+ </Handler>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPInputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw98.iu.xsede.org"/>
+ <property name="inputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ </InHandlers>
+ <OutHandlers>
+ <Handler class="org.apache.airavata.job.TestOutHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw111.iu.xsede.org"/>
+ <property name="outputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ </OutHandlers>
+ </Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-ec2/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ec2/pom.xml b/modules/gfac/gfac-ec2/pom.xml
index 6851658..344792a 100644
--- a/modules/gfac/gfac-ec2/pom.xml
+++ b/modules/gfac/gfac-ec2/pom.xml
@@ -18,8 +18,8 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>airavata-gfac-ec2</artifactId>
- <name>Airavata GFac Core</name>
- <description>The core GFAC functionality independent from any webservice implementation.</description>
+ <name>Airavata GFac EC2 Implementation</name>
+ <description>The core GFAC EC2 implementation using the framework features</description>
<url>http://airavata.apache.org/</url>
<dependencies>
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-ec2/src/main/java/org/apache/airavata/gfac/ec2/EC2Provider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ec2/src/main/java/org/apache/airavata/gfac/ec2/EC2Provider.java b/modules/gfac/gfac-ec2/src/main/java/org/apache/airavata/gfac/ec2/EC2Provider.java
index e773fe6..1ade760 100644
--- a/modules/gfac/gfac-ec2/src/main/java/org/apache/airavata/gfac/ec2/EC2Provider.java
+++ b/modules/gfac/gfac-ec2/src/main/java/org/apache/airavata/gfac/ec2/EC2Provider.java
@@ -36,8 +36,8 @@ import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.ec2.util.AmazonEC2Util;
import org.apache.airavata.gfac.ec2.util.EC2ProviderUtil;
+import org.apache.airavata.gfac.provider.AbstractProvider;
import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.provider.impl.AbstractProvider;
import org.apache.airavata.gfac.provider.utils.ProviderUtils;
import org.apache.airavata.gfac.utils.GFacUtils;
import org.apache.airavata.model.workspace.experiment.JobState;
@@ -131,7 +131,7 @@ public class EC2Provider extends AbstractProvider {
new BasicAWSCredentials(amazonSecurityContext.getAccessKey(), amazonSecurityContext.getSecretKey());
AmazonEC2Client ec2client = new AmazonEC2Client(credential);
taskID = jobExecutionContext.getTaskData().getTaskID();
- GFacUtils.saveJobStatus(details, JobState.SETUP, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.SETUP);
initEc2Environment(jobExecutionContext, ec2client);
checkConnection(instance, ec2client);
}
@@ -184,7 +184,7 @@ public class EC2Provider extends AbstractProvider {
// Authenticate
int result = sshClient.authenticate(publicKeyAuth);
if(result== AuthenticationProtocolState.FAILED) {
- GFacUtils.saveJobStatus(details, JobState.FAILED, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.FAILED);
throw new GFacProviderException("The authentication failed");
} else if(result==AuthenticationProtocolState.PARTIAL) {
throw new GFacProviderException("The authentication succeeded but another"
@@ -192,12 +192,12 @@ public class EC2Provider extends AbstractProvider {
} else if(result==AuthenticationProtocolState.COMPLETE) {
log.info("ssh client authentication is complete...");
}
- GFacUtils.saveJobStatus(details, JobState.SUBMITTED, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.SUBMITTED);
SessionChannelClient session = sshClient.openSessionChannel();
log.info("ssh session successfully opened...");
session.requestPseudoTerminal("vt100", 80, 25, 0, 0, "");
session.startShell();
- GFacUtils.saveJobStatus(details, JobState.ACTIVE, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.ACTIVE);
session.getOutputStream().write(shellCmd.getBytes());
@@ -228,7 +228,7 @@ public class EC2Provider extends AbstractProvider {
((StringParameterType) outParam.getType()).setValue(executionResult);
jobExecutionContext.getOutMessageContext().addParameter(paramName, outParam);
}
- GFacUtils.saveJobStatus(details, JobState.COMPLETE, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.COMPLETE);
} catch (InvalidSshKeyException e) {
throw new GFacProviderException("Invalid SSH key", e);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-ec2/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ec2/src/test/resources/gfac-config.xml b/modules/gfac/gfac-ec2/src/test/resources/gfac-config.xml
index 61dca4f..fec698b 100644
--- a/modules/gfac/gfac-ec2/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-ec2/src/test/resources/gfac-config.xml
@@ -19,72 +19,8 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
- </InHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
- <property name="name" value="value"/>
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler">
- <property name="name" value="value"/>
- </Handler>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Provider>
-
<Provider class="org.apache.airavata.gfac.ec2.EC2Provider" host="org.apache.airavata.schemas.gfac.impl.Ec2HostTypeImpl">
<InHandlers/>
<OutHandlers/>
</Provider>
-
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
- </InHandlers>
- <OutHandlers/>
- </Provider>
-
- <Application name="UltraScan">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Application>
-
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
- </OutHandlers>
- </Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
index 3eb020f..d8f58a1 100644
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
@@ -29,7 +29,6 @@ import org.apache.airavata.gfac.AbstractSecurityContext;
import org.apache.airavata.gfac.Constants;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.RequestData;
-import org.apache.airavata.gsi.ssh.api.Cluster;
import org.globus.gsi.X509Credential;
import org.globus.gsi.gssapi.GlobusGSSCredentialImpl;
import org.globus.gsi.provider.GlobusProvider;
@@ -61,8 +60,6 @@ public class GSISecurityContext extends AbstractSecurityContext {
private GSSCredential gssCredentials = null;
- private Cluster pbsCluster = null;
-
// Set trusted cert path and add provider
static {
Security.addProvider(new GlobusProvider());
@@ -108,9 +105,6 @@ public class GSISecurityContext extends AbstractSecurityContext {
}
- public GSISecurityContext(Cluster pbsCluster) {
- this.setPbsCluster(pbsCluster);
- }
/**
* Gets GSSCredentials. The process is as follows;
@@ -278,11 +272,4 @@ public class GSISecurityContext extends AbstractSecurityContext {
}
}
- public Cluster getPbsCluster() {
- return pbsCluster;
- }
-
- public void setPbsCluster(Cluster pbsCluster) {
- this.pbsCluster = pbsCluster;
- }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java
index feadd72..2a91afd 100644
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java
@@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory;
public class GramDirectorySetupHandler extends AbstractHandler {
private static final Logger log = LoggerFactory.getLogger(GramDirectorySetupHandler.class);
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException,GFacException {
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
log.info("Invoking GramDirectorySetupHandler ...");
super.invoke(jobExecutionContext);
String[] gridFTPEndpointArray = null;
@@ -119,17 +119,19 @@ public class GramDirectorySetupHandler extends AbstractHandler {
}
}
if (success == false) {
- GFacUtils.saveErrorDetails(pe.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, pe.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
throw pe;
}
} catch (SecurityException e) {
throw new GFacHandlerException(e.getMessage(), e);
} catch (ApplicationSettingsException e1) {
throw new GFacHandlerException(e1.getMessage(), e1);
- }
+ } catch (GFacException e) {
+ throw new GFacHandlerException(e);
+ }
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java
index 4b1beab..ffbacdd 100644
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java
@@ -63,7 +63,7 @@ import org.slf4j.LoggerFactory;
public class GridFTPInputHandler extends AbstractHandler {
private static final Logger log = LoggerFactory.getLogger(AppDescriptorCheckHandler.class);
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException,GFacException {
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
log.info("Invoking GridFTPInputHandler ...");
super.invoke(jobExecutionContext);
DataTransferDetails detail = new DataTransferDetails();
@@ -101,7 +101,7 @@ public class GridFTPInputHandler extends AbstractHandler {
status.setTransferState(TransferState.FAILED);
detail.setTransferStatus(status);
registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
} catch (Exception e1) {
throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
}
@@ -196,7 +196,7 @@ public class GridFTPInputHandler extends AbstractHandler {
return destURI.getPath();
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java
index e0cb0f8..33e549f 100644
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java
@@ -75,7 +75,7 @@ public class GridFTPOutputHandler extends AbstractHandler {
private Registry registry;
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException,GFacException {
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
log.info("Invoking GridFTPOutputHandler ...");
super.invoke(jobExecutionContext);
@@ -224,7 +224,7 @@ public class GridFTPOutputHandler extends AbstractHandler {
status.setTransferState(TransferState.FAILED);
detail.setTransferStatus(status);
registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
} catch (Exception e1) {
throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
}
@@ -341,7 +341,7 @@ public class GridFTPOutputHandler extends AbstractHandler {
return outputFileStagingPath + File.separator + fileName;
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java
index 0cf413c..2ba4b4b 100644
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java
@@ -37,6 +37,7 @@ import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.security.GSISecurityContext;
import org.apache.airavata.gfac.notification.events.JobIDEvent;
import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
import org.apache.airavata.gfac.provider.GFacProviderException;
import org.apache.airavata.gfac.util.GramProviderUtils;
import org.apache.airavata.gfac.utils.GFacUtils;
@@ -56,7 +57,7 @@ import org.ietf.jgss.GSSException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class GramProvider extends AbstractProvider{
+public class GramProvider extends AbstractProvider {
private static final Logger log = LoggerFactory.getLogger(GramJobSubmissionListener.class);
private GramJob job;
@@ -197,7 +198,7 @@ public class GramProvider extends AbstractProvider{
details.setJobID(jobID);
details.setJobDescription(job.getRSL());
jobExecutionContext.setJobDetails(details);
- GFacUtils.saveJobStatus(details, JobState.UN_SUBMITTED, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.UN_SUBMITTED);
applicationSaved=true;
String jobStatusMessage = "Un-submitted JobID= " + jobID;
@@ -218,34 +219,34 @@ public class GramProvider extends AbstractProvider{
log.error("Error while submitting commit request - Credentials provided are invalid. Job Id - "
+ job.getIDAsString(), e);
log.info("Attempting to renew credentials and re-submit commit signal...");
- GFacUtils.saveErrorDetails(gssException.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
renewCredentials(jobExecutionContext);
try {
job.signal(GramJob.SIGNAL_COMMIT_REQUEST);
} catch (GramException e1) {
- GFacUtils.saveErrorDetails(gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
throw new GFacException("Error while sending commit request. Job Id - "
+ job.getIDAsString(), e1);
} catch (GSSException e1) {
- GFacUtils.saveErrorDetails(gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
throw new GFacException("Error while sending commit request. Job Id - "
+ job.getIDAsString() + ". Credentials provided invalid", e1);
}
}
- GFacUtils.updateJobStatus(details, JobState.SUBMITTED);
+ GFacUtils.updateJobStatus(jobExecutionContext, details, JobState.SUBMITTED);
jobStatusMessage = "Submitted JobID= " + job.getIDAsString();
log.info(jobStatusMessage);
jobExecutionContext.getNotifier().publish(new JobIDEvent(jobStatusMessage));
} catch (GSSException e) {
// Renew credentials and re-submit
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
reSubmitJob(gateKeeper, jobExecutionContext, globusHostType, e);
} catch (GramException e) {
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
throw new GFacException("An error occurred while submitting a job, job id = " + job.getIDAsString(), e);
}
@@ -263,10 +264,10 @@ public class GramProvider extends AbstractProvider{
renewCredentialsAttempt = false;
} catch (GramException e) {
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
throw new GFacException("An error occurred while submitting a job, job id = " + job.getIDAsString(), e);
} catch (GSSException e) {
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
// Renew credentials and re-submit
reSubmitJob(gateKeeper, jobExecutionContext, globusHostType, e);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java
index 5500853..a77864d 100644
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java
@@ -70,7 +70,7 @@ public class GramJobSubmissionListener implements GramJobListener{
private void setStatus(int status, int error) {
try {
- GFacUtils.updateJobStatus(context.getJobDetails(), GramProviderUtils.getApplicationJobStatus(status));
+ GFacUtils.updateJobStatus(context, context.getJobDetails(), GramProviderUtils.getApplicationJobStatus(status));
} catch (GFacException e) {
log.error("Error persisting status" + e.getLocalizedMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gsissh/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/pom.xml b/modules/gfac/gfac-gsissh/pom.xml
index b05bddf..402619b 100644
--- a/modules/gfac/gfac-gsissh/pom.xml
+++ b/modules/gfac/gfac-gsissh/pom.xml
@@ -30,12 +30,17 @@
<artifactId>slf4j-api</artifactId>
</dependency>
- <!-- GFAC schemas -->
+ <!-- GFAC dependencies -->
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata-gfac-core</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-hpc-monitor</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- Credential Store -->
<dependency>
<groupId>org.apache.airavata</groupId>
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java
index 66b8094..1b9741d 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java
@@ -39,21 +39,24 @@ import java.util.Map;
public class GSISSHDirectorySetupHandler extends AbstractHandler{
private static final Logger log = LoggerFactory.getLogger(GSISSHDirectorySetupHandler.class);
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacException {
- if(jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null){
- try {
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ try {
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
}
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ } catch (GFacException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
- log.info("Setup SSH job directorties");
- super.invoke(jobExecutionContext);
- makeDirectory(jobExecutionContext);
+ log.info("Setup SSH job directorties");
+ super.invoke(jobExecutionContext);
+ makeDirectory(jobExecutionContext);
}
- private void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacException {
+ private void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ try {
Cluster cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
if (cluster == null) {
throw new GFacHandlerException("Security context is not set properly");
@@ -61,7 +64,7 @@ public class GSISSHDirectorySetupHandler extends AbstractHandler{
log.info("Successfully retrieved the Security Context");
}
ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- try {
+
String workingDirectory = app.getScratchWorkingDirectory();
cluster.makeDirectory(workingDirectory);
cluster.makeDirectory(app.getScratchWorkingDirectory());
@@ -84,7 +87,7 @@ public class GSISSHDirectorySetupHandler extends AbstractHandler{
detail.setTransferStatus(status);
try {
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
} catch (Exception e1) {
throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
}
@@ -92,7 +95,7 @@ public class GSISSHDirectorySetupHandler extends AbstractHandler{
}
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java
index 80f30f0..6f4fafd 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java
@@ -45,27 +45,26 @@ import java.io.IOException;
import java.util.*;
public class GSISSHInputHandler extends AbstractHandler {
-
private static final Logger log = LoggerFactory.getLogger(GSISSHInputHandler.class);
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException, GFacException {
- if(jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null){
- try {
- GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- }
- }
- log.info("Invoking SCPInputHandler");
- super.invoke(jobExecutionContext);
-
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ MessageContext inputNew = new MessageContext();
DataTransferDetails detail = new DataTransferDetails();
TransferStatus status = new TransferStatus();
-
- MessageContext inputNew = new MessageContext();
try {
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
+ try {
+ GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ }
+ }
+ log.info("Invoking SCPInputHandler");
+ super.invoke(jobExecutionContext);
+
+
MessageContext input = jobExecutionContext.getInMessageContext();
Set<String> parameters = input.getParameters().keySet();
for (String paramName : parameters) {
@@ -82,8 +81,8 @@ public class GSISSHInputHandler extends AbstractHandler {
status.setTransferState(TransferState.UPLOAD);
detail.setTransferStatus(status);
detail.setTransferDescription("Input Data Staged: " + stageInputFiles);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
- newFiles.add(stageInputFiles);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ newFiles.add(stageInputFiles);
}
((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
}
@@ -94,11 +93,11 @@ public class GSISSHInputHandler extends AbstractHandler {
status.setTransferState(TransferState.FAILED);
detail.setTransferStatus(status);
try {
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE, jobExecutionContext.getTaskData().getTaskID());
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
}
jobExecutionContext.setInMessageContext(inputNew);
@@ -121,7 +120,7 @@ public class GSISSHInputHandler extends AbstractHandler {
String substring = paramValue.substring(i + 1);
try {
String targetFile = app.getInputDataDirectory() + File.separator + substring;
- if(paramValue.startsWith("file")){
+ if (paramValue.startsWith("file")) {
paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
}
cluster.scpTo(targetFile, paramValue);
@@ -131,7 +130,7 @@ public class GSISSHInputHandler extends AbstractHandler {
}
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
index fd80d56..3855248 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
@@ -56,7 +56,7 @@ import java.util.Set;
public class GSISSHOutputHandler extends AbstractHandler{
private static final Logger log = LoggerFactory.getLogger(GSISSHOutputHandler.class);
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException, GFacException {
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context
// so attempting to get it from the registry
if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) {
@@ -87,14 +87,17 @@ public class GSISSHOutputHandler extends AbstractHandler{
}
}
}
+ try {
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
- if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
- try {
GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
}
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ } catch (GFacException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
}
super.invoke(jobExecutionContext);
DataTransferDetails detail = new DataTransferDetails();
@@ -201,7 +204,7 @@ public class GSISSHOutputHandler extends AbstractHandler{
status.setTransferState(TransferState.FAILED);
detail.setTransferStatus(status);
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
} catch (Exception e1) {
throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
}
@@ -210,7 +213,7 @@ public class GSISSHOutputHandler extends AbstractHandler{
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
index 06e2f4a..3cfe599 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
@@ -20,11 +20,17 @@
*/
package org.apache.airavata.gfac.provider.impl;
+import org.apache.airavata.gfac.Constants;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.security.GSISecurityContext;
+import org.apache.airavata.gfac.cpi.GFacImpl;
+import org.apache.airavata.gfac.handler.ThreadedHandler;
+import org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler;
import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.util.GFACGSISSHUtils;
import org.apache.airavata.gfac.utils.GFacUtils;
import org.apache.airavata.gsi.ssh.api.Cluster;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
@@ -33,14 +39,16 @@ import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
import org.apache.airavata.model.workspace.experiment.ErrorCategory;
import org.apache.airavata.model.workspace.experiment.JobDetails;
import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.schemas.gfac.GsisshHostType;
import org.apache.airavata.schemas.gfac.HostDescriptionType;
import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
import java.util.Map;
-public class GSISSHProvider extends AbstractProvider{
+public class GSISSHProvider extends AbstractProvider {
private static final Logger log = LoggerFactory.getLogger(GSISSHProvider.class);
public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
@@ -71,7 +79,7 @@ public class GSISSHProvider extends AbstractProvider{
log.info("Successfully retrieved the Security Context");
}
// This installed path is a mandetory field, because this could change based on the computing resource
- JobDescriptor jobDescriptor = GFacUtils.createJobDescriptor(jobExecutionContext, app, cluster);
+ JobDescriptor jobDescriptor = GFACGSISSHUtils.createJobDescriptor(jobExecutionContext, app, cluster);
log.info(jobDescriptor.toXML());
@@ -81,25 +89,43 @@ public class GSISSHProvider extends AbstractProvider{
jobExecutionContext.setJobDetails(jobDetails);
if(jobID == null){
jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobDetails, JobState.FAILED, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
}else{
jobDetails.setJobID(jobID);
- GFacUtils.saveJobStatus(jobDetails, JobState.SUBMITTED, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.SUBMITTED);
}
+
+ // Now job has submitted to the resource, its up to the Provider to parse the information to daemon handler
+ // to perform monitoring, daemon handlers can be accessed from anywhere
+ List<ThreadedHandler> daemonHandlers = GFacImpl.getDaemonHandlers();
+ GridPullMonitorHandler pullMonitorHandler = null;
+ for(ThreadedHandler threadedHandler:daemonHandlers){
+ if(threadedHandler instanceof GridPullMonitorHandler){
+ pullMonitorHandler = (GridPullMonitorHandler)threadedHandler;
+ }
+ }
+ // we know this hos is type GsiSSHHostType
+ String monitorMode = ((GsisshHostType) host).getMonitorMode();
+ if("".equals(monitorMode) || monitorMode == null || org.apache.airavata.common.utils.Constants.PULL.equals(monitorMode)){
+ log.info("Job is launched successfully now parsing it to monitoring in pull mode, JobID Returned: " + jobID);
+ pullMonitorHandler.invoke(jobExecutionContext);
+ }else{
+ log.error("Currently we only support Pull monitoring");
+ }
} catch (SSHApiException e) {
String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
log.error(error);
jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobDetails,JobState.FAILED,taskID);
- GFacUtils.saveErrorDetails(error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails,JobState.FAILED);
+ GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
throw new GFacProviderException(error, e);
} catch (Exception e) {
String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
log.error(error);
jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobDetails,JobState.FAILED,taskID);
- GFacUtils.saveErrorDetails(error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR, taskID);
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails,JobState.FAILED);
+ GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
throw new GFacProviderException(error, e);
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java
index d0f92c1..0351d00 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java
@@ -21,24 +21,33 @@
package org.apache.airavata.gfac.util;
import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
import org.apache.airavata.commons.gfac.type.HostDescription;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
import org.apache.airavata.gfac.Constants;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.RequestData;
import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
import org.apache.airavata.gfac.context.security.GSISecurityContext;
import org.apache.airavata.gsi.ssh.api.Cluster;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
import org.apache.airavata.gsi.ssh.api.ServerInfo;
import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
import org.apache.airavata.gsi.ssh.util.CommonUtils;
+import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
import org.apache.airavata.schemas.gfac.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+
public class GFACGSISSHUtils {
private final static Logger logger = LoggerFactory.getLogger(GFACGSISSHUtils.class);
@@ -95,4 +104,89 @@ public class GFACGSISSHUtils {
}
jobExecutionContext.addSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT,context);
}
+ public static JobDescriptor createJobDescriptor(JobExecutionContext jobExecutionContext,
+ ApplicationDeploymentDescriptionType app, Cluster cluster) {
+ JobDescriptor jobDescriptor = new JobDescriptor();
+ // this is common for any application descriptor
+ jobDescriptor.setInputDirectory(app.getInputDataDirectory());
+ jobDescriptor.setOutputDirectory(app.getOutputDataDirectory());
+ jobDescriptor.setExecutablePath(app.getExecutableLocation());
+ jobDescriptor.setStandardOutFile(app.getStandardOutput());
+ jobDescriptor.setStandardErrorFile(app.getStandardError());
+ Random random = new Random();
+ int i = random.nextInt();
+ jobDescriptor.setJobName(app.getApplicationName().getStringValue() + String.valueOf(i));
+ jobDescriptor.setWorkingDirectory(app.getStaticWorkingDirectory());
+
+
+ List<String> inputValues = new ArrayList<String>();
+ MessageContext input = jobExecutionContext.getInMessageContext();
+ Map<String, Object> inputs = input.getParameters();
+ Set<String> keys = inputs.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
+ if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
+ || "FileArray".equals(actualParameter.getType().getType().toString())) {
+ String[] values = null;
+ if (actualParameter.getType() instanceof URIArrayType) {
+ values = ((URIArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof StringArrayType) {
+ values = ((StringArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof FileArrayType) {
+ values = ((FileArrayType) actualParameter.getType()).getValueArray();
+ }
+ String value = StringUtil.createDelimiteredString(values, " ");
+ inputValues.add(value);
+ } else {
+ String paramValue = MappingFactory.toString(actualParameter);
+ inputValues.add(paramValue);
+ }
+ }
+ jobDescriptor.setInputValues(inputValues);
+
+ // this part will fill out the hpcApplicationDescriptor
+ if (app instanceof HpcApplicationDeploymentType) {
+ HpcApplicationDeploymentType applicationDeploymentType
+ = (HpcApplicationDeploymentType) app;
+ jobDescriptor.setShellName("/bin/bash");
+ jobDescriptor.setAllEnvExport(true);
+ jobDescriptor.setMailOptions("n");
+ jobDescriptor.setNodes(applicationDeploymentType.getNodeCount());
+ jobDescriptor.setProcessesPerNode(applicationDeploymentType.getProcessorsPerNode());
+ jobDescriptor.setMaxWallTime(String.valueOf(applicationDeploymentType.getMaxWallTime()));
+ jobDescriptor.setJobSubmitter(applicationDeploymentType.getJobSubmitterCommand());
+ if (applicationDeploymentType.getProjectAccount() != null) {
+ if (applicationDeploymentType.getProjectAccount().getProjectAccountNumber() != null) {
+ jobDescriptor.setAcountString(applicationDeploymentType.getProjectAccount().getProjectAccountNumber());
+ }
+ }
+ if (applicationDeploymentType.getQueue() != null) {
+ if (applicationDeploymentType.getQueue().getQueueName() != null) {
+ jobDescriptor.setQueueName(applicationDeploymentType.getQueue().getQueueName());
+ }
+ }
+ jobDescriptor.setOwner(((PBSCluster) cluster).getServerInfo().getUserName());
+ TaskDetails taskData = jobExecutionContext.getTaskData();
+ if (taskData != null && taskData.isSetTaskScheduling()) {
+ ComputationalResourceScheduling computionnalResource = taskData.getTaskScheduling();
+ if (computionnalResource.getNodeCount() > 0) {
+ jobDescriptor.setNodes(computionnalResource.getNodeCount());
+ }
+ if (computionnalResource.getComputationalProjectAccount() != null) {
+ jobDescriptor.setAcountString(computionnalResource.getComputationalProjectAccount());
+ }
+ if (computionnalResource.getQueueName() != null) {
+ jobDescriptor.setQueueName(computionnalResource.getQueueName());
+ }
+ if (computionnalResource.getTotalCPUCount() > 0) {
+ jobDescriptor.setProcessesPerNode(computionnalResource.getTotalCPUCount());
+ }
+ if (computionnalResource.getWallTimeLimit() > 0) {
+ jobDescriptor.setMaxWallTime(String.valueOf(computionnalResource.getWallTimeLimit()));
+ }
+ }
+
+ }
+ return jobDescriptor;
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
index aafcdec..49eadf4 100644
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
@@ -21,7 +21,6 @@
package org.apache.airavata.gfac.handler;
-import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
import org.apache.airavata.gfac.provider.utils.HadoopUtils;
@@ -96,7 +95,7 @@ public class HDFSDataMovementHandler implements GFacHandler {
private void handleOutPath(JobExecutionContext jobExecutionContext){}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
index f72afd1..c7de3f9 100644
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
@@ -23,7 +23,6 @@ package org.apache.airavata.gfac.handler;
import com.google.common.io.Files;
import org.apache.airavata.commons.gfac.type.HostDescription;
-import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
import org.apache.airavata.schemas.gfac.HadoopHostType;
@@ -71,6 +70,7 @@ public class HadoopDeploymentHandler implements GFacHandler {
* configuration provided. If there is a already deployed hadoop cluster this will skip
* deployment.
*
+ *
* @param jobExecutionContext job execution context containing all the required configurations
* and runtime information.
* @throws GFacHandlerException
@@ -268,7 +268,7 @@ public class HadoopDeploymentHandler implements GFacHandler {
return ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory();
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
index c20e2ea..0fc8b47 100644
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
@@ -31,6 +31,7 @@ import org.apache.airavata.commons.gfac.type.ActualParameter;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.provider.AbstractProvider;
import org.apache.airavata.gfac.provider.GFacProviderException;
import org.apache.airavata.gfac.provider.utils.HadoopUtils;
import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType;
@@ -55,7 +56,7 @@ import sun.reflect.generics.reflectiveObjects.NotImplementedException;
* Executes hadoop job using the cluster configuration provided by handlers in
* in-flow.
*/
-public class HadoopProvider extends AbstractProvider{
+public class HadoopProvider extends AbstractProvider {
private static final Logger logger = LoggerFactory.getLogger(HadoopProvider.class);
private boolean isWhirrBasedDeployment = false;
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
index 7c7e704..f43a5fb 100644
--- a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
@@ -19,37 +19,6 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
- </InHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
- <property name="name" value="value"/>
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler">
- <property name="name" value="value"/>
- </Handler>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Provider>
-
- <Provider class="org.apache.airavata.gfac.ec2.EC2Provider" host="org.apache.airavata.schemas.gfac.impl.Ec2HostTypeImpl">
- <InHandlers/>
- <OutHandlers/>
- </Provider>
<Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
<InHandlers>
@@ -68,23 +37,4 @@
<Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Application>
-
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHOutputHandler"/>
- </OutHandlers>
- </Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/553caa08/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
index 8174a3d..8a11c2a 100644
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
@@ -21,7 +21,6 @@
package org.apache.airavata.gfac.handler;
import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
import org.apache.airavata.schemas.gfac.HostDescriptionType;
@@ -56,7 +55,7 @@ public class LocalDirectorySetupHandler implements GFacHandler{
}
}
- public void initProperties(Map<String, String> properties) throws GFacHandlerException, GFacException {
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
}
}