You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2015/03/23 20:38:28 UTC
[01/15] airavata git commit: implementing passive gfac submitter
using rabbbitmq
Repository: airavata
Updated Branches:
refs/heads/master c05d5e6c8 -> aa27ce109
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
new file mode 100644
index 0000000..8029a0c
--- /dev/null
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
@@ -0,0 +1,88 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.messaging.core.impl;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ThriftUtils;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.MessagingConstants;
+import org.apache.airavata.messaging.core.Publisher;
+import org.apache.airavata.model.messaging.event.*;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RabbitMQTaskLaunchPublisher implements Publisher{
+ private final static Logger log = LoggerFactory.getLogger(RabbitMQTaskLaunchPublisher.class);
+ public static final String LAUNCH_TASK = "launch.task";
+ public static final String TERMINATE_TASK = "teminate.task";
+
+ private RabbitMQProducer rabbitMQProducer;
+
+ public RabbitMQTaskLaunchPublisher() throws Exception {
+ String brokerUrl;
+ String exchangeName;
+ try {
+ brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_EXCHANGE_NAME);
+ } catch (ApplicationSettingsException e) {
+ String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
+ log.error(message, e);
+ throw new AiravataException(message, e);
+ }
+ rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName);
+ rabbitMQProducer.open();
+ }
+
+ public void publish(MessageContext msgCtx) throws AiravataException {
+ try {
+ log.info("Publishing to lauch queue ...");
+ byte[] body = ThriftUtils.serializeThriftObject(msgCtx.getEvent());
+ Message message = new Message();
+ message.setEvent(body);
+ message.setMessageId(msgCtx.getMessageId());
+ message.setMessageType(msgCtx.getType());
+ message.setUpdatedTime(msgCtx.getUpdatedTime().getTime());
+ String routingKey = null;
+ if (msgCtx.getType().equals(MessageType.LAUNCHTASK)){
+ TaskSubmitEvent event = (TaskSubmitEvent) msgCtx.getEvent();
+ routingKey = LAUNCH_TASK + "."+event.getExperimentId() + "." +
+ event.getTaskId() + "." + event.getGatewayId();
+ }else if(msgCtx.getType().equals(MessageType.TERMINATETASK)){
+ TaskTerminateEvent event = (TaskTerminateEvent) msgCtx.getEvent();
+ routingKey = TERMINATE_TASK + "."+event.getExperimentId() + "." +
+ event.getTaskId();
+ }
+ byte[] messageBody = ThriftUtils.serializeThriftObject(message);
+ rabbitMQProducer.send(messageBody, routingKey);
+ } catch (TException e) {
+ String msg = "Error while deserializing the object";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ } catch (Exception e) {
+ String msg = "Error while sending to rabbitmq";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
index e88945d..b200468 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
@@ -103,7 +103,7 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
// registering with zk
try {
if (ServerSettings.isRabbitMqPublishEnabled()) {
- publisher = PublisherFactory.createPublisher();
+ publisher = PublisherFactory.createActivityPublisher();
}
String zkhostPort = AiravataZKUtils.getZKhostPort();
String airavataServerHostPort = ServerSettings
@@ -156,6 +156,7 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
registry = RegistryFactory.getDefaultRegistry();
orchestrator.initialize();
orchestrator.getOrchestratorContext().setZk(this.zk);
+ orchestrator.getOrchestratorContext().setPublisher(this.publisher);
} catch (OrchestratorException e) {
log.error(e.getMessage(), e);
throw new OrchestratorException("Error while initializing orchestrator service", e);
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/orchestrator/orchestrator-core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/pom.xml b/modules/orchestrator/orchestrator-core/pom.xml
index 0dffaee..23863fb 100644
--- a/modules/orchestrator/orchestrator-core/pom.xml
+++ b/modules/orchestrator/orchestrator-core/pom.xml
@@ -121,11 +121,21 @@ the License. -->
<artifactId>airavata-server-configuration</artifactId>
<scope>test</scope>
</dependency>
- <!-- zookeeper dependencies -->
+ <!-- zookeeper and curator dependencies -->
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <version>3.4.0</version>
+ <version>${zk.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
</dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
index 5d465ff..b77087b 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.airavata.gfac.client.GFACInstance;
+import org.apache.airavata.messaging.core.Publisher;
import org.apache.airavata.orchestrator.core.OrchestratorConfiguration;
import org.apache.airavata.registry.cpi.Registry;
import org.apache.zookeeper.ZooKeeper;
@@ -39,6 +40,8 @@ public class OrchestratorContext {
private Registry newRegistry;
private static ZooKeeper zk; // this instance can be accessed by the Validators and other components
+
+ private Publisher publisher;
public OrchestratorContext(List<GFACInstance> gfacInstanceList) {
if (gfacInstanceList != null) {
@@ -48,6 +51,14 @@ public class OrchestratorContext {
}
}
+ public Publisher getPublisher() {
+ return publisher;
+ }
+
+ public void setPublisher(Publisher publisher) {
+ this.publisher = publisher;
+ }
+
public OrchestratorContext() {
this(null);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
new file mode 100644
index 0000000..58ac982
--- /dev/null
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
@@ -0,0 +1,247 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.orchestrator.core.impl;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.AiravataZKUtils;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.credential.store.store.CredentialReader;
+import org.apache.airavata.gfac.client.GFACInstance;
+import org.apache.airavata.gfac.client.GFacClientFactory;
+import org.apache.airavata.gfac.core.utils.GFacUtils;
+import org.apache.airavata.gfac.cpi.GfacService;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.Publisher;
+import org.apache.airavata.messaging.core.PublisherFactory;
+import org.apache.airavata.model.messaging.event.MessageType;
+import org.apache.airavata.model.messaging.event.TaskSubmitEvent;
+import org.apache.airavata.orchestrator.core.context.OrchestratorContext;
+import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
+import org.apache.airavata.orchestrator.core.job.JobSubmitter;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+/**
+ * This class can be used to do the communication between orchestrator and gfac to handle using a queue
+ */
+public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
+ private final static Logger logger = LoggerFactory.getLogger(GFACPassiveJobSubmitter.class);
+
+ public static final String IP = "ip";
+
+ private OrchestratorContext orchestratorContext;
+
+ private static Integer mutex = -1;
+
+ private Publisher publisher;
+
+ public void initialize(OrchestratorContext orchestratorContext) throws OrchestratorException {
+ this.orchestratorContext = orchestratorContext;
+ if(orchestratorContext.getPublisher()!=null){ // use the same publisher this will be empty if rabbitmq.publish is not enabled in the configuraiton
+ this.publisher = orchestratorContext.getPublisher();
+ }else {
+ try {
+ this.publisher = PublisherFactory.createTaskLaunchPublisher();
+ } catch (AiravataException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException("Cannot initialize " + GFACPassiveJobSubmitter.class + " need to start Rabbitmq server to use " + GFACPassiveJobSubmitter.class);
+ }
+ }
+ }
+
+ public GFACInstance selectGFACInstance() throws OrchestratorException {
+ // currently we only support one instance but future we have to pick an
+ // instance
+ return null;
+ }
+
+ public boolean submit(String experimentID, String taskID) throws OrchestratorException {
+ return submit(experimentID, taskID, null);
+ }
+
+ /**
+ * Submit the job to a shared launch.queue accross multiple gfac instances
+ *
+ * @param experimentID
+ * @param taskID
+ * @param tokenId
+ * @return
+ * @throws OrchestratorException
+ */
+ public boolean submit(String experimentID, String taskID, String tokenId) throws OrchestratorException {
+
+ ZooKeeper zk = orchestratorContext.getZk();
+ GfacService.Client gfacClient = null;
+ try {
+ if (zk == null || !zk.getState().isConnected()) {
+ String zkhostPort = AiravataZKUtils.getZKhostPort();
+ zk = new ZooKeeper(zkhostPort, 6000, this);
+ synchronized (mutex) {
+ mutex.wait();
+ }
+ }
+ String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
+ String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ List<String> children = zk.getChildren(gfacServer, this);
+
+ if (children.size() == 0) {
+ // Zookeeper data need cleaning
+ throw new OrchestratorException("There is no active GFac instance to route the request");
+ } else {
+ String pickedChild = children.get(new Random().nextInt(Integer.MAX_VALUE) % children.size());
+ // here we are not using an index because the getChildren does not return the same order everytime
+ String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
+ logger.info("GFAC instance node data: " + gfacNodeData);
+ String[] split = gfacNodeData.split(":");
+ gfacClient = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
+ if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
+ // before submitting the job we check again the state of the node
+ if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, tokenId)) {
+ String gatewayId = null;
+ CredentialReader credentialReader = GFacUtils.getCredentialReader();
+ if (credentialReader != null) {
+ try {
+ gatewayId = credentialReader.getGatewayID(tokenId);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage());
+ }
+ }
+ if(gatewayId == null || gatewayId.isEmpty()){
+ gatewayId = ServerSettings.getDefaultUserGateway();
+ }
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, gatewayId);
+ MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK,"LAUNCH.TASK-"+ UUID.randomUUID().toString(),gatewayId);
+ publisher.publish(messageContext);
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (KeeperException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (ApplicationSettingsException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ }finally {
+ gfacClient.getOutputProtocol().getTransport().close();
+ }
+ return true;
+
+ }
+
+ /**
+ * Submit the experiment the terminate.queue job queue and remove the experiment from shared launch.queue
+ * @param experimentID
+ * @param taskID
+ * @return
+ * @throws OrchestratorException
+ */
+ public boolean terminate(String experimentID, String taskID) throws OrchestratorException {
+ ZooKeeper zk = orchestratorContext.getZk();
+ GfacService.Client localhost = null;
+ try {
+ if (zk == null || !zk.getState().isConnected()) {
+ String zkhostPort = AiravataZKUtils.getZKhostPort();
+ zk = new ZooKeeper(zkhostPort, 6000, this);
+ synchronized (mutex) {
+ mutex.wait();
+ }
+ }
+ String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
+ String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ List<String> children = zk.getChildren(gfacServer, this);
+
+ if (children.size() == 0) {
+ // Zookeeper data need cleaning
+ throw new OrchestratorException("There is no active GFac instance to route the request");
+ } else {
+ String pickedChild = children.get(new Random().nextInt(Integer.MAX_VALUE) % children.size());
+ // here we are not using an index because the getChildren does not return the same order everytime
+ String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
+ logger.info("GFAC instance node data: " + gfacNodeData);
+ String[] split = gfacNodeData.split(":");
+ localhost = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
+ if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
+ // before submitting the job we check again the state of the node
+ if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, null);
+ MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK,"LAUNCH.TERMINATE-"+ UUID.randomUUID().toString(),null);
+ publisher.publish(messageContext);
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (KeeperException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (ApplicationSettingsException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ }finally {
+
+ }
+ return false;
+
+ }
+
+ synchronized public void process(WatchedEvent event) {
+ synchronized (mutex) {
+ switch (event.getState()) {
+ case SyncConnected:
+ mutex.notify();
+ }
+ switch (event.getType()) {
+ case NodeCreated:
+ mutex.notify();
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
new file mode 100644
index 0000000..54339a2
--- /dev/null
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
@@ -0,0 +1,212 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.orchestrator.core.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.AiravataZKUtils;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.credential.store.store.CredentialReader;
+import org.apache.airavata.gfac.client.GFACInstance;
+import org.apache.airavata.gfac.client.GFacClientFactory;
+import org.apache.airavata.gfac.core.utils.GFacUtils;
+import org.apache.airavata.gfac.cpi.GfacService;
+import org.apache.airavata.orchestrator.core.context.OrchestratorContext;
+import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
+import org.apache.airavata.orchestrator.core.job.JobSubmitter;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * this class is responsible for submitting a job to gfac in service mode,
+ * it will select a gfac instance based on the incoming request and submit to that
+ * gfac instance.
+ */
+public class GFACRPCJobSubmitter implements JobSubmitter, Watcher {
+ private final static Logger logger = LoggerFactory.getLogger(GFACRPCJobSubmitter.class);
+ public static final String IP = "ip";
+
+ private OrchestratorContext orchestratorContext;
+
+ private static Integer mutex = -1;
+
+ public void initialize(OrchestratorContext orchestratorContext) throws OrchestratorException {
+ this.orchestratorContext = orchestratorContext;
+ }
+
+ public GFACInstance selectGFACInstance() throws OrchestratorException {
+ // currently we only support one instance but future we have to pick an
+ // instance
+ return null;
+ }
+
+ public boolean submit(String experimentID, String taskID) throws OrchestratorException {
+ return this.submit(experimentID, taskID, null);
+ }
+
+ public boolean submit(String experimentID, String taskID, String tokenId) throws OrchestratorException {
+ ZooKeeper zk = orchestratorContext.getZk();
+ GfacService.Client gfacClient = null;
+ try {
+ if (zk == null || !zk.getState().isConnected()) {
+ String zkhostPort = AiravataZKUtils.getZKhostPort();
+ zk = new ZooKeeper(zkhostPort, 6000, this);
+ synchronized (mutex) {
+ mutex.wait();
+ }
+ }
+ String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
+ String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ List<String> children = zk.getChildren(gfacServer, this);
+
+ if (children.size() == 0) {
+ // Zookeeper data need cleaning
+ throw new OrchestratorException("There is no active GFac instance to route the request");
+ } else {
+ String pickedChild = children.get(new Random().nextInt(Integer.MAX_VALUE) % children.size());
+ // here we are not using an index because the getChildren does not return the same order everytime
+ String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
+ logger.info("GFAC instance node data: " + gfacNodeData);
+ String[] split = gfacNodeData.split(":");
+ gfacClient = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
+ if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
+ // before submitting the job we check again the state of the node
+ if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, tokenId)) {
+ String gatewayId = null;
+ CredentialReader credentialReader = GFacUtils.getCredentialReader();
+ if (credentialReader != null) {
+ try {
+ gatewayId = credentialReader.getGatewayID(tokenId);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage());
+ }
+ }
+ if(gatewayId == null || gatewayId.isEmpty()){
+ gatewayId = ServerSettings.getDefaultUserGateway();
+ }
+ return gfacClient.submitJob(experimentID, taskID, gatewayId);
+ }
+ }
+ }
+ } catch (TException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (KeeperException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (ApplicationSettingsException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ }finally {
+ gfacClient.getOutputProtocol().getTransport().close();
+ }
+ return false;
+ }
+
+ public boolean terminate(String experimentID, String taskID) throws OrchestratorException {
+ ZooKeeper zk = orchestratorContext.getZk();
+ GfacService.Client localhost = null;
+ try {
+ if (zk == null || !zk.getState().isConnected()) {
+ String zkhostPort = AiravataZKUtils.getZKhostPort();
+ zk = new ZooKeeper(zkhostPort, 6000, this);
+ synchronized (mutex) {
+ mutex.wait();
+ }
+ }
+ String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
+ String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ List<String> children = zk.getChildren(gfacServer, this);
+
+ if (children.size() == 0) {
+ // Zookeeper data need cleaning
+ throw new OrchestratorException("There is no active GFac instance to route the request");
+ } else {
+ String pickedChild = children.get(new Random().nextInt(Integer.MAX_VALUE) % children.size());
+ // here we are not using an index because the getChildren does not return the same order everytime
+ String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
+ logger.info("GFAC instance node data: " + gfacNodeData);
+ String[] split = gfacNodeData.split(":");
+ localhost = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
+ if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
+ // before submitting the job we check again the state of the node
+ if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
+ return localhost.cancelJob(experimentID, taskID);
+ }
+ }
+ }
+ } catch (TException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (KeeperException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (ApplicationSettingsException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new OrchestratorException(e);
+ }finally {
+
+ }
+ return false;
+ }
+
+ synchronized public void process(WatchedEvent event) {
+ synchronized (mutex) {
+ switch (event.getState()) {
+ case SyncConnected:
+ mutex.notify();
+ }
+ switch (event.getType()) {
+ case NodeCreated:
+ mutex.notify();
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
deleted file mode 100644
index 3bbc588..0000000
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.airavata.orchestrator.core.impl;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.AiravataZKUtils;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.gfac.client.GFACInstance;
-import org.apache.airavata.gfac.client.GFacClientFactory;
-import org.apache.airavata.gfac.core.utils.GFacUtils;
-import org.apache.airavata.gfac.cpi.GfacService;
-import org.apache.airavata.orchestrator.core.context.OrchestratorContext;
-import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
-import org.apache.airavata.orchestrator.core.job.JobSubmitter;
-import org.apache.thrift.TException;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/*
- * this class is responsible for submitting a job to gfac in service mode,
- * it will select a gfac instance based on the incoming request and submit to that
- * gfac instance.
- */
-public class GFACServiceJobSubmitter implements JobSubmitter, Watcher {
- private final static Logger logger = LoggerFactory.getLogger(GFACServiceJobSubmitter.class);
- public static final String IP = "ip";
-
- private OrchestratorContext orchestratorContext;
-
- private static Integer mutex = -1;
-
- public void initialize(OrchestratorContext orchestratorContext) throws OrchestratorException {
- this.orchestratorContext = orchestratorContext;
- }
-
- public GFACInstance selectGFACInstance() throws OrchestratorException {
- // currently we only support one instance but future we have to pick an
- // instance
- return null;
- }
-
- public boolean submit(String experimentID, String taskID) throws OrchestratorException {
- return this.submit(experimentID, taskID, null);
- }
-
- public boolean submit(String experimentID, String taskID, String tokenId) throws OrchestratorException {
- ZooKeeper zk = orchestratorContext.getZk();
- GfacService.Client gfacClient = null;
- try {
- if (zk == null || !zk.getState().isConnected()) {
- String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
- synchronized (mutex) {
- mutex.wait();
- }
- }
- String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
- String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
- List<String> children = zk.getChildren(gfacServer, this);
-
- if (children.size() == 0) {
- // Zookeeper data need cleaning
- throw new OrchestratorException("There is no active GFac instance to route the request");
- } else {
- String pickedChild = children.get(new Random().nextInt(Integer.MAX_VALUE) % children.size());
- // here we are not using an index because the getChildren does not return the same order everytime
- String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
- logger.info("GFAC instance node data: " + gfacNodeData);
- String[] split = gfacNodeData.split(":");
- gfacClient = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
- if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
- // before submitting the job we check again the state of the node
- if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, tokenId)) {
- String gatewayId = null;
- CredentialReader credentialReader = GFacUtils.getCredentialReader();
- if (credentialReader != null) {
- try {
- gatewayId = credentialReader.getGatewayID(tokenId);
- } catch (Exception e) {
- logger.error(e.getLocalizedMessage());
- }
- }
- if(gatewayId == null || gatewayId.isEmpty()){
- gatewayId = ServerSettings.getDefaultUserGateway();
- }
- return gfacClient.submitJob(experimentID, taskID, gatewayId);
- }
- }
- }
- } catch (TException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (InterruptedException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (KeeperException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (ApplicationSettingsException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- }finally {
- gfacClient.getOutputProtocol().getTransport().close();
- }
- return false;
- }
-
- public boolean terminate(String experimentID, String taskID) throws OrchestratorException {
- ZooKeeper zk = orchestratorContext.getZk();
- GfacService.Client localhost = null;
- try {
- if (zk == null || !zk.getState().isConnected()) {
- String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
- synchronized (mutex) {
- mutex.wait();
- }
- }
- String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
- String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
- List<String> children = zk.getChildren(gfacServer, this);
-
- if (children.size() == 0) {
- // Zookeeper data need cleaning
- throw new OrchestratorException("There is no active GFac instance to route the request");
- } else {
- String pickedChild = children.get(new Random().nextInt(Integer.MAX_VALUE) % children.size());
- // here we are not using an index because the getChildren does not return the same order everytime
- String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
- logger.info("GFAC instance node data: " + gfacNodeData);
- String[] split = gfacNodeData.split(":");
- localhost = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
- if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
- // before submitting the job we check again the state of the node
- if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
- return localhost.cancelJob(experimentID, taskID);
- }
- }
- }
- } catch (TException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (InterruptedException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (KeeperException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (ApplicationSettingsException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
- }finally {
-
- }
- return false;
- }
-
- synchronized public void process(WatchedEvent event) {
- synchronized (mutex) {
- switch (event.getState()) {
- case SyncConnected:
- mutex.notify();
- }
- switch (event.getType()) {
- case NodeCreated:
- mutex.notify();
- break;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/workflow-model/workflow-engine/src/main/java/org/apache/airavata/workflow/engine/WorkflowEngineImpl.java
----------------------------------------------------------------------
diff --git a/modules/workflow-model/workflow-engine/src/main/java/org/apache/airavata/workflow/engine/WorkflowEngineImpl.java b/modules/workflow-model/workflow-engine/src/main/java/org/apache/airavata/workflow/engine/WorkflowEngineImpl.java
index a8ee98b..0af8881 100644
--- a/modules/workflow-model/workflow-engine/src/main/java/org/apache/airavata/workflow/engine/WorkflowEngineImpl.java
+++ b/modules/workflow-model/workflow-engine/src/main/java/org/apache/airavata/workflow/engine/WorkflowEngineImpl.java
@@ -25,7 +25,7 @@ import org.airavata.appcatalog.cpi.WorkflowCatalog;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.messaging.core.Publisher;
-import org.apache.airavata.messaging.core.impl.RabbitMQPublisher;
+import org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher;
import org.apache.airavata.model.error.AiravataClientConnectException;
import org.apache.airavata.model.workspace.experiment.Experiment;
import org.apache.airavata.orchestrator.client.OrchestratorClientFactory;
@@ -47,7 +47,7 @@ public class WorkflowEngineImpl implements WorkflowEngine {
private Publisher rabbitMQPublisher;
WorkflowEngineImpl() {
try {
- rabbitMQPublisher = new RabbitMQPublisher();
+ rabbitMQPublisher = new RabbitMQStatusPublisher();
} catch (Exception e) {
logger.error("Failed to instantiate RabbitMQPublisher", e);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 82ea635..447bc97 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,6 +75,7 @@
<axiom.version>1.2.8</axiom.version>
<surefire.version>2.12</surefire.version>
<junit.version>4.7</junit.version>
+ <curator.version>2.7.1</curator.version>
<jcr.version>2.0</jcr.version>
<xmlbeans.version>2.5.0</xmlbeans.version>
<xpp3.version>1.1.6</xpp3.version>
@@ -97,6 +98,7 @@
<mysql.connector.version>5.1.31</mysql.connector.version>
<skipTests>false</skipTests>
<google.gson.version>2.3</google.gson.version>
+ <zk.version>3.4.0</zk.version>
</properties>
<developers>
@@ -408,6 +410,11 @@
<artifactId>gson</artifactId>
<version>${google.gson.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zk.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
[06/15] airavata git commit: initial version of passive job submission
Posted by la...@apache.org.
initial version of passive job submission
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/60788efe
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/60788efe
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/60788efe
Branch: refs/heads/master
Commit: 60788efec3a808274a19e9687a669cabad84e89b
Parents: a486b67
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Tue Feb 17 22:09:15 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Tue Feb 17 22:09:15 2015 -0500
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 2 +-
.../airavata/gfac/leader/CuratorClient.java | 79 ------------------
.../gfac/leader/LeaderSelectorExample.java | 80 ------------------
.../airavata/gfac/server/GfacServerHandler.java | 85 ++++++++++----------
.../core/impl/RabbitMQTaskLaunchConsumer.java | 1 -
.../core/impl/GFACPassiveJobSubmitter.java | 5 --
6 files changed, 43 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/60788efe/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index b7121b9..6937c25 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -95,7 +95,7 @@ public class CreateLaunchExperiment {
public static void createAndLaunchExp() throws TException {
// final String expId = createEchoExperimentForFSD(airavataClient);
try {
- for (int i = 0; i < 1; i++) {
+ for (int i = 0; i < 10; i++) {
// final String expId = createExperimentForSSHHost(airavata);
// final String expId = createEchoExperimentForFSD(airavataClient);
// final String expId = createMPIExperimentForFSD(airavataClient);
http://git-wip-us.apache.org/repos/asf/airavata/blob/60788efe/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java
deleted file mode 100644
index 2db9a6f..0000000
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.airavata.gfac.leader;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
-import org.apache.curator.framework.recipes.leader.LeaderSelector;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * An example leader selector client. Note that {@link LeaderSelectorListenerAdapter} which
- * has the recommended handling for connection state issues
- */
-public class CuratorClient extends LeaderSelectorListenerAdapter implements Closeable {
- private final String name;
- private final LeaderSelector leaderSelector;
- private final AtomicInteger leaderCount = new AtomicInteger();
-
- public CuratorClient(CuratorFramework client, String path, String name) {
- this.name = name;
-
- // create a leader selector using the given path for management
- // all participants in a given leader selection must use the same path
- // ExampleClient here is also a LeaderSelectorListener but this isn't required
- leaderSelector = new LeaderSelector(client, path, this);
-
- // for most cases you will want your instance to requeue when it relinquishes leadership
- leaderSelector.autoRequeue();
- }
-
- public void start() throws IOException {
- // the selection for this instance doesn't start until the leader selector is started
- // leader selection is done in the background so this call to leaderSelector.start() returns immediately
- leaderSelector.start();
- }
-
- @Override
- public void close() throws IOException {
- leaderSelector.close();
- }
-
- @Override
- public void takeLeadership(CuratorFramework client) throws Exception {
- // we are now the leader. This method should not return until we want to relinquish leadership
-
- final int waitSeconds = (int) (5 * Math.random()) + 1;
-
- System.out.println(name + " is now the leader. Waiting " + waitSeconds + " seconds...");
- System.out.println(name + " has been leader " + leaderCount.getAndIncrement() + " time(s) before.");
- try {
- Thread.sleep(TimeUnit.SECONDS.toMillis(waitSeconds));
- } catch (InterruptedException e) {
- System.err.println(name + " was interrupted.");
- Thread.currentThread().interrupt();
- } finally {
- System.out.println(name + " relinquishing leadership.\n");
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/60788efe/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java
deleted file mode 100644
index ad02641..0000000
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.leader;
-
-import com.google.common.collect.Lists;
-import org.apache.airavata.common.utils.AiravataZKUtils;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.curator.utils.CloseableUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.util.List;
-
-public class LeaderSelectorExample {
- private final static Logger logger = LoggerFactory.getLogger(LeaderSelectorExample.class);
- private static final int CLIENT_QTY = 10;
-
- private static final String PATH = "/examples/leader";
-
- public static void main(String[] args) throws Exception
- {
- // all of the useful sample code is in ExampleClient.java
-
- System.out.println("Create " + CLIENT_QTY + " clients, have each negotiate for leadership and then wait a random number of seconds before letting another leader election occur.");
- System.out.println("Notice that leader election is fair: all clients will become leader and will do so the same number of times.");
-
- try
- {
- for ( int i = 0; i < CLIENT_QTY; ++i )
- {
- CuratorFramework client = CuratorFrameworkFactory.newClient(AiravataZKUtils.getZKhostPort(), new ExponentialBackoffRetry(1000, 3));
-
- CuratorClient example = new CuratorClient(client, PATH, "Client #" + i);
-
- client.start();
- example.start();
- }
-
- System.out.println("Press enter/return to quit\n");
- new BufferedReader(new InputStreamReader(System.in)).readLine();
- }
- finally
- {
- System.out.println("Shutting down...");
-
- /*for ( CuratorClient exampleClient : examples )
- {
- CloseableUtils.closeQuietly(exampleClient);
- }
- for ( CuratorFramework client : clients )
- {
- CloseableUtils.closeQuietly(client);
- }*/
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/60788efe/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index c838703..679a5ee 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -35,7 +35,6 @@ import org.apache.airavata.gfac.core.utils.GFacUtils;
import org.apache.airavata.gfac.core.utils.InputHandlerWorker;
import org.apache.airavata.gfac.cpi.GfacService;
import org.apache.airavata.gfac.cpi.gfac_cpi_serviceConstants;
-import org.apache.airavata.gfac.leader.CuratorClient;
import org.apache.airavata.messaging.core.MessageContext;
import org.apache.airavata.messaging.core.MessageHandler;
import org.apache.airavata.messaging.core.MessagingConstants;
@@ -60,7 +59,9 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -93,11 +94,15 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private List<Future> inHandlerFutures;
- private RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer;
+ private String nodeName = null;
- CuratorFramework curatorFramework = null;
+ private CuratorFramework curatorFramework = null;
+ private BlockingQueue<TaskSubmitEvent> taskSubmitEvents;
+ private BlockingQueue<TaskTerminateEvent> taskTerminateEvents;
+
+ private CuratorClient curatorClient;
public GfacServerHandler() throws Exception{
// registering with zk
try {
@@ -107,6 +112,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
zk = new ZooKeeper(zkhostPort, 6000, this); // no watcher is required, this will only use to store some data
gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
gfacExperiments = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ nodeName = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME);
synchronized (mutex) {
mutex.wait(); // waiting for the syncConnected event
}
@@ -121,10 +127,14 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
BetterGfacImpl.startStatusUpdators(registry, zk, publisher);
inHandlerFutures = new ArrayList<Future>();
- if(ServerSettings.isGFacPassiveMode()) {
- rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
- rabbitMQTaskLaunchConsumer.listen(new TaskLaunchMessageHandler());
+ if (ServerSettings.isGFacPassiveMode()) {
+ taskSubmitEvents = new LinkedBlockingDeque<TaskSubmitEvent>();
+ taskTerminateEvents = new LinkedBlockingDeque<TaskTerminateEvent>();
curatorFramework = CuratorFrameworkFactory.newClient(AiravataZKUtils.getZKhostPort(), new ExponentialBackoffRetry(1000, 3));
+ curatorClient = new CuratorClient(curatorFramework, nodeName);
+
+ curatorFramework.start();
+ curatorClient.start();
}
@@ -296,51 +306,37 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
}
private class TaskLaunchMessageHandler implements MessageHandler {
- private String experimentId;
-
- private String nodeName;
-
+ public static final String LAUNCH_TASK = "launch.task";
+ public static final String TERMINATE_TASK = "teminate.task";
public TaskLaunchMessageHandler(){
- try {
- nodeName = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME);
- } catch (ApplicationSettingsException e) {
- logger.error(e.getMessage(), e);
- }
+
}
public Map<String, Object> getProperties() {
Map<String, Object> props = new HashMap<String, Object>();
- try {
- props.put(MessagingConstants.RABBIT_ROUTING_KEY, ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME));
- } catch (ApplicationSettingsException e) {
- // if we cannot find gfac node name configured we set a random id
- logger.error("airavata-server.properties should configure: " + Constants.ZOOKEEPER_GFAC_SERVER_NAME + " value.");
- logger.error("listening to a random generated routing key");
- props.put(MessagingConstants.RABBIT_ROUTING_KEY, UUID.randomUUID().toString());
- }
+ ArrayList<String> keys = new ArrayList<String>();
+ keys.add(LAUNCH_TASK);
+ keys.add(TERMINATE_TASK);
+ props.put(MessagingConstants.RABBIT_ROUTING_KEY, keys);
return props;
}
public void onMessage(MessageContext message) {
- if (message.getType().equals(MessageType.LAUNCHTASK)){
+ if (message.getType().equals(MessageType.LAUNCHTASK)) {
try {
TaskSubmitEvent event = new TaskSubmitEvent();
TBase messageEvent = message.getEvent();
byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
ThriftUtils.createThriftFromBytes(bytes, event);
- CuratorClient curatorClient = new CuratorClient(curatorFramework, event, nodeName);
- try {
- curatorClient.start();
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- }
+ taskSubmitEvents.add(event);
- System.out.println(" Message Received with message id '" + message.getMessageId()
- + "' and with message type '" + message.getType());
- } catch (TException e) {
+
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType());
+ } catch (TException e) {
logger.error(e.getMessage(), e); //nobody is listening so nothing to throw
}
- }else if(message.getType().equals(MessageType.TERMINATETASK)){
+ } else if (message.getType().equals(MessageType.TERMINATETASK)) {
try {
TaskTerminateEvent event = new TaskTerminateEvent();
TBase messageEvent = message.getEvent();
@@ -361,18 +357,16 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private final LeaderSelector leaderSelector;
private final AtomicInteger leaderCount = new AtomicInteger();
private final String path;
- private TaskSubmitEvent event;
private String experimentNode;
- public CuratorClient(CuratorFramework client, TaskSubmitEvent taskSubmitEvent, String name) {
+ public CuratorClient(CuratorFramework client, String name) {
this.name = name;
- this.event = taskSubmitEvent;
- this.path = File.separator + event.getExperimentId() + "-" + event.getTaskId() + "-" + event.getGatewayId();
// create a leader selector using the given path for management
// all participants in a given leader selection must use the same path
// ExampleClient here is also a LeaderSelectorListener but this isn't required
- leaderSelector = new LeaderSelector(client, path, this);
experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ path = experimentNode + File.separator + "leader";
+ leaderSelector = new LeaderSelector(client, path, this);
// for most cases you will want your instance to requeue when it relinquishes leadership
leaderSelector.autoRequeue();
}
@@ -393,18 +387,23 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
// we are now the leader. This method should not return until we want to relinquish leadership
final int waitSeconds = (int) (5 * Math.random()) + 1;
- System.out.println(name + " is now the leader. Waiting " + waitSeconds + " seconds...");
- System.out.println(name + " has been leader " + leaderCount.getAndIncrement() + " time(s) before.");
+ logger.info(name + " is now the leader. Waiting " + waitSeconds + " seconds...");
+ logger.info(name + " has been leader " + leaderCount.getAndIncrement() + " time(s) before.");
+ RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
+ String listenId = rabbitMQTaskLaunchConsumer.listen(new TaskLaunchMessageHandler());
+ TaskSubmitEvent event = taskSubmitEvents.take();
try {
GFacUtils.createExperimentEntryForRPC(event.getExperimentId(),event.getTaskId(),client.getZookeeperClient().getZooKeeper(),experimentNode,name,event.getTokenId());
submitJob(event.getExperimentId(), event.getTaskId(), event.getGatewayId());
Thread.sleep(TimeUnit.SECONDS.toMillis(waitSeconds));
} catch (InterruptedException e) {
- System.err.println(name + " was interrupted.");
+ logger.error(name + " was interrupted.");
Thread.currentThread().interrupt();
} finally {
- System.out.println(name + " relinquishing leadership.\n");
+ Thread.sleep(5);
+ logger.info(name + " relinquishing leadership.: "+ new Date().toString());
+ rabbitMQTaskLaunchConsumer.stopListen(listenId);
}
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/60788efe/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
index 056dcac..4bc7468 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
@@ -190,7 +190,6 @@ public class RabbitMQTaskLaunchConsumer {
for (String key : details.getRoutingKeys()) {
channel.queueUnbind(details.getQueueName(), taskLaunchExchangeName, key);
}
- channel.queueDelete(details.getQueueName(), true, true);
} catch (IOException e) {
String msg = "could not un-bind queue: " + details.getQueueName() + " for exchange " + taskLaunchExchangeName;
log.debug(msg);
http://git-wip-us.apache.org/repos/asf/airavata/blob/60788efe/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
index 78cc6b7..b5e25b1 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
@@ -102,7 +102,6 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
public boolean submit(String experimentID, String taskID, String tokenId) throws OrchestratorException {
ZooKeeper zk = orchestratorContext.getZk();
- GfacService.Client gfacClient = null;
try {
if (zk == null || !zk.getState().isConnected()) {
String zkhostPort = AiravataZKUtils.getZKhostPort();
@@ -151,8 +150,6 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw new OrchestratorException(e);
- }finally {
- gfacClient.getOutputProtocol().getTransport().close();
}
return true;
@@ -167,7 +164,6 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
*/
public boolean terminate(String experimentID, String taskID) throws OrchestratorException {
ZooKeeper zk = orchestratorContext.getZk();
- GfacService.Client localhost = null;
try {
if (zk == null || !zk.getState().isConnected()) {
String zkhostPort = AiravataZKUtils.getZKhostPort();
@@ -189,7 +185,6 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
logger.info("GFAC instance node data: " + gfacNodeData);
String[] split = gfacNodeData.split(":");
- localhost = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
// before submitting the job we check again the state of the node
if (GFacUtils.createExperimentEntryForRPC(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
[09/15] airavata git commit: adding support to proper acking for
messages
Posted by la...@apache.org.
adding support to proper acking for messages
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/1231c014
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/1231c014
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/1231c014
Branch: refs/heads/master
Commit: 1231c014bebd1d23c2bdd340b7d721abe279d45a
Parents: ffbb1b9
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Wed Feb 25 00:59:09 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Wed Feb 25 00:59:09 2015 -0500
----------------------------------------------------------------------
.../airavata/api/server/AiravataAPIServer.java | 1 +
.../client/samples/CreateLaunchExperiment.java | 23 +++--
.../airavata/gfac/server/GfacServerHandler.java | 48 +++++++++--
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 1 +
.../core/monitor/GfacInternalStatusUpdator.java | 3 +
.../airavata/gfac/core/utils/GFacUtils.java | 21 +++--
.../handlers/GridPullMonitorHandler.java | 1 +
.../messaging/client/RabbitMQListner.java | 4 +-
.../airavata/messaging/core/MessageContext.java | 17 ++++
.../core/impl/RabbitMQTaskLaunchConsumer.java | 10 ++-
.../server/OrchestratorServerHandler.java | 90 ++++++++++----------
.../util/OrchestratorRecoveryHandler.java | 1 +
.../core/impl/GFACPassiveJobSubmitter.java | 10 +--
13 files changed, 151 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
index 0e6da90..da42ce0 100644
--- a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
+++ b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
@@ -299,6 +299,7 @@ public class AiravataAPIServer implements IServer, Watcher{
@Override
synchronized public void process(WatchedEvent watchedEvent) {
+ logger.info(watchedEvent.getPath());
synchronized (mutex) {
Event.KeeperState state = watchedEvent.getState();
logger.info(state.name());
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index c4c303f..78c2d71 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -47,17 +47,17 @@ import java.util.*;
public class CreateLaunchExperiment {
//FIXME: Read from a config file
-// public static final String THRIFT_SERVER_HOST = "localhost";
-// public static final int THRIFT_SERVER_PORT = 8930;
- public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org";
- public static final int THRIFT_SERVER_PORT = 9930;
+ public static final String THRIFT_SERVER_HOST = "localhost";
+ public static final int THRIFT_SERVER_PORT = 8930;
+// public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org";
+// public static final int THRIFT_SERVER_PORT = 9930;
private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
private static final String DEFAULT_USER = "default.registry.user";
private static final String DEFAULT_GATEWAY = "default.registry.gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_a8fc8511-7b8e-431a-ad0f-de5eb1a9c576";
+ private static String echoAppId = "Echo_1365a7fd-eae1-4575-b447-99afb4d79c82";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_42124128-628b-484c-829d-aff8b584eb00";
@@ -93,7 +93,7 @@ public class CreateLaunchExperiment {
// final String expId = createEchoExperimentForFSD(airavataClient);
List<String> experimentIds = new ArrayList<String>();
try {
- for (int i = 0; i < 100; i++) {
+ for (int i = 0; i < 1; i++) {
// final String expId = createExperimentForSSHHost(airavata);
// final String expId = createEchoExperimentForFSD(airavataClient);
// final String expId = createMPIExperimentForFSD(airavataClient);
@@ -120,12 +120,11 @@ public class CreateLaunchExperiment {
launchExperiment(airavataClient, expId);
}
- Thread.sleep(10000);
-
- for(String exId:experimentIds) {
- Experiment experiment = airavataClient.getExperiment(exId);
- System.out.println(experiment.getExperimentStatus().toString());
- }
+ Thread.sleep(100);
+ for (String exId : experimentIds) {
+ Experiment experiment = airavataClient.getExperiment(exId);
+ System.out.println(experiment.getExperimentStatus().toString());
+ }
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index 1c0f095..cca793e 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -65,8 +65,6 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
private Registry registry;
private AppCatalog appCatalog;
- private String registryURL;
-
private String gatewayName;
private String airavataUserName;
@@ -144,12 +142,13 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
CreateMode.PERSISTENT);
}
String instanceId = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME);
- String instantNode = gfacServer + File.separator + instanceId;
- zkStat = zk.exists(instantNode, true);
+ String instanceNode = gfacServer + File.separator + instanceId;
+ zkStat = zk.exists(instanceNode, true);
if (zkStat == null) {
- zk.create(instantNode,
+ zk.create(instanceNode,
airavataServerHostPort.getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL); // other component will watch these childeren creation deletion to monitor the status of the node
+ zk.getChildren(instanceNode, true);
}
zkStat = zk.exists(gfacExperiments, false);
if (zkStat == null) {
@@ -168,6 +167,8 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
}
synchronized public void process(WatchedEvent watchedEvent) {
+ logger.info(watchedEvent.getPath());
+ logger.info(watchedEvent.getType().toString());
synchronized (mutex) {
Event.KeeperState state = watchedEvent.getState();
logger.info(state.name());
@@ -191,10 +192,39 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
} catch (KeeperException e) {
logger.error(e.getMessage(), e);
}
+ } else if (Event.EventType.NodeDeleted.equals(watchedEvent.getType())) {
+ String path = watchedEvent.getPath();
+ String experimentNode = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ if (path.startsWith(experimentNode)) {
+ // we got a watch when experiment is removed
+ String deliveryPath = path + GFacUtils.DELIVERY_TAG_POSTFIX;
+ try {
+ Stat exists = zk.exists(deliveryPath, false);
+ byte[] data = zk.getData(path + GFacUtils.DELIVERY_TAG_POSTFIX, false, exists);
+ long value = ByateArrayToLong(data);
+ logger.info("ExperimentId+taskId" + path);
+ logger.info("Sending Ack back to the Queue, because task is over");
+ rabbitMQTaskLaunchConsumer.sendAck(value);
+ ZKUtil.deleteRecursive(zk,deliveryPath);
+ } catch (KeeperException e) {
+ logger.error(e.getMessage(), e);
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
}
}
}
+ private long ByateArrayToLong(byte[] data) {
+ long value = 0;
+ for (int i = 0; i < data.length; i++)
+ {
+ value += ((long) data[i] & 0xffL) << (8 * i);
+ }
+ return value;
+ }
+
public String getGFACServiceVersion() throws TException {
return gfac_cpi_serviceConstants.GFAC_CPI_VERSION;
}
@@ -314,12 +344,18 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
try {
- GFacUtils.createExperimentEntryForRPC(event.getExperimentId(), event.getTaskId(), zk, experimentNode, nodeName, event.getTokenId());
+ GFacUtils.createExperimentEntryForPassive(event.getExperimentId(), event.getTaskId(), zk, experimentNode, nodeName, event.getTokenId(), message.getDeliveryTag());
+ AiravataZKUtils.getExpStatePath(event.getExperimentId(),event.getTaskId());
submitJob(event.getExperimentId(), event.getTaskId(), event.getGatewayId());
} catch (KeeperException e) {
logger.error(nodeName + " was interrupted.");
+ rabbitMQTaskLaunchConsumer.sendAck(message.getDeliveryTag());
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
+ rabbitMQTaskLaunchConsumer.sendAck(message.getDeliveryTag());
+ } catch (ApplicationSettingsException e) {
+ logger.error(e.getMessage(), e);
+ rabbitMQTaskLaunchConsumer.sendAck(message.getDeliveryTag());
}
System.out.println(" Message Received with message id '" + message.getMessageId()
+ "' and with message type '" + message.getType());
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index bb612a6..00930e5 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -1158,6 +1158,7 @@ public class BetterGfacImpl implements GFac,Watcher {
}
public void process(WatchedEvent watchedEvent) {
+ log.info(watchedEvent.getPath());
if(Event.EventType.NodeDataChanged.equals(watchedEvent.getType())){
// node data is changed, this means node is cancelled.
log.info("Experiment is cancelled with this path:"+watchedEvent.getPath());
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
index 7818da0..26902e7 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
@@ -29,6 +29,7 @@ import org.apache.airavata.common.utils.Constants;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.common.utils.listener.AbstractActivityListener;
import org.apache.airavata.gfac.core.monitor.state.GfacExperimentStateChangeRequest;
+import org.apache.airavata.gfac.core.utils.GFacUtils;
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
@@ -49,6 +50,7 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
MonitorID monitorID = statusChangeRequest.getMonitorID();
String experimentPath = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments") +
File.separator + ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME) + File.separator + statusChangeRequest.getMonitorID().getExperimentID() + "+" + monitorID.getTaskID();
+ String deliveryTagPath = experimentPath + GFacUtils.DELIVERY_TAG_POSTFIX;
Stat exists = null;
try {
if (!zk.getState().isConnected()) {
@@ -107,6 +109,7 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
}
public void process(WatchedEvent watchedEvent) {
+ logger.info(watchedEvent.getPath());
synchronized (mutex) {
Event.KeeperState state = watchedEvent.getState();
if (state == Event.KeeperState.SyncConnected) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
index 9f104fa..c825ffd 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
@@ -60,12 +60,14 @@ import java.io.*;
import java.net.InetAddress;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
import java.util.*;
//import org.apache.airavata.commons.gfac.type.ActualParameter;
public class GFacUtils {
private final static Logger log = LoggerFactory.getLogger(GFacUtils.class);
+ public static final String DELIVERY_TAG_POSTFIX = "-deliveryTag";
private GFacUtils() {
}
@@ -1156,7 +1158,7 @@ public class GFacUtils {
// This method is dangerous because of moving the experiment data
public static boolean createExperimentEntryForPassive(String experimentID,
String taskID, ZooKeeper zk, String experimentNode,
- String pickedChild, String tokenId) throws KeeperException,
+ String pickedChild, String tokenId,long deliveryTag) throws KeeperException,
InterruptedException {
String experimentPath = experimentNode + File.separator + pickedChild;
String newExpNode = experimentPath + File.separator + experimentID
@@ -1165,15 +1167,14 @@ public class GFacUtils {
String experimentEntry = GFacUtils.findExperimentEntry(experimentID, taskID, zk);
String foundExperimentPath = null;
if (exists1 == null && experimentEntry == null) { // this means this is a very new experiment
- List<String> runningGfacNodeNames = AiravataZKUtils
- .getAllGfacNodeNames(zk); // here we take old gfac servers
- // too
+ List<String> runningGfacNodeNames = AiravataZKUtils.getAllGfacNodeNames(zk); // here we take old gfac servers
+
for (String gfacServerNode : runningGfacNodeNames) {
if (!gfacServerNode.equals(pickedChild)) {
foundExperimentPath = experimentNode + File.separator
+ gfacServerNode + File.separator + experimentID
+ "+" + taskID;
- exists1 = zk.exists(foundExperimentPath, false);
+ exists1 = zk.exists(foundExperimentPath, true);
if (exists1 != null) { // when the experiment is found we
// break the loop
break;
@@ -1183,21 +1184,23 @@ public class GFacUtils {
if (exists1 == null) { // OK this is a pretty new experiment so we
// are going to create a new node
log.info("This is a new Job, so creating all the experiment docs from the scratch");
+ Stat expParent = zk.exists(newExpNode, false);
zk.create(newExpNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
- Stat expParent = zk.exists(newExpNode, false);
if (tokenId != null && expParent != null) {
zk.setData(newExpNode, tokenId.getBytes(),
expParent.getVersion());
}
- zk.create(newExpNode + File.separator + "state", String
+ String s = zk.create(newExpNode + File.separator + "state", String
.valueOf(GfacExperimentState.LAUNCHED.getValue())
.getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
- zk.create(newExpNode + File.separator + "operation","submit".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ String s1 = zk.create(newExpNode + File.separator + "operation", "submit".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ zk.exists(s1, true);// we want to know when this node get deleted
+ String s2 = zk.create(newExpNode + DELIVERY_TAG_POSTFIX, ByteBuffer.allocate(8).putLong(deliveryTag).array(), ZooDefs.Ids.OPEN_ACL_UNSAFE, // here we store the value of delivery message
CreateMode.PERSISTENT);
-
} else {
// ohhh this node exists in some other failed gfac folder, we
// have to move it to this gfac experiment list,safely
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
index e64f596..d5f9f90 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
@@ -125,6 +125,7 @@ public class GridPullMonitorHandler extends ThreadedHandler implements Watcher{
public void process(WatchedEvent watchedEvent) {
+ logger.info(watchedEvent.getPath());
if(Event.EventType.NodeDataChanged.equals(watchedEvent.getType())){
// node data is changed, this means node is cancelled.
logger.info("Experiment is cancelled with this path:"+watchedEvent.getPath());
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListner.java
----------------------------------------------------------------------
diff --git a/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListner.java b/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListner.java
index 601497a..48edbe8 100644
--- a/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListner.java
+++ b/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListner.java
@@ -28,7 +28,7 @@ import org.apache.airavata.common.utils.ThriftUtils;
import org.apache.airavata.messaging.core.MessageContext;
import org.apache.airavata.messaging.core.MessageHandler;
import org.apache.airavata.messaging.core.MessagingConstants;
-import org.apache.airavata.messaging.core.impl.RabbitMQConsumer;
+import org.apache.airavata.messaging.core.impl.RabbitMQStatusConsumer;
import org.apache.airavata.model.messaging.event.*;
import org.apache.airavata.model.workspace.experiment.ExperimentState;
import org.apache.commons.cli.*;
@@ -67,7 +67,7 @@ public class RabbitMQListner {
String brokerUrl = ServerSettings.getSetting(RABBITMQ_BROKER_URL);
System.out.println("broker url " + brokerUrl);
final String exchangeName = ServerSettings.getSetting(RABBITMQ_EXCHANGE_NAME);
- RabbitMQConsumer consumer = new RabbitMQConsumer(brokerUrl, exchangeName);
+ RabbitMQStatusConsumer consumer = new RabbitMQStatusConsumer(brokerUrl, exchangeName);
consumer.listen(new MessageHandler() {
@Override
public Map<String, Object> getProperties() {
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessageContext.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessageContext.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessageContext.java
index 0a39d92..272f413 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessageContext.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessageContext.java
@@ -32,6 +32,7 @@ public class MessageContext {
private final String messageId;
private final String gatewayId;
private Timestamp updatedTime;
+ private long deliveryTag;
public MessageContext(TBase message, MessageType type, String messageId, String gatewayId) {
@@ -41,6 +42,14 @@ public class MessageContext {
this.gatewayId = gatewayId;
}
+ public MessageContext(TBase event, MessageType type, String messageId, String gatewayId, long deliveryTag) {
+ this.event = event;
+ this.type = type;
+ this.messageId = messageId;
+ this.gatewayId = gatewayId;
+ this.deliveryTag = deliveryTag;
+ }
+
public TBase getEvent() {
return event;
}
@@ -64,4 +73,12 @@ public class MessageContext {
public String getGatewayId() {
return gatewayId;
}
+
+ public long getDeliveryTag() {
+ return deliveryTag;
+ }
+
+ public void setDeliveryTag(long deliveryTag) {
+ this.deliveryTag = deliveryTag;
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
index 1c7b0e8..7c88a25 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
@@ -165,7 +165,7 @@ public class RabbitMQTaskLaunchConsumer {
event = taskTerminateEvent;
gatewayId = null;
}
- MessageContext messageContext = new MessageContext(event, message.getMessageType(), message.getMessageId(), gatewayId);
+ MessageContext messageContext = new MessageContext(event, message.getMessageType(), message.getMessageId(), gatewayId,deliveryTag);
messageContext.setUpdatedTime(AiravataUtils.getTime(message.getUpdatedTime()));
handler.onMessage(messageContext);
try {
@@ -241,4 +241,12 @@ public class RabbitMQTaskLaunchConsumer {
}
}
}
+
+ public void sendAck(long deliveryTag){
+ try {
+ channel.basicAck(deliveryTag,false); //todo move this logic to monitoring component to ack when the job is done
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
index b200468..f430bc9 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
@@ -292,43 +292,45 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
* This method gracefully handler gfac node failures
*/
synchronized public void process(WatchedEvent watchedEvent) {
+ log.info(watchedEvent.getPath());
synchronized (mutex) {
try {
Event.KeeperState state = watchedEvent.getState();
switch (state) {
- case SyncConnected:
- mutex.notify();
- break;
- case Expired:case Disconnected:
- try {
- zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, this);
- synchronized (mutex) {
- mutex.wait(); // waiting for the syncConnected event
- }
- String airavataServerHostPort = ServerSettings
- .getSetting(Constants.ORCHESTRATOR_SERVER_HOST)
- + ":"
- + ServerSettings
- .getSetting(Constants.ORCHESTRATOR_SERVER_PORT);
- String OrchServer = ServerSettings
- .getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_ORCHESTRATOR_SERVER_NODE);
- registerOrchestratorService(airavataServerHostPort, OrchServer);
- } catch (IOException e) {
- e.printStackTrace();
- } catch (ApplicationSettingsException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- } catch (KeeperException e) {
- e.printStackTrace();
- }
- break;
- }
+ case SyncConnected:
+ mutex.notify();
+ break;
+ case Expired:
+ case Disconnected:
+ try {
+ zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, this);
+ synchronized (mutex) {
+ mutex.wait(); // waiting for the syncConnected event
+ }
+ String airavataServerHostPort = ServerSettings
+ .getSetting(Constants.ORCHESTRATOR_SERVER_HOST)
+ + ":"
+ + ServerSettings
+ .getSetting(Constants.ORCHESTRATOR_SERVER_PORT);
+ String OrchServer = ServerSettings
+ .getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_ORCHESTRATOR_SERVER_NODE);
+ registerOrchestratorService(airavataServerHostPort, OrchServer);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (ApplicationSettingsException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (KeeperException e) {
+ e.printStackTrace();
+ }
+ break;
+ }
if (watchedEvent.getPath() != null
&& watchedEvent.getPath().startsWith(
- ServerSettings.getSetting(
- Constants.ZOOKEEPER_GFAC_SERVER_NODE,
- "/gfac-server"))) {
+ ServerSettings.getSetting(
+ Constants.ZOOKEEPER_GFAC_SERVER_NODE,
+ "/gfac-server"))) {
List<String> children = zk.getChildren(ServerSettings
.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE,
"/gfac-server"), true);
@@ -340,18 +342,18 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
+ File.separator + gfacNodes, this);
}
switch (watchedEvent.getType()) {
- case NodeCreated:
- mutex.notify();
- break;
- case NodeDeleted:
- // here we have to handle gfac node shutdown case
- if (children.size() == 0) {
- log.error("There are not gfac instances to route failed jobs");
- return;
- }
- // we recover one gfac node at a time
- final WatchedEvent event = watchedEvent;
- final OrchestratorServerHandler handler = this;
+ case NodeCreated:
+ mutex.notify();
+ break;
+ case NodeDeleted:
+ // here we have to handle gfac node shutdown case
+ if (children.size() == 0) {
+ log.error("There are not gfac instances to route failed jobs");
+ return;
+ }
+ // we recover one gfac node at a time
+ final WatchedEvent event = watchedEvent;
+ final OrchestratorServerHandler handler = this;
/*(new Thread() { // disabling ft implementation with zk
public void run() {
int retry = 0;
@@ -372,7 +374,7 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
}
}).start();*/
- break;
+ break;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
index fb3bd51..f19b949 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
@@ -95,6 +95,7 @@ public class OrchestratorRecoveryHandler implements Watcher {
}
synchronized public void process(WatchedEvent watchedEvent) {
+ log.info(watchedEvent.getPath());
synchronized (mutex) {
Event.KeeperState state = watchedEvent.getState();
switch (state) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/1231c014/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
index b5e25b1..8066113 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
@@ -187,11 +187,9 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
String[] split = gfacNodeData.split(":");
if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
// before submitting the job we check again the state of the node
- if (GFacUtils.createExperimentEntryForRPC(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
- TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, null,null);
- MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK,"LAUNCH.TERMINATE-"+ UUID.randomUUID().toString(),null);
- publisher.publish(messageContext);
- }
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, null, null);
+ MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.TERMINATETASK, "LAUNCH.TERMINATE-" + UUID.randomUUID().toString(), null);
+ publisher.publish(messageContext);
}
}
} catch (InterruptedException e) {
@@ -217,6 +215,8 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
}
synchronized public void process(WatchedEvent event) {
+ logger.info(getClass().getName() + event.getPath());
+ logger.info(getClass().getName()+event.getType());
synchronized (mutex) {
switch (event.getState()) {
case SyncConnected:
[12/15] airavata git commit: adding more fixes to worker based
submission
Posted by la...@apache.org.
adding more fixes to worker based submission
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/cb6c4ccf
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/cb6c4ccf
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/cb6c4ccf
Branch: refs/heads/master
Commit: cb6c4ccf267c165e64d6858584b2300eaaa37df4
Parents: 48be39f
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Thu Mar 19 16:08:29 2015 -0400
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Thu Mar 19 16:08:29 2015 -0400
----------------------------------------------------------------------
.../airavata/api/server/AiravataAPIServer.java | 4 +-
.../client/samples/CreateLaunchExperiment.java | 432 ++++++++++---------
.../airavata/common/utils/AiravataZKUtils.java | 5 +
.../apache/airavata/common/utils/Constants.java | 1 +
.../main/resources/airavata-server.properties | 10 +-
.../main/resources/airavata-server.properties | 1 +
.../airavata/gfac/server/GfacServerHandler.java | 14 +-
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 5 +-
.../core/monitor/GfacInternalStatusUpdator.java | 32 +-
.../airavata/gfac/core/monitor/MonitorID.java | 3 +-
.../airavata/gfac/core/utils/GFacUtils.java | 127 +++---
.../gfac/local/provider/impl/LocalProvider.java | 4 +-
.../airavata/gfac/monitor/util/CommonUtils.java | 2 +-
.../core/impl/RabbitMQTaskLaunchConsumer.java | 5 +-
.../server/OrchestratorServerHandler.java | 4 +-
.../util/OrchestratorRecoveryHandler.java | 2 +-
.../core/impl/GFACPassiveJobSubmitter.java | 44 +-
.../core/impl/GFACRPCJobSubmitter.java | 4 +-
18 files changed, 369 insertions(+), 330 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
index 2556df1..159e0e4 100644
--- a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
+++ b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
@@ -127,7 +127,7 @@ public class AiravataAPIServer implements IServer, Watcher{
String rabbitMqBrokerURL = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.RABBITMQ_BROKER_URL);
String rabbitMqExchange = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.RABBITMQ_EXCHANGE);
String rabbitMq = rabbitMqBrokerURL + File.separator + rabbitMqExchange;
- zk = new ZooKeeper(zkhostPort, 6000, this); // no watcher is required, this will only use to store some data
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this); // no watcher is required, this will only use to store some data
String apiServer = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_API_SERVER_NODE, "/airavata-server");
String OrchServer = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_ORCHESTRATOR_SERVER_NODE, "/orchestrator-server");
String gfacServer = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
@@ -309,7 +309,7 @@ public class AiravataAPIServer implements IServer, Watcher{
case Expired:case Disconnected:
try {
mutex = -1;
- zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, this);
+ zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait(); // waiting for the syncConnected event
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index f499345..231da87 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -52,13 +52,13 @@ public class CreateLaunchExperiment {
public static final int THRIFT_SERVER_PORT = 8930;
// public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org";
// public static final int THRIFT_SERVER_PORT = 9930;
-
+
private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
private static final String DEFAULT_USER = "default.registry.user";
private static final String DEFAULT_GATEWAY = "php_reference_gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_1365a7fd-eae1-4575-b447-99afb4d79c82";
+ private static String echoAppId = "Echo_78c785c6-3748-4a93-8569-19fee56581bc";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_aa083c86-4680-4002-b3ef-fad93c181926";
@@ -78,27 +78,26 @@ public class CreateLaunchExperiment {
private static String gatewayId;
- // unicore service endpoint url
+ // unicore service endpoint url
private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";
-
-
+
+
public static void main(String[] args) throws Exception {
airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
System.out.println("API version is " + airavataClient.getAPIVersion());
- getAvailableAppInterfaceComputeResources("Echo_4fb76cb3-6bf6-409e-aa17-7cc7ee6e41af");
// createGateway();
// getGateway("testGatewayId");
// registerApplications(); // run this only the first time
-// createAndLaunchExp();
+ createAndLaunchExp();
}
-
+
private static String fsdResourceId;
public static void getAvailableAppInterfaceComputeResources(String appInterfaceId) {
try {
Map<String, String> availableAppInterfaceComputeResources = airavataClient.getAvailableAppInterfaceComputeResources(appInterfaceId);
- for (String key : availableAppInterfaceComputeResources.keySet()){
+ for (String key : availableAppInterfaceComputeResources.keySet()) {
System.out.println("id : " + key);
System.out.println("name : " + availableAppInterfaceComputeResources.get(key));
}
@@ -115,7 +114,7 @@ public class CreateLaunchExperiment {
}
- public static void createGateway(){
+ public static void createGateway() {
try {
Gateway gateway = new Gateway();
gateway.setGatewayId("testGatewayId2");
@@ -134,14 +133,14 @@ public class CreateLaunchExperiment {
}
- public static void getGateway(String gatewayId){
+ public static void getGateway(String gatewayId) {
try {
Gateway gateway = airavataClient.getGateway(gatewayId);
gateway.setDomain("testDomain");
airavataClient.updateGateway(gatewayId, gateway);
List<Gateway> allGateways = airavataClient.getAllGateways();
System.out.println(allGateways.size());
- if (airavataClient.isGatewayExist(gatewayId)){
+ if (airavataClient.isGatewayExist(gatewayId)) {
Gateway gateway1 = airavataClient.getGateway(gatewayId);
System.out.println(gateway1.getGatewayName());
}
@@ -161,10 +160,9 @@ public class CreateLaunchExperiment {
public static void createAndLaunchExp() throws TException {
-// final String expId = createEchoExperimentForFSD(airavataClient);
List<String> experimentIds = new ArrayList<String>();
try {
- for (int i = 0; i < 1; i++) {
+ for (int i = 0; i < 100; i++) {
// final String expId = createExperimentForSSHHost(airavata);
// final String expId = createEchoExperimentForFSD(airavataClient);
// final String expId = createMPIExperimentForFSD(airavataClient);
@@ -192,11 +190,11 @@ public class CreateLaunchExperiment {
launchExperiment(airavataClient, expId);
}
- Thread.sleep(100);
- for (String exId : experimentIds) {
- Experiment experiment = airavataClient.getExperiment(exId);
- System.out.println(experiment.getExperimentStatus().toString());
- }
+ Thread.sleep(10000);
+ for (String exId : experimentIds) {
+ Experiment experiment = airavataClient.getExperiment(exId);
+ System.out.println(experiment.getExperimentID() + " " + experiment.getExperimentStatus().getExperimentState().name());
+ }
} catch (Exception e) {
@@ -206,8 +204,7 @@ public class CreateLaunchExperiment {
}
}
-
-
+
public static void registerApplications() {
RegisterSampleApplications registerSampleApplications = new RegisterSampleApplications(airavataClient);
@@ -233,25 +230,25 @@ public class CreateLaunchExperiment {
}
public static String registerUnicoreEndpoint(String hostName, String hostDesc, JobSubmissionProtocol protocol, SecurityProtocol securityProtocol) throws TException {
-
- ComputeResourceDescription computeResourceDescription = RegisterSampleApplicationsUtils
- .createComputeResourceDescription(hostName, hostDesc, null, null);
-
- fsdResourceId = airavataClient.registerComputeResource(computeResourceDescription);
-
- if (fsdResourceId.isEmpty())
- throw new AiravataClientException();
-
- System.out.println("FSD Compute ResourceID: "+fsdResourceId);
-
- JobSubmissionInterface jobSubmission = RegisterSampleApplicationsUtils.createJobSubmissionInterface(fsdResourceId, protocol, 2);
- UnicoreJobSubmission ucrJobSubmission = new UnicoreJobSubmission();
- ucrJobSubmission.setSecurityProtocol(securityProtocol);
- ucrJobSubmission.setUnicoreEndPointURL(unicoreEndPointURL);
-
- return jobSubmission.getJobSubmissionInterfaceId();
- }
-
+
+ ComputeResourceDescription computeResourceDescription = RegisterSampleApplicationsUtils
+ .createComputeResourceDescription(hostName, hostDesc, null, null);
+
+ fsdResourceId = airavataClient.registerComputeResource(computeResourceDescription);
+
+ if (fsdResourceId.isEmpty())
+ throw new AiravataClientException();
+
+ System.out.println("FSD Compute ResourceID: " + fsdResourceId);
+
+ JobSubmissionInterface jobSubmission = RegisterSampleApplicationsUtils.createJobSubmissionInterface(fsdResourceId, protocol, 2);
+ UnicoreJobSubmission ucrJobSubmission = new UnicoreJobSubmission();
+ ucrJobSubmission.setSecurityProtocol(securityProtocol);
+ ucrJobSubmission.setUnicoreEndPointURL(unicoreEndPointURL);
+
+ return jobSubmission.getJobSubmissionInterfaceId();
+ }
+
public static String createEchoExperimentForTrestles(Airavata.Client client) throws TException {
try {
List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId);
@@ -299,28 +296,27 @@ public class CreateLaunchExperiment {
}
return null;
}
-
-
+
+
public static String createEchoExperimentForFSD(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId);
for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) {
- inputDataObjectType.setValue("Hello World");
- }else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo2")) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) {
+ inputDataObjectType.setValue("Hello World");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo2")) {
inputDataObjectType.setValue("http://www.textfiles.com/100/ad.txt");
- }else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo3")) {
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo3")) {
inputDataObjectType.setValue("file:///tmp/test.txt");
- }
+ }
}
List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId);
- Experiment simpleExperiment =
+ Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "echoExperiment", "SimpleEcho2", echoAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
-
-
-
+
+
Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(echoAppId);
if (computeResources != null && computeResources.size() != 0) {
for (String id : computeResources.keySet()) {
@@ -331,13 +327,13 @@ public class CreateLaunchExperiment {
userConfigurationData.setAiravataAutoSchedule(false);
userConfigurationData.setOverrideManualScheduledParams(false);
userConfigurationData.setComputationalResourceScheduling(scheduling);
-
+
// set output directory
AdvancedOutputDataHandling dataHandling = new AdvancedOutputDataHandling();
- dataHandling.setOutputDataDir("/tmp/airavata/output/"+UUID.randomUUID().toString()+"/");
+ dataHandling.setOutputDataDir("/tmp/airavata/output/" + UUID.randomUUID().toString() + "/");
userConfigurationData.setAdvanceOutputDataHandling(dataHandling);
simpleExperiment.setUserConfigurationData(userConfigurationData);
-
+
return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment);
}
}
@@ -357,25 +353,24 @@ public class CreateLaunchExperiment {
}
return null;
}
-
-
+
+
public static String createMPIExperimentForFSD(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(mpiAppId);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(mpiAppId);
for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Sample_Input")) {
- inputDataObjectType.setValue("");
- }
+ if (inputDataObjectType.getName().equalsIgnoreCase("Sample_Input")) {
+ inputDataObjectType.setValue("");
+ }
}
List<OutputDataObjectType> exOut = client.getApplicationOutputs(mpiAppId);
-
- Experiment simpleExperiment =
+
+ Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "mpiExperiment", "HelloMPI", mpiAppId, null);
- simpleExperiment.setExperimentOutputs(exOut);
-
-
-
+ simpleExperiment.setExperimentOutputs(exOut);
+
+
Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(mpiAppId);
if (computeResources != null && computeResources.size() != 0) {
for (String id : computeResources.keySet()) {
@@ -386,13 +381,13 @@ public class CreateLaunchExperiment {
userConfigurationData.setAiravataAutoSchedule(false);
userConfigurationData.setOverrideManualScheduledParams(false);
userConfigurationData.setComputationalResourceScheduling(scheduling);
-
+
// set output directory
AdvancedOutputDataHandling dataHandling = new AdvancedOutputDataHandling();
- dataHandling.setOutputDataDir("/tmp/airavata/output/"+UUID.randomUUID().toString()+"/");
+ dataHandling.setOutputDataDir("/tmp/airavata/output/" + UUID.randomUUID().toString() + "/");
userConfigurationData.setAdvanceOutputDataHandling(dataHandling);
simpleExperiment.setUserConfigurationData(userConfigurationData);
-
+
return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment);
}
}
@@ -413,15 +408,13 @@ public class CreateLaunchExperiment {
return null;
}
-
-
-
+
public static String createExperimentWRFStampede(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(wrfAppId);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(wrfAppId);
setWRFInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(wrfAppId);
-
+
Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "WRFExperiment", "Testing", wrfAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -458,26 +451,26 @@ public class CreateLaunchExperiment {
}
- private static void setWRFInputs(List<InputDataObjectType> exInputs) {
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Config_Namelist_File")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/WRF_FILES/namelist.input");
- }else if (inputDataObjectType.getName().equalsIgnoreCase("WRF_Initial_Conditions")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/WRF_FILES/wrfinput_d01");
- }else if (inputDataObjectType.getName().equalsIgnoreCase("WRF_Boundary_File")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/WRF_FILES/wrfbdy_d01");
- }
- }
- }
+ private static void setWRFInputs(List<InputDataObjectType> exInputs) {
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Config_Namelist_File")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/WRF_FILES/namelist.input");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("WRF_Initial_Conditions")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/WRF_FILES/wrfinput_d01");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("WRF_Boundary_File")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/WRF_FILES/wrfbdy_d01");
+ }
+ }
+ }
public static String createExperimentGROMACSStampede(Airavata.Client client) throws TException {
try {
-
- List<InputDataObjectType> exInputs = client.getApplicationInputs(gromacsAppId);
- setGROMACSInputs(exInputs);
+
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(gromacsAppId);
+ setGROMACSInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(gromacsAppId);
-
+
Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "GromacsExperiment", "Testing", gromacsAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -514,20 +507,21 @@ public class CreateLaunchExperiment {
}
private static void setGROMACSInputs(List<InputDataObjectType> exInputs) {
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("GROMOS_Coordinate_File")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/GROMMACS_FILES/pdb1y6l-EM-vacuum.gro");
- }else if (inputDataObjectType.getName().equalsIgnoreCase("Portable_Input_Binary_File")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/GROMMACS_FILES/pdb1y6l-EM-vacuum.tpr");
- }
- }
- }
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("GROMOS_Coordinate_File")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/GROMMACS_FILES/pdb1y6l-EM-vacuum.gro");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Portable_Input_Binary_File")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/GROMMACS_FILES/pdb1y6l-EM-vacuum.tpr");
+ }
+ }
+ }
+
public static String createExperimentESPRESSOStampede(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(espressoAppId);
- setESPRESSOInputs(exInputs);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(espressoAppId);
+ setESPRESSOInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(espressoAppId);
-
+
Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "EspressoExperiment", "Testing", espressoAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -562,22 +556,23 @@ public class CreateLaunchExperiment {
}
return null;
}
+
private static void setESPRESSOInputs(List<InputDataObjectType> exInputs) {
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("AI_Pseudopotential_File")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/ESPRESSO_FILES/Al.sample.in");
- }else if (inputDataObjectType.getName().equalsIgnoreCase("AI_Primitive_Cell")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/ESPRESSO_FILES/Al.pz-vbc.UPF");
- }
- }
-}
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("AI_Pseudopotential_File")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/ESPRESSO_FILES/Al.sample.in");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("AI_Primitive_Cell")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/ESPRESSO_FILES/Al.pz-vbc.UPF");
+ }
+ }
+ }
public static String createExperimentTRINITYStampede(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(trinityAppId);
- setTRINITYInputs(exInputs);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(trinityAppId);
+ setTRINITYInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(trinityAppId);
-
+
Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "TrinityExperiment", "Testing", trinityAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -612,21 +607,23 @@ public class CreateLaunchExperiment {
}
return null;
}
+
private static void setTRINITYInputs(List<InputDataObjectType> exInputs) {
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("RNA_Seq_Left_Input")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/TRINITY_FILES/reads.left.fq");
- }else if (inputDataObjectType.getName().equalsIgnoreCase("RNA_Seq_Right_Input")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/TRINITY_FILES/reads.right.fq");
- }
- }
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("RNA_Seq_Left_Input")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/TRINITY_FILES/reads.left.fq");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("RNA_Seq_Right_Input")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/TRINITY_FILES/reads.right.fq");
+ }
+ }
}
+
public static String createExperimentLAMMPSStampede(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(lammpsAppId);
- setLAMMPSInputs(exInputs);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(lammpsAppId);
+ setLAMMPSInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(lammpsAppId);
-
+
Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "LAMMPSExperiment", "Testing", lammpsAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -661,17 +658,19 @@ public class CreateLaunchExperiment {
}
return null;
}
+
private static void setLAMMPSInputs(List<InputDataObjectType> exInputs) {
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Friction_Simulation_Input")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/LAMMPS_FILES/in.friction");
- }
- }
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Friction_Simulation_Input")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/LAMMPS_FILES/in.friction");
+ }
+ }
}
+
public static String createExperimentNWCHEMStampede(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(nwchemAppId);
- setNWCHEMInputs(exInputs);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(nwchemAppId);
+ setNWCHEMInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(nwchemAppId);
Experiment simpleExperiment =
@@ -708,17 +707,19 @@ public class CreateLaunchExperiment {
}
return null;
}
+
private static void setNWCHEMInputs(List<InputDataObjectType> exInputs) {
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Water_Molecule_Input")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/NWCHEM_FILES/water.nw");
- }
- }
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Water_Molecule_Input")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/NWCHEM_FILES/water.nw");
+ }
+ }
}
+
public static String createExperimentAUTODOCKStampede(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(nwchemAppId);
- setAUTODOCKInputs(exInputs);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(nwchemAppId);
+ setAUTODOCKInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(nwchemAppId);
Experiment simpleExperiment =
@@ -755,43 +756,45 @@ public class CreateLaunchExperiment {
}
return null;
}
+
private static void setAUTODOCKInputs(List<InputDataObjectType> exInputs) {
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("AD4_parameters.dat")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/AD4_parameters.dat");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.A.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.A.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.C.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.C.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.d.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.d.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.e.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.e.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.HD.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.HD.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.maps.fld")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.maps.fld");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.NA.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.NA.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.N.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.N.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.OA.map")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.OA.map");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("ind.dpf")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/ind.dpf");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("ind.pdbqt")) {
- inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/ind.pdbqt");
- }
- }
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("AD4_parameters.dat")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/AD4_parameters.dat");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.A.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.A.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.C.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.C.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.d.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.d.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.e.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.e.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.HD.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.HD.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.maps.fld")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.maps.fld");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.NA.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.NA.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.N.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.N.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("hsg1.OA.map")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/hsg1.OA.map");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("ind.dpf")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/ind.dpf");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("ind.pdbqt")) {
+ inputDataObjectType.setValue("/Users/shameera/Downloads/PHP-Gateway-Scripts/appScripts/AUTODOCK_FILES/ind.pdbqt");
+ }
+ }
}
+
public static String createExperimentWRFTrestles(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(wrfAppId);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(wrfAppId);
setWRFInputs(exInputs);
List<OutputDataObjectType> exOut = client.getApplicationOutputs(wrfAppId);
-
- Experiment simpleExperiment =
+
+ Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "WRFExperiment", "Testing", wrfAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -952,11 +955,11 @@ public class CreateLaunchExperiment {
try {
List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId);
for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) {
- inputDataObjectType.setValue("Hello World");
- }
- }
- List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId);
+ if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) {
+ inputDataObjectType.setValue("Hello World");
+ }
+ }
+ List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId);
Project project = ProjectModelUtil.createProject("default", "admin", "test project");
String projectId = client.createProject(DEFAULT_GATEWAY, project);
@@ -1068,13 +1071,13 @@ public class CreateLaunchExperiment {
public static String createExperimentForBR2(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId);
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) {
- inputDataObjectType.setValue("Hello World");
- }
- }
- List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId);
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) {
+ inputDataObjectType.setValue("Hello World");
+ }
+ }
+ List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId);
Project project = ProjectModelUtil.createProject("default", "lahiru", "test project");
@@ -1163,6 +1166,7 @@ public class CreateLaunchExperiment {
}
return null;
}
+
public static String createExperimentLAMMPSForLSF(Airavata.Client client) throws TException {
try {
List<InputDataObjectType> exInputs = client.getApplicationInputs(lammpsAppId);
@@ -1225,7 +1229,7 @@ public class CreateLaunchExperiment {
public static String createExperimentForBR2Amber(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId);
// for (InputDataObjectType inputDataObjectType : exInputs) {
// if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
// inputDataObjectType.setValue("/Users/raminder/Documents/Sample/Amber/02_Heat.rst");
@@ -1236,17 +1240,17 @@ public class CreateLaunchExperiment {
// }
//
// }
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
- inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/02_Heat.rst");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) {
- inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/03_Prod.in");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) {
- inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/prmtop");
- }
- }
-
- List<OutputDataObjectType> exOut = client.getApplicationOutputs(amberAppId);
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
+ inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/02_Heat.rst");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) {
+ inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/03_Prod.in");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) {
+ inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/prmtop");
+ }
+ }
+
+ List<OutputDataObjectType> exOut = client.getApplicationOutputs(amberAppId);
Project project = ProjectModelUtil.createProject("default", "admin", "test project");
String projectId = client.createProject(DEFAULT_GATEWAY, project);
@@ -1289,7 +1293,7 @@ public class CreateLaunchExperiment {
public static String createExperimentForStampedeAmber(Airavata.Client client) throws TException {
try {
- List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId);
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId);
// for (InputDataObjectType inputDataObjectType : exInputs) {
// if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
// inputDataObjectType.setValue("/Users/raminder/Documents/Sample/Amber/02_Heat.rst");
@@ -1300,17 +1304,17 @@ public class CreateLaunchExperiment {
// }
//
// }
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
- inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/02_Heat.rst");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) {
- inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/03_Prod.in");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) {
- inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/prmtop");
- }
- }
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
+ inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/02_Heat.rst");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) {
+ inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/03_Prod.in");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) {
+ inputDataObjectType.setValue("file://root@test-drive.airavata.org:/var/www/experimentData/admin101a290e6330f15a91349159553ae8b6bb1/prmtop");
+ }
+ }
- List<OutputDataObjectType> exOut = client.getApplicationOutputs(amberAppId);
+ List<OutputDataObjectType> exOut = client.getApplicationOutputs(amberAppId);
Project project = ProjectModelUtil.createProject("default", "admin", "test project");
@@ -1353,8 +1357,8 @@ public class CreateLaunchExperiment {
public static String createExperimentForTrestlesAmber(Airavata.Client client) throws TException {
try {
-
- List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId);
+
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId);
// for (InputDataObjectType inputDataObjectType : exInputs) {
// if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
// inputDataObjectType.setValue("/Users/raminder/Documents/Sample/Amber/02_Heat.rst");
@@ -1365,16 +1369,16 @@ public class CreateLaunchExperiment {
// }
//
// }
- for (InputDataObjectType inputDataObjectType : exInputs) {
- if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
- inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/02_Heat.rst");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) {
- inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/03_Prod.in");
- } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) {
- inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/prmtop");
- }
- }
- List<OutputDataObjectType> exOut = client.getApplicationOutputs(amberAppId);
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
+ inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/02_Heat.rst");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) {
+ inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/03_Prod.in");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) {
+ inputDataObjectType.setValue("/Users/chathuri/dev/airavata/source/php/inputs/AMBER_FILES/prmtop");
+ }
+ }
+ List<OutputDataObjectType> exOut = client.getApplicationOutputs(amberAppId);
Project project = ProjectModelUtil.createProject("default", "admin", "test project");
String projectId = client.createProject(DEFAULT_GATEWAY, project);
@@ -1418,7 +1422,7 @@ public class CreateLaunchExperiment {
public static void launchExperiment(Airavata.Client client, String expId)
throws TException {
try {
- String tokenId ="-0bbb-403b-a88a-42b6dbe198e9";
+ String tokenId = "-0bbb-403b-a88a-42b6dbe198e9";
client.launchExperiment(expId, tokenId);
} catch (ExperimentNotFoundException e) {
logger.error("Error occured while launching the experiment...", e.getMessage());
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
index 46f06f1..a0cc142 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
@@ -64,6 +64,10 @@ public class AiravataZKUtils {
+ ":" + ServerSettings.getSetting(Constants.ZOOKEEPER_SERVER_PORT,"2181");
}
+ public static int getZKTimeout()throws ApplicationSettingsException {
+ return Integer.parseInt(ServerSettings.getSetting(Constants.ZOOKEEPER_TIMEOUT,"30000"));
+ }
+
public static String getExpStatePath(String experimentId, String taskId) throws ApplicationSettingsException {
return AiravataZKUtils.getExpZnodePath(experimentId, taskId) +
File.separator +
@@ -88,6 +92,7 @@ public class AiravataZKUtils {
return null;
}
+
public static int getExpStateValueWithGivenPath(ZooKeeper zk,String fullPath)throws ApplicationSettingsException,
KeeperException, InterruptedException {
Stat exists = zk.exists(fullPath, false);
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/Constants.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/Constants.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/Constants.java
index 154bea1..391a3c6 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/Constants.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/Constants.java
@@ -57,4 +57,5 @@ public final class Constants {
public static final String ZOOKEEPER_API_SERVER_NAME = "api-server-name";
public static final String STAT = "stat";
public static final String JOB = "job";
+ public static final String ZOOKEEPER_TIMEOUT = "zookeeper.timeout";
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index c65feeb..2d8bc48 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -214,20 +214,19 @@ amqp.hosts=info1.dyn.teragrid.org,info2.dyn.teragrid.org
proxy.file.path=/Users/lahirugunathilake/Downloads/x509up_u503876
connection.name=xsede
#publisher
+rabbitmq.broker.url=amqp://localhost:5672
+
activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
publish.rabbitmq=false
-<<<<<<< HEAD
status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
-rabbitmq.broker.url=amqp://localhost:5672
rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
-=======
+
+
activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQPublisher
-rabbitmq.broker.url=amqp://gw111.iu.xsede.org:5672
rabbitmq.exchange.name=airavata_rabbitmq_exchange
->>>>>>> master
###########################################################################
# Orchestrator module Configuration
@@ -252,6 +251,7 @@ embedded.zk=true
zookeeper.server.host=localhost
zookeeper.server.port=2181
airavata-server=/api-server
+zookeeper.timeout=30000
orchestrator-server=/orchestrator-server
gfac-server=/gfac-server
gfac-experiments=/gfac-experiments
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties b/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
index 64e0160..f2e8d82 100644
--- a/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
+++ b/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
@@ -228,6 +228,7 @@ orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
embedded.zk=true
zookeeper.server.host=localhost
zookeeper.server.port=2181
+zookeeper.timeout=30000
airavata-server=/api-server
orchestrator-server=/orchestrator-server
gfac-server=/gfac-server
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index 88979a4..d45710e 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -66,6 +66,8 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
private RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer;
+ private static int requestCount=0;
+
private Registry registry;
private AppCatalog appCatalog;
@@ -96,7 +98,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
String zkhostPort = AiravataZKUtils.getZKhostPort();
airavataServerHostPort = ServerSettings.getSetting(Constants.GFAC_SERVER_HOST)
+ ":" + ServerSettings.getSetting(Constants.GFAC_SERVER_PORT);
- zk = new ZooKeeper(zkhostPort, 6000, this); // no watcher is required, this will only use to store some data
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this); // no watcher is required, this will only use to store some data
gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
gfacExperiments = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
synchronized (mutex) {
@@ -110,14 +112,14 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
appCatalog = AppCatalogFactory.getAppCatalog();
setGatewayProperties();
BetterGfacImpl.startDaemonHandlers();
- BetterGfacImpl.startStatusUpdators(registry, zk, publisher);
- inHandlerFutures = new ArrayList<Future>();
if (ServerSettings.isGFacPassiveMode()) {
rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
rabbitMQTaskLaunchConsumer.listen(new TaskLaunchMessageHandler());
-
}
+ BetterGfacImpl.startStatusUpdators(registry, zk, publisher, rabbitMQTaskLaunchConsumer);
+ inHandlerFutures = new ArrayList<Future>();
+
} catch (ApplicationSettingsException e) {
logger.error("Error initialising GFAC", e);
throw new Exception("Error initialising GFAC", e);
@@ -182,7 +184,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
break;
case Expired:case Disconnected:
try {
- zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, this);
+ zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait(); // waiting for the syncConnected event
}
@@ -231,6 +233,8 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
* @param gatewayId
*/
public boolean submitJob(String experimentId, String taskId, String gatewayId) throws TException {
+ requestCount++;
+ logger.info("-----------------------------------------------------" + requestCount+"-----------------------------------------------------");
logger.infoId(experimentId, "GFac Received submit jog request for the Experiment: {} TaskId: {}", experimentId, taskId);
GFac gfac = getGfac();
InputHandlerWorker inputHandlerWorker = new InputHandlerWorker(gfac, experimentId, taskId, gatewayId);
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index 7559878..3cd07c6 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -49,6 +49,7 @@ import org.apache.airavata.gfac.core.states.GfacPluginState;
import org.apache.airavata.gfac.core.utils.GFacUtils;
import org.apache.airavata.messaging.core.Publisher;
import org.apache.airavata.messaging.core.PublisherFactory;
+import org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchConsumer;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription;
import org.apache.airavata.model.appcatalog.appinterface.DataType;
@@ -111,7 +112,7 @@ public class BetterGfacImpl implements GFac,Watcher {
this.appCatalog = appCatalog;
}
- public static void startStatusUpdators(Registry registry, ZooKeeper zk, MonitorPublisher publisher) {
+ public static void startStatusUpdators(Registry registry, ZooKeeper zk, MonitorPublisher publisher,RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer) {
try {
String[] listenerClassList = ServerSettings.getActivityListeners();
Publisher rabbitMQPublisher = null;
@@ -122,7 +123,7 @@ public class BetterGfacImpl implements GFac,Watcher {
Class<? extends AbstractActivityListener> aClass = Class.forName(listenerClass).asSubclass(AbstractActivityListener.class);
AbstractActivityListener abstractActivityListener = aClass.newInstance();
activityListeners.add(abstractActivityListener);
- abstractActivityListener.setup(publisher, registry, zk, rabbitMQPublisher);
+ abstractActivityListener.setup(publisher, registry, zk, rabbitMQPublisher,rabbitMQTaskLaunchConsumer);
log.info("Registering listener: " + listenerClass);
publisher.registerListener(abstractActivityListener);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
index 26902e7..eaa3c5f 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
@@ -30,6 +30,9 @@ import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.common.utils.listener.AbstractActivityListener;
import org.apache.airavata.gfac.core.monitor.state.GfacExperimentStateChangeRequest;
import org.apache.airavata.gfac.core.utils.GFacUtils;
+import org.apache.airavata.messaging.core.Publisher;
+import org.apache.airavata.messaging.core.impl.RabbitMQProducer;
+import org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchConsumer;
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
@@ -44,19 +47,21 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
private static Integer mutex = -1;
+ private RabbitMQTaskLaunchConsumer consumer;
+
@Subscribe
public void updateZK(GfacExperimentStateChangeRequest statusChangeRequest) throws Exception {
logger.info("Gfac internal state changed to: " + statusChangeRequest.getState().toString());
MonitorID monitorID = statusChangeRequest.getMonitorID();
- String experimentPath = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments") +
- File.separator + ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME) + File.separator + statusChangeRequest.getMonitorID().getExperimentID() + "+" + monitorID.getTaskID();
- String deliveryTagPath = experimentPath + GFacUtils.DELIVERY_TAG_POSTFIX;
+ String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ String experimentPath = experimentNode + File.separator + ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME)
+ + File.separator + statusChangeRequest.getMonitorID().getExperimentID() + "+" + monitorID.getTaskID();
Stat exists = null;
try {
if (!zk.getState().isConnected()) {
String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
- synchronized (mutex){
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this);
+ synchronized (mutex) {
mutex.wait();
}
}
@@ -77,11 +82,11 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
throw new Exception(e.getMessage(), e);
}
Stat state = zk.exists(experimentPath + File.separator + AiravataZKUtils.ZK_EXPERIMENT_STATE_NODE, false);
- if(state == null) {
+ if (state == null) {
// state znode has to be created
zk.create(experimentPath + File.separator + AiravataZKUtils.ZK_EXPERIMENT_STATE_NODE,
String.valueOf(statusChangeRequest.getState().getValue()).getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }else {
+ } else {
zk.setData(experimentPath + File.separator + AiravataZKUtils.ZK_EXPERIMENT_STATE_NODE,
String.valueOf(statusChangeRequest.getState().getValue()).getBytes(), state.getVersion());
}
@@ -89,12 +94,20 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
case COMPLETED:
logger.info("Experiment Completed, So removing the ZK entry for the experiment" + monitorID.getExperimentID());
logger.info("Zookeeper experiment Path: " + experimentPath);
+ if (ServerSettings.isGFacPassiveMode()) {
+ consumer.sendAck(GFacUtils.getDeliveryTag(statusChangeRequest.getMonitorID().getExperimentID(),
+ monitorID.getTaskID(), zk, experimentNode, ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME)));
+ }
ZKUtil.deleteRecursive(zk, experimentPath);
break;
case FAILED:
logger.info("Experiment Failed, So removing the ZK entry for the experiment" + monitorID.getExperimentID());
logger.info("Zookeeper experiment Path: " + experimentPath);
- ZKUtil.deleteRecursive(zk,experimentPath);
+ if (ServerSettings.isGFacPassiveMode()) {
+ consumer.sendAck(GFacUtils.getDeliveryTag(statusChangeRequest.getMonitorID().getExperimentID(),
+ monitorID.getTaskID(), zk, experimentNode, ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME)));
+ }
+ ZKUtil.deleteRecursive(zk, experimentPath);
break;
default:
}
@@ -105,6 +118,9 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
if (configuration instanceof ZooKeeper) {
this.zk = (ZooKeeper) configuration;
}
+ if (configuration instanceof RabbitMQTaskLaunchConsumer) {
+ this.consumer = (RabbitMQTaskLaunchConsumer) configuration;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/MonitorID.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/MonitorID.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/MonitorID.java
index 55da288..aefe490 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/MonitorID.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/MonitorID.java
@@ -94,10 +94,11 @@ public class MonitorID {
experimentID = jobExecutionContext.getExperiment().getExperimentID();
workflowNodeID = jobExecutionContext.getWorkflowNodeDetails().getNodeInstanceId();// at this point we only have one node todo: fix this
try {
- jobID = jobExecutionContext.getJobDetails().getJobID();
jobName = jobExecutionContext.getJobDetails().getJobName();
+ jobID = jobExecutionContext.getJobDetails().getJobID();
}catch(NullPointerException e){
logger.error("There is not job created at this point");
+ // this is not a big deal we create MonitorId before having a jobId or job Name
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
index 83928c3..707cf97 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
@@ -1159,29 +1159,13 @@ public class GFacUtils {
public static boolean createExperimentEntryForPassive(String experimentID,
String taskID, ZooKeeper zk, String experimentNode,
String pickedChild, String tokenId,long deliveryTag) throws KeeperException,
- InterruptedException {
+ InterruptedException, ApplicationSettingsException {
String experimentPath = experimentNode + File.separator + pickedChild;
String newExpNode = experimentPath + File.separator + experimentID
+ "+" + taskID;
Stat exists1 = zk.exists(newExpNode, false);
String experimentEntry = GFacUtils.findExperimentEntry(experimentID, taskID, zk);
- String foundExperimentPath = null;
if (exists1 == null && experimentEntry == null) { // this means this is a very new experiment
- List<String> runningGfacNodeNames = AiravataZKUtils.getAllGfacNodeNames(zk); // here we take old gfac servers
-
- for (String gfacServerNode : runningGfacNodeNames) {
- if (!gfacServerNode.equals(pickedChild)) {
- foundExperimentPath = experimentNode + File.separator
- + gfacServerNode + File.separator + experimentID
- + "+" + taskID;
- exists1 = zk.exists(foundExperimentPath, true);
- if (exists1 != null) { // when the experiment is found we
- // break the loop
- break;
- }
- }
- }
- if (exists1 == null) { // OK this is a pretty new experiment so we
// are going to create a new node
log.info("This is a new Job, so creating all the experiment docs from the scratch");
Stat expParent = zk.exists(newExpNode, false);
@@ -1199,34 +1183,35 @@ public class GFacUtils {
String s1 = zk.create(newExpNode + File.separator + "operation", "submit".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
zk.exists(s1, true);// we want to know when this node get deleted
- String s2 = zk.create(newExpNode + DELIVERY_TAG_POSTFIX, ByteBuffer.allocate(8).putLong(deliveryTag).array(), ZooDefs.Ids.OPEN_ACL_UNSAFE, // here we store the value of delivery message
+ String s2 = zk.create(newExpNode + DELIVERY_TAG_POSTFIX, longToBytes(deliveryTag), ZooDefs.Ids.OPEN_ACL_UNSAFE, // here we store the value of delivery message
CreateMode.PERSISTENT);
- } else {
- // ohhh this node exists in some other failed gfac folder, we
- // have to move it to this gfac experiment list,safely
+ }else if(experimentEntry != null && GFacUtils.isCancelled(experimentID,taskID,zk) ){
+ // this happens when a cancel request comes to a differnt gfac node, in this case we do not move gfac experiment
+ // node to gfac node specific location, because original request execution will fail with errors
+ log.error("This experiment is already cancelled and its already executing the cancel operation so cannot submit again !");
+ return false;
+ } else if(experimentEntry != null && !GFacUtils.isCancelled(experimentID,taskID,zk)){
+ if(ServerSettings.isGFacPassiveMode()){
+ log.error("ExperimentID: " + experimentID + " taskID: " + taskID
+ + " was running by some Gfac instance,but it failed");
log.info("This is an old Job, so copying data from old experiment location");
zk.create(newExpNode,
- zk.getData(foundExperimentPath, false, exists1),
+ zk.getData(experimentEntry, false, exists1),
ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- List<String> children = zk.getChildren(foundExperimentPath,
+ List<String> children = zk.getChildren(experimentEntry,
false);
for (String childNode1 : children) {
- String level1 = foundExperimentPath + File.separator
+ String level1 = experimentEntry + File.separator
+ childNode1;
- Stat exists2 = zk.exists(level1, false); // no need to check
- // exists
+ Stat exists2 = zk.exists(level1, false); // no need to check exists
String newLeve1 = newExpNode + File.separator + childNode1;
- log.info("Creating new znode: " + newLeve1); // these has to
- // be info
- // logs
+ log.info("Creating new znode: " + newLeve1); // these has to be info logs
zk.create(newLeve1, zk.getData(level1, false, exists2),
ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
for (String childNode2 : zk.getChildren(level1, false)) {
String level2 = level1 + File.separator + childNode2;
- Stat exists3 = zk.exists(level2, false); // no need to
- // check
- // exists
+ Stat exists3 = zk.exists(level2, false); // no need to check exists
String newLeve2 = newLeve1 + File.separator
+ childNode2;
log.info("Creating new znode: " + newLeve2);
@@ -1239,40 +1224,47 @@ public class GFacUtils {
// old experiment,otherwise we do
// not delete a single file
log.info("After a successful copying of experiment data for an old experiment we delete the old data");
- log.info("Deleting experiment data: " + foundExperimentPath);
- ZKUtil.deleteRecursive(zk, foundExperimentPath);
- }
- }else if(experimentEntry != null && GFacUtils.isCancelled(experimentID,taskID,zk) ){
- // this happens when a cancel request comes to a differnt gfac node, in this case we do not move gfac experiment
- // node to gfac node specific location, because original request execution will fail with errors
- log.error("This experiment is already cancelled and its already executing the cancel operation so cannot submit again !");
- return false;
- } else {
- log.error("ExperimentID: " + experimentID + " taskID: " + taskID
- + " is already running by this Gfac instance");
- List<String> runningGfacNodeNames = AiravataZKUtils
- .getAllGfacNodeNames(zk); // here we take old gfac servers
- // too
- for (String gfacServerNode : runningGfacNodeNames) {
- if (!gfacServerNode.equals(pickedChild)) {
- foundExperimentPath = experimentNode + File.separator
- + gfacServerNode + File.separator + experimentID
- + "+" + taskID;
- break;
+ log.info("Deleting experiment data: " + experimentEntry);
+ ZKUtil.deleteRecursive(zk, experimentEntry);
+ }else {
+ log.error("ExperimentID: " + experimentID + " taskID: " + taskID
+ + " is already running by this Gfac instance");
+ List<String> runningGfacNodeNames = AiravataZKUtils
+ .getAllGfacNodeNames(zk); // here we take old gfac servers
+ // too
+ for (String gfacServerNode : runningGfacNodeNames) {
+ if (!gfacServerNode.equals(pickedChild)) {
+ experimentEntry = experimentNode + File.separator
+ + gfacServerNode + File.separator + experimentID
+ + "+" + taskID;
+ break;
+ }
+ }
+ if(experimentEntry!=null) {
+ ZKUtil.deleteRecursive(zk, experimentEntry);
}
}
- ZKUtil.deleteRecursive(zk, foundExperimentPath);
+
}
return true;
}
+ /**
+ * This will return a value if the server is down because we iterate through exisiting experiment nodes, not
+ * through gfac-server nodes
+ * @param experimentID
+ * @param taskID
+ * @param zk
+ * @return
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
public static String findExperimentEntry(String experimentID,
String taskID, ZooKeeper zk
) throws KeeperException,
InterruptedException {
- String gfacServer = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
String experimentNode = ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
- List<String> children = zk.getChildren(gfacServer, false);
+ List<String> children = zk.getChildren(experimentNode, false);
for(String pickedChild:children) {
String experimentPath = experimentNode + File.separator + pickedChild;
String newExpNode = experimentPath + File.separator + experimentID
@@ -1341,6 +1333,18 @@ public class GFacUtils {
}
}
+ public static long getDeliveryTag(String experimentID,
+ String taskID, ZooKeeper zk, String experimentNode,
+ String pickedChild) throws KeeperException, InterruptedException,GFacException {
+ String experimentPath = experimentNode + File.separator + pickedChild;
+ String deliveryTagPath = experimentPath + File.separator + experimentID
+ + "+" + taskID + DELIVERY_TAG_POSTFIX;
+ Stat exists = zk.exists(deliveryTagPath, false);
+ if(exists==null) {
+ throw new GFacException("Cannot find delivery Tag for this experiment");
+ }
+ return bytesToLong(zk.getData(deliveryTagPath, false, exists));
+ }
public static String getPluginData(JobExecutionContext jobExecutionContext,
String className) throws ApplicationSettingsException,
KeeperException, InterruptedException {
@@ -1452,4 +1456,17 @@ public class GFacUtils {
return sb.toString();
}
+
+ public static byte[] longToBytes(long x) {
+ ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
+ buffer.putLong(x);
+ return buffer.array();
+ }
+
+ public static long bytesToLong(byte[] bytes) {
+ ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
+ buffer.put(bytes);
+ buffer.flip();//need flip
+ return buffer.getLong();
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java
index 9f055e9..d62d3d7 100644
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java
@@ -124,9 +124,9 @@ public class LocalProvider extends AbstractProvider {
// log info
log.info("Command = " + InputUtils.buildCommand(cmdList));
log.info("Working dir = " + builder.directory());
- for (String key : builder.environment().keySet()) {
+ /*for (String key : builder.environment().keySet()) {
log.info("Env[" + key + "] = " + builder.environment().get(key));
- }
+ }*/
}
public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
index cbac726..15b7241 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
@@ -199,7 +199,7 @@ public class CommonUtils {
if (zk == null || !zk.getState().isConnected()) {
try {
final CountDownLatch countDownLatch = new CountDownLatch(1);
- zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, new Watcher() {
+ zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), AiravataZKUtils.getZKTimeout(), new Watcher() {
@Override
public void process(WatchedEvent event) {
countDownLatch.countDown();
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
index 7c88a25..9cad924 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
@@ -165,14 +165,15 @@ public class RabbitMQTaskLaunchConsumer {
event = taskTerminateEvent;
gatewayId = null;
}
+ System.out.println("*deliveryTag:"+deliveryTag);
MessageContext messageContext = new MessageContext(event, message.getMessageType(), message.getMessageId(), gatewayId,deliveryTag);
messageContext.setUpdatedTime(AiravataUtils.getTime(message.getUpdatedTime()));
handler.onMessage(messageContext);
- try {
+ /*try {
channel.basicAck(deliveryTag,false); //todo move this logic to monitoring component to ack when the job is done
} catch (IOException e) {
logger.error(e.getMessage(), e);
- }
+ }*/
} catch (TException e) {
String msg = "Failed to de-serialize the thrift message, from routing keys and queueName " + id;
log.warn(msg, e);
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
index f430bc9..63f0d9c 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
@@ -115,7 +115,7 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
// setGatewayName(ServerSettings.getDefaultUserGateway());
setAiravataUserName(ServerSettings.getDefaultUser());
try {
- zk = new ZooKeeper(zkhostPort, 6000, this); // no watcher is
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this); // no watcher is
// required, this
// will only use to
// store some data
@@ -303,7 +303,7 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
case Expired:
case Disconnected:
try {
- zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, this);
+ zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait(); // waiting for the syncConnected event
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
index f19b949..993a303 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
@@ -70,7 +70,7 @@ public class OrchestratorRecoveryHandler implements Watcher {
*/
public void recover() throws OrchestratorException, ApplicationSettingsException, IOException, KeeperException, InterruptedException {
String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait();
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
index 8066113..c17638b 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
@@ -105,42 +105,30 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
try {
if (zk == null || !zk.getState().isConnected()) {
String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait();
}
}
- String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
- List<String> children = zk.getChildren(gfacServer, this);
-
- if (children.size() == 0) {
- // Zookeeper data need cleaning
- throw new OrchestratorException("There is no active GFac instance to route the request");
- } else {
- String gatewayId = null;
- CredentialReader credentialReader = GFacUtils.getCredentialReader();
- if (credentialReader != null) {
- try {
- gatewayId = credentialReader.getGatewayID(tokenId);
- } catch (Exception e) {
- logger.error(e.getLocalizedMessage());
- }
+ String gatewayId = null;
+ CredentialReader credentialReader = GFacUtils.getCredentialReader();
+ if (credentialReader != null) {
+ try {
+ gatewayId = credentialReader.getGatewayID(tokenId);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage());
}
- if(gatewayId == null || gatewayId.isEmpty()){
- gatewayId = ServerSettings.getDefaultUserGateway();
- }
-
- TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, gatewayId,tokenId);
- MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK, "LAUNCH.TASK-" + UUID.randomUUID().toString(), gatewayId);
- messageContext.setUpdatedTime(AiravataUtils.getCurrentTimestamp());
- publisher.publish(messageContext);
}
+ if (gatewayId == null || gatewayId.isEmpty()) {
+ gatewayId = ServerSettings.getDefaultUserGateway();
+ }
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, gatewayId, tokenId);
+ MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK, "LAUNCH.TASK-" + UUID.randomUUID().toString(), gatewayId);
+ messageContext.setUpdatedTime(AiravataUtils.getCurrentTimestamp());
+ publisher.publish(messageContext);
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
throw new OrchestratorException(e);
- } catch (KeeperException e) {
- logger.error(e.getMessage(), e);
- throw new OrchestratorException(e);
} catch (ApplicationSettingsException e) {
logger.error(e.getMessage(), e);
throw new OrchestratorException(e);
@@ -167,7 +155,7 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
try {
if (zk == null || !zk.getState().isConnected()) {
String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait();
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/cb6c4ccf/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
index b855de2..5a1be5a 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
@@ -78,7 +78,7 @@ public class GFACRPCJobSubmitter implements JobSubmitter, Watcher {
try {
if (zk == null || !zk.getState().isConnected()) {
String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait();
}
@@ -146,7 +146,7 @@ public class GFACRPCJobSubmitter implements JobSubmitter, Watcher {
try {
if (zk == null || !zk.getState().isConnected()) {
String zkhostPort = AiravataZKUtils.getZKhostPort();
- zk = new ZooKeeper(zkhostPort, 6000, this);
+ zk = new ZooKeeper(zkhostPort, AiravataZKUtils.getZKTimeout(), this);
synchronized (mutex) {
mutex.wait();
}
[13/15] airavata git commit: deleting delivery token
Posted by la...@apache.org.
deleting delivery token
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/5310bb4b
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/5310bb4b
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/5310bb4b
Branch: refs/heads/master
Commit: 5310bb4b570bd0b09208e710f121e88418261c5f
Parents: cb6c4cc
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Fri Mar 20 16:14:07 2015 -0400
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Fri Mar 20 16:14:07 2015 -0400
----------------------------------------------------------------------
.../airavata/gfac/server/GfacServerHandler.java | 3 +
.../core/monitor/GfacInternalStatusUpdator.java | 2 +
.../airavata/gfac/core/utils/GFacUtils.java | 134 ++++++++-----------
3 files changed, 61 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/5310bb4b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index d45710e..855bfc5 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -59,6 +59,7 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Future;
+import java.util.concurrent.locks.Lock;
public class GfacServerHandler implements GfacService.Iface, Watcher {
@@ -79,6 +80,8 @@ public class GfacServerHandler implements GfacService.Iface, Watcher {
private static Integer mutex = -1;
+ private static Lock lock;
+
private MonitorPublisher publisher;
private String gfacServer;
http://git-wip-us.apache.org/repos/asf/airavata/blob/5310bb4b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
index eaa3c5f..6c456b0 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/GfacInternalStatusUpdator.java
@@ -98,6 +98,7 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
consumer.sendAck(GFacUtils.getDeliveryTag(statusChangeRequest.getMonitorID().getExperimentID(),
monitorID.getTaskID(), zk, experimentNode, ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME)));
}
+ ZKUtil.deleteRecursive(zk,experimentPath+GFacUtils.DELIVERY_TAG_POSTFIX);
ZKUtil.deleteRecursive(zk, experimentPath);
break;
case FAILED:
@@ -107,6 +108,7 @@ public class GfacInternalStatusUpdator implements AbstractActivityListener, Watc
consumer.sendAck(GFacUtils.getDeliveryTag(statusChangeRequest.getMonitorID().getExperimentID(),
monitorID.getTaskID(), zk, experimentNode, ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME)));
}
+ ZKUtil.deleteRecursive(zk,experimentPath+GFacUtils.DELIVERY_TAG_POSTFIX);
ZKUtil.deleteRecursive(zk, experimentPath);
break;
default:
http://git-wip-us.apache.org/repos/asf/airavata/blob/5310bb4b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
index 707cf97..a0fc2cb 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
@@ -1157,94 +1157,72 @@ public class GFacUtils {
// This method is dangerous because of moving the experiment data
public static boolean createExperimentEntryForPassive(String experimentID,
- String taskID, ZooKeeper zk, String experimentNode,
- String pickedChild, String tokenId,long deliveryTag) throws KeeperException,
+ String taskID, ZooKeeper zk, String experimentNode,
+ String pickedChild, String tokenId, long deliveryTag) throws KeeperException,
InterruptedException, ApplicationSettingsException {
String experimentPath = experimentNode + File.separator + pickedChild;
String newExpNode = experimentPath + File.separator + experimentID
+ "+" + taskID;
Stat exists1 = zk.exists(newExpNode, false);
String experimentEntry = GFacUtils.findExperimentEntry(experimentID, taskID, zk);
- if (exists1 == null && experimentEntry == null) { // this means this is a very new experiment
- // are going to create a new node
- log.info("This is a new Job, so creating all the experiment docs from the scratch");
- Stat expParent = zk.exists(newExpNode, false);
- zk.create(newExpNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
-
- if (tokenId != null && expParent != null) {
- zk.setData(newExpNode, tokenId.getBytes(),
- expParent.getVersion());
- }
- String s = zk.create(newExpNode + File.separator + "state", String
- .valueOf(GfacExperimentState.LAUNCHED.getValue())
- .getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
- String s1 = zk.create(newExpNode + File.separator + "operation", "submit".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
- zk.exists(s1, true);// we want to know when this node get deleted
- String s2 = zk.create(newExpNode + DELIVERY_TAG_POSTFIX, longToBytes(deliveryTag), ZooDefs.Ids.OPEN_ACL_UNSAFE, // here we store the value of delivery message
- CreateMode.PERSISTENT);
- }else if(experimentEntry != null && GFacUtils.isCancelled(experimentID,taskID,zk) ){
- // this happens when a cancel request comes to a differnt gfac node, in this case we do not move gfac experiment
- // node to gfac node specific location, because original request execution will fail with errors
- log.error("This experiment is already cancelled and its already executing the cancel operation so cannot submit again !");
+ if (exists1 != null) {
+ log.error("This request is wrong because its already running in the same instance");
return false;
- } else if(experimentEntry != null && !GFacUtils.isCancelled(experimentID,taskID,zk)){
- if(ServerSettings.isGFacPassiveMode()){
- log.error("ExperimentID: " + experimentID + " taskID: " + taskID
- + " was running by some Gfac instance,but it failed");
- log.info("This is an old Job, so copying data from old experiment location");
- zk.create(newExpNode,
- zk.getData(experimentEntry, false, exists1),
- ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } else if (experimentEntry == null) { // this means this is a very new experiment
+ // are going to create a new node
+ log.info("This is a new Job, so creating all the experiment docs from the scratch");
+ Stat expParent = zk.exists(newExpNode, false);
+ zk.create(newExpNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
- List<String> children = zk.getChildren(experimentEntry,
- false);
- for (String childNode1 : children) {
- String level1 = experimentEntry + File.separator
- + childNode1;
- Stat exists2 = zk.exists(level1, false); // no need to check exists
- String newLeve1 = newExpNode + File.separator + childNode1;
- log.info("Creating new znode: " + newLeve1); // these has to be info logs
- zk.create(newLeve1, zk.getData(level1, false, exists2),
- ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- for (String childNode2 : zk.getChildren(level1, false)) {
- String level2 = level1 + File.separator + childNode2;
- Stat exists3 = zk.exists(level2, false); // no need to check exists
- String newLeve2 = newLeve1 + File.separator
- + childNode2;
- log.info("Creating new znode: " + newLeve2);
- zk.create(newLeve2, zk.getData(level2, false, exists3),
- ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
- }
- }
- // After all the files are successfully transfered we delete the
- // old experiment,otherwise we do
- // not delete a single file
- log.info("After a successful copying of experiment data for an old experiment we delete the old data");
- log.info("Deleting experiment data: " + experimentEntry);
- ZKUtil.deleteRecursive(zk, experimentEntry);
- }else {
- log.error("ExperimentID: " + experimentID + " taskID: " + taskID
- + " is already running by this Gfac instance");
- List<String> runningGfacNodeNames = AiravataZKUtils
- .getAllGfacNodeNames(zk); // here we take old gfac servers
- // too
- for (String gfacServerNode : runningGfacNodeNames) {
- if (!gfacServerNode.equals(pickedChild)) {
- experimentEntry = experimentNode + File.separator
- + gfacServerNode + File.separator + experimentID
- + "+" + taskID;
- break;
- }
- }
- if(experimentEntry!=null) {
- ZKUtil.deleteRecursive(zk, experimentEntry);
- }
+ if (tokenId != null && expParent != null) {
+ zk.setData(newExpNode, tokenId.getBytes(),
+ expParent.getVersion());
}
+ String s = zk.create(newExpNode + File.separator + "state", String
+ .valueOf(GfacExperimentState.LAUNCHED.getValue())
+ .getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ String s1 = zk.create(newExpNode + File.separator + "operation", "submit".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ zk.exists(s1, true);// we want to know when this node get deleted
+ String s2 = zk.create(newExpNode + DELIVERY_TAG_POSTFIX, longToBytes(deliveryTag), ZooDefs.Ids.OPEN_ACL_UNSAFE, // here we store the value of delivery message
+ CreateMode.PERSISTENT);
+ } else {
+ log.error("ExperimentID: " + experimentID + " taskID: " + taskID
+ + " was running by some Gfac instance,but it failed");
+ log.info("This is an old Job, so copying data from old experiment location");
+ zk.create(newExpNode,
+ zk.getData(experimentEntry, false, exists1),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ List<String> children = zk.getChildren(experimentEntry,
+ false);
+ for (String childNode1 : children) {
+ String level1 = experimentEntry + File.separator
+ + childNode1;
+ Stat exists2 = zk.exists(level1, false); // no need to check exists
+ String newLeve1 = newExpNode + File.separator + childNode1;
+ log.info("Creating new znode: " + newLeve1); // these has to be info logs
+ zk.create(newLeve1, zk.getData(level1, false, exists2),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ for (String childNode2 : zk.getChildren(level1, false)) {
+ String level2 = level1 + File.separator + childNode2;
+ Stat exists3 = zk.exists(level2, false); // no need to check exists
+ String newLeve2 = newLeve1 + File.separator
+ + childNode2;
+ log.info("Creating new znode: " + newLeve2);
+ zk.create(newLeve2, zk.getData(level2, false, exists3),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ }
+ }
+ // After all the files are successfully transfered we delete the
+ // old experiment,otherwise we do
+ // not delete a single file
+ log.info("After a successful copying of experiment data for an old experiment we delete the old data");
+ log.info("Deleting experiment data: " + experimentEntry);
+ ZKUtil.deleteRecursive(zk, experimentEntry);
}
return true;
}
[07/15] airavata git commit: implementing queue submission without
curator
Posted by la...@apache.org.
implementing queue submission without curator
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/840e627b
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/840e627b
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/840e627b
Branch: refs/heads/master
Commit: 840e627b4e24baef8dbf62df8da1042380cb8af1
Parents: 60788ef
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Tue Feb 17 23:58:04 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Tue Feb 17 23:58:04 2015 -0500
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 17 ++-
.../airavata/common/utils/AiravataZKUtils.java | 22 +++
.../airavata/gfac/server/GfacServerHandler.java | 133 +++++--------------
.../messaging/core/impl/RabbitMQProducer.java | 18 ++-
.../core/impl/RabbitMQTaskLaunchConsumer.java | 22 +--
.../core/impl/RabbitMQTaskLaunchPublisher.java | 5 +-
6 files changed, 102 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/840e627b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 6937c25..1e9d983 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -94,14 +94,16 @@ public class CreateLaunchExperiment {
public static void createAndLaunchExp() throws TException {
// final String expId = createEchoExperimentForFSD(airavataClient);
+ List<String> experimentIds = new ArrayList<String>();
try {
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < 100; i++) {
// final String expId = createExperimentForSSHHost(airavata);
// final String expId = createEchoExperimentForFSD(airavataClient);
// final String expId = createMPIExperimentForFSD(airavataClient);
// final String expId = createEchoExperimentForStampede(airavataClient);
// final String expId = createEchoExperimentForTrestles(airavataClient);
final String expId = createExperimentEchoForLocalHost(airavataClient);
+ experimentIds.add(expId);
// final String expId = createExperimentWRFTrestles(airavataClient);
// final String expId = createExperimentForBR2(airavataClient);
// final String expId = createExperimentForBR2Amber(airavataClient);
@@ -115,11 +117,20 @@ public class CreateLaunchExperiment {
// final String expId = createExperimentTRINITYStampede(airavataClient);
// final String expId = createExperimentAUTODOCKStampede(airavataClient); // this is not working , we need to register AutoDock app on stampede
// final String expId = "Ultrascan_ln_eb029947-391a-4ccf-8ace-9bafebe07cc0";
- System.out.println("Experiment ID : " + expId);
+ System.out.println("Experiment ID : " + expId);
// updateExperiment(airavata, expId);
-
+
launchExperiment(airavataClient, expId);
}
+
+ Thread.sleep(10000);
+
+ for(String exId:experimentIds) {
+ Experiment experiment = airavataClient.getExperiment(exId);
+ System.out.println(experiment.getExperimentStatus().toString());
+ }
+
+
} catch (Exception e) {
logger.error("Error while connecting with server", e.getMessage());
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/airavata/blob/840e627b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
index f91fc3c..46f06f1 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
@@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.nio.ByteBuffer;
import java.util.List;
public class AiravataZKUtils {
@@ -172,4 +173,25 @@ public class AiravataZKUtils {
logger.info("Skipping Zookeeper embedded startup ...");
}
}
+
+ public static void storeDeliveryTag(ZooKeeper zk,String newExpNode,Double deliveryTag) throws KeeperException, InterruptedException {
+ String s = zk.create(newExpNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+
+ Stat expParent = zk.exists(newExpNode, false);
+ if (expParent != null) {
+ zk.setData(newExpNode, toByteArray(deliveryTag),
+ expParent.getVersion());
+ }
+ }
+
+ public static byte[] toByteArray(double value) {
+ byte[] bytes = new byte[8];
+ ByteBuffer.wrap(bytes).putDouble(value);
+ return bytes;
+ }
+
+ public static double toDouble(byte[] bytes) {
+ return ByteBuffer.wrap(bytes).getDouble();
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/840e627b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index 679a5ee..1c0f095 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -45,30 +45,23 @@ import org.apache.airavata.model.messaging.event.TaskTerminateEvent;
import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
import org.apache.airavata.registry.cpi.Registry;
import org.apache.airavata.registry.cpi.RegistryException;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.recipes.leader.LeaderSelector;
-import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
-import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
-import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-public class GfacServerHandler implements GfacService.Iface, Watcher{
+public class GfacServerHandler implements GfacService.Iface, Watcher {
private final static AiravataLogger logger = AiravataLoggerFactory.getLogger(GfacServerHandler.class);
+ private RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer;
+
private Registry registry;
private AppCatalog appCatalog;
@@ -80,8 +73,6 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private ZooKeeper zk;
- private boolean connected = false;
-
private static Integer mutex = -1;
private MonitorPublisher publisher;
@@ -94,16 +85,10 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private List<Future> inHandlerFutures;
- private String nodeName = null;
-
- private CuratorFramework curatorFramework = null;
private BlockingQueue<TaskSubmitEvent> taskSubmitEvents;
- private BlockingQueue<TaskTerminateEvent> taskTerminateEvents;
-
- private CuratorClient curatorClient;
- public GfacServerHandler() throws Exception{
+ public GfacServerHandler() throws Exception {
// registering with zk
try {
String zkhostPort = AiravataZKUtils.getZKhostPort();
@@ -112,7 +97,6 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
zk = new ZooKeeper(zkhostPort, 6000, this); // no watcher is required, this will only use to store some data
gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
gfacExperiments = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
- nodeName = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME);
synchronized (mutex) {
mutex.wait(); // waiting for the syncConnected event
}
@@ -128,17 +112,11 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
inHandlerFutures = new ArrayList<Future>();
if (ServerSettings.isGFacPassiveMode()) {
- taskSubmitEvents = new LinkedBlockingDeque<TaskSubmitEvent>();
- taskTerminateEvents = new LinkedBlockingDeque<TaskTerminateEvent>();
- curatorFramework = CuratorFrameworkFactory.newClient(AiravataZKUtils.getZKhostPort(), new ExponentialBackoffRetry(1000, 3));
- curatorClient = new CuratorClient(curatorFramework, nodeName);
+ rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
+ rabbitMQTaskLaunchConsumer.listen(new TaskLaunchMessageHandler());
- curatorFramework.start();
- curatorClient.start();
}
-
-
- } catch (ApplicationSettingsException e) {
+ } catch (ApplicationSettingsException e) {
logger.error("Error initialising GFAC", e);
throw new Exception("Error initialising GFAC", e);
} catch (InterruptedException e) {
@@ -184,7 +162,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
zk.create(gfacExperiments + File.separator + instanceId,
airavataServerHostPort.getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
- }else{
+ } else {
logger.error(" Zookeeper is inconsistent state !!!!!");
}
}
@@ -195,9 +173,8 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
logger.info(state.name());
if (state == Event.KeeperState.SyncConnected) {
mutex.notify();
- connected = true;
- } else if(state == Event.KeeperState.Expired ||
- state == Event.KeeperState.Disconnected){
+ } else if (state == Event.KeeperState.Expired ||
+ state == Event.KeeperState.Disconnected) {
try {
mutex = -1;
zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, this);
@@ -292,24 +269,29 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
public void setAiravataUserName(String airavataUserName) {
this.airavataUserName = airavataUserName;
}
+
protected void setGatewayProperties() throws ApplicationSettingsException {
- setAiravataUserName(ServerSettings.getDefaultUser());
- setGatewayName(ServerSettings.getDefaultUserGateway());
- }
+ setAiravataUserName(ServerSettings.getDefaultUser());
+ setGatewayName(ServerSettings.getDefaultUserGateway());
+ }
- private GFac getGfac()throws TException{
+ private GFac getGfac() throws TException {
try {
- return new BetterGfacImpl(registry, appCatalog, zk,publisher);
+ return new BetterGfacImpl(registry, appCatalog, zk, publisher);
} catch (Exception e) {
- throw new TException("Error initializing gfac instance",e);
+ throw new TException("Error initializing gfac instance", e);
}
}
private class TaskLaunchMessageHandler implements MessageHandler {
public static final String LAUNCH_TASK = "launch.task";
public static final String TERMINATE_TASK = "teminate.task";
- public TaskLaunchMessageHandler(){
+ private String experimentNode;
+ private String nodeName;
+ public TaskLaunchMessageHandler() {
+ experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ nodeName = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME,"gfac-node0");
}
public Map<String, Object> getProperties() {
@@ -318,6 +300,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
keys.add(LAUNCH_TASK);
keys.add(TERMINATE_TASK);
props.put(MessagingConstants.RABBIT_ROUTING_KEY, keys);
+ props.put(MessagingConstants.RABBIT_QUEUE, LAUNCH_TASK);
return props;
}
@@ -328,9 +311,16 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
TBase messageEvent = message.getEvent();
byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
ThriftUtils.createThriftFromBytes(bytes, event);
- taskSubmitEvents.add(event);
-
-
+ experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+
+ try {
+ GFacUtils.createExperimentEntryForRPC(event.getExperimentId(), event.getTaskId(), zk, experimentNode, nodeName, event.getTokenId());
+ submitJob(event.getExperimentId(), event.getTaskId(), event.getGatewayId());
+ } catch (KeeperException e) {
+ logger.error(nodeName + " was interrupted.");
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ }
System.out.println(" Message Received with message id '" + message.getMessageId()
+ "' and with message type '" + message.getType());
} catch (TException e) {
@@ -351,61 +341,4 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
}
}
}
-
- public class CuratorClient extends LeaderSelectorListenerAdapter implements Closeable {
- private final String name;
- private final LeaderSelector leaderSelector;
- private final AtomicInteger leaderCount = new AtomicInteger();
- private final String path;
- private String experimentNode;
-
- public CuratorClient(CuratorFramework client, String name) {
- this.name = name;
- // create a leader selector using the given path for management
- // all participants in a given leader selection must use the same path
- // ExampleClient here is also a LeaderSelectorListener but this isn't required
- experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
- path = experimentNode + File.separator + "leader";
- leaderSelector = new LeaderSelector(client, path, this);
- // for most cases you will want your instance to requeue when it relinquishes leadership
- leaderSelector.autoRequeue();
- }
-
- public void start() throws IOException {
- // the selection for this instance doesn't start until the leader selector is started
- // leader selection is done in the background so this call to leaderSelector.start() returns immediately
- leaderSelector.start();
- }
-
- @Override
- public void close() throws IOException {
- leaderSelector.close();
- }
-
- @Override
- public void takeLeadership(CuratorFramework client) throws Exception {
- // we are now the leader. This method should not return until we want to relinquish leadership
- final int waitSeconds = (int) (5 * Math.random()) + 1;
-
- logger.info(name + " is now the leader. Waiting " + waitSeconds + " seconds...");
- logger.info(name + " has been leader " + leaderCount.getAndIncrement() + " time(s) before.");
- RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
- String listenId = rabbitMQTaskLaunchConsumer.listen(new TaskLaunchMessageHandler());
-
- TaskSubmitEvent event = taskSubmitEvents.take();
- try {
- GFacUtils.createExperimentEntryForRPC(event.getExperimentId(),event.getTaskId(),client.getZookeeperClient().getZooKeeper(),experimentNode,name,event.getTokenId());
- submitJob(event.getExperimentId(), event.getTaskId(), event.getGatewayId());
- Thread.sleep(TimeUnit.SECONDS.toMillis(waitSeconds));
- } catch (InterruptedException e) {
- logger.error(name + " was interrupted.");
- Thread.currentThread().interrupt();
- } finally {
- Thread.sleep(5);
- logger.info(name + " relinquishing leadership.: "+ new Date().toString());
- rabbitMQTaskLaunchConsumer.stopListen(listenId);
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/840e627b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
index 570b17f..fffeece 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
@@ -113,8 +113,10 @@ public class RabbitMQProducer {
log.info("setting basic.qos / prefetch count to " + prefetchCount + " for " + exchangeName);
channel.basicQos(prefetchCount);
}
- channel.exchangeDeclare(exchangeName, getExchangeType, false);
- } catch (Exception e) {
+ if(exchangeName!=null) {
+ channel.exchangeDeclare(exchangeName, getExchangeType, false);
+ }
+ } catch (Exception e) {
reset();
String msg = "could not open channel for exchange " + exchangeName;
log.error(msg);
@@ -132,6 +134,18 @@ public class RabbitMQProducer {
}
}
+ public void sendToWorkerQueue(byte []message, String routingKey) throws Exception {
+ try {
+ channel.basicPublish( "", routingKey,
+ MessageProperties.PERSISTENT_TEXT_PLAIN,
+ message);
+ } catch (IOException e) {
+ String msg = "Failed to publish message to exchange: " + exchangeName;
+ log.error(msg, e);
+ throw new Exception(msg, e);
+ }
+ }
+
private Connection createConnection() throws IOException {
try {
ConnectionFactory connectionFactory = new ConnectionFactory();
http://git-wip-us.apache.org/repos/asf/airavata/blob/840e627b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
index 4bc7468..1c7b0e8 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
@@ -24,6 +24,7 @@ import com.rabbitmq.client.*;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.AiravataUtils;
+import org.apache.airavata.common.utils.AiravataZKUtils;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.common.utils.ThriftUtils;
import org.apache.airavata.messaging.core.MessageContext;
@@ -82,7 +83,7 @@ public class RabbitMQTaskLaunchConsumer {
log.info("connected to rabbitmq: " + connection + " for " + taskLaunchExchangeName);
channel = connection.createChannel();
- channel.exchangeDeclare(taskLaunchExchangeName, "fanout");
+// channel.exchangeDeclare(taskLaunchExchangeName, "fanout");
} catch (Exception e) {
String msg = "could not open channel for exchange " + taskLaunchExchangeName;
@@ -98,7 +99,6 @@ public class RabbitMQTaskLaunchConsumer {
if (routing == null) {
throw new IllegalArgumentException("The routing key must be present");
}
-
List<String> keys = new ArrayList<String>();
if (routing instanceof List) {
for (Object o : (List)routing) {
@@ -113,7 +113,7 @@ public class RabbitMQTaskLaunchConsumer {
if (queueName == null) {
if (!channel.isOpen()) {
channel = connection.createChannel();
- channel.exchangeDeclare(taskLaunchExchangeName, "fanout");
+// channel.exchangeDeclare(taskLaunchExchangeName, "fanout");
}
queueName = channel.queueDeclare().getQueue();
} else {
@@ -131,11 +131,11 @@ public class RabbitMQTaskLaunchConsumer {
}
// bind all the routing keys
- for (String routingKey : keys) {
- channel.queueBind(queueName, taskLaunchExchangeName, routingKey);
- }
-
- channel.basicConsume(queueName, true, consumerTag, new DefaultConsumer(channel) {
+// for (String routingKey : keys) {
+// channel.queueBind(queueName, taskLaunchExchangeName, routingKey);
+// }
+ // autoAck=false, we will ack after task is done
+ channel.basicConsume(queueName, false, consumerTag, new QueueingConsumer(channel) {
@Override
public void handleDelivery(String consumerTag,
Envelope envelope,
@@ -147,6 +147,7 @@ public class RabbitMQTaskLaunchConsumer {
ThriftUtils.createThriftFromBytes(body, message);
TBase event = null;
String gatewayId = null;
+ long deliveryTag = envelope.getDeliveryTag(); //todo store this in zookeeper, once job is done we can ack
if(message.getMessageType().equals(MessageType.LAUNCHTASK)) {
TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent();
ThriftUtils.createThriftFromBytes(message.getEvent(), taskSubmitEvent);
@@ -167,6 +168,11 @@ public class RabbitMQTaskLaunchConsumer {
MessageContext messageContext = new MessageContext(event, message.getMessageType(), message.getMessageId(), gatewayId);
messageContext.setUpdatedTime(AiravataUtils.getTime(message.getUpdatedTime()));
handler.onMessage(messageContext);
+ try {
+ channel.basicAck(deliveryTag,false); //todo move this logic to monitoring component to ack when the job is done
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
} catch (TException e) {
String msg = "Failed to de-serialize the thrift message, from routing keys and queueName " + id;
log.warn(msg, e);
http://git-wip-us.apache.org/repos/asf/airavata/blob/840e627b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
index 23b2379..0f95fbf 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
@@ -20,6 +20,7 @@
*/
package org.apache.airavata.messaging.core.impl;
+import com.rabbitmq.client.MessageProperties;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ServerSettings;
@@ -50,7 +51,7 @@ public class RabbitMQTaskLaunchPublisher implements Publisher{
log.error(message, e);
throw new AiravataException(message, e);
}
- rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName,"fanout");
+ rabbitMQProducer = new RabbitMQProducer(brokerUrl, null,null);
rabbitMQProducer.open();
}
@@ -70,7 +71,7 @@ public class RabbitMQTaskLaunchPublisher implements Publisher{
routingKey = TERMINATE_TASK;
}
byte[] messageBody = ThriftUtils.serializeThriftObject(message);
- rabbitMQProducer.send(messageBody, routingKey);
+ rabbitMQProducer.sendToWorkerQueue(messageBody, routingKey);
} catch (TException e) {
String msg = "Error while deserializing the object";
log.error(msg, e);
[02/15] airavata git commit: implementing passive gfac submitter
using rabbbitmq
Posted by la...@apache.org.
implementing passive gfac submitter using rabbbitmq
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/88d27d95
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/88d27d95
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/88d27d95
Branch: refs/heads/master
Commit: 88d27d9574f9b077d334eabde147a0787b186899
Parents: 30aefc4
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Tue Feb 10 14:04:13 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Tue Feb 10 14:04:13 2015 -0500
----------------------------------------------------------------------
.../server/handler/AiravataServerHandler.java | 2 +-
.../lib/airavata/messagingEvents_types.cpp | 186 +++++-
.../lib/airavata/messagingEvents_types.h | 99 +++-
.../Airavata/Model/Messaging/Event/Types.php | 208 +++++++
.../model/messaging/event/MessageType.java | 8 +-
.../model/messaging/event/TaskSubmitEvent.java | 588 +++++++++++++++++++
.../messaging/event/TaskTerminateEvent.java | 492 ++++++++++++++++
airavata-api/generate-thrift-files.sh | 22 +-
.../messagingEvents.thrift | 15 +-
.../airavata/common/utils/ServerSettings.java | 5 +
.../main/resources/airavata-server.properties | 4 +-
.../main/resources/airavata-server.properties | 4 +-
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 2 +-
.../messaging/core/PublisherFactory.java | 21 +-
.../messaging/core/impl/RabbitMQPublisher.java | 99 ----
.../core/impl/RabbitMQStatusPublisher.java | 99 ++++
.../core/impl/RabbitMQTaskLaunchPublisher.java | 88 +++
.../server/OrchestratorServerHandler.java | 3 +-
modules/orchestrator/orchestrator-core/pom.xml | 14 +-
.../core/context/OrchestratorContext.java | 11 +
.../core/impl/GFACPassiveJobSubmitter.java | 247 ++++++++
.../core/impl/GFACRPCJobSubmitter.java | 212 +++++++
.../core/impl/GFACServiceJobSubmitter.java | 212 -------
.../workflow/engine/WorkflowEngineImpl.java | 4 +-
pom.xml | 7 +
25 files changed, 2314 insertions(+), 338 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
index 8e2ca17..8d1cd75 100644
--- a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
+++ b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
@@ -118,7 +118,7 @@ public class AiravataServerHandler implements Airavata.Iface {
public AiravataServerHandler() {
try {
if (ServerSettings.isRabbitMqPublishEnabled()) {
- publisher = PublisherFactory.createPublisher();
+ publisher = PublisherFactory.createActivityPublisher();
}
} catch (ApplicationSettingsException e) {
logger.error("Error occured while reading airavata-server properties..", e);
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
index 1f839f3..a2e72f5 100644
--- a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
+++ b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
@@ -45,15 +45,19 @@ int _kMessageTypeValues[] = {
MessageType::EXPERIMENT,
MessageType::TASK,
MessageType::WORKFLOWNODE,
- MessageType::JOB
+ MessageType::JOB,
+ MessageType::LAUNCHTASK,
+ MessageType::TERMINATETASK
};
const char* _kMessageTypeNames[] = {
"EXPERIMENT",
"TASK",
"WORKFLOWNODE",
- "JOB"
+ "JOB",
+ "LAUNCHTASK",
+ "TERMINATETASK"
};
-const std::map<int, const char*> _MessageType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kMessageTypeValues, _kMessageTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*> _MessageType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(6, _kMessageTypeValues, _kMessageTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
const char* ExperimentStatusChangeEvent::ascii_fingerprint = "38C252E94E93B69D04EB3A6EE2F9EDFB";
const uint8_t ExperimentStatusChangeEvent::binary_fingerprint[16] = {0x38,0xC2,0x52,0xE9,0x4E,0x93,0xB6,0x9D,0x04,0xEB,0x3A,0x6E,0xE2,0xF9,0xED,0xFB};
@@ -835,6 +839,182 @@ void swap(JobIdentifier &a, JobIdentifier &b) {
swap(a.gatewayId, b.gatewayId);
}
+const char* TaskSubmitEvent::ascii_fingerprint = "AB879940BD15B6B25691265F7384B271";
+const uint8_t TaskSubmitEvent::binary_fingerprint[16] = {0xAB,0x87,0x99,0x40,0xBD,0x15,0xB6,0xB2,0x56,0x91,0x26,0x5F,0x73,0x84,0xB2,0x71};
+
+uint32_t TaskSubmitEvent::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+ bool isset_experimentId = false;
+ bool isset_taskId = false;
+ bool isset_gatewayId = false;
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->experimentId);
+ isset_experimentId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->taskId);
+ isset_taskId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->gatewayId);
+ isset_gatewayId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ if (!isset_experimentId)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ if (!isset_taskId)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ if (!isset_gatewayId)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ return xfer;
+}
+
+uint32_t TaskSubmitEvent::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ xfer += oprot->writeStructBegin("TaskSubmitEvent");
+
+ xfer += oprot->writeFieldBegin("experimentId", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->experimentId);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("taskId", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->taskId);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("gatewayId", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->gatewayId);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+}
+
+void swap(TaskSubmitEvent &a, TaskSubmitEvent &b) {
+ using ::std::swap;
+ swap(a.experimentId, b.experimentId);
+ swap(a.taskId, b.taskId);
+ swap(a.gatewayId, b.gatewayId);
+}
+
+const char* TaskTerminateEvent::ascii_fingerprint = "07A9615F837F7D0A952B595DD3020972";
+const uint8_t TaskTerminateEvent::binary_fingerprint[16] = {0x07,0xA9,0x61,0x5F,0x83,0x7F,0x7D,0x0A,0x95,0x2B,0x59,0x5D,0xD3,0x02,0x09,0x72};
+
+uint32_t TaskTerminateEvent::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+ bool isset_experimentId = false;
+ bool isset_taskId = false;
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->experimentId);
+ isset_experimentId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->taskId);
+ isset_taskId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ if (!isset_experimentId)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ if (!isset_taskId)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ return xfer;
+}
+
+uint32_t TaskTerminateEvent::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ xfer += oprot->writeStructBegin("TaskTerminateEvent");
+
+ xfer += oprot->writeFieldBegin("experimentId", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->experimentId);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("taskId", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->taskId);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+}
+
+void swap(TaskTerminateEvent &a, TaskTerminateEvent &b) {
+ using ::std::swap;
+ swap(a.experimentId, b.experimentId);
+ swap(a.taskId, b.taskId);
+}
+
const char* JobStatusChangeEvent::ascii_fingerprint = "062775D589B60D1687103FD465B0F5E8";
const uint8_t JobStatusChangeEvent::binary_fingerprint[16] = {0x06,0x27,0x75,0xD5,0x89,0xB6,0x0D,0x16,0x87,0x10,0x3F,0xD4,0x65,0xB0,0xF5,0xE8};
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
index 572a8bd..f063fc2 100644
--- a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
+++ b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
@@ -52,7 +52,9 @@ struct MessageType {
EXPERIMENT = 0,
TASK = 1,
WORKFLOWNODE = 2,
- JOB = 3
+ JOB = 3,
+ LAUNCHTASK = 4,
+ TERMINATETASK = 5
};
};
@@ -460,6 +462,101 @@ class JobIdentifier {
void swap(JobIdentifier &a, JobIdentifier &b);
+class TaskSubmitEvent {
+ public:
+
+ static const char* ascii_fingerprint; // = "AB879940BD15B6B25691265F7384B271";
+ static const uint8_t binary_fingerprint[16]; // = {0xAB,0x87,0x99,0x40,0xBD,0x15,0xB6,0xB2,0x56,0x91,0x26,0x5F,0x73,0x84,0xB2,0x71};
+
+ TaskSubmitEvent() : experimentId(), taskId(), gatewayId() {
+ }
+
+ virtual ~TaskSubmitEvent() throw() {}
+
+ std::string experimentId;
+ std::string taskId;
+ std::string gatewayId;
+
+ void __set_experimentId(const std::string& val) {
+ experimentId = val;
+ }
+
+ void __set_taskId(const std::string& val) {
+ taskId = val;
+ }
+
+ void __set_gatewayId(const std::string& val) {
+ gatewayId = val;
+ }
+
+ bool operator == (const TaskSubmitEvent & rhs) const
+ {
+ if (!(experimentId == rhs.experimentId))
+ return false;
+ if (!(taskId == rhs.taskId))
+ return false;
+ if (!(gatewayId == rhs.gatewayId))
+ return false;
+ return true;
+ }
+ bool operator != (const TaskSubmitEvent &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const TaskSubmitEvent & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+void swap(TaskSubmitEvent &a, TaskSubmitEvent &b);
+
+
+class TaskTerminateEvent {
+ public:
+
+ static const char* ascii_fingerprint; // = "07A9615F837F7D0A952B595DD3020972";
+ static const uint8_t binary_fingerprint[16]; // = {0x07,0xA9,0x61,0x5F,0x83,0x7F,0x7D,0x0A,0x95,0x2B,0x59,0x5D,0xD3,0x02,0x09,0x72};
+
+ TaskTerminateEvent() : experimentId(), taskId() {
+ }
+
+ virtual ~TaskTerminateEvent() throw() {}
+
+ std::string experimentId;
+ std::string taskId;
+
+ void __set_experimentId(const std::string& val) {
+ experimentId = val;
+ }
+
+ void __set_taskId(const std::string& val) {
+ taskId = val;
+ }
+
+ bool operator == (const TaskTerminateEvent & rhs) const
+ {
+ if (!(experimentId == rhs.experimentId))
+ return false;
+ if (!(taskId == rhs.taskId))
+ return false;
+ return true;
+ }
+ bool operator != (const TaskTerminateEvent &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const TaskTerminateEvent & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+void swap(TaskTerminateEvent &a, TaskTerminateEvent &b);
+
+
class JobStatusChangeEvent {
public:
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php b/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
index d20392a..40810d3 100644
--- a/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
+++ b/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
@@ -35,11 +35,15 @@ final class MessageType {
const TASK = 1;
const WORKFLOWNODE = 2;
const JOB = 3;
+ const LAUNCHTASK = 4;
+ const TERMINATETASK = 5;
static public $__names = array(
0 => 'EXPERIMENT',
1 => 'TASK',
2 => 'WORKFLOWNODE',
3 => 'JOB',
+ 4 => 'LAUNCHTASK',
+ 5 => 'TERMINATETASK',
);
}
@@ -967,6 +971,210 @@ class JobIdentifier {
}
+class TaskSubmitEvent {
+ static $_TSPEC;
+
+ public $experimentId = null;
+ public $taskId = null;
+ public $gatewayId = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'experimentId',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'taskId',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'gatewayId',
+ 'type' => TType::STRING,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['experimentId'])) {
+ $this->experimentId = $vals['experimentId'];
+ }
+ if (isset($vals['taskId'])) {
+ $this->taskId = $vals['taskId'];
+ }
+ if (isset($vals['gatewayId'])) {
+ $this->gatewayId = $vals['gatewayId'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'TaskSubmitEvent';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->experimentId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->taskId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->gatewayId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('TaskSubmitEvent');
+ if ($this->experimentId !== null) {
+ $xfer += $output->writeFieldBegin('experimentId', TType::STRING, 1);
+ $xfer += $output->writeString($this->experimentId);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->taskId !== null) {
+ $xfer += $output->writeFieldBegin('taskId', TType::STRING, 2);
+ $xfer += $output->writeString($this->taskId);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->gatewayId !== null) {
+ $xfer += $output->writeFieldBegin('gatewayId', TType::STRING, 3);
+ $xfer += $output->writeString($this->gatewayId);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+}
+
+class TaskTerminateEvent {
+ static $_TSPEC;
+
+ public $experimentId = null;
+ public $taskId = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'experimentId',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'taskId',
+ 'type' => TType::STRING,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['experimentId'])) {
+ $this->experimentId = $vals['experimentId'];
+ }
+ if (isset($vals['taskId'])) {
+ $this->taskId = $vals['taskId'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'TaskTerminateEvent';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->experimentId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->taskId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('TaskTerminateEvent');
+ if ($this->experimentId !== null) {
+ $xfer += $output->writeFieldBegin('experimentId', TType::STRING, 1);
+ $xfer += $output->writeString($this->experimentId);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->taskId !== null) {
+ $xfer += $output->writeFieldBegin('taskId', TType::STRING, 2);
+ $xfer += $output->writeString($this->taskId);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+}
+
class JobStatusChangeEvent {
static $_TSPEC;
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/MessageType.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/MessageType.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/MessageType.java
index d00f404..230b87b 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/MessageType.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/MessageType.java
@@ -32,7 +32,9 @@ import org.apache.thrift.TEnum;
EXPERIMENT(0),
TASK(1),
WORKFLOWNODE(2),
- JOB(3);
+ JOB(3),
+ LAUNCHTASK(4),
+ TERMINATETASK(5);
private final int value;
@@ -61,6 +63,10 @@ import org.apache.thrift.TEnum;
return WORKFLOWNODE;
case 3:
return JOB;
+ case 4:
+ return LAUNCHTASK;
+ case 5:
+ return TERMINATETASK;
default:
return null;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java
new file mode 100644
index 0000000..c813c76
--- /dev/null
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java
@@ -0,0 +1,588 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.1)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.airavata.model.messaging.event;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.thrift.server.AbstractNonblockingServer.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class TaskSubmitEvent implements org.apache.thrift.TBase<TaskSubmitEvent, TaskSubmitEvent._Fields>, java.io.Serializable, Cloneable, Comparable<TaskSubmitEvent> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TaskSubmitEvent");
+
+ private static final org.apache.thrift.protocol.TField EXPERIMENT_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("experimentId", org.apache.thrift.protocol.TType.STRING, (short)1);
+ private static final org.apache.thrift.protocol.TField TASK_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("taskId", org.apache.thrift.protocol.TType.STRING, (short)2);
+ private static final org.apache.thrift.protocol.TField GATEWAY_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("gatewayId", org.apache.thrift.protocol.TType.STRING, (short)3);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TaskSubmitEventStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TaskSubmitEventTupleSchemeFactory());
+ }
+
+ private String experimentId; // required
+ private String taskId; // required
+ private String gatewayId; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ EXPERIMENT_ID((short)1, "experimentId"),
+ TASK_ID((short)2, "taskId"),
+ GATEWAY_ID((short)3, "gatewayId");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // EXPERIMENT_ID
+ return EXPERIMENT_ID;
+ case 2: // TASK_ID
+ return TASK_ID;
+ case 3: // GATEWAY_ID
+ return GATEWAY_ID;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.EXPERIMENT_ID, new org.apache.thrift.meta_data.FieldMetaData("experimentId", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.TASK_ID, new org.apache.thrift.meta_data.FieldMetaData("taskId", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.GATEWAY_ID, new org.apache.thrift.meta_data.FieldMetaData("gatewayId", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TaskSubmitEvent.class, metaDataMap);
+ }
+
+ public TaskSubmitEvent() {
+ }
+
+ public TaskSubmitEvent(
+ String experimentId,
+ String taskId,
+ String gatewayId)
+ {
+ this();
+ this.experimentId = experimentId;
+ this.taskId = taskId;
+ this.gatewayId = gatewayId;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TaskSubmitEvent(TaskSubmitEvent other) {
+ if (other.isSetExperimentId()) {
+ this.experimentId = other.experimentId;
+ }
+ if (other.isSetTaskId()) {
+ this.taskId = other.taskId;
+ }
+ if (other.isSetGatewayId()) {
+ this.gatewayId = other.gatewayId;
+ }
+ }
+
+ public TaskSubmitEvent deepCopy() {
+ return new TaskSubmitEvent(this);
+ }
+
+ @Override
+ public void clear() {
+ this.experimentId = null;
+ this.taskId = null;
+ this.gatewayId = null;
+ }
+
+ public String getExperimentId() {
+ return this.experimentId;
+ }
+
+ public void setExperimentId(String experimentId) {
+ this.experimentId = experimentId;
+ }
+
+ public void unsetExperimentId() {
+ this.experimentId = null;
+ }
+
+ /** Returns true if field experimentId is set (has been assigned a value) and false otherwise */
+ public boolean isSetExperimentId() {
+ return this.experimentId != null;
+ }
+
+ public void setExperimentIdIsSet(boolean value) {
+ if (!value) {
+ this.experimentId = null;
+ }
+ }
+
+ public String getTaskId() {
+ return this.taskId;
+ }
+
+ public void setTaskId(String taskId) {
+ this.taskId = taskId;
+ }
+
+ public void unsetTaskId() {
+ this.taskId = null;
+ }
+
+ /** Returns true if field taskId is set (has been assigned a value) and false otherwise */
+ public boolean isSetTaskId() {
+ return this.taskId != null;
+ }
+
+ public void setTaskIdIsSet(boolean value) {
+ if (!value) {
+ this.taskId = null;
+ }
+ }
+
+ public String getGatewayId() {
+ return this.gatewayId;
+ }
+
+ public void setGatewayId(String gatewayId) {
+ this.gatewayId = gatewayId;
+ }
+
+ public void unsetGatewayId() {
+ this.gatewayId = null;
+ }
+
+ /** Returns true if field gatewayId is set (has been assigned a value) and false otherwise */
+ public boolean isSetGatewayId() {
+ return this.gatewayId != null;
+ }
+
+ public void setGatewayIdIsSet(boolean value) {
+ if (!value) {
+ this.gatewayId = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case EXPERIMENT_ID:
+ if (value == null) {
+ unsetExperimentId();
+ } else {
+ setExperimentId((String)value);
+ }
+ break;
+
+ case TASK_ID:
+ if (value == null) {
+ unsetTaskId();
+ } else {
+ setTaskId((String)value);
+ }
+ break;
+
+ case GATEWAY_ID:
+ if (value == null) {
+ unsetGatewayId();
+ } else {
+ setGatewayId((String)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case EXPERIMENT_ID:
+ return getExperimentId();
+
+ case TASK_ID:
+ return getTaskId();
+
+ case GATEWAY_ID:
+ return getGatewayId();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case EXPERIMENT_ID:
+ return isSetExperimentId();
+ case TASK_ID:
+ return isSetTaskId();
+ case GATEWAY_ID:
+ return isSetGatewayId();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TaskSubmitEvent)
+ return this.equals((TaskSubmitEvent)that);
+ return false;
+ }
+
+ public boolean equals(TaskSubmitEvent that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_experimentId = true && this.isSetExperimentId();
+ boolean that_present_experimentId = true && that.isSetExperimentId();
+ if (this_present_experimentId || that_present_experimentId) {
+ if (!(this_present_experimentId && that_present_experimentId))
+ return false;
+ if (!this.experimentId.equals(that.experimentId))
+ return false;
+ }
+
+ boolean this_present_taskId = true && this.isSetTaskId();
+ boolean that_present_taskId = true && that.isSetTaskId();
+ if (this_present_taskId || that_present_taskId) {
+ if (!(this_present_taskId && that_present_taskId))
+ return false;
+ if (!this.taskId.equals(that.taskId))
+ return false;
+ }
+
+ boolean this_present_gatewayId = true && this.isSetGatewayId();
+ boolean that_present_gatewayId = true && that.isSetGatewayId();
+ if (this_present_gatewayId || that_present_gatewayId) {
+ if (!(this_present_gatewayId && that_present_gatewayId))
+ return false;
+ if (!this.gatewayId.equals(that.gatewayId))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public int compareTo(TaskSubmitEvent other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = Boolean.valueOf(isSetExperimentId()).compareTo(other.isSetExperimentId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetExperimentId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.experimentId, other.experimentId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetTaskId()).compareTo(other.isSetTaskId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTaskId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.taskId, other.taskId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetGatewayId()).compareTo(other.isSetGatewayId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetGatewayId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.gatewayId, other.gatewayId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TaskSubmitEvent(");
+ boolean first = true;
+
+ sb.append("experimentId:");
+ if (this.experimentId == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.experimentId);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("taskId:");
+ if (this.taskId == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.taskId);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("gatewayId:");
+ if (this.gatewayId == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.gatewayId);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ if (!isSetExperimentId()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'experimentId' is unset! Struct:" + toString());
+ }
+
+ if (!isSetTaskId()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'taskId' is unset! Struct:" + toString());
+ }
+
+ if (!isSetGatewayId()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'gatewayId' is unset! Struct:" + toString());
+ }
+
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TaskSubmitEventStandardSchemeFactory implements SchemeFactory {
+ public TaskSubmitEventStandardScheme getScheme() {
+ return new TaskSubmitEventStandardScheme();
+ }
+ }
+
+ private static class TaskSubmitEventStandardScheme extends StandardScheme<TaskSubmitEvent> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, TaskSubmitEvent struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // EXPERIMENT_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.experimentId = iprot.readString();
+ struct.setExperimentIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // TASK_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.taskId = iprot.readString();
+ struct.setTaskIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 3: // GATEWAY_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.gatewayId = iprot.readString();
+ struct.setGatewayIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, TaskSubmitEvent struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.experimentId != null) {
+ oprot.writeFieldBegin(EXPERIMENT_ID_FIELD_DESC);
+ oprot.writeString(struct.experimentId);
+ oprot.writeFieldEnd();
+ }
+ if (struct.taskId != null) {
+ oprot.writeFieldBegin(TASK_ID_FIELD_DESC);
+ oprot.writeString(struct.taskId);
+ oprot.writeFieldEnd();
+ }
+ if (struct.gatewayId != null) {
+ oprot.writeFieldBegin(GATEWAY_ID_FIELD_DESC);
+ oprot.writeString(struct.gatewayId);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TaskSubmitEventTupleSchemeFactory implements SchemeFactory {
+ public TaskSubmitEventTupleScheme getScheme() {
+ return new TaskSubmitEventTupleScheme();
+ }
+ }
+
+ private static class TaskSubmitEventTupleScheme extends TupleScheme<TaskSubmitEvent> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, TaskSubmitEvent struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ oprot.writeString(struct.experimentId);
+ oprot.writeString(struct.taskId);
+ oprot.writeString(struct.gatewayId);
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, TaskSubmitEvent struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ struct.experimentId = iprot.readString();
+ struct.setExperimentIdIsSet(true);
+ struct.taskId = iprot.readString();
+ struct.setTaskIdIsSet(true);
+ struct.gatewayId = iprot.readString();
+ struct.setGatewayIdIsSet(true);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskTerminateEvent.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskTerminateEvent.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskTerminateEvent.java
new file mode 100644
index 0000000..59b9f85
--- /dev/null
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskTerminateEvent.java
@@ -0,0 +1,492 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.1)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.airavata.model.messaging.event;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.thrift.server.AbstractNonblockingServer.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class TaskTerminateEvent implements org.apache.thrift.TBase<TaskTerminateEvent, TaskTerminateEvent._Fields>, java.io.Serializable, Cloneable, Comparable<TaskTerminateEvent> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TaskTerminateEvent");
+
+ private static final org.apache.thrift.protocol.TField EXPERIMENT_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("experimentId", org.apache.thrift.protocol.TType.STRING, (short)1);
+ private static final org.apache.thrift.protocol.TField TASK_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("taskId", org.apache.thrift.protocol.TType.STRING, (short)2);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TaskTerminateEventStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TaskTerminateEventTupleSchemeFactory());
+ }
+
+ private String experimentId; // required
+ private String taskId; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ EXPERIMENT_ID((short)1, "experimentId"),
+ TASK_ID((short)2, "taskId");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // EXPERIMENT_ID
+ return EXPERIMENT_ID;
+ case 2: // TASK_ID
+ return TASK_ID;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.EXPERIMENT_ID, new org.apache.thrift.meta_data.FieldMetaData("experimentId", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.TASK_ID, new org.apache.thrift.meta_data.FieldMetaData("taskId", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TaskTerminateEvent.class, metaDataMap);
+ }
+
+ public TaskTerminateEvent() {
+ }
+
+ public TaskTerminateEvent(
+ String experimentId,
+ String taskId)
+ {
+ this();
+ this.experimentId = experimentId;
+ this.taskId = taskId;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TaskTerminateEvent(TaskTerminateEvent other) {
+ if (other.isSetExperimentId()) {
+ this.experimentId = other.experimentId;
+ }
+ if (other.isSetTaskId()) {
+ this.taskId = other.taskId;
+ }
+ }
+
+ public TaskTerminateEvent deepCopy() {
+ return new TaskTerminateEvent(this);
+ }
+
+ @Override
+ public void clear() {
+ this.experimentId = null;
+ this.taskId = null;
+ }
+
+ public String getExperimentId() {
+ return this.experimentId;
+ }
+
+ public void setExperimentId(String experimentId) {
+ this.experimentId = experimentId;
+ }
+
+ public void unsetExperimentId() {
+ this.experimentId = null;
+ }
+
+ /** Returns true if field experimentId is set (has been assigned a value) and false otherwise */
+ public boolean isSetExperimentId() {
+ return this.experimentId != null;
+ }
+
+ public void setExperimentIdIsSet(boolean value) {
+ if (!value) {
+ this.experimentId = null;
+ }
+ }
+
+ public String getTaskId() {
+ return this.taskId;
+ }
+
+ public void setTaskId(String taskId) {
+ this.taskId = taskId;
+ }
+
+ public void unsetTaskId() {
+ this.taskId = null;
+ }
+
+ /** Returns true if field taskId is set (has been assigned a value) and false otherwise */
+ public boolean isSetTaskId() {
+ return this.taskId != null;
+ }
+
+ public void setTaskIdIsSet(boolean value) {
+ if (!value) {
+ this.taskId = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case EXPERIMENT_ID:
+ if (value == null) {
+ unsetExperimentId();
+ } else {
+ setExperimentId((String)value);
+ }
+ break;
+
+ case TASK_ID:
+ if (value == null) {
+ unsetTaskId();
+ } else {
+ setTaskId((String)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case EXPERIMENT_ID:
+ return getExperimentId();
+
+ case TASK_ID:
+ return getTaskId();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case EXPERIMENT_ID:
+ return isSetExperimentId();
+ case TASK_ID:
+ return isSetTaskId();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TaskTerminateEvent)
+ return this.equals((TaskTerminateEvent)that);
+ return false;
+ }
+
+ public boolean equals(TaskTerminateEvent that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_experimentId = true && this.isSetExperimentId();
+ boolean that_present_experimentId = true && that.isSetExperimentId();
+ if (this_present_experimentId || that_present_experimentId) {
+ if (!(this_present_experimentId && that_present_experimentId))
+ return false;
+ if (!this.experimentId.equals(that.experimentId))
+ return false;
+ }
+
+ boolean this_present_taskId = true && this.isSetTaskId();
+ boolean that_present_taskId = true && that.isSetTaskId();
+ if (this_present_taskId || that_present_taskId) {
+ if (!(this_present_taskId && that_present_taskId))
+ return false;
+ if (!this.taskId.equals(that.taskId))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public int compareTo(TaskTerminateEvent other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = Boolean.valueOf(isSetExperimentId()).compareTo(other.isSetExperimentId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetExperimentId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.experimentId, other.experimentId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetTaskId()).compareTo(other.isSetTaskId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTaskId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.taskId, other.taskId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TaskTerminateEvent(");
+ boolean first = true;
+
+ sb.append("experimentId:");
+ if (this.experimentId == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.experimentId);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("taskId:");
+ if (this.taskId == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.taskId);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ if (!isSetExperimentId()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'experimentId' is unset! Struct:" + toString());
+ }
+
+ if (!isSetTaskId()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'taskId' is unset! Struct:" + toString());
+ }
+
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TaskTerminateEventStandardSchemeFactory implements SchemeFactory {
+ public TaskTerminateEventStandardScheme getScheme() {
+ return new TaskTerminateEventStandardScheme();
+ }
+ }
+
+ private static class TaskTerminateEventStandardScheme extends StandardScheme<TaskTerminateEvent> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, TaskTerminateEvent struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // EXPERIMENT_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.experimentId = iprot.readString();
+ struct.setExperimentIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // TASK_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.taskId = iprot.readString();
+ struct.setTaskIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, TaskTerminateEvent struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.experimentId != null) {
+ oprot.writeFieldBegin(EXPERIMENT_ID_FIELD_DESC);
+ oprot.writeString(struct.experimentId);
+ oprot.writeFieldEnd();
+ }
+ if (struct.taskId != null) {
+ oprot.writeFieldBegin(TASK_ID_FIELD_DESC);
+ oprot.writeString(struct.taskId);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TaskTerminateEventTupleSchemeFactory implements SchemeFactory {
+ public TaskTerminateEventTupleScheme getScheme() {
+ return new TaskTerminateEventTupleScheme();
+ }
+ }
+
+ private static class TaskTerminateEventTupleScheme extends TupleScheme<TaskTerminateEvent> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, TaskTerminateEvent struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ oprot.writeString(struct.experimentId);
+ oprot.writeString(struct.taskId);
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, TaskTerminateEvent struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ struct.experimentId = iprot.readString();
+ struct.setExperimentIdIsSet(true);
+ struct.taskId = iprot.readString();
+ struct.setTaskIdIsSet(true);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/generate-thrift-files.sh
----------------------------------------------------------------------
diff --git a/airavata-api/generate-thrift-files.sh b/airavata-api/generate-thrift-files.sh
index 4cd3288..bd823e4 100755
--- a/airavata-api/generate-thrift-files.sh
+++ b/airavata-api/generate-thrift-files.sh
@@ -27,7 +27,7 @@ DATAMODEL_SRC_DIR='airavata-data-models/src/main/java'
JAVA_API_SDK_DIR='airavata-api-stubs/src/main/java'
CPP_SDK_DIR='airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/'
PHP_SDK_DIR='airavata-client-sdks/airavata-php-sdk/src/main/resources/lib'
-
+THRIFT_EXEC=thrift
# The Function fail prints error messages on failure and quits the script.
fail() {
echo $@
@@ -96,7 +96,7 @@ copy_changed_files() {
# Generation of thrift files will require installing Apache Thrift. Please add thrift to your path.
# Verify is thrift is installed, is in the path is at a specified version.
-VERSION=$(thrift -version 2>/dev/null | grep -F "${REQUIRED_THRIFT_VERSION}" | wc -l)
+VERSION=$($THRIFT_EXEC -version 2>/dev/null | grep -F "${REQUIRED_THRIFT_VERSION}" | wc -l)
if [ "$VERSION" -ne 1 ] ; then
echo "****************************************************"
echo "*** thrift is not installed or is not in the path"
@@ -125,11 +125,11 @@ rm -rf ${JAVA_BEAN_GEN_DIR}
# Generate the Airavata Data Model using thrift Java Beans generator. This will take generate the classes in bean style
# with members being private and setters returning voids.
# The airavataDataModel.thrift includes rest of data models.
-thrift ${THRIFT_ARGS} --gen java:beans ${THRIFT_IDL_DIR}/airavataDataModel.thrift || fail unable to generate java bean thrift classes on base data model
+$THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${THRIFT_IDL_DIR}/airavataDataModel.thrift || fail unable to generate java bean thrift classes on base data model
-thrift ${THRIFT_ARGS} --gen java:beans ${THRIFT_IDL_DIR}/appCatalogModels.thrift || fail unable to generate java bean thrift classes on app catalog data models
+$THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${THRIFT_IDL_DIR}/appCatalogModels.thrift || fail unable to generate java bean thrift classes on app catalog data models
-thrift ${THRIFT_ARGS} --gen java:beans ${THRIFT_IDL_DIR}/workflowDataModel.thrift || fail unable to generate java bean thrift classes on app workflow data models
+$THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${THRIFT_IDL_DIR}/workflowDataModel.thrift || fail unable to generate java bean thrift classes on app workflow data models
# For the generated java beans add the ASF V2 License header
add_license_header $JAVA_BEAN_GEN_DIR
@@ -150,9 +150,9 @@ rm -rf ${JAVA_GEN_DIR}
# Using thrift Java generator, generate the java classes based on Airavata API. This
# The airavataAPI.thrift includes rest of data models.
-thrift ${THRIFT_ARGS} --gen java ${THRIFT_IDL_DIR}/airavataAPI.thrift || fail unable to generate java thrift classes on AiravataAPI
+$THRIFT_EXEC ${THRIFT_ARGS} --gen java ${THRIFT_IDL_DIR}/airavataAPI.thrift || fail unable to generate java thrift classes on AiravataAPI
-#thrift ${THRIFT_ARGS} --gen java ${THRIFT_IDL_DIR}/workflowAPI.thrift || fail unable to generate java thrift classes on WorkflowAPI
+#$THRIFT_EXEC ${THRIFT_ARGS} --gen java ${THRIFT_IDL_DIR}/workflowAPI.thrift || fail unable to generate java thrift classes on WorkflowAPI
# For the generated java classes add the ASF V2 License header
add_license_header $JAVA_GEN_DIR
@@ -173,9 +173,9 @@ rm -rf ${CPP_GEN_DIR}
# Using thrift Java generator, generate the java classes based on Airavata API. This
# The airavataAPI.thrift includes rest of data models.
-thrift ${THRIFT_ARGS} --gen cpp ${THRIFT_IDL_DIR}/airavataAPI.thrift || fail unable to generate C++ thrift classes
+/usr/local/Cellar/thrift/0.9.1/bin/thrift ${THRIFT_ARGS} --gen cpp ${THRIFT_IDL_DIR}/airavataAPI.thrift || fail unable to generate C++ thrift classes
-#thrift ${THRIFT_ARGS} --gen cpp ${THRIFT_IDL_DIR}/workflowAPI.thrift || fail unable to generate C++ thrift classes for WorkflowAPI
+#$THRIFT_EXEC ${THRIFT_ARGS} --gen cpp ${THRIFT_IDL_DIR}/workflowAPI.thrift || fail unable to generate C++ thrift classes for WorkflowAPI
# For the generated CPP classes add the ASF V2 License header
add_license_header $CPP_GEN_DIR
@@ -195,9 +195,9 @@ rm -rf ${PHP_GEN_DIR}
# Using thrift Java generator, generate the java classes based on Airavata API. This
# The airavataAPI.thrift includes rest of data models.
-thrift ${THRIFT_ARGS} --gen php:autoload ${THRIFT_IDL_DIR}/airavataAPI.thrift || fail unable to generate PHP thrift classes
+$THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${THRIFT_IDL_DIR}/airavataAPI.thrift || fail unable to generate PHP thrift classes
-#thrift ${THRIFT_ARGS} --gen php:autoload ${THRIFT_IDL_DIR}/workflowAPI.thrift || fail unable to generate PHP thrift classes for WorkflowAPI
+#$THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${THRIFT_IDL_DIR}/workflowAPI.thrift || fail unable to generate PHP thrift classes for WorkflowAPI
# For the generated java classes add the ASF V2 License header
## TODO Write PHP license parser
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
----------------------------------------------------------------------
diff --git a/airavata-api/thrift-interface-descriptions/messagingEvents.thrift b/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
index c9f3808..d736701 100644
--- a/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
+++ b/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
@@ -38,7 +38,9 @@ enum MessageType {
EXPERIMENT,
TASK,
WORKFLOWNODE,
- JOB
+ JOB,
+ LAUNCHTASK,
+ TERMINATETASK
}
struct ExperimentStatusChangeEvent {
@@ -100,6 +102,17 @@ struct JobIdentifier {
// //8:
// }
+struct TaskSubmitEvent{
+ 1: required string experimentId,
+ 2: required string taskId,
+ 3: required string gatewayId
+}
+
+struct TaskTerminateEvent{
+ 1: required string experimentId,
+ 2: required string taskId,
+}
+
struct JobStatusChangeEvent {
1: required experimentModel.JobState state;
2: required JobIdentifier jobIdentity;
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index 988ad3d..4ea0b44 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -52,6 +52,7 @@ public class ServerSettings extends ApplicationSettings {
private static final String MY_PROXY_PASSWORD = "myproxy.password";
private static final String MY_PROXY_LIFETIME = "myproxy.life";
private static final String ACTIVITY_PUBLISHER = "activity.publisher";
+ private static final String TASK_LAUNCH_PUBLISHER = "task.launch.publisher";
private static final String ACTIVITY_LISTENERS = "activity.listeners";
public static final String PUBLISH_RABBITMQ = "publish.rabbitmq";
public static final String JOB_NOTIFICATION_ENABLE = "job.notification.enable";
@@ -154,6 +155,10 @@ public class ServerSettings extends ApplicationSettings {
return getSetting(ACTIVITY_PUBLISHER);
}
+ public static String getTaskLaunchPublisher() throws ApplicationSettingsException{
+ return getSetting(TASK_LAUNCH_PUBLISHER);
+ }
+
public static boolean isRabbitMqPublishEnabled() throws ApplicationSettingsException{
String setting = getSetting(PUBLISH_RABBITMQ);
return Boolean.parseBoolean(setting);
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index c73e61a..c90fab1 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -216,6 +216,7 @@ connection.name=xsede
activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
publish.rabbitmq=false
activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQPublisher
+task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
rabbitmq.broker.url=amqp://localhost:5672
rabbitmq.exchange.name=airavata_rabbitmq_exchange
@@ -224,7 +225,8 @@ rabbitmq.exchange.name=airavata_rabbitmq_exchange
###########################################################################
#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
-job.submitter=org.apache.airavata.orchestrator.core.impl.GFACServiceJobSubmitter
+#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
+job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
submitter.interval=10000
threadpool.size=10
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties b/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
index fb02901..d6be51a 100644
--- a/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
+++ b/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
@@ -201,6 +201,7 @@ connection.name=xsede
activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
publish.rabbitmq=false
activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQPublisher
+task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
rabbitmq.broker.url=amqp://localhost:5672
rabbitmq.exchange.name=airavata_rabbitmq_exchange
@@ -209,7 +210,8 @@ rabbitmq.exchange.name=airavata_rabbitmq_exchange
###########################################################################
#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
-job.submitter=org.apache.airavata.orchestrator.core.impl.GFACServiceJobSubmitter
+#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
+job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
submitter.interval=10000
threadpool.size=10
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index 00d313c..bb612a6 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -116,7 +116,7 @@ public class BetterGfacImpl implements GFac,Watcher {
String[] listenerClassList = ServerSettings.getActivityListeners();
Publisher rabbitMQPublisher = null;
if (ServerSettings.isRabbitMqPublishEnabled()){
- rabbitMQPublisher = PublisherFactory.createPublisher();
+ rabbitMQPublisher = PublisherFactory.createActivityPublisher();
}
for (String listenerClass : listenerClassList) {
Class<? extends AbstractActivityListener> aClass = Class.forName(listenerClass).asSubclass(AbstractActivityListener.class);
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
index 2080cc6..59cdbdf 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory;
public class PublisherFactory {
private static Logger log = LoggerFactory.getLogger(PublisherFactory.class);
- public static Publisher createPublisher() throws AiravataException {
+ public static Publisher createActivityPublisher() throws AiravataException {
String activityPublisher = ServerSettings.getActivityPublisher();
if (activityPublisher == null) {
@@ -47,4 +47,23 @@ public class PublisherFactory {
throw new AiravataException(msg, e);
}
}
+
+ public static Publisher createTaskLaunchPublisher() throws AiravataException {
+ String taskLaunchPublisher = ServerSettings.getTaskLaunchPublisher();
+
+ if (taskLaunchPublisher == null) {
+ String s = "Task launch publisher is not specified";
+ log.error(s);
+ throw new AiravataException(s);
+ }
+
+ try {
+ Class<? extends Publisher> aPublisher = Class.forName(taskLaunchPublisher).asSubclass(Publisher.class);
+ return aPublisher.newInstance();
+ } catch (Exception e) {
+ String msg = "Failed to load the publisher from the publisher class property: " + taskLaunchPublisher;
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java
deleted file mode 100644
index ff14a8c..0000000
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQPublisher.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.messaging.core.impl;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.common.utils.ThriftUtils;
-import org.apache.airavata.messaging.core.MessageContext;
-import org.apache.airavata.messaging.core.MessagingConstants;
-import org.apache.airavata.messaging.core.Publisher;
-import org.apache.airavata.model.messaging.event.*;
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class RabbitMQPublisher implements Publisher {
-
- private static Logger log = LoggerFactory.getLogger(RabbitMQPublisher.class);
-
- private RabbitMQProducer rabbitMQProducer;
-
-
- public RabbitMQPublisher() throws Exception {
- String brokerUrl;
- String exchangeName;
- try {
- brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
- exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_EXCHANGE_NAME);
- } catch (ApplicationSettingsException e) {
- String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
- log.error(message, e);
- throw new AiravataException(message, e);
- }
- rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName);
- rabbitMQProducer.open();
- }
-
- public void publish(MessageContext msgCtx) throws AiravataException {
- try {
- log.info("Publishing status to rabbitmq...");
- byte[] body = ThriftUtils.serializeThriftObject(msgCtx.getEvent());
- Message message = new Message();
- message.setEvent(body);
- message.setMessageId(msgCtx.getMessageId());
- message.setMessageType(msgCtx.getType());
- message.setUpdatedTime(msgCtx.getUpdatedTime().getTime());
- String routingKey = null;
- if (msgCtx.getType().equals(MessageType.EXPERIMENT)){
- ExperimentStatusChangeEvent event = (ExperimentStatusChangeEvent) msgCtx.getEvent();
- routingKey = event.getExperimentId();
- } else if (msgCtx.getType().equals(MessageType.TASK)) {
- TaskStatusChangeEvent event = (TaskStatusChangeEvent) msgCtx.getEvent();
- routingKey = event.getTaskIdentity().getExperimentId() + "." +
- event.getTaskIdentity().getWorkflowNodeId() + "." + event.getTaskIdentity().getTaskId();
- }else if (msgCtx.getType().equals(MessageType.WORKFLOWNODE)){
- WorkflowNodeStatusChangeEvent event = (WorkflowNodeStatusChangeEvent) msgCtx.getEvent();
- WorkflowIdentifier workflowNodeIdentity = event.getWorkflowNodeIdentity();
- routingKey = workflowNodeIdentity.getExperimentId() + "." + workflowNodeIdentity.getWorkflowNodeId();
- }else if (msgCtx.getType().equals(MessageType.JOB)){
- JobStatusChangeEvent event = (JobStatusChangeEvent)msgCtx.getEvent();
- JobIdentifier identity = event.getJobIdentity();
- routingKey = identity.getExperimentId() + "." +
- identity.getWorkflowNodeId() + "." +
- identity.getTaskId() + "." +
- identity.getJobId();
- }
- byte[] messageBody = ThriftUtils.serializeThriftObject(message);
- rabbitMQProducer.send(messageBody, routingKey);
- } catch (TException e) {
- String msg = "Error while deserializing the object";
- log.error(msg, e);
- throw new AiravataException(msg, e);
- } catch (Exception e) {
- String msg = "Error while sending to rabbitmq";
- log.error(msg, e);
- throw new AiravataException(msg, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/88d27d95/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
new file mode 100644
index 0000000..a4b4d1a
--- /dev/null
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
@@ -0,0 +1,99 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.messaging.core.impl;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ThriftUtils;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.MessagingConstants;
+import org.apache.airavata.messaging.core.Publisher;
+import org.apache.airavata.model.messaging.event.*;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RabbitMQStatusPublisher implements Publisher {
+
+ private static Logger log = LoggerFactory.getLogger(RabbitMQStatusPublisher.class);
+
+ private RabbitMQProducer rabbitMQProducer;
+
+
+ public RabbitMQStatusPublisher() throws Exception {
+ String brokerUrl;
+ String exchangeName;
+ try {
+ brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_EXCHANGE_NAME);
+ } catch (ApplicationSettingsException e) {
+ String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
+ log.error(message, e);
+ throw new AiravataException(message, e);
+ }
+ rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName);
+ rabbitMQProducer.open();
+ }
+
+ public void publish(MessageContext msgCtx) throws AiravataException {
+ try {
+ log.info("Publishing status to rabbitmq...");
+ byte[] body = ThriftUtils.serializeThriftObject(msgCtx.getEvent());
+ Message message = new Message();
+ message.setEvent(body);
+ message.setMessageId(msgCtx.getMessageId());
+ message.setMessageType(msgCtx.getType());
+ message.setUpdatedTime(msgCtx.getUpdatedTime().getTime());
+ String routingKey = null;
+ if (msgCtx.getType().equals(MessageType.EXPERIMENT)){
+ ExperimentStatusChangeEvent event = (ExperimentStatusChangeEvent) msgCtx.getEvent();
+ routingKey = event.getExperimentId();
+ } else if (msgCtx.getType().equals(MessageType.TASK)) {
+ TaskStatusChangeEvent event = (TaskStatusChangeEvent) msgCtx.getEvent();
+ routingKey = event.getTaskIdentity().getExperimentId() + "." +
+ event.getTaskIdentity().getWorkflowNodeId() + "." + event.getTaskIdentity().getTaskId();
+ }else if (msgCtx.getType().equals(MessageType.WORKFLOWNODE)){
+ WorkflowNodeStatusChangeEvent event = (WorkflowNodeStatusChangeEvent) msgCtx.getEvent();
+ WorkflowIdentifier workflowNodeIdentity = event.getWorkflowNodeIdentity();
+ routingKey = workflowNodeIdentity.getExperimentId() + "." + workflowNodeIdentity.getWorkflowNodeId();
+ }else if (msgCtx.getType().equals(MessageType.JOB)){
+ JobStatusChangeEvent event = (JobStatusChangeEvent)msgCtx.getEvent();
+ JobIdentifier identity = event.getJobIdentity();
+ routingKey = identity.getExperimentId() + "." +
+ identity.getWorkflowNodeId() + "." +
+ identity.getTaskId() + "." +
+ identity.getJobId();
+ }
+ byte[] messageBody = ThriftUtils.serializeThriftObject(message);
+ rabbitMQProducer.send(messageBody, routingKey);
+ } catch (TException e) {
+ String msg = "Error while deserializing the object";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ } catch (Exception e) {
+ String msg = "Error while sending to rabbitmq";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ }
+ }
+}
[11/15] airavata git commit: merging with master
Posted by la...@apache.org.
merging with master
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/48be39fe
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/48be39fe
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/48be39fe
Branch: refs/heads/master
Commit: 48be39fea6f1df43d0097364a937428198fcc9af
Parents: 93ed077 73f371d
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Thu Mar 19 11:02:15 2015 -0400
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Thu Mar 19 11:02:15 2015 -0400
----------------------------------------------------------------------
.../airavata/api/server/AiravataAPIServer.java | 38 +-
.../server/handler/AiravataServerHandler.java | 412 +-
.../api/server/util/DataModelUtils.java | 8 +-
.../java/org/apache/airavata/api/Airavata.java | 28098 +++++++++++++----
.../airavata/api/airavataAPIConstants.java | 2 +-
.../main/resources/lib/airavata/Airavata.cpp | 8232 +++--
.../src/main/resources/lib/airavata/Airavata.h | 2388 +-
.../lib/airavata/Airavata_server.skeleton.cpp | 95 +-
.../lib/airavata/airavataAPI_constants.cpp | 2 +-
.../lib/airavata/computeResourceModel_types.cpp | 12 +-
.../lib/airavata/computeResourceModel_types.h | 5 +-
.../lib/airavata/experimentModel_types.cpp | 62 +-
.../lib/airavata/experimentModel_types.h | 61 +-
.../gatewayResourceProfileModel_constants.cpp | 2 -
.../gatewayResourceProfileModel_constants.h | 1 -
.../gatewayResourceProfileModel_types.cpp | 72 +-
.../gatewayResourceProfileModel_types.h | 45 +-
.../lib/airavata/workspaceModel_types.cpp | 51 +-
.../lib/airavata/workspaceModel_types.h | 45 +-
.../resources/lib/Airavata/API/Airavata.php | 7240 +++--
.../main/resources/lib/Airavata/API/Types.php | 2 +-
.../Model/AppCatalog/ComputeResource/Types.php | 10 +-
.../Model/AppCatalog/GatewayProfile/Types.php | 92 +-
.../Model/Workspace/Experiment/Types.php | 60 +
.../lib/Airavata/Model/Workspace/Types.php | 58 +-
.../client/samples/CreateLaunchExperiment.java | 822 +-
.../samples/CreateLaunchExperimentUS3.java | 20 +-
.../client/samples/RegisterSampleData.java | 37 +-
.../samples/TestCreateLaunchExperiment.java | 10 +-
.../tools/RegisterOGCEUS3Application.java | 10 +-
.../tools/RegisterSampleApplications.java | 335 +-
.../client/tools/RegisterUS3Application.java | 14 +-
.../computeresource/AuthenticationMode.java | 70 +
.../computeresource/ResourceJobManagerType.java | 42 +-
.../computeresource/UnicoreJobSubmission.java | 2 +
.../ComputeResourcePreference.java | 168 +-
.../gatewayprofile/GatewayResourceProfile.java | 226 +-
.../airavata/model/workspace/Gateway.java | 334 +-
.../ComputationalResourceScheduling.java | 109 +-
.../experiment/UserConfigurationData.java | 205 +-
.../airavataAPI.thrift | 169 +-
.../computeResourceModel.thrift | 23 +-
.../experimentModel.thrift | 11 +-
.../gatewayResourceProfileModel.thrift | 28 +-
.../workspaceModel.thrift | 6 +-
.../appcatalog/cpi/ApplicationDeployment.java | 4 +-
.../appcatalog/cpi/ApplicationInterface.java | 8 +-
.../appcatalog/cpi/WorkflowCatalog.java | 4 +-
.../data/impl/ApplicationDeploymentImpl.java | 6 +-
.../data/impl/ApplicationInterfaceImpl.java | 12 +-
.../data/impl/GwyResourceProfileImpl.java | 30 +-
.../catalog/data/impl/WorkflowCatalogImpl.java | 6 +-
.../data/model/ApplicationDeployment.java | 10 +
.../data/model/ApplicationInterface.java | 10 +
.../catalog/data/model/ApplicationModule.java | 10 +
.../data/model/ComputeResourcePreference.java | 10 +
.../catalog/data/model/GatewayProfile.java | 19 -
.../data/model/UnicoreJobSubmission.java | 5 +-
.../catalog/data/model/Workflow.java | 10 +
.../data/resources/AbstractResource.java | 6 +-
.../data/resources/AppDeploymentResource.java | 12 +
.../data/resources/AppInterfaceResource.java | 12 +
.../data/resources/AppModuleResource.java | 12 +
.../ComputeHostPreferenceResource.java | 11 +
.../data/resources/GatewayProfileResource.java | 30 +-
.../resources/UnicoreJobSubmissionResource.java | 16 +-
.../data/resources/WorkflowResource.java | 12 +
.../catalog/data/util/AppCatalogJPAUtils.java | 7 +-
.../data/util/AppCatalogThriftConversion.java | 4 +-
.../src/main/resources/appcatalog-derby.sql | 211 +-
.../src/main/resources/appcatalog-mysql.sql | 209 +-
.../app/catalog/test/AppDeploymentTest.java | 11 +-
.../app/catalog/test/AppInterfaceTest.java | 12 +-
.../app/catalog/test/GatewayProfileTest.java | 5 +-
.../src/test/resources/appcatalog-derby.sql | 211 +-
.../apache/airavata/common/utils/Constants.java | 4 +-
.../common/utils/DatabaseTestCases.java | 2 +-
.../main/resources/airavata-client.properties | 2 +-
.../server/src/main/resources/LSFTemplate.xslt | 92 +
.../server/src/main/resources/PBSTemplate.xslt | 22 +-
.../server/src/main/resources/SGETemplate.xslt | 18 +-
.../src/main/resources/SLURMTemplate.xslt | 22 +-
.../main/resources/airavata-server.properties | 6 +-
.../credential-store-webapp/pom.xml | 158 -
.../basic/BasicAccessAuthenticator.java | 226 -
.../credentialstore/local/LocalUserStore.java | 339 -
.../session/HttpAuthenticatorFilter.java | 191 -
.../session/ServletRequestHelper.java | 129 -
.../main/resources/airavata-server.properties | 237 -
.../main/resources/credential-store/client.xml | 36 -
.../credential-store/oauth-privkey.pk8 | 28 -
.../resources/credential-store/oauth-pubkey.pem | 9 -
.../src/main/webapp/WEB-INF/web.xml | 130 -
.../src/main/webapp/acs/index.jsp | 44 -
.../src/main/webapp/credential-store/error.jsp | 53 -
.../credential-store/password-credentials.jsp | 33 -
.../webapp/credential-store/show-redirect.jsp | 44 -
.../main/webapp/credential-store/success.jsp | 25 -
.../src/main/webapp/gateway/acs.jsp | 62 -
.../src/main/webapp/gateway/callback.jsp | 78 -
.../src/main/webapp/gateway/list_users.jsp | 78 -
.../src/main/webapp/gateway/logout.jsp | 35 -
.../src/main/webapp/gateway/user.jsp | 102 -
.../src/main/webapp/images/airavata-logo-2.png | Bin 4314 -> 0 bytes
.../src/main/webapp/index.jsp | 26 -
.../src/main/webapp/user-store/add.jsp | 142 -
.../src/main/webapp/user-store/index.jsp | 138 -
.../src/main/webapp/user-store/password.jsp | 157 -
.../credential-store/pom.xml | 154 -
.../scripts/credential-store-h2.sql | 42 -
.../scripts/credential-store-mysql.sql | 42 -
.../credential/store/client/TestSSLClient.java | 140 -
.../store/cpi/CredentialStoreService.java | 6888 ----
.../store/cpi/cs_cpi_serviceConstants.java | 55 -
.../credential/store/credential/AuditInfo.java | 53 -
.../store/credential/CommunityUser.java | 71 -
.../credential/store/credential/Credential.java | 62 -
.../impl/certificate/CertificateAuditInfo.java | 101 -
.../impl/certificate/CertificateCredential.java | 102 -
.../impl/password/PasswordCredential.java | 53 -
.../credential/impl/ssh/SSHCredential.java | 88 -
.../impl/ssh/SSHCredentialGenerator.java | 103 -
.../store/datamodel/CertificateCredential.java | 1104 -
.../store/datamodel/CommunityUser.java | 589 -
.../store/datamodel/PasswordCredential.java | 698 -
.../store/datamodel/SSHCredential.java | 998 -
.../store/datamodel/csDataModelConstants.java | 55 -
.../exception/CredentialStoreException.java | 397 -
.../store/notifier/CredentialStoreNotifier.java | 42 -
.../store/notifier/NotificationMessage.java | 46 -
.../store/notifier/NotifierBootstrap.java | 144 -
.../notifier/impl/EmailNotificationMessage.java | 58 -
.../store/notifier/impl/EmailNotifier.java | 71 -
.../impl/EmailNotifierConfiguration.java | 84 -
.../store/server/CredentialStoreServer.java | 158 -
.../server/CredentialStoreServerHandler.java | 201 -
.../store/servlet/CredentialBootstrapper.java | 49 -
.../servlet/CredentialStoreCallbackServlet.java | 272 -
.../servlet/CredentialStoreStartServlet.java | 183 -
.../store/store/CredentialReader.java | 112 -
.../store/store/CredentialReaderFactory.java | 54 -
.../store/store/CredentialStoreException.java | 40 -
.../store/store/CredentialWriter.java | 39 -
.../store/impl/CertificateCredentialWriter.java | 121 -
.../store/store/impl/CredentialReaderImpl.java | 162 -
.../store/store/impl/SSHCredentialWriter.java | 87 -
.../store/store/impl/db/CommunityUserDAO.java | 257 -
.../store/store/impl/db/CredentialsDAO.java | 458 -
.../store/store/impl/db/ParentDAO.java | 37 -
.../store/util/ConfigurationReader.java | 121 -
.../store/util/CredentialStoreConstants.java | 37 -
.../credential/store/util/PrivateKeyStore.java | 70 -
.../credential/store/util/TokenGenerator.java | 57 -
.../airavata/credential/store/util/Utility.java | 110 -
.../store/notifier/impl/EmailNotifierTest.java | 56 -
.../store/impl/db/CommunityUserDAOTest.java | 207 -
.../store/store/impl/db/CredentialsDAOTest.java | 421 -
.../store/util/ConfigurationReaderTest.java | 58 -
.../store/util/TokenGeneratorTest.java | 42 -
.../test/resources/credential-store/client.xml | 35 -
.../src/test/resources/keystore.jks | Bin 2230 -> 0 bytes
.../src/test/resources/mykeystore.jks | Bin 498 -> 0 bytes
.../credentialStoreErrors.thrift | 32 -
.../cs-thrift-description/cs.cpi.service.thrift | 50 -
.../cs-thrift-description/csDataModel.thrift | 61 -
.../cs-thrift-description/generate-cs-stubs.sh | 134 -
modules/credential-store-service/pom.xml | 42 -
.../credential-store-service/pom.xml | 166 +
.../scripts/credential-store-h2.sql | 42 +
.../scripts/credential-store-mysql.sql | 42 +
.../credential/store/credential/AuditInfo.java | 53 +
.../store/credential/CommunityUser.java | 71 +
.../credential/store/credential/Credential.java | 62 +
.../impl/certificate/CertificateAuditInfo.java | 101 +
.../impl/certificate/CertificateCredential.java | 102 +
.../impl/password/PasswordCredential.java | 53 +
.../credential/impl/ssh/SSHCredential.java | 88 +
.../impl/ssh/SSHCredentialGenerator.java | 103 +
.../store/notifier/CredentialStoreNotifier.java | 42 +
.../store/notifier/NotificationMessage.java | 46 +
.../store/notifier/NotifierBootstrap.java | 144 +
.../notifier/impl/EmailNotificationMessage.java | 58 +
.../store/notifier/impl/EmailNotifier.java | 71 +
.../impl/EmailNotifierConfiguration.java | 84 +
.../store/server/CredentialStoreServer.java | 158 +
.../server/CredentialStoreServerHandler.java | 202 +
.../store/servlet/CredentialBootstrapper.java | 49 +
.../servlet/CredentialStoreCallbackServlet.java | 272 +
.../servlet/CredentialStoreStartServlet.java | 183 +
.../store/store/CredentialReader.java | 112 +
.../store/store/CredentialReaderFactory.java | 54 +
.../store/store/CredentialStoreException.java | 40 +
.../store/store/CredentialWriter.java | 39 +
.../store/impl/CertificateCredentialWriter.java | 121 +
.../store/store/impl/CredentialReaderImpl.java | 162 +
.../store/store/impl/SSHCredentialWriter.java | 87 +
.../store/store/impl/db/CommunityUserDAO.java | 257 +
.../store/store/impl/db/CredentialsDAO.java | 458 +
.../store/store/impl/db/ParentDAO.java | 37 +
.../store/util/ConfigurationReader.java | 121 +
.../store/util/CredentialStoreConstants.java | 37 +
.../credential/store/util/PrivateKeyStore.java | 70 +
.../credential/store/util/TokenGenerator.java | 57 +
.../airavata/credential/store/util/Utility.java | 110 +
.../store/notifier/impl/EmailNotifierTest.java | 56 +
.../store/impl/db/CommunityUserDAOTest.java | 207 +
.../store/store/impl/db/CredentialsDAOTest.java | 421 +
.../store/store/impl/db/SSHCredentialTest.java | 92 +
.../store/util/ConfigurationReaderTest.java | 58 +
.../store/util/TokenGeneratorTest.java | 42 +
.../test/resources/airavata-server.properties | 254 +
.../test/resources/credential-store/client.xml | 35 +
.../src/test/resources/keystore.jks | Bin 0 -> 2230 bytes
.../src/test/resources/mykeystore.jks | Bin 0 -> 498 bytes
.../credential-store-stubs/pom.xml | 50 +
.../credential/store/client/TestSSLClient.java | 140 +
.../store/cpi/CredentialStoreService.java | 6888 ++++
.../store/cpi/credentialStoreCPIConstants.java | 55 +
.../store/datamodel/CertificateCredential.java | 1104 +
.../store/datamodel/CommunityUser.java | 589 +
.../store/datamodel/PasswordCredential.java | 698 +
.../store/datamodel/SSHCredential.java | 998 +
.../credentialStoreDataModelConstants.java | 55 +
.../exception/CredentialStoreException.java | 397 +
.../credential-store-webapp/pom.xml | 158 +
.../basic/BasicAccessAuthenticator.java | 226 +
.../credentialstore/local/LocalUserStore.java | 339 +
.../session/HttpAuthenticatorFilter.java | 191 +
.../session/ServletRequestHelper.java | 129 +
.../main/resources/airavata-server.properties | 237 +
.../main/resources/credential-store/client.xml | 36 +
.../credential-store/oauth-privkey.pk8 | 28 +
.../resources/credential-store/oauth-pubkey.pem | 9 +
.../src/main/webapp/WEB-INF/web.xml | 130 +
.../src/main/webapp/acs/index.jsp | 44 +
.../src/main/webapp/credential-store/error.jsp | 53 +
.../credential-store/password-credentials.jsp | 33 +
.../webapp/credential-store/show-redirect.jsp | 44 +
.../main/webapp/credential-store/success.jsp | 25 +
.../src/main/webapp/gateway/acs.jsp | 62 +
.../src/main/webapp/gateway/callback.jsp | 78 +
.../src/main/webapp/gateway/list_users.jsp | 78 +
.../src/main/webapp/gateway/logout.jsp | 35 +
.../src/main/webapp/gateway/user.jsp | 102 +
.../src/main/webapp/images/airavata-logo-2.png | Bin 0 -> 4314 bytes
.../src/main/webapp/index.jsp | 26 +
.../src/main/webapp/user-store/add.jsp | 142 +
.../src/main/webapp/user-store/index.jsp | 138 +
.../src/main/webapp/user-store/password.jsp | 157 +
.../credentialStoreCPI.thrift | 61 +
.../credentialStoreDataModel.thrift | 61 +
.../credentialStoreErrors.thrift | 32 +
.../cs-thrift-descriptions/generate-cs-stubs.sh | 134 +
modules/credential-store/pom.xml | 43 +
modules/distribution/server/pom.xml | 1211 +-
.../server/src/main/assembly/bin-assembly.xml | 1 +
.../server/src/main/resources/bin/data.sql | 141 -
.../airavata/gfac/server/GfacServerHandler.java | 54 +-
.../airavata/gfac/client/util/Initialize.java | 4 +-
modules/gfac/gfac-bes/pom.xml | 4 +-
.../gfac/bes/provider/impl/BESProvider.java | 10 +-
.../org/apache/airavata/gfac/RequestData.java | 2 +
.../gfac/core/context/JobExecutionContext.java | 9 +
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 26 +-
.../airavata/gfac/core/utils/GFacUtils.java | 2 +-
.../gfac/core/utils/OutHandlerWorker.java | 11 +-
.../gfac/gsissh/util/GFACGSISSHUtils.java | 10 +-
.../monitor/impl/pull/qstat/HPCPullMonitor.java | 42 +-
.../impl/pull/qstat/ResourceConnection.java | 16 +-
.../airavata/gfac/monitor/util/CommonUtils.java | 1 +
.../gfac/ssh/provider/impl/SSHProvider.java | 1 +
.../gfac/ssh/security/TokenizedSSHAuthInfo.java | 5 +-
.../airavata/gfac/ssh/util/GFACSSHUtils.java | 53 +-
.../airavata/gfac/ssh/util/HandleOutputs.java | 5 +-
.../apache/airavata/integration/BaseCaseIT.java | 4 +-
.../airavata/integration/DataRetrievalIT.java | 8 +-
.../airavata/integration/SimpleEchoIT.java | 4 +-
.../SingleAppIntegrationTestBase.java | 6 +-
.../WorkflowIntegrationTestBase.java | 2 +-
.../integration/tools/DocumentCreatorNew.java | 75 +-
modules/messaging/client/pom.xml | 2 +-
.../messaging/client/RabbitMQListener.java | 228 +
.../messaging/client/RabbitMQListner.java | 230 -
.../airavata/messaging/client/TestReader.java | 50 +
.../orchestrator/util/DataModelUtils.java | 10 +-
.../orchestrator/client/util/Initialize.java | 4 +-
.../airavata/orchestrator/cpi/Orchestrator.java | 2 +-
.../orchestrator/core/util/Initialize.java | 4 +-
.../persistance/registry/jpa/ResourceType.java | 2 -
.../persistance/registry/jpa/ResourceUtils.java | 69 +-
.../registry/jpa/impl/ExperimentRegistry.java | 16 +-
.../registry/jpa/impl/GatewayRegistry.java | 76 +
.../registry/jpa/impl/LoggingRegistryImpl.java | 35 +-
.../registry/jpa/impl/ProjectRegistry.java | 16 +-
.../registry/jpa/impl/RegistryFactory.java | 17 +
.../registry/jpa/impl/RegistryImpl.java | 28 +-
.../Computational_Resource_Scheduling.java | 10 +
.../registry/jpa/model/Experiment.java | 14 +-
.../jpa/model/ExperimentConfigData.java | 20 +
.../persistance/registry/jpa/model/Gateway.java | 32 +-
.../registry/jpa/model/Gateway_Worker.java | 14 +-
.../registry/jpa/model/Gateway_Worker_PK.java | 14 +-
.../persistance/registry/jpa/model/Project.java | 13 +-
.../registry/jpa/model/Published_Workflow.java | 124 -
.../jpa/model/Published_Workflow_PK.java | 64 -
.../registry/jpa/model/User_Workflow.java | 122 -
.../registry/jpa/model/User_Workflow_PK.java | 74 -
.../jpa/resources/AbstractResource.java | 33 +-
.../ComputationSchedulingResource.java | 10 +
.../jpa/resources/ConfigDataResource.java | 22 +
.../jpa/resources/ExperimentResource.java | 6 +-
.../registry/jpa/resources/GatewayResource.java | 159 +-
.../registry/jpa/resources/ProjectResource.java | 4 +-
.../jpa/resources/PublishWorkflowResource.java | 282 -
.../jpa/resources/UserWorkflowResource.java | 174 -
.../registry/jpa/resources/Utils.java | 72 +-
.../registry/jpa/resources/WorkerResource.java | 100 +-
.../jpa/utils/ThriftDataModelConversion.java | 22 +
.../src/main/resources/META-INF/persistence.xml | 2 -
.../src/main/resources/registry-derby.sql | 58 +-
.../src/main/resources/registry-mysql.sql | 56 +-
.../registry/jpa/GatewayResourceTest.java | 26 +-
.../jpa/PublishWorkflowResourceTest.java | 62 -
.../registry/jpa/UserWorkflowResourceTest.java | 66 -
.../registry/jpa/util/Initialize.java | 4 +-
.../src/test/resources/registry-derby.sql | 58 +-
.../airavata/registry/cpi/ParentDataType.java | 3 +-
.../apache/airavata/registry/cpi/Registry.java | 2 +-
.../registry/cpi/RegistryModelType.java | 1 +
.../airavata/registry/cpi/utils/Constants.java | 3 +-
.../src/test/resources/jdbc-authenticator.xml | 2 +-
.../test/resources/session-authenticator.xml | 2 +-
modules/test-suite/pom.xml | 116 -
.../tests/LeadCallbackHandlerTest.java | 173 -
.../tests/LeadNotificationManagerTest.java | 49 -
.../tests/MultipleSubscriptionTest.java | 105 -
.../tests/RenewSubscriptionTest.java | 155 -
.../tests/ThreadMessagePassingCallback.java | 27 -
.../tests/impl/publish/Test.java | 40 -
.../tests/impl/publish/TestWSMPublisher.java | 119 -
.../tests/messagebox/MessagePullerTest.java | 140 -
.../MultipleSubscriptionForMessageBoxTest.java | 118 -
.../tests/messagebox/RenewSubscriptionTest.java | 109 -
.../tests/messagebox/SubscriberThread.java | 96 -
.../restart/MessageBoxClientRestartTest.java | 131 -
.../restart/MessageBoxCreateThread.java | 83 -
.../tests/samples/workflow/SimpleTest.java | 202 -
.../workflow/SimpleWorkflowExecution.java | 473 -
.../workflow/WorkflowNotificationListener.java | 127 -
.../tests/util/CommonUtils.java | 26 -
.../tests/util/SubscriberThread.java | 91 -
.../tests/util/TestConfigKeys.java | 34 -
.../src/test/resources/gram.properties.template | 70 -
.../src/test/resources/unit_test.properties | 26 -
.../airavata/workflow/engine/WorkflowUtils.java | 10 +-
.../registry/JCRComponentRegistry.java | 3 +-
.../dialogs/workflow/WorkflowImportWindow.java | 3 +-
.../ui/experiment/LaunchApplicationWindow.java | 8 +-
.../RegistryWorkflowPublisherWindow.java | 3 +-
.../WorkflowInterpreterLaunchWindow.java | 9 +-
pom.xml | 6 +-
.../apache/airavata/gsi/ssh/GSSContextX509.java | 11 +-
.../airavata/gsi/ssh/api/job/JobDescriptor.java | 14 +
.../ssh/api/job/JobManagerConfiguration.java | 9 +-
.../gsi/ssh/api/job/LSFJobConfiguration.java | 116 +
.../gsi/ssh/api/job/LSFOutputParser.java | 110 +
.../airavata/gsi/ssh/api/job/OutputParser.java | 14 +-
.../gsi/ssh/api/job/PBSJobConfiguration.java | 15 +
.../gsi/ssh/api/job/PBSOutputParser.java | 8 +-
.../gsi/ssh/api/job/SGEOutputParser.java | 8 +-
.../gsi/ssh/api/job/SlurmJobConfiguration.java | 15 +
.../gsi/ssh/api/job/SlurmOutputParser.java | 10 +-
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 74 +-
.../apache/airavata/gsi/ssh/impl/JobStatus.java | 20 +-
.../airavata/gsi/ssh/impl/PBSCluster.java | 4 +
.../airavata/gsi/ssh/impl/RawCommandInfo.java | 8 +-
.../gsi/ssh/listener/JobSubmissionListener.java | 2 +-
.../airavata/gsi/ssh/util/CommonUtils.java | 13 +
.../gsissh/src/main/resources/LSFTemplate.xslt | 93 +
.../main/resources/schemas/PBSJobDescriptor.xsd | 3 +-
.../impl/DefaultSSHApiTestWithMyProxyAuth.java | 25 +-
.../gsi/ssh/impl/VanilaTestWithSSHAuth.java | 48 +-
tools/registry-tool/README | 2 +-
383 files changed, 59870 insertions(+), 35784 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/AiravataAPIServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --cc airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 90b8e6d,ed141b9..f499345
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@@ -54,16 -55,16 +55,16 @@@ public class CreateLaunchExperiment
private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
private static final String DEFAULT_USER = "default.registry.user";
- private static final String DEFAULT_GATEWAY = "default.registry.gateway";
+ private static final String DEFAULT_GATEWAY = "php_reference_gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_61988d1f-7ca9-47ba-9212-a0ac2e973cf1";
+ private static String echoAppId = "Echo_1365a7fd-eae1-4575-b447-99afb4d79c82";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
- private static String amberAppId = "Amber_9e4f28b6-7a5d-4fe1-b07f-2053f8f0deb3";
+ private static String amberAppId = "Amber_aa083c86-4680-4002-b3ef-fad93c181926";
private static String gromacsAppId = "GROMACS_05622038-9edd-4cb1-824e-0b7cb993364b";
private static String espressoAppId = "ESPRESSO_10cc2820-5d0b-4c63-9546-8a8b595593c1";
- private static String lammpsAppId = "LAMMPS_10893eb5-3840-438c-8446-d26c7ecb001f";
+ private static String lammpsAppId = "LAMMPS_2472685b-8acf-497e-aafe-cc66fe5f4cb6";
private static String nwchemAppId = "NWChem_2c8fee64-acf9-4a89-b6d3-91eb53c7640c";
private static String trinityAppId = "Trinity_e894acf5-9bca-46e8-a1bd-7e2d5155191a";
private static String autodockAppId = "AutoDock_43d9fdd0-c404-49f4-b913-3abf9080a8c9";
@@@ -89,9 -95,73 +95,74 @@@
private static String fsdResourceId;
+ public static void getAvailableAppInterfaceComputeResources(String appInterfaceId) {
+ try {
+ Map<String, String> availableAppInterfaceComputeResources = airavataClient.getAvailableAppInterfaceComputeResources(appInterfaceId);
+ for (String key : availableAppInterfaceComputeResources.keySet()){
+ System.out.println("id : " + key);
+ System.out.println("name : " + availableAppInterfaceComputeResources.get(key));
+ }
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ public static void createGateway(){
+ try {
+ Gateway gateway = new Gateway();
+ gateway.setGatewayId("testGatewayId2");
+ gateway.setGatewayName("testGateway2");
+ gatewayId = airavataClient.addGateway(gateway);
+ System.out.println(gatewayId);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ public static void getGateway(String gatewayId){
+ try {
+ Gateway gateway = airavataClient.getGateway(gatewayId);
+ gateway.setDomain("testDomain");
+ airavataClient.updateGateway(gatewayId, gateway);
+ List<Gateway> allGateways = airavataClient.getAllGateways();
+ System.out.println(allGateways.size());
+ if (airavataClient.isGatewayExist(gatewayId)){
+ Gateway gateway1 = airavataClient.getGateway(gatewayId);
+ System.out.println(gateway1.getGatewayName());
+ }
+ boolean b = airavataClient.deleteGateway("testGatewayId2");
+ System.out.println(b);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
public static void createAndLaunchExp() throws TException {
// final String expId = createEchoExperimentForFSD(airavataClient);
+ List<String> experimentIds = new ArrayList<String>();
try {
for (int i = 0; i < 1; i++) {
// final String expId = createExperimentForSSHHost(airavata);
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --cc modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
index 0000000,badf28d..64e0160
mode 000000,100644..100644
--- a/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
+++ b/modules/credential-store/credential-store-webapp/src/main/resources/airavata-server.properties
@@@ -1,0 -1,234 +1,237 @@@
+ #
+ #
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+ #
+
+ ###########################################################################
+ #
+ # This properties file provides configuration for all Airavata Services:
+ # API Server, Registry, Workflow Interpreter, GFac, Orchestrator
+ #
+ ###########################################################################
+
+ ###########################################################################
+ # API Server Registry Configuration
+ ###########################################################################
+
+ #for derby [AiravataJPARegistry]
+ registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+ registry.jdbc.url=jdbc:derby://localhost:1527/experiment_catalog;create=true;user=airavata;password=airavata
+ # MySql database configuration
+ #registry.jdbc.driver=com.mysql.jdbc.Driver
+ #registry.jdbc.url=jdbc:mysql://localhost:3306/persistent_data
+ registry.jdbc.user=airavata
+ registry.jdbc.password=airavata
+ start.derby.server.mode=true
+ validationQuery=SELECT 1 from CONFIGURATION
+ jpa.cache.size=5000
+ #jpa.connection.properties=MaxActive=10,MaxIdle=5,MinIdle=2,MaxWait=60000,testWhileIdle=true,testOnBorrow=true
+
+ # Properties for default user mode
+ default.registry.user=admin
+ default.registry.password=admin
+ default.registry.password.hash.method=SHA
+ default.registry.gateway=default
+
+ #ip=127.0.0.1
+
+ ###########################################################################
+ # Application Catalog DB Configuration
+ ###########################################################################
+ #for derby [AiravataJPARegistry]
+ appcatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+ appcatalog.jdbc.url=jdbc:derby://localhost:1527/app_catalog;create=true;user=airavata;password=airavata
+ # MySql database configuration
+ #appcatalog.jdbc.driver=com.mysql.jdbc.Driver
+ #appcatalog.jdbc.url=jdbc:mysql://localhost:3306/app_catalog
+ appcatalog.jdbc.user=airavata
+ appcatalog.jdbc.password=airavata
+ appcatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+ ###########################################################################
+ # Server module Configuration
+ ###########################################################################
+
+ servers=apiserver,orchestrator,gfac,workflowserver
+ #shutdown.trategy=NONE
+ shutdown.trategy=SELF_TERMINATE
+
+
+ apiserver.server.host=localhost
+ apiserver.server.port=8930
+ apiserver.server.min.threads=50
+ workflow.server.host=localhost
+ workflow.server.port=8931
+ orchestrator.server.host=localhost
+ orchestrator.server.port=8940
+ gfac.server.host=localhost
+ gfac.server.port=8950
+ orchestrator.server.min.threads=50
+
+ ###########################################################################
+ # Credential Store module Configuration
+ ###########################################################################
+ credential.store.keystore.url=/Users/lahirugunathilake/Downloads/airavata_sym.jks
+ credential.store.keystore.alias=airavata
+ credential.store.keystore.password=airavata
+ credential.store.jdbc.url=jdbc:derby://localhost:1527/experiment_catalog;create=true;user=airavata;password=airavata
+ credential.store.jdbc.user=airavata
+ credential.store.jdbc.password=airavata
+ credential.store.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+
+ notifier.enabled=false
+ #period in milliseconds
+ notifier.duration=5000
+
+ email.server=smtp.googlemail.com
+ email.server.port=465
+ email.user=airavata
+ email.password=xxx
+ email.ssl=true
+ email.from=airavata@apache.org
+
+ ###########################################################################
+ # Airavata GFac MyProxy GSI credentials to access Grid Resources.
+ ###########################################################################
+ #
+ # Security Configuration used by Airavata Generic Factory Service
+ # to interact with Computational Resources.
+ #
+ gfac=org.apache.airavata.gfac.server.GfacServer
+ myproxy.server=myproxy.teragrid.org
+ myproxy.username=ogce
+ myproxy.password=
+ myproxy.life=3600
+ # XSEDE Trusted certificates can be downloaded from https://software.xsede.org/security/xsede-certs.tar.gz
+ trusted.cert.location=/Users/lahirugunathilake/Downloads/certificates
+ # SSH PKI key pair or ssh password can be used SSH based authentication is used.
+ # if user specify both password authentication gets the higher preference
+
+ ################# ---------- For ssh key pair authentication ------------------- ################
+ #public.ssh.key=/path to public key for ssh
+ #ssh.username=username for ssh connection
+ #private.ssh.key=/path to private key file for ssh
+ #ssh.keypass=passphrase for the private key
+
+
+ ################# ---------- For ssh key pair authentication ------------------- ################
+ #ssh.username=username for ssh connection
+ #ssh.password=Password for ssh connection
+
+
+
+ ###########################################################################
+ # Airavata Workflow Interpreter Configurations
+ ###########################################################################
+
+ #runInThread=true
+ #provenance=true
+ #provenanceWriterThreadPoolSize=20
+ #gfac.embedded=true
+ #workflowserver=org.apache.airavata.api.server.WorkflowServer
+
+
+ ###########################################################################
+ # API Server module Configuration
+ ###########################################################################
+ apiserver=org.apache.airavata.api.server.AiravataAPIServer
+
+ ###########################################################################
+ # Workflow Server module Configuration
+ ###########################################################################
+
+ workflowserver=org.apache.airavata.api.server.WorkflowServer
+
+ ###########################################################################
+ # Advance configuration to change service implementations
+ ###########################################################################
+ # If false, disables two phase commit when submitting jobs
+ TwoPhase=true
+ #
+ # Class which implemented HostScheduler interface. It will determine the which host to submit the request
+ #
+ host.scheduler=org.apache.airavata.gfac.core.scheduler.impl.SimpleHostScheduler
+
+ ###########################################################################
+ # Monitoring module Configuration
+ ###########################################################################
+
+ #This will be the primary monitoring tool which runs in airavata, in future there will be multiple monitoring
+ #mechanisms and one would be able to start a monitor
+ monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor
+
+
+ ###########################################################################
+ # AMQP Notification Configuration
+ ###########################################################################
+
+
+ amqp.notification.enable=1
+
+ amqp.broker.host=localhost
+ amqp.broker.port=5672
+ amqp.broker.username=guest
+ amqp.broker.password=guest
+
+ amqp.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPSenderImpl
+ amqp.topic.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPTopicSenderImpl
+ amqp.broadcast.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPBroadcastSenderImpl
+
+ #,org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor
+ #This is the amqp related configuration and this lists down the Rabbitmq host, this is an xsede specific configuration
+ amqp.hosts=info1.dyn.teragrid.org,info2.dyn.teragrid.org
+ proxy.file.path=/Users/lahirugunathilake/Downloads/x509up_u503876
+ connection.name=xsede
+ #publisher
+ activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
+ publish.rabbitmq=false
-activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQPublisher
++status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
++task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
+ rabbitmq.broker.url=amqp://localhost:5672
-rabbitmq.exchange.name=airavata_rabbitmq_exchange
++rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
++rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
+
+ ###########################################################################
+ # Orchestrator module Configuration
+ ###########################################################################
+
+ #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
-job.submitter=org.apache.airavata.orchestrator.core.impl.GFACServiceJobSubmitter
++#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
++job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
+ job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
+ submitter.interval=10000
+ threadpool.size=10
+ start.submitter=true
+ embedded.mode=true
+ enable.validation=true
+ orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
+
+ ###########################################################################
+ # Zookeeper Server Configuration
+ ###########################################################################
+
+ embedded.zk=true
+ zookeeper.server.host=localhost
+ zookeeper.server.port=2181
+ airavata-server=/api-server
+ orchestrator-server=/orchestrator-server
+ gfac-server=/gfac-server
+ gfac-experiments=/gfac-experiments
+ gfac-server-name=gfac-node0
+ orchestrator-server-name=orch-node0
+ airavata-server-name=api-node0
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --cc modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index cca793e,4973a41..88979a4
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@@ -27,7 -27,10 +27,11 @@@ import org.apache.aiaravata.application
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.logger.AiravataLogger;
import org.apache.airavata.common.logger.AiravataLoggerFactory;
+import org.apache.airavata.common.utils.*;
+ import org.apache.airavata.common.utils.AiravataZKUtils;
+ import org.apache.airavata.common.utils.Constants;
+ import org.apache.airavata.common.utils.MonitorPublisher;
+ import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
import org.apache.airavata.gfac.core.cpi.GFac;
import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java
----------------------------------------------------------------------
diff --cc modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java
index 0000000,53d08d3..3f876ae
mode 000000,100644..100644
--- a/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java
+++ b/modules/messaging/client/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java
@@@ -1,0 -1,228 +1,228 @@@
+ /*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+ package org.apache.airavata.messaging.client;
+
+ import org.apache.airavata.common.exception.ApplicationSettingsException;
+ import org.apache.airavata.common.utils.AiravataUtils;
+ import org.apache.airavata.common.utils.ServerSettings;
+ import org.apache.airavata.common.utils.ThriftUtils;
+ import org.apache.airavata.messaging.core.MessageContext;
+ import org.apache.airavata.messaging.core.MessageHandler;
+ import org.apache.airavata.messaging.core.MessagingConstants;
-import org.apache.airavata.messaging.core.impl.RabbitMQConsumer;
++import org.apache.airavata.messaging.core.impl.RabbitMQStatusConsumer;
+ import org.apache.airavata.model.messaging.event.*;
+ import org.apache.commons.cli.*;
+ import org.apache.thrift.TBase;
+ import org.apache.thrift.TException;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.*;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+
+
+ public class RabbitMQListener {
+ public static final String RABBITMQ_BROKER_URL = "rabbitmq.broker.url";
+ public static final String RABBITMQ_EXCHANGE_NAME = "rabbitmq.exchange.name";
+ private final static Logger logger = LoggerFactory.getLogger(RabbitMQListener.class);
+ private static String gatewayId = "*";
+ private static boolean gatewayLevelMessages = false;
+ private static boolean experimentLevelMessages = false;
+ private static boolean jobLevelMessages = false;
+ private static String experimentId = "*";
+ private static String jobId = "*";
+ private static boolean allMessages = false;
+
+ public static void main(String[] args) {
+ File file = new File("/tmp/latency_client");
+ parseArguments(args);
+ try {
+ FileOutputStream fos = new FileOutputStream(file, false);
+ final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos));
+ AiravataUtils.setExecutionAsServer();
+ String brokerUrl = ServerSettings.getSetting(RABBITMQ_BROKER_URL);
+ System.out.println("broker url " + brokerUrl);
+ final String exchangeName = ServerSettings.getSetting(RABBITMQ_EXCHANGE_NAME);
- RabbitMQConsumer consumer = new RabbitMQConsumer(brokerUrl, exchangeName);
++ RabbitMQStatusConsumer consumer = new RabbitMQStatusConsumer(brokerUrl, exchangeName);
+ consumer.listen(new MessageHandler() {
+ @Override
+ public Map<String, Object> getProperties() {
+ Map<String, Object> props = new HashMap<String, Object>();
+ List<String> routingKeys = new ArrayList<String>();
+ if (allMessages){
+ routingKeys.add("*");
+ routingKeys.add("*.*");
+ routingKeys.add("*.*.*");
+ routingKeys.add("*.*.*.*");
+ routingKeys.add("*.*.*.*.*");
+ }else {
+ if (gatewayLevelMessages){
+ routingKeys.add(gatewayId);
+ routingKeys.add(gatewayId + ".*");
+ routingKeys.add(gatewayId + ".*.*");
+ routingKeys.add(gatewayId + ".*.*.*");
+ routingKeys.add(gatewayId + ".*.*.*.*");
+ }else if (experimentLevelMessages){
+ routingKeys.add(gatewayId);
+ routingKeys.add(gatewayId + "." + experimentId);
+ routingKeys.add(gatewayId + "." + experimentId+ ".*");
+ routingKeys.add(gatewayId + "." + experimentId+ ".*.*");
+ routingKeys.add(gatewayId + "." + experimentId+ ".*.*.*");
+ }else if (jobLevelMessages){
+ routingKeys.add(gatewayId);
+ routingKeys.add(gatewayId + "." + experimentId);
+ routingKeys.add(gatewayId + "." + experimentId+ ".*");
+ routingKeys.add(gatewayId + "." + experimentId+ ".*.*");
+ routingKeys.add(gatewayId + "." + experimentId+ ".*." + jobId);
+ }
+ }
+ props.put(MessagingConstants.RABBIT_ROUTING_KEY, routingKeys);
+ return props;
+ }
+
+ @Override
+ public void onMessage(MessageContext message) {
+ try {
+ long latency = System.currentTimeMillis() - message.getUpdatedTime().getTime();
+ bw.write(message.getMessageId() + " :" + latency);
+ bw.newLine();
+ bw.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (message.getType().equals(MessageType.EXPERIMENT)){
+ try {
+ ExperimentStatusChangeEvent event = new ExperimentStatusChangeEvent();
+ TBase messageEvent = message.getEvent();
+ byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
+ ThriftUtils.createThriftFromBytes(bytes, event);
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType() + "' and with state : '" + event.getState().toString() +
+ " for Gateway " + event.getGatewayId());
+ } catch (TException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }else if (message.getType().equals(MessageType.WORKFLOWNODE)){
+ try {
+ WorkflowNodeStatusChangeEvent event = new WorkflowNodeStatusChangeEvent();
+ TBase messageEvent = message.getEvent();
+ byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
+ ThriftUtils.createThriftFromBytes(bytes, event);
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType() + "' and with state : '" + event.getState().toString() +
+ " for Gateway " + event.getWorkflowNodeIdentity().getGatewayId());
+ } catch (TException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }else if (message.getType().equals(MessageType.TASK)){
+ try {
+ TaskStatusChangeEvent event = new TaskStatusChangeEvent();
+ TBase messageEvent = message.getEvent();
+ byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
+ ThriftUtils.createThriftFromBytes(bytes, event);
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType() + "' and with state : '" + event.getState().toString() +
+ " for Gateway " + event.getTaskIdentity().getGatewayId());
+ } catch (TException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }else if (message.getType().equals(MessageType.JOB)){
+ try {
+ JobStatusChangeEvent event = new JobStatusChangeEvent();
+ TBase messageEvent = message.getEvent();
+ byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
+ ThriftUtils.createThriftFromBytes(bytes, event);
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType() + "' and with state : '" + event.getState().toString() +
+ " for Gateway " + event.getJobIdentity().getGatewayId());
+ } catch (TException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ }
+ });
+ } catch (ApplicationSettingsException e) {
+ logger.error("Error reading airavata server properties", e);
+ }catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+
+ }
+
+ public static void parseArguments(String[] args) {
+ try{
+ Options options = new Options();
+
+ options.addOption("gId", true , "Gateway ID");
+ options.addOption("eId", true, "Experiment ID");
+ options.addOption("jId", true, "Job ID");
+ options.addOption("a", false, "All Notifications");
+
+ CommandLineParser parser = new PosixParser();
+ CommandLine cmd = parser.parse( options, args);
+ if (cmd.getOptions() == null || cmd.getOptions().length == 0){
+ logger.info("You have not specified any options. We assume you need to listen to all the messages...");
+ allMessages = true;
+ gatewayId = "*";
+ }
+ if (cmd.hasOption("a")){
+ logger.info("Listening to all the messages...");
+ allMessages = true;
+ gatewayId = "*";
+ }else {
+ gatewayId = cmd.getOptionValue("gId");
+ if (gatewayId == null){
+ gatewayId = "*";
+ logger.info("You have not specified a gateway id. We assume you need to listen to all the messages...");
+ } else {
+ gatewayLevelMessages = true;
+ }
+ experimentId = cmd.getOptionValue("eId");
+ if (experimentId == null && !gatewayId.equals("*")){
+ experimentId = "*";
+ logger.info("You have not specified a experiment id. We assume you need to listen to all the messages for the gateway with id " + gatewayId);
+ } else if (experimentId == null && gatewayId.equals("*")) {
+ experimentId = "*";
+ logger.info("You have not specified a experiment id and a gateway id. We assume you need to listen to all the messages...");
+ }else {
+ experimentLevelMessages = true;
+ }
+ jobId = cmd.getOptionValue("jId");
+ if (jobId == null && !gatewayId.equals("*") && !experimentId.equals("*")){
+ jobId = "*";
+ logger.info("You have not specified a job id. We assume you need to listen to all the messages for the gateway with id " + gatewayId
+ + " with experiment id : " + experimentId );
+ } else if (jobId == null && gatewayId.equals("*") && experimentId.equals("*")) {
+ jobId = "*";
+ logger.info("You have not specified a job Id or experiment Id or a gateway Id. We assume you need to listen to all the messages...");
+ }else {
+ jobLevelMessages = true;
+ }
+ }
+ } catch (ParseException e) {
+ logger.error("Error while reading command line parameters" , e);
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/airavata/blob/48be39fe/pom.xml
----------------------------------------------------------------------
[04/15] airavata git commit: wrapping up working version of queue
based communication between orchestrator and gfac
Posted by la...@apache.org.
wrapping up working version of queue based communication between orchestrator and gfac
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/b6bf782d
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/b6bf782d
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/b6bf782d
Branch: refs/heads/master
Commit: b6bf782db30b0e7555f78852da3efcfdbfd530e4
Parents: 0149c1a
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Thu Feb 12 11:36:02 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Thu Feb 12 11:36:02 2015 -0500
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 8 ++++----
.../apache/airavata/gfac/server/GfacServerHandler.java | 13 ++++---------
.../airavata/messaging/core/impl/RabbitMQProducer.java | 11 ++++++++++-
.../core/impl/RabbitMQTaskLaunchPublisher.java | 4 ++--
.../core/impl/GFACPassiveJobSubmitter.java | 4 +++-
5 files changed, 23 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/b6bf782d/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index b90e0ff..8483da7 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -60,7 +60,7 @@ public class CreateLaunchExperiment {
private static final String DEFAULT_GATEWAY = "default.registry.gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_2e539083-665d-40fd-aaa2-4a751028326b";
+ private static String echoAppId = "Echo_78e34255-39f3-4c07-add6-a1a672c80104";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_eda074ea-223d-49d7-a942-6c8742249f36";
@@ -85,7 +85,7 @@ public class CreateLaunchExperiment {
public static void main(String[] args) throws Exception {
airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
System.out.println("API version is " + airavataClient.getAPIVersion());
-// registerApplications(); // run this only the first time
+// registerApplications(); // run this only the first time
createAndLaunchExp();
}
@@ -101,13 +101,13 @@ public class CreateLaunchExperiment {
// final String expId = createMPIExperimentForFSD(airavataClient);
// final String expId = createEchoExperimentForStampede(airavataClient);
// final String expId = createEchoExperimentForTrestles(airavataClient);
-// final String expId = createExperimentEchoForLocalHost(airavataClient);
+ final String expId = createExperimentEchoForLocalHost(airavataClient);
// final String expId = createExperimentWRFTrestles(airavataClient);
// final String expId = createExperimentForBR2(airavataClient);
// final String expId = createExperimentForBR2Amber(airavataClient);
// final String expId = createExperimentWRFStampede(airavataClient);
// final String expId = createExperimentForStampedeAmber(airavataClient);
- final String expId = createExperimentForTrestlesAmber(airavataClient);
+// final String expId = createExperimentForTrestlesAmber(airavataClient);
// final String expId = createExperimentGROMACSStampede(airavataClient);
// final String expId = createExperimentESPRESSOStampede(airavataClient);
// final String expId = createExperimentLAMMPSStampede(airavataClient);
http://git-wip-us.apache.org/repos/asf/airavata/blob/b6bf782d/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index d428d9c..c8f1100 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -113,6 +113,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
if(ServerSettings.isGFacPassiveMode()) {
rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
+ rabbitMQTaskLaunchConsumer.listen(new TaskLaunchMessageHandler());
}
@@ -291,19 +292,13 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
}
}
- private class NotificationMessageHandler implements MessageHandler {
+ private class TaskLaunchMessageHandler implements MessageHandler {
private String experimentId;
- private NotificationMessageHandler(String experimentId) {
- this.experimentId = experimentId;
- }
-
+
public Map<String, Object> getProperties() {
Map<String, Object> props = new HashMap<String, Object>();
- List<String> routingKeys = new ArrayList<String>();
- routingKeys.add(experimentId);
- routingKeys.add(experimentId + ".*");
- props.put(MessagingConstants.RABBIT_ROUTING_KEY, routingKeys);
+ props.put(MessagingConstants.RABBIT_ROUTING_KEY, UUID.randomUUID().toString());
return props;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b6bf782d/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
index b4a6d46..570b17f 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQProducer.java
@@ -48,6 +48,15 @@ public class RabbitMQProducer {
private String url;
+ private String getExchangeType = "topic";
+
+
+ public RabbitMQProducer(String url, String exchangeName,String getExchangeType) {
+ this.exchangeName = exchangeName;
+ this.url = url;
+ this.getExchangeType = getExchangeType;
+ }
+
public RabbitMQProducer(String url, String exchangeName) {
this.exchangeName = exchangeName;
this.url = url;
@@ -104,7 +113,7 @@ public class RabbitMQProducer {
log.info("setting basic.qos / prefetch count to " + prefetchCount + " for " + exchangeName);
channel.basicQos(prefetchCount);
}
- channel.exchangeDeclare(exchangeName, "topic", false);
+ channel.exchangeDeclare(exchangeName, getExchangeType, false);
} catch (Exception e) {
reset();
String msg = "could not open channel for exchange " + exchangeName;
http://git-wip-us.apache.org/repos/asf/airavata/blob/b6bf782d/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
index fe58042..23b2379 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
@@ -44,13 +44,13 @@ public class RabbitMQTaskLaunchPublisher implements Publisher{
String exchangeName;
try {
brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
- exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_STATUS_EXCHANGE_NAME);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_TASK_LAUNCH_EXCHANGE_NAME);
} catch (ApplicationSettingsException e) {
String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
log.error(message, e);
throw new AiravataException(message, e);
}
- rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName);
+ rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName,"fanout");
rabbitMQProducer.open();
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/b6bf782d/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
index 58ac982..bfe2b16 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
@@ -22,6 +22,7 @@ package org.apache.airavata.orchestrator.core.impl;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.AiravataUtils;
import org.apache.airavata.common.utils.AiravataZKUtils;
import org.apache.airavata.common.utils.Constants;
import org.apache.airavata.common.utils.ServerSettings;
@@ -141,7 +142,8 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
gatewayId = ServerSettings.getDefaultUserGateway();
}
TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, gatewayId);
- MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK,"LAUNCH.TASK-"+ UUID.randomUUID().toString(),gatewayId);
+ MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK, "LAUNCH.TASK-" + UUID.randomUUID().toString(), gatewayId);
+ messageContext.setUpdatedTime(AiravataUtils.getCurrentTimestamp());
publisher.publish(messageContext);
}
}
[05/15] airavata git commit: adding curator leader election logic
Posted by la...@apache.org.
adding curator leader election logic
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/a486b67d
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/a486b67d
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/a486b67d
Branch: refs/heads/master
Commit: a486b67d9187294bae14dd15d4d5b90a84484c73
Parents: b6bf782
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Mon Feb 16 22:38:41 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Mon Feb 16 22:38:41 2015 -0500
----------------------------------------------------------------------
.../lib/airavata/messagingEvents_types.cpp | 20 +++-
.../lib/airavata/messagingEvents_types.h | 13 ++-
.../Airavata/Model/Messaging/Event/Types.php | 20 ++++
.../client/samples/CreateLaunchExperiment.java | 2 +-
.../model/messaging/event/TaskSubmitEvent.java | 100 +++++++++++++++-
airavata-api/generate-thrift-files.sh | 2 +-
.../messagingEvents.thrift | 3 +-
modules/gfac/airavata-gfac-service/pom.xml | 10 ++
.../airavata/gfac/leader/CuratorClient.java | 79 +++++++++++++
.../gfac/leader/LeaderSelectorExample.java | 80 +++++++++++++
.../airavata/gfac/server/GfacServerHandler.java | 112 +++++++++++++++---
modules/gfac/gfac-core/pom.xml | 1 +
.../airavata/gfac/core/utils/GFacUtils.java | 116 ++++++++++++++++++-
modules/orchestrator/orchestrator-core/pom.xml | 10 --
.../core/impl/GFACPassiveJobSubmitter.java | 46 +++-----
.../core/impl/GFACRPCJobSubmitter.java | 4 +-
16 files changed, 547 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
index a2e72f5..71f45be 100644
--- a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
+++ b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.cpp
@@ -839,8 +839,8 @@ void swap(JobIdentifier &a, JobIdentifier &b) {
swap(a.gatewayId, b.gatewayId);
}
-const char* TaskSubmitEvent::ascii_fingerprint = "AB879940BD15B6B25691265F7384B271";
-const uint8_t TaskSubmitEvent::binary_fingerprint[16] = {0xAB,0x87,0x99,0x40,0xBD,0x15,0xB6,0xB2,0x56,0x91,0x26,0x5F,0x73,0x84,0xB2,0x71};
+const char* TaskSubmitEvent::ascii_fingerprint = "C93D890311F28844166CF6E571EB3AC2";
+const uint8_t TaskSubmitEvent::binary_fingerprint[16] = {0xC9,0x3D,0x89,0x03,0x11,0xF2,0x88,0x44,0x16,0x6C,0xF6,0xE5,0x71,0xEB,0x3A,0xC2};
uint32_t TaskSubmitEvent::read(::apache::thrift::protocol::TProtocol* iprot) {
@@ -856,6 +856,7 @@ uint32_t TaskSubmitEvent::read(::apache::thrift::protocol::TProtocol* iprot) {
bool isset_experimentId = false;
bool isset_taskId = false;
bool isset_gatewayId = false;
+ bool isset_tokenId = false;
while (true)
{
@@ -889,6 +890,14 @@ uint32_t TaskSubmitEvent::read(::apache::thrift::protocol::TProtocol* iprot) {
xfer += iprot->skip(ftype);
}
break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->tokenId);
+ isset_tokenId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -904,6 +913,8 @@ uint32_t TaskSubmitEvent::read(::apache::thrift::protocol::TProtocol* iprot) {
throw TProtocolException(TProtocolException::INVALID_DATA);
if (!isset_gatewayId)
throw TProtocolException(TProtocolException::INVALID_DATA);
+ if (!isset_tokenId)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
return xfer;
}
@@ -923,6 +934,10 @@ uint32_t TaskSubmitEvent::write(::apache::thrift::protocol::TProtocol* oprot) co
xfer += oprot->writeString(this->gatewayId);
xfer += oprot->writeFieldEnd();
+ xfer += oprot->writeFieldBegin("tokenId", ::apache::thrift::protocol::T_STRING, 4);
+ xfer += oprot->writeString(this->tokenId);
+ xfer += oprot->writeFieldEnd();
+
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -933,6 +948,7 @@ void swap(TaskSubmitEvent &a, TaskSubmitEvent &b) {
swap(a.experimentId, b.experimentId);
swap(a.taskId, b.taskId);
swap(a.gatewayId, b.gatewayId);
+ swap(a.tokenId, b.tokenId);
}
const char* TaskTerminateEvent::ascii_fingerprint = "07A9615F837F7D0A952B595DD3020972";
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
index f063fc2..c7e2bb5 100644
--- a/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
+++ b/airavata-api/airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/messagingEvents_types.h
@@ -465,10 +465,10 @@ void swap(JobIdentifier &a, JobIdentifier &b);
class TaskSubmitEvent {
public:
- static const char* ascii_fingerprint; // = "AB879940BD15B6B25691265F7384B271";
- static const uint8_t binary_fingerprint[16]; // = {0xAB,0x87,0x99,0x40,0xBD,0x15,0xB6,0xB2,0x56,0x91,0x26,0x5F,0x73,0x84,0xB2,0x71};
+ static const char* ascii_fingerprint; // = "C93D890311F28844166CF6E571EB3AC2";
+ static const uint8_t binary_fingerprint[16]; // = {0xC9,0x3D,0x89,0x03,0x11,0xF2,0x88,0x44,0x16,0x6C,0xF6,0xE5,0x71,0xEB,0x3A,0xC2};
- TaskSubmitEvent() : experimentId(), taskId(), gatewayId() {
+ TaskSubmitEvent() : experimentId(), taskId(), gatewayId(), tokenId() {
}
virtual ~TaskSubmitEvent() throw() {}
@@ -476,6 +476,7 @@ class TaskSubmitEvent {
std::string experimentId;
std::string taskId;
std::string gatewayId;
+ std::string tokenId;
void __set_experimentId(const std::string& val) {
experimentId = val;
@@ -489,6 +490,10 @@ class TaskSubmitEvent {
gatewayId = val;
}
+ void __set_tokenId(const std::string& val) {
+ tokenId = val;
+ }
+
bool operator == (const TaskSubmitEvent & rhs) const
{
if (!(experimentId == rhs.experimentId))
@@ -497,6 +502,8 @@ class TaskSubmitEvent {
return false;
if (!(gatewayId == rhs.gatewayId))
return false;
+ if (!(tokenId == rhs.tokenId))
+ return false;
return true;
}
bool operator != (const TaskSubmitEvent &rhs) const {
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php b/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
index 40810d3..b0d7676 100644
--- a/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
+++ b/airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib/Airavata/Model/Messaging/Event/Types.php
@@ -977,6 +977,7 @@ class TaskSubmitEvent {
public $experimentId = null;
public $taskId = null;
public $gatewayId = null;
+ public $tokenId = null;
public function __construct($vals=null) {
if (!isset(self::$_TSPEC)) {
@@ -993,6 +994,10 @@ class TaskSubmitEvent {
'var' => 'gatewayId',
'type' => TType::STRING,
),
+ 4 => array(
+ 'var' => 'tokenId',
+ 'type' => TType::STRING,
+ ),
);
}
if (is_array($vals)) {
@@ -1005,6 +1010,9 @@ class TaskSubmitEvent {
if (isset($vals['gatewayId'])) {
$this->gatewayId = $vals['gatewayId'];
}
+ if (isset($vals['tokenId'])) {
+ $this->tokenId = $vals['tokenId'];
+ }
}
}
@@ -1048,6 +1056,13 @@ class TaskSubmitEvent {
$xfer += $input->skip($ftype);
}
break;
+ case 4:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->tokenId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -1076,6 +1091,11 @@ class TaskSubmitEvent {
$xfer += $output->writeString($this->gatewayId);
$xfer += $output->writeFieldEnd();
}
+ if ($this->tokenId !== null) {
+ $xfer += $output->writeFieldBegin('tokenId', TType::STRING, 4);
+ $xfer += $output->writeString($this->tokenId);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 8483da7..b7121b9 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -60,7 +60,7 @@ public class CreateLaunchExperiment {
private static final String DEFAULT_GATEWAY = "default.registry.gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_78e34255-39f3-4c07-add6-a1a672c80104";
+ private static String echoAppId = "Echo_a8fc8511-7b8e-431a-ad0f-de5eb1a9c576";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_eda074ea-223d-49d7-a942-6c8742249f36";
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java
index c813c76..71d497e 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/messaging/event/TaskSubmitEvent.java
@@ -55,6 +55,7 @@ import org.slf4j.LoggerFactory;
private static final org.apache.thrift.protocol.TField EXPERIMENT_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("experimentId", org.apache.thrift.protocol.TType.STRING, (short)1);
private static final org.apache.thrift.protocol.TField TASK_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("taskId", org.apache.thrift.protocol.TType.STRING, (short)2);
private static final org.apache.thrift.protocol.TField GATEWAY_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("gatewayId", org.apache.thrift.protocol.TType.STRING, (short)3);
+ private static final org.apache.thrift.protocol.TField TOKEN_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("tokenId", org.apache.thrift.protocol.TType.STRING, (short)4);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@@ -65,12 +66,14 @@ import org.slf4j.LoggerFactory;
private String experimentId; // required
private String taskId; // required
private String gatewayId; // required
+ private String tokenId; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
@SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
EXPERIMENT_ID((short)1, "experimentId"),
TASK_ID((short)2, "taskId"),
- GATEWAY_ID((short)3, "gatewayId");
+ GATEWAY_ID((short)3, "gatewayId"),
+ TOKEN_ID((short)4, "tokenId");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -91,6 +94,8 @@ import org.slf4j.LoggerFactory;
return TASK_ID;
case 3: // GATEWAY_ID
return GATEWAY_ID;
+ case 4: // TOKEN_ID
+ return TOKEN_ID;
default:
return null;
}
@@ -140,6 +145,8 @@ import org.slf4j.LoggerFactory;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.GATEWAY_ID, new org.apache.thrift.meta_data.FieldMetaData("gatewayId", org.apache.thrift.TFieldRequirementType.REQUIRED,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.TOKEN_ID, new org.apache.thrift.meta_data.FieldMetaData("tokenId", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TaskSubmitEvent.class, metaDataMap);
}
@@ -150,12 +157,14 @@ import org.slf4j.LoggerFactory;
public TaskSubmitEvent(
String experimentId,
String taskId,
- String gatewayId)
+ String gatewayId,
+ String tokenId)
{
this();
this.experimentId = experimentId;
this.taskId = taskId;
this.gatewayId = gatewayId;
+ this.tokenId = tokenId;
}
/**
@@ -171,6 +180,9 @@ import org.slf4j.LoggerFactory;
if (other.isSetGatewayId()) {
this.gatewayId = other.gatewayId;
}
+ if (other.isSetTokenId()) {
+ this.tokenId = other.tokenId;
+ }
}
public TaskSubmitEvent deepCopy() {
@@ -182,6 +194,7 @@ import org.slf4j.LoggerFactory;
this.experimentId = null;
this.taskId = null;
this.gatewayId = null;
+ this.tokenId = null;
}
public String getExperimentId() {
@@ -253,6 +266,29 @@ import org.slf4j.LoggerFactory;
}
}
+ public String getTokenId() {
+ return this.tokenId;
+ }
+
+ public void setTokenId(String tokenId) {
+ this.tokenId = tokenId;
+ }
+
+ public void unsetTokenId() {
+ this.tokenId = null;
+ }
+
+ /** Returns true if field tokenId is set (has been assigned a value) and false otherwise */
+ public boolean isSetTokenId() {
+ return this.tokenId != null;
+ }
+
+ public void setTokenIdIsSet(boolean value) {
+ if (!value) {
+ this.tokenId = null;
+ }
+ }
+
public void setFieldValue(_Fields field, Object value) {
switch (field) {
case EXPERIMENT_ID:
@@ -279,6 +315,14 @@ import org.slf4j.LoggerFactory;
}
break;
+ case TOKEN_ID:
+ if (value == null) {
+ unsetTokenId();
+ } else {
+ setTokenId((String)value);
+ }
+ break;
+
}
}
@@ -293,6 +337,9 @@ import org.slf4j.LoggerFactory;
case GATEWAY_ID:
return getGatewayId();
+ case TOKEN_ID:
+ return getTokenId();
+
}
throw new IllegalStateException();
}
@@ -310,6 +357,8 @@ import org.slf4j.LoggerFactory;
return isSetTaskId();
case GATEWAY_ID:
return isSetGatewayId();
+ case TOKEN_ID:
+ return isSetTokenId();
}
throw new IllegalStateException();
}
@@ -354,6 +403,15 @@ import org.slf4j.LoggerFactory;
return false;
}
+ boolean this_present_tokenId = true && this.isSetTokenId();
+ boolean that_present_tokenId = true && that.isSetTokenId();
+ if (this_present_tokenId || that_present_tokenId) {
+ if (!(this_present_tokenId && that_present_tokenId))
+ return false;
+ if (!this.tokenId.equals(that.tokenId))
+ return false;
+ }
+
return true;
}
@@ -400,6 +458,16 @@ import org.slf4j.LoggerFactory;
return lastComparison;
}
}
+ lastComparison = Boolean.valueOf(isSetTokenId()).compareTo(other.isSetTokenId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTokenId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tokenId, other.tokenId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -443,6 +511,14 @@ import org.slf4j.LoggerFactory;
sb.append(this.gatewayId);
}
first = false;
+ if (!first) sb.append(", ");
+ sb.append("tokenId:");
+ if (this.tokenId == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.tokenId);
+ }
+ first = false;
sb.append(")");
return sb.toString();
}
@@ -461,6 +537,10 @@ import org.slf4j.LoggerFactory;
throw new org.apache.thrift.protocol.TProtocolException("Required field 'gatewayId' is unset! Struct:" + toString());
}
+ if (!isSetTokenId()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'tokenId' is unset! Struct:" + toString());
+ }
+
// check for sub-struct validity
}
@@ -522,6 +602,14 @@ import org.slf4j.LoggerFactory;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 4: // TOKEN_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.tokenId = iprot.readString();
+ struct.setTokenIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -550,6 +638,11 @@ import org.slf4j.LoggerFactory;
oprot.writeString(struct.gatewayId);
oprot.writeFieldEnd();
}
+ if (struct.tokenId != null) {
+ oprot.writeFieldBegin(TOKEN_ID_FIELD_DESC);
+ oprot.writeString(struct.tokenId);
+ oprot.writeFieldEnd();
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -570,6 +663,7 @@ import org.slf4j.LoggerFactory;
oprot.writeString(struct.experimentId);
oprot.writeString(struct.taskId);
oprot.writeString(struct.gatewayId);
+ oprot.writeString(struct.tokenId);
}
@Override
@@ -581,6 +675,8 @@ import org.slf4j.LoggerFactory;
struct.setTaskIdIsSet(true);
struct.gatewayId = iprot.readString();
struct.setGatewayIdIsSet(true);
+ struct.tokenId = iprot.readString();
+ struct.setTokenIdIsSet(true);
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/airavata-api/generate-thrift-files.sh
----------------------------------------------------------------------
diff --git a/airavata-api/generate-thrift-files.sh b/airavata-api/generate-thrift-files.sh
index bd823e4..c8a000d 100755
--- a/airavata-api/generate-thrift-files.sh
+++ b/airavata-api/generate-thrift-files.sh
@@ -27,7 +27,7 @@ DATAMODEL_SRC_DIR='airavata-data-models/src/main/java'
JAVA_API_SDK_DIR='airavata-api-stubs/src/main/java'
CPP_SDK_DIR='airavata-client-sdks/airavata-cpp-sdk/src/main/resources/lib/airavata/'
PHP_SDK_DIR='airavata-client-sdks/airavata-php-sdk/src/main/resources/lib'
-THRIFT_EXEC=thrift
+THRIFT_EXEC=/usr/local/Cellar/thrift/0.9.1/bin/thrift
# The Function fail prints error messages on failure and quits the script.
fail() {
echo $@
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
----------------------------------------------------------------------
diff --git a/airavata-api/thrift-interface-descriptions/messagingEvents.thrift b/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
index d736701..d9e85d4 100644
--- a/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
+++ b/airavata-api/thrift-interface-descriptions/messagingEvents.thrift
@@ -105,7 +105,8 @@ struct JobIdentifier {
struct TaskSubmitEvent{
1: required string experimentId,
2: required string taskId,
- 3: required string gatewayId
+ 3: required string gatewayId,
+ 4: required string tokenId
}
struct TaskTerminateEvent{
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/gfac/airavata-gfac-service/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/pom.xml b/modules/gfac/airavata-gfac-service/pom.xml
index 0884942..5a178bd 100644
--- a/modules/gfac/airavata-gfac-service/pom.xml
+++ b/modules/gfac/airavata-gfac-service/pom.xml
@@ -80,6 +80,16 @@
<artifactId>airavata-server-configuration</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
</dependencies>
<properties>
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java
new file mode 100644
index 0000000..2db9a6f
--- /dev/null
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/CuratorClient.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.gfac.leader;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
+import org.apache.curator.framework.recipes.leader.LeaderSelector;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * An example leader selector client. Note that {@link LeaderSelectorListenerAdapter} which
+ * has the recommended handling for connection state issues
+ */
+public class CuratorClient extends LeaderSelectorListenerAdapter implements Closeable {
+ private final String name;
+ private final LeaderSelector leaderSelector;
+ private final AtomicInteger leaderCount = new AtomicInteger();
+
+ public CuratorClient(CuratorFramework client, String path, String name) {
+ this.name = name;
+
+ // create a leader selector using the given path for management
+ // all participants in a given leader selection must use the same path
+ // ExampleClient here is also a LeaderSelectorListener but this isn't required
+ leaderSelector = new LeaderSelector(client, path, this);
+
+ // for most cases you will want your instance to requeue when it relinquishes leadership
+ leaderSelector.autoRequeue();
+ }
+
+ public void start() throws IOException {
+ // the selection for this instance doesn't start until the leader selector is started
+ // leader selection is done in the background so this call to leaderSelector.start() returns immediately
+ leaderSelector.start();
+ }
+
+ @Override
+ public void close() throws IOException {
+ leaderSelector.close();
+ }
+
+ @Override
+ public void takeLeadership(CuratorFramework client) throws Exception {
+ // we are now the leader. This method should not return until we want to relinquish leadership
+
+ final int waitSeconds = (int) (5 * Math.random()) + 1;
+
+ System.out.println(name + " is now the leader. Waiting " + waitSeconds + " seconds...");
+ System.out.println(name + " has been leader " + leaderCount.getAndIncrement() + " time(s) before.");
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(waitSeconds));
+ } catch (InterruptedException e) {
+ System.err.println(name + " was interrupted.");
+ Thread.currentThread().interrupt();
+ } finally {
+ System.out.println(name + " relinquishing leadership.\n");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java
new file mode 100644
index 0000000..ad02641
--- /dev/null
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/leader/LeaderSelectorExample.java
@@ -0,0 +1,80 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.leader;
+
+import com.google.common.collect.Lists;
+import org.apache.airavata.common.utils.AiravataZKUtils;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.utils.CloseableUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.List;
+
+public class LeaderSelectorExample {
+ private final static Logger logger = LoggerFactory.getLogger(LeaderSelectorExample.class);
+ private static final int CLIENT_QTY = 10;
+
+ private static final String PATH = "/examples/leader";
+
+ public static void main(String[] args) throws Exception
+ {
+ // all of the useful sample code is in ExampleClient.java
+
+ System.out.println("Create " + CLIENT_QTY + " clients, have each negotiate for leadership and then wait a random number of seconds before letting another leader election occur.");
+ System.out.println("Notice that leader election is fair: all clients will become leader and will do so the same number of times.");
+
+ try
+ {
+ for ( int i = 0; i < CLIENT_QTY; ++i )
+ {
+ CuratorFramework client = CuratorFrameworkFactory.newClient(AiravataZKUtils.getZKhostPort(), new ExponentialBackoffRetry(1000, 3));
+
+ CuratorClient example = new CuratorClient(client, PATH, "Client #" + i);
+
+ client.start();
+ example.start();
+ }
+
+ System.out.println("Press enter/return to quit\n");
+ new BufferedReader(new InputStreamReader(System.in)).readLine();
+ }
+ finally
+ {
+ System.out.println("Shutting down...");
+
+ /*for ( CuratorClient exampleClient : examples )
+ {
+ CloseableUtils.closeQuietly(exampleClient);
+ }
+ for ( CuratorFramework client : clients )
+ {
+ CloseableUtils.closeQuietly(client);
+ }*/
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index c8f1100..c838703 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -31,31 +31,38 @@ import org.apache.airavata.common.utils.*;
import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
import org.apache.airavata.gfac.core.cpi.GFac;
import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
+import org.apache.airavata.gfac.core.utils.GFacUtils;
import org.apache.airavata.gfac.core.utils.InputHandlerWorker;
import org.apache.airavata.gfac.cpi.GfacService;
import org.apache.airavata.gfac.cpi.gfac_cpi_serviceConstants;
+import org.apache.airavata.gfac.leader.CuratorClient;
import org.apache.airavata.messaging.core.MessageContext;
import org.apache.airavata.messaging.core.MessageHandler;
import org.apache.airavata.messaging.core.MessagingConstants;
-import org.apache.airavata.messaging.core.impl.RabbitMQStatusConsumer;
import org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchConsumer;
-import org.apache.airavata.model.messaging.event.ExperimentStatusChangeEvent;
import org.apache.airavata.model.messaging.event.MessageType;
import org.apache.airavata.model.messaging.event.TaskSubmitEvent;
import org.apache.airavata.model.messaging.event.TaskTerminateEvent;
-import org.apache.airavata.model.workspace.experiment.ExperimentState;
import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
import org.apache.airavata.registry.cpi.Registry;
import org.apache.airavata.registry.cpi.RegistryException;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.recipes.leader.LeaderSelector;
+import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
+import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
+import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
public class GfacServerHandler implements GfacService.Iface, Watcher{
@@ -88,6 +95,9 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer;
+ CuratorFramework curatorFramework = null;
+
+
public GfacServerHandler() throws Exception{
// registering with zk
try {
@@ -114,6 +124,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
if(ServerSettings.isGFacPassiveMode()) {
rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
rabbitMQTaskLaunchConsumer.listen(new TaskLaunchMessageHandler());
+ curatorFramework = CuratorFrameworkFactory.newClient(AiravataZKUtils.getZKhostPort(), new ExponentialBackoffRetry(1000, 3));
}
@@ -229,14 +240,6 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
inHandlerFutures.add(GFacThreadPoolExecutor.getFixedThreadPool().submit(inputHandlerWorker));
// we immediately return when we have a threadpool
return true;
-// }else{
-// logger.error(experimentId, "Failed to submit job to the GFac implementation, experiment {}, task {}, " +
-// "gateway {}", experimentId, taskId, gatewayId);
-// return false;
-// }
-// } catch (GFacException e) {
-// throw new TException("Error launching the experiment : " + e.getMessage(), e);
-// }
}
public boolean cancelJob(String experimentId, String taskId) throws TException {
@@ -295,10 +298,26 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private class TaskLaunchMessageHandler implements MessageHandler {
private String experimentId;
-
+ private String nodeName;
+
+ public TaskLaunchMessageHandler(){
+ try {
+ nodeName = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME);
+ } catch (ApplicationSettingsException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
public Map<String, Object> getProperties() {
Map<String, Object> props = new HashMap<String, Object>();
- props.put(MessagingConstants.RABBIT_ROUTING_KEY, UUID.randomUUID().toString());
+ try {
+ props.put(MessagingConstants.RABBIT_ROUTING_KEY, ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NAME));
+ } catch (ApplicationSettingsException e) {
+ // if we cannot find gfac node name configured we set a random id
+ logger.error("airavata-server.properties should configure: " + Constants.ZOOKEEPER_GFAC_SERVER_NAME + " value.");
+ logger.error("listening to a random generated routing key");
+ props.put(MessagingConstants.RABBIT_ROUTING_KEY, UUID.randomUUID().toString());
+ }
return props;
}
@@ -309,10 +328,16 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
TBase messageEvent = message.getEvent();
byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
ThriftUtils.createThriftFromBytes(bytes, event);
- submitJob(event.getExperimentId(), event.getTaskId(), event.getGatewayId());
- System.out.println(" Message Received with message id '" + message.getMessageId()
- + "' and with message type '" + message.getType());
- } catch (TException e) {
+ CuratorClient curatorClient = new CuratorClient(curatorFramework, event, nodeName);
+ try {
+ curatorClient.start();
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType());
+ } catch (TException e) {
logger.error(e.getMessage(), e); //nobody is listening so nothing to throw
}
}else if(message.getType().equals(MessageType.TERMINATETASK)){
@@ -331,4 +356,57 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
}
}
+ public class CuratorClient extends LeaderSelectorListenerAdapter implements Closeable {
+ private final String name;
+ private final LeaderSelector leaderSelector;
+ private final AtomicInteger leaderCount = new AtomicInteger();
+ private final String path;
+ private TaskSubmitEvent event;
+ private String experimentNode;
+
+ public CuratorClient(CuratorFramework client, TaskSubmitEvent taskSubmitEvent, String name) {
+ this.name = name;
+ this.event = taskSubmitEvent;
+ this.path = File.separator + event.getExperimentId() + "-" + event.getTaskId() + "-" + event.getGatewayId();
+ // create a leader selector using the given path for management
+ // all participants in a given leader selection must use the same path
+ // ExampleClient here is also a LeaderSelectorListener but this isn't required
+ leaderSelector = new LeaderSelector(client, path, this);
+ experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
+ // for most cases you will want your instance to requeue when it relinquishes leadership
+ leaderSelector.autoRequeue();
+ }
+
+ public void start() throws IOException {
+ // the selection for this instance doesn't start until the leader selector is started
+ // leader selection is done in the background so this call to leaderSelector.start() returns immediately
+ leaderSelector.start();
+ }
+
+ @Override
+ public void close() throws IOException {
+ leaderSelector.close();
+ }
+
+ @Override
+ public void takeLeadership(CuratorFramework client) throws Exception {
+ // we are now the leader. This method should not return until we want to relinquish leadership
+ final int waitSeconds = (int) (5 * Math.random()) + 1;
+
+ System.out.println(name + " is now the leader. Waiting " + waitSeconds + " seconds...");
+ System.out.println(name + " has been leader " + leaderCount.getAndIncrement() + " time(s) before.");
+
+ try {
+ GFacUtils.createExperimentEntryForRPC(event.getExperimentId(),event.getTaskId(),client.getZookeeperClient().getZooKeeper(),experimentNode,name,event.getTokenId());
+ submitJob(event.getExperimentId(), event.getTaskId(), event.getGatewayId());
+ Thread.sleep(TimeUnit.SECONDS.toMillis(waitSeconds));
+ } catch (InterruptedException e) {
+ System.err.println(name + " was interrupted.");
+ Thread.currentThread().interrupt();
+ } finally {
+ System.out.println(name + " relinquishing leadership.\n");
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/gfac/gfac-core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/pom.xml b/modules/gfac/gfac-core/pom.xml
index 4fc2a15..2a42503 100644
--- a/modules/gfac/gfac-core/pom.xml
+++ b/modules/gfac/gfac-core/pom.xml
@@ -131,6 +131,7 @@
<artifactId>zookeeper</artifactId>
<version>3.4.0</version>
</dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
index cbbce48..9f104fa 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
@@ -1044,9 +1044,9 @@ public class GFacUtils {
}
// This method is dangerous because of moving the experiment data
- public static boolean createExperimentEntry(String experimentID,
- String taskID, ZooKeeper zk, String experimentNode,
- String pickedChild, String tokenId) throws KeeperException,
+ public static boolean createExperimentEntryForRPC(String experimentID,
+ String taskID, ZooKeeper zk, String experimentNode,
+ String pickedChild, String tokenId) throws KeeperException,
InterruptedException {
String experimentPath = experimentNode + File.separator + pickedChild;
String newExpNode = experimentPath + File.separator + experimentID
@@ -1153,6 +1153,116 @@ public class GFacUtils {
return true;
}
+ // This method is dangerous because of moving the experiment data
+ public static boolean createExperimentEntryForPassive(String experimentID,
+ String taskID, ZooKeeper zk, String experimentNode,
+ String pickedChild, String tokenId) throws KeeperException,
+ InterruptedException {
+ String experimentPath = experimentNode + File.separator + pickedChild;
+ String newExpNode = experimentPath + File.separator + experimentID
+ + "+" + taskID;
+ Stat exists1 = zk.exists(newExpNode, false);
+ String experimentEntry = GFacUtils.findExperimentEntry(experimentID, taskID, zk);
+ String foundExperimentPath = null;
+ if (exists1 == null && experimentEntry == null) { // this means this is a very new experiment
+ List<String> runningGfacNodeNames = AiravataZKUtils
+ .getAllGfacNodeNames(zk); // here we take old gfac servers
+ // too
+ for (String gfacServerNode : runningGfacNodeNames) {
+ if (!gfacServerNode.equals(pickedChild)) {
+ foundExperimentPath = experimentNode + File.separator
+ + gfacServerNode + File.separator + experimentID
+ + "+" + taskID;
+ exists1 = zk.exists(foundExperimentPath, false);
+ if (exists1 != null) { // when the experiment is found we
+ // break the loop
+ break;
+ }
+ }
+ }
+ if (exists1 == null) { // OK this is a pretty new experiment so we
+ // are going to create a new node
+ log.info("This is a new Job, so creating all the experiment docs from the scratch");
+ zk.create(newExpNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+
+ Stat expParent = zk.exists(newExpNode, false);
+ if (tokenId != null && expParent != null) {
+ zk.setData(newExpNode, tokenId.getBytes(),
+ expParent.getVersion());
+ }
+ zk.create(newExpNode + File.separator + "state", String
+ .valueOf(GfacExperimentState.LAUNCHED.getValue())
+ .getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ zk.create(newExpNode + File.separator + "operation","submit".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+
+ } else {
+ // ohhh this node exists in some other failed gfac folder, we
+ // have to move it to this gfac experiment list,safely
+ log.info("This is an old Job, so copying data from old experiment location");
+ zk.create(newExpNode,
+ zk.getData(foundExperimentPath, false, exists1),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+
+ List<String> children = zk.getChildren(foundExperimentPath,
+ false);
+ for (String childNode1 : children) {
+ String level1 = foundExperimentPath + File.separator
+ + childNode1;
+ Stat exists2 = zk.exists(level1, false); // no need to check
+ // exists
+ String newLeve1 = newExpNode + File.separator + childNode1;
+ log.info("Creating new znode: " + newLeve1); // these has to
+ // be info
+ // logs
+ zk.create(newLeve1, zk.getData(level1, false, exists2),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ for (String childNode2 : zk.getChildren(level1, false)) {
+ String level2 = level1 + File.separator + childNode2;
+ Stat exists3 = zk.exists(level2, false); // no need to
+ // check
+ // exists
+ String newLeve2 = newLeve1 + File.separator
+ + childNode2;
+ log.info("Creating new znode: " + newLeve2);
+ zk.create(newLeve2, zk.getData(level2, false, exists3),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ }
+ }
+ // After all the files are successfully transfered we delete the
+ // old experiment,otherwise we do
+ // not delete a single file
+ log.info("After a successful copying of experiment data for an old experiment we delete the old data");
+ log.info("Deleting experiment data: " + foundExperimentPath);
+ ZKUtil.deleteRecursive(zk, foundExperimentPath);
+ }
+ }else if(experimentEntry != null && GFacUtils.isCancelled(experimentID,taskID,zk) ){
+ // this happens when a cancel request comes to a differnt gfac node, in this case we do not move gfac experiment
+ // node to gfac node specific location, because original request execution will fail with errors
+ log.error("This experiment is already cancelled and its already executing the cancel operation so cannot submit again !");
+ return false;
+ } else {
+ log.error("ExperimentID: " + experimentID + " taskID: " + taskID
+ + " is already running by this Gfac instance");
+ List<String> runningGfacNodeNames = AiravataZKUtils
+ .getAllGfacNodeNames(zk); // here we take old gfac servers
+ // too
+ for (String gfacServerNode : runningGfacNodeNames) {
+ if (!gfacServerNode.equals(pickedChild)) {
+ foundExperimentPath = experimentNode + File.separator
+ + gfacServerNode + File.separator + experimentID
+ + "+" + taskID;
+ break;
+ }
+ }
+ ZKUtil.deleteRecursive(zk, foundExperimentPath);
+ }
+ return true;
+ }
+
public static String findExperimentEntry(String experimentID,
String taskID, ZooKeeper zk
) throws KeeperException,
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/orchestrator/orchestrator-core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/pom.xml b/modules/orchestrator/orchestrator-core/pom.xml
index 23863fb..99c0abb 100644
--- a/modules/orchestrator/orchestrator-core/pom.xml
+++ b/modules/orchestrator/orchestrator-core/pom.xml
@@ -127,16 +127,6 @@ the License. -->
<artifactId>zookeeper</artifactId>
<version>${zk.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
index bfe2b16..78cc6b7 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACPassiveJobSubmitter.java
@@ -39,7 +39,6 @@ import org.apache.airavata.model.messaging.event.TaskSubmitEvent;
import org.apache.airavata.orchestrator.core.context.OrchestratorContext;
import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
import org.apache.airavata.orchestrator.core.job.JobSubmitter;
-import org.apache.thrift.TException;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@@ -113,40 +112,29 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
}
}
String gfacServer = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_SERVER_NODE, "/gfac-server");
- String experimentNode = ServerSettings.getSetting(Constants.ZOOKEEPER_GFAC_EXPERIMENT_NODE, "/gfac-experiments");
List<String> children = zk.getChildren(gfacServer, this);
if (children.size() == 0) {
// Zookeeper data need cleaning
throw new OrchestratorException("There is no active GFac instance to route the request");
} else {
- String pickedChild = children.get(new Random().nextInt(Integer.MAX_VALUE) % children.size());
- // here we are not using an index because the getChildren does not return the same order everytime
- String gfacNodeData = new String(zk.getData(gfacServer + File.separator + pickedChild, false, null));
- logger.info("GFAC instance node data: " + gfacNodeData);
- String[] split = gfacNodeData.split(":");
- gfacClient = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
- if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
- // before submitting the job we check again the state of the node
- if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, tokenId)) {
- String gatewayId = null;
- CredentialReader credentialReader = GFacUtils.getCredentialReader();
- if (credentialReader != null) {
- try {
- gatewayId = credentialReader.getGatewayID(tokenId);
- } catch (Exception e) {
- logger.error(e.getLocalizedMessage());
- }
- }
- if(gatewayId == null || gatewayId.isEmpty()){
- gatewayId = ServerSettings.getDefaultUserGateway();
- }
- TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, gatewayId);
- MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK, "LAUNCH.TASK-" + UUID.randomUUID().toString(), gatewayId);
- messageContext.setUpdatedTime(AiravataUtils.getCurrentTimestamp());
- publisher.publish(messageContext);
+ String gatewayId = null;
+ CredentialReader credentialReader = GFacUtils.getCredentialReader();
+ if (credentialReader != null) {
+ try {
+ gatewayId = credentialReader.getGatewayID(tokenId);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage());
}
}
+ if(gatewayId == null || gatewayId.isEmpty()){
+ gatewayId = ServerSettings.getDefaultUserGateway();
+ }
+
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, gatewayId,tokenId);
+ MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK, "LAUNCH.TASK-" + UUID.randomUUID().toString(), gatewayId);
+ messageContext.setUpdatedTime(AiravataUtils.getCurrentTimestamp());
+ publisher.publish(messageContext);
}
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
@@ -204,8 +192,8 @@ public class GFACPassiveJobSubmitter implements JobSubmitter,Watcher {
localhost = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
// before submitting the job we check again the state of the node
- if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
- TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, null);
+ if (GFacUtils.createExperimentEntryForRPC(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent(experimentID, taskID, null,null);
MessageContext messageContext = new MessageContext(taskSubmitEvent, MessageType.LAUNCHTASK,"LAUNCH.TERMINATE-"+ UUID.randomUUID().toString(),null);
publisher.publish(messageContext);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/a486b67d/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
index 54339a2..b855de2 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACRPCJobSubmitter.java
@@ -99,7 +99,7 @@ public class GFACRPCJobSubmitter implements JobSubmitter, Watcher {
gfacClient = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
// before submitting the job we check again the state of the node
- if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, tokenId)) {
+ if (GFacUtils.createExperimentEntryForRPC(experimentID, taskID, zk, experimentNode, pickedChild, tokenId)) {
String gatewayId = null;
CredentialReader credentialReader = GFacUtils.getCredentialReader();
if (credentialReader != null) {
@@ -167,7 +167,7 @@ public class GFACRPCJobSubmitter implements JobSubmitter, Watcher {
localhost = GFacClientFactory.createGFacClient(split[0], Integer.parseInt(split[1]));
if (zk.exists(gfacServer + File.separator + pickedChild, false) != null) {
// before submitting the job we check again the state of the node
- if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
+ if (GFacUtils.createExperimentEntryForRPC(experimentID, taskID, zk, experimentNode, pickedChild, null)) {
return localhost.cancelJob(experimentID, taskID);
}
}
[03/15] airavata git commit: adding consumers to gfac without leader
election
Posted by la...@apache.org.
adding consumers to gfac without leader election
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/0149c1af
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/0149c1af
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/0149c1af
Branch: refs/heads/master
Commit: 0149c1afd477ab1d86d19374bcb9824520d3a3bc
Parents: 88d27d9
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Wed Feb 11 15:48:40 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Wed Feb 11 15:48:40 2015 -0500
----------------------------------------------------------------------
.../airavata/common/utils/ServerSettings.java | 21 +-
.../main/resources/airavata-server.properties | 7 +-
.../main/resources/airavata-server.properties | 5 +-
.../airavata/gfac/server/GfacServerHandler.java | 75 ++++-
.../messaging/core/MessagingConstants.java | 3 +-
.../messaging/core/PublisherFactory.java | 2 +-
.../airavata/messaging/core/TestClient.java | 5 +-
.../messaging/core/impl/RabbitMQConsumer.java | 258 -----------------
.../core/impl/RabbitMQStatusConsumer.java | 274 +++++++++++++++++++
.../core/impl/RabbitMQStatusPublisher.java | 2 +-
.../core/impl/RabbitMQTaskLaunchConsumer.java | 239 ++++++++++++++++
.../core/impl/RabbitMQTaskLaunchPublisher.java | 12 +-
.../core/utils/OrchestratorConstants.java | 1 -
.../airavata/xbaya/messaging/Monitor.java | 5 +-
14 files changed, 614 insertions(+), 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index 4ea0b44..b076e6a 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -28,7 +28,7 @@ import org.apache.airavata.common.exception.ApplicationSettingsException;
public class ServerSettings extends ApplicationSettings {
- private static final String DEFAULT_USER = "default.registry.user";
+ private static final String DEFAULT_USER = "default.registry.user";
private static final String DEFAULT_USER_PASSWORD = "default.registry.password";
private static final String DEFAULT_USER_GATEWAY = "default.registry.gateway";
@@ -51,13 +51,15 @@ public class ServerSettings extends ApplicationSettings {
private static final String MY_PROXY_USER = "myproxy.user";
private static final String MY_PROXY_PASSWORD = "myproxy.password";
private static final String MY_PROXY_LIFETIME = "myproxy.life";
- private static final String ACTIVITY_PUBLISHER = "activity.publisher";
+ private static final String STATUS_PUBLISHER = "status.publisher";
private static final String TASK_LAUNCH_PUBLISHER = "task.launch.publisher";
private static final String ACTIVITY_LISTENERS = "activity.listeners";
public static final String PUBLISH_RABBITMQ = "publish.rabbitmq";
public static final String JOB_NOTIFICATION_ENABLE = "job.notification.enable";
public static final String JOB_NOTIFICATION_EMAILIDS = "job.notification.emailids";
public static final String JOB_NOTIFICATION_FLAGS = "job.notification.flags";
+ public static final String GFAC_PASSIVE = "gfac.passive"; // by default this is desabled
+
private static boolean stopAllThreads = false;
@@ -73,7 +75,7 @@ public class ServerSettings extends ApplicationSettings {
return getSetting(DEFAULT_USER_GATEWAY);
}
- public static String getServerContextRoot() {
+ public static String getServerContextRoot() {
return getSetting(SERVER_CONTEXT_ROOT, "axis2");
}
@@ -151,19 +153,24 @@ public class ServerSettings extends ApplicationSettings {
return getSetting(ACTIVITY_LISTENERS).split(",");
}
- public static String getActivityPublisher() throws ApplicationSettingsException{
- return getSetting(ACTIVITY_PUBLISHER);
+ public static String getStatusPublisher() throws ApplicationSettingsException {
+ return getSetting(STATUS_PUBLISHER);
}
- public static String getTaskLaunchPublisher() throws ApplicationSettingsException{
+ public static String getTaskLaunchPublisher() throws ApplicationSettingsException {
return getSetting(TASK_LAUNCH_PUBLISHER);
}
- public static boolean isRabbitMqPublishEnabled() throws ApplicationSettingsException{
+ public static boolean isRabbitMqPublishEnabled() throws ApplicationSettingsException {
String setting = getSetting(PUBLISH_RABBITMQ);
return Boolean.parseBoolean(setting);
}
+ public static boolean isGFacPassiveMode()throws ApplicationSettingsException {
+ String setting = getSetting(GFAC_PASSIVE);
+ return Boolean.parseBoolean(setting);
+ }
+
public static boolean isEmbeddedZK() {
return Boolean.parseBoolean(getSetting(EMBEDDED_ZK, "true"));
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index c90fab1..e309901 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -137,6 +137,7 @@ myproxy.password=
myproxy.life=3600
# XSEDE Trusted certificates can be downloaded from https://software.xsede.org/security/xsede-certs.tar.gz
trusted.cert.location=/Users/lahirugunathilake/Downloads/certificates
+gfac.passive=false
# SSH PKI key pair or ssh password can be used SSH based authentication is used.
# if user specify both password authentication gets the higher preference
@@ -215,10 +216,12 @@ connection.name=xsede
#publisher
activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
publish.rabbitmq=false
-activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQPublisher
+status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
rabbitmq.broker.url=amqp://localhost:5672
-rabbitmq.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
+
###########################################################################
# Orchestrator module Configuration
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties b/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
index d6be51a..2ecdeb6 100644
--- a/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
+++ b/modules/credential-store-service/credential-store-webapp/src/main/resources/airavata-server.properties
@@ -200,10 +200,11 @@ connection.name=xsede
#publisher
activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
publish.rabbitmq=false
-activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQPublisher
+status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
rabbitmq.broker.url=amqp://localhost:5672
-rabbitmq.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
###########################################################################
# Orchestrator module Configuration
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index 583ec07..d428d9c 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -27,28 +27,34 @@ import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.logger.AiravataLogger;
import org.apache.airavata.common.logger.AiravataLoggerFactory;
-import org.apache.airavata.common.utils.AiravataZKUtils;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.common.utils.*;
import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
import org.apache.airavata.gfac.core.cpi.GFac;
import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
import org.apache.airavata.gfac.core.utils.InputHandlerWorker;
import org.apache.airavata.gfac.cpi.GfacService;
import org.apache.airavata.gfac.cpi.gfac_cpi_serviceConstants;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.MessageHandler;
+import org.apache.airavata.messaging.core.MessagingConstants;
+import org.apache.airavata.messaging.core.impl.RabbitMQStatusConsumer;
+import org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchConsumer;
+import org.apache.airavata.model.messaging.event.ExperimentStatusChangeEvent;
+import org.apache.airavata.model.messaging.event.MessageType;
+import org.apache.airavata.model.messaging.event.TaskSubmitEvent;
+import org.apache.airavata.model.messaging.event.TaskTerminateEvent;
+import org.apache.airavata.model.workspace.experiment.ExperimentState;
import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
import org.apache.airavata.registry.cpi.Registry;
import org.apache.airavata.registry.cpi.RegistryException;
+import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.Future;
@@ -80,6 +86,8 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private List<Future> inHandlerFutures;
+ private RabbitMQTaskLaunchConsumer rabbitMQTaskLaunchConsumer;
+
public GfacServerHandler() throws Exception{
// registering with zk
try {
@@ -102,7 +110,13 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
BetterGfacImpl.startDaemonHandlers();
BetterGfacImpl.startStatusUpdators(registry, zk, publisher);
inHandlerFutures = new ArrayList<Future>();
- } catch (ApplicationSettingsException e) {
+
+ if(ServerSettings.isGFacPassiveMode()) {
+ rabbitMQTaskLaunchConsumer = new RabbitMQTaskLaunchConsumer();
+ }
+
+
+ } catch (ApplicationSettingsException e) {
logger.error("Error initialising GFAC", e);
throw new Exception("Error initialising GFAC", e);
} catch (InterruptedException e) {
@@ -277,4 +291,49 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
}
}
+ private class NotificationMessageHandler implements MessageHandler {
+ private String experimentId;
+
+ private NotificationMessageHandler(String experimentId) {
+ this.experimentId = experimentId;
+ }
+
+ public Map<String, Object> getProperties() {
+ Map<String, Object> props = new HashMap<String, Object>();
+ List<String> routingKeys = new ArrayList<String>();
+ routingKeys.add(experimentId);
+ routingKeys.add(experimentId + ".*");
+ props.put(MessagingConstants.RABBIT_ROUTING_KEY, routingKeys);
+ return props;
+ }
+
+ public void onMessage(MessageContext message) {
+ if (message.getType().equals(MessageType.LAUNCHTASK)){
+ try {
+ TaskSubmitEvent event = new TaskSubmitEvent();
+ TBase messageEvent = message.getEvent();
+ byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
+ ThriftUtils.createThriftFromBytes(bytes, event);
+ submitJob(event.getExperimentId(), event.getTaskId(), event.getGatewayId());
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType());
+ } catch (TException e) {
+ logger.error(e.getMessage(), e); //nobody is listening so nothing to throw
+ }
+ }else if(message.getType().equals(MessageType.TERMINATETASK)){
+ try {
+ TaskTerminateEvent event = new TaskTerminateEvent();
+ TBase messageEvent = message.getEvent();
+ byte[] bytes = ThriftUtils.serializeThriftObject(messageEvent);
+ ThriftUtils.createThriftFromBytes(bytes, event);
+ cancelJob(event.getExperimentId(), event.getTaskId());
+ System.out.println(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getType());
+ } catch (TException e) {
+ logger.error(e.getMessage(), e); //nobody is listening so nothing to throw
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessagingConstants.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessagingConstants.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessagingConstants.java
index 7458d81..07b39e7 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessagingConstants.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/MessagingConstants.java
@@ -23,7 +23,8 @@ package org.apache.airavata.messaging.core;
public abstract class MessagingConstants {
public static final String RABBITMQ_BROKER_URL = "rabbitmq.broker.url";
- public static final String RABBITMQ_EXCHANGE_NAME = "rabbitmq.exchange.name";
+ public static final String RABBITMQ_STATUS_EXCHANGE_NAME = "rabbitmq.status.exchange.name";
+ public static final String RABBITMQ_TASK_LAUNCH_EXCHANGE_NAME = "rabbitmq.task.launch.exchange.name";
public static final String RABBIT_ROUTING_KEY = "routingKey";
public static final String RABBIT_QUEUE= "queue";
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
index 59cdbdf..2e560a3 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/PublisherFactory.java
@@ -30,7 +30,7 @@ public class PublisherFactory {
private static Logger log = LoggerFactory.getLogger(PublisherFactory.class);
public static Publisher createActivityPublisher() throws AiravataException {
- String activityPublisher = ServerSettings.getActivityPublisher();
+ String activityPublisher = ServerSettings.getStatusPublisher();
if (activityPublisher == null) {
String s = "Activity publisher is not specified";
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/TestClient.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/TestClient.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/TestClient.java
index 362f3f2..aea561f 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/TestClient.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/TestClient.java
@@ -25,9 +25,8 @@ import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.AiravataUtils;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.common.utils.ThriftUtils;
-import org.apache.airavata.messaging.core.impl.RabbitMQConsumer;
+import org.apache.airavata.messaging.core.impl.RabbitMQStatusConsumer;
import org.apache.airavata.model.messaging.event.ExperimentStatusChangeEvent;
-import org.apache.airavata.model.messaging.event.Message;
import org.apache.airavata.model.messaging.event.MessageType;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
@@ -51,7 +50,7 @@ public class TestClient {
AiravataUtils.setExecutionAsServer();
String brokerUrl = ServerSettings.getSetting(RABBITMQ_BROKER_URL);
final String exchangeName = ServerSettings.getSetting(RABBITMQ_EXCHANGE_NAME);
- RabbitMQConsumer consumer = new RabbitMQConsumer(brokerUrl, exchangeName);
+ RabbitMQStatusConsumer consumer = new RabbitMQStatusConsumer(brokerUrl, exchangeName);
consumer.listen(new MessageHandler() {
@Override
public Map<String, Object> getProperties() {
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQConsumer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQConsumer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQConsumer.java
deleted file mode 100644
index 1f13496..0000000
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQConsumer.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.messaging.core.impl;
-
-
-import com.rabbitmq.client.*;
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.AiravataUtils;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.common.utils.ThriftUtils;
-import org.apache.airavata.messaging.core.Consumer;
-import org.apache.airavata.messaging.core.MessageContext;
-import org.apache.airavata.messaging.core.MessageHandler;
-import org.apache.airavata.messaging.core.MessagingConstants;
-import org.apache.airavata.model.messaging.event.*;
-import org.apache.thrift.TBase;
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class RabbitMQConsumer implements Consumer {
- private static Logger log = LoggerFactory.getLogger(RabbitMQConsumer.class);
-
- private String exchangeName;
- private String url;
- private Connection connection;
- private Channel channel;
- private Map<String, QueueDetails> queueDetailsMap = new HashMap<String, QueueDetails>();
-
- public RabbitMQConsumer() throws AiravataException {
- try {
- url = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
- exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_EXCHANGE_NAME);
-
- createConnection();
- } catch (ApplicationSettingsException e) {
- String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
- log.error(message, e);
- throw new AiravataException(message, e);
- }
- }
-
- public RabbitMQConsumer(String brokerUrl, String exchangeName) throws AiravataException {
- this.exchangeName = exchangeName;
- this.url = brokerUrl;
-
- createConnection();
- }
-
- private void createConnection() throws AiravataException {
- try {
- ConnectionFactory connectionFactory = new ConnectionFactory();
- connectionFactory.setUri(url);
- connection = connectionFactory.newConnection();
- connection.addShutdownListener(new ShutdownListener() {
- public void shutdownCompleted(ShutdownSignalException cause) {
- }
- });
- log.info("connected to rabbitmq: " + connection + " for " + exchangeName);
-
- channel = connection.createChannel();
- channel.exchangeDeclare(exchangeName, "topic", false);
-
- } catch (Exception e) {
- String msg = "could not open channel for exchange " + exchangeName;
- log.error(msg);
- throw new AiravataException(msg, e);
- }
- }
-
- public String listen(final MessageHandler handler) throws AiravataException {
- try {
- Map<String, Object> props = handler.getProperties();
- final Object routing = props.get(MessagingConstants.RABBIT_ROUTING_KEY);
- if (routing == null) {
- throw new IllegalArgumentException("The routing key must be present");
- }
-
- List<String> keys = new ArrayList<String>();
- if (routing instanceof List) {
- for (Object o : (List)routing) {
- keys.add(o.toString());
- }
- } else if (routing instanceof String) {
- keys.add((String) routing);
- }
-
- String queueName = (String) props.get(MessagingConstants.RABBIT_QUEUE);
- String consumerTag = (String) props.get(MessagingConstants.RABBIT_CONSUMER_TAG);
- if (queueName == null) {
- if (!channel.isOpen()) {
- channel = connection.createChannel();
- channel.exchangeDeclare(exchangeName, "topic", false);
- }
- queueName = channel.queueDeclare().getQueue();
- } else {
- channel.queueDeclare(queueName, true, false, false, null);
- }
-
- final String id = getId(keys, queueName);
- if (queueDetailsMap.containsKey(id)) {
- throw new IllegalStateException("This subscriber is already defined for this Consumer, " +
- "cannot define the same subscriber twice");
- }
-
- if (consumerTag == null) {
- consumerTag = "default";
- }
-
- // bind all the routing keys
- for (String routingKey : keys) {
- channel.queueBind(queueName, exchangeName, routingKey);
- }
-
- channel.basicConsume(queueName, true, consumerTag, new DefaultConsumer(channel) {
- @Override
- public void handleDelivery(String consumerTag,
- Envelope envelope,
- AMQP.BasicProperties properties,
- byte[] body) {
- Message message = new Message();
-
- try {
- ThriftUtils.createThriftFromBytes(body, message);
- TBase event = null;
- String gatewayId = null;
- if (message.getMessageType().equals(MessageType.EXPERIMENT)) {
- ExperimentStatusChangeEvent experimentStatusChangeEvent = new ExperimentStatusChangeEvent();
- ThriftUtils.createThriftFromBytes(message.getEvent(), experimentStatusChangeEvent);
- log.debug(" Message Received with message id '" + message.getMessageId()
- + "' and with message type '" + message.getMessageType() + "' with status " +
- experimentStatusChangeEvent.getState());
- event = experimentStatusChangeEvent;
- gatewayId = experimentStatusChangeEvent.getGatewayId();
- } else if (message.getMessageType().equals(MessageType.WORKFLOWNODE)) {
- WorkflowNodeStatusChangeEvent wfnStatusChangeEvent = new WorkflowNodeStatusChangeEvent();
- ThriftUtils.createThriftFromBytes(message.getEvent(), wfnStatusChangeEvent);
- log.debug(" Message Received with message id '" + message.getMessageId()
- + "' and with message type '" + message.getMessageType() + "' with status " +
- wfnStatusChangeEvent.getState());
- event = wfnStatusChangeEvent;
- gatewayId = wfnStatusChangeEvent.getWorkflowNodeIdentity().getGatewayId();
- } else if (message.getMessageType().equals(MessageType.TASK)) {
- TaskStatusChangeEvent taskStatusChangeEvent = new TaskStatusChangeEvent();
- ThriftUtils.createThriftFromBytes(message.getEvent(), taskStatusChangeEvent);
- log.debug(" Message Received with message id '" + message.getMessageId()
- + "' and with message type '" + message.getMessageType() + "' with status " +
- taskStatusChangeEvent.getState());
- event = taskStatusChangeEvent;
- gatewayId = taskStatusChangeEvent.getTaskIdentity().getGatewayId();
- } else if (message.getMessageType().equals(MessageType.JOB)) {
- JobStatusChangeEvent jobStatusChangeEvent = new JobStatusChangeEvent();
- ThriftUtils.createThriftFromBytes(message.getEvent(), jobStatusChangeEvent);
- log.debug(" Message Received with message id '" + message.getMessageId()
- + "' and with message type '" + message.getMessageType() + "' with status " +
- jobStatusChangeEvent.getState());
- event = jobStatusChangeEvent;
- gatewayId = jobStatusChangeEvent.getJobIdentity().getGatewayId();
- }
- MessageContext messageContext = new MessageContext(event, message.getMessageType(), message.getMessageId(), gatewayId);
- messageContext.setUpdatedTime(AiravataUtils.getTime(message.getUpdatedTime()));
- handler.onMessage(messageContext);
- } catch (TException e) {
- String msg = "Failed to de-serialize the thrift message, from routing keys and queueName " + id;
- log.warn(msg, e);
- }
- }
- });
- // save the name for deleting the queue
- queueDetailsMap.put(id, new QueueDetails(queueName, keys));
- return id;
- } catch (Exception e) {
- String msg = "could not open channel for exchange " + exchangeName;
- log.error(msg);
- throw new AiravataException(msg, e);
- }
- }
-
- public void stopListen(final String id) throws AiravataException {
- QueueDetails details = queueDetailsMap.get(id);
- if (details != null) {
- try {
- for (String key : details.getRoutingKeys()) {
- channel.queueUnbind(details.getQueueName(), exchangeName, key);
- }
- channel.queueDelete(details.getQueueName(), true, true);
- } catch (IOException e) {
- String msg = "could not un-bind queue: " + details.getQueueName() + " for exchange " + exchangeName;
- log.debug(msg);
- }
- }
- }
-
- /**
- * Private class for holding some information about the consumers registered
- */
- private class QueueDetails {
- String queueName;
-
- List<String> routingKeys;
-
- private QueueDetails(String queueName, List<String> routingKeys) {
- this.queueName = queueName;
- this.routingKeys = routingKeys;
- }
-
- public String getQueueName() {
- return queueName;
- }
-
- public List<String> getRoutingKeys() {
- return routingKeys;
- }
- }
-
- private String getId(List<String> routingKeys, String queueName) {
- String id = "";
- for (String key : routingKeys) {
- id = id + "_" + key;
- }
- return id + "_" + queueName;
- }
-
- public void close() {
- if (connection != null) {
- try {
- connection.close();
- } catch (IOException ignore) {
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusConsumer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusConsumer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusConsumer.java
new file mode 100644
index 0000000..d5e8c72
--- /dev/null
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusConsumer.java
@@ -0,0 +1,274 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.messaging.core.impl;
+
+
+import com.rabbitmq.client.*;
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.AiravataUtils;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ThriftUtils;
+import org.apache.airavata.messaging.core.Consumer;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.MessageHandler;
+import org.apache.airavata.messaging.core.MessagingConstants;
+import org.apache.airavata.model.messaging.event.*;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RabbitMQStatusConsumer implements Consumer {
+ private static Logger log = LoggerFactory.getLogger(RabbitMQStatusConsumer.class);
+
+ private String exchangeName;
+ private String url;
+ private Connection connection;
+ private Channel channel;
+ private Map<String, QueueDetails> queueDetailsMap = new HashMap<String, QueueDetails>();
+
+ public RabbitMQStatusConsumer() throws AiravataException {
+ try {
+ url = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_STATUS_EXCHANGE_NAME);
+
+ createConnection();
+ } catch (ApplicationSettingsException e) {
+ String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
+ log.error(message, e);
+ throw new AiravataException(message, e);
+ }
+ }
+
+ public RabbitMQStatusConsumer(String brokerUrl, String exchangeName) throws AiravataException {
+ this.exchangeName = exchangeName;
+ this.url = brokerUrl;
+
+ createConnection();
+ }
+
+ private void createConnection() throws AiravataException {
+ try {
+ ConnectionFactory connectionFactory = new ConnectionFactory();
+ connectionFactory.setUri(url);
+ connection = connectionFactory.newConnection();
+ connection.addShutdownListener(new ShutdownListener() {
+ public void shutdownCompleted(ShutdownSignalException cause) {
+ }
+ });
+ log.info("connected to rabbitmq: " + connection + " for " + exchangeName);
+
+ channel = connection.createChannel();
+ channel.exchangeDeclare(exchangeName, "topic", false);
+
+ } catch (Exception e) {
+ String msg = "could not open channel for exchange " + exchangeName;
+ log.error(msg);
+ throw new AiravataException(msg, e);
+ }
+ }
+
+ public String listen(final MessageHandler handler) throws AiravataException {
+ try {
+ Map<String, Object> props = handler.getProperties();
+ final Object routing = props.get(MessagingConstants.RABBIT_ROUTING_KEY);
+ if (routing == null) {
+ throw new IllegalArgumentException("The routing key must be present");
+ }
+
+ List<String> keys = new ArrayList<String>();
+ if (routing instanceof List) {
+ for (Object o : (List)routing) {
+ keys.add(o.toString());
+ }
+ } else if (routing instanceof String) {
+ keys.add((String) routing);
+ }
+
+ String queueName = (String) props.get(MessagingConstants.RABBIT_QUEUE);
+ String consumerTag = (String) props.get(MessagingConstants.RABBIT_CONSUMER_TAG);
+ if (queueName == null) {
+ if (!channel.isOpen()) {
+ channel = connection.createChannel();
+ channel.exchangeDeclare(exchangeName, "topic", false);
+ }
+ queueName = channel.queueDeclare().getQueue();
+ } else {
+ channel.queueDeclare(queueName, true, false, false, null);
+ }
+
+ final String id = getId(keys, queueName);
+ if (queueDetailsMap.containsKey(id)) {
+ throw new IllegalStateException("This subscriber is already defined for this Consumer, " +
+ "cannot define the same subscriber twice");
+ }
+
+ if (consumerTag == null) {
+ consumerTag = "default";
+ }
+
+ // bind all the routing keys
+ for (String routingKey : keys) {
+ channel.queueBind(queueName, exchangeName, routingKey);
+ }
+
+ channel.basicConsume(queueName, true, consumerTag, new DefaultConsumer(channel) {
+ @Override
+ public void handleDelivery(String consumerTag,
+ Envelope envelope,
+ AMQP.BasicProperties properties,
+ byte[] body) {
+ Message message = new Message();
+
+ try {
+ ThriftUtils.createThriftFromBytes(body, message);
+ TBase event = null;
+ String gatewayId = null;
+ if (message.getMessageType().equals(MessageType.EXPERIMENT)) {
+ ExperimentStatusChangeEvent experimentStatusChangeEvent = new ExperimentStatusChangeEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), experimentStatusChangeEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' with status " +
+ experimentStatusChangeEvent.getState());
+ event = experimentStatusChangeEvent;
+ gatewayId = experimentStatusChangeEvent.getGatewayId();
+ } else if (message.getMessageType().equals(MessageType.WORKFLOWNODE)) {
+ WorkflowNodeStatusChangeEvent wfnStatusChangeEvent = new WorkflowNodeStatusChangeEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), wfnStatusChangeEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' with status " +
+ wfnStatusChangeEvent.getState());
+ event = wfnStatusChangeEvent;
+ gatewayId = wfnStatusChangeEvent.getWorkflowNodeIdentity().getGatewayId();
+ } else if (message.getMessageType().equals(MessageType.TASK)) {
+ TaskStatusChangeEvent taskStatusChangeEvent = new TaskStatusChangeEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), taskStatusChangeEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' with status " +
+ taskStatusChangeEvent.getState());
+ event = taskStatusChangeEvent;
+ gatewayId = taskStatusChangeEvent.getTaskIdentity().getGatewayId();
+ } else if (message.getMessageType().equals(MessageType.JOB)) {
+ JobStatusChangeEvent jobStatusChangeEvent = new JobStatusChangeEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), jobStatusChangeEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' with status " +
+ jobStatusChangeEvent.getState());
+ event = jobStatusChangeEvent;
+ gatewayId = jobStatusChangeEvent.getJobIdentity().getGatewayId();
+ }else if(message.getMessageType().equals(MessageType.LAUNCHTASK)) {
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), taskSubmitEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' for experimentId: " +
+ taskSubmitEvent.getExperimentId() + "and taskId: " + taskSubmitEvent.getTaskId());
+ event = taskSubmitEvent;
+ gatewayId = taskSubmitEvent.getGatewayId();
+ }else if(message.getMessageType().equals(MessageType.TERMINATETASK)) {
+ TaskTerminateEvent taskTerminateEvent = new TaskTerminateEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), taskTerminateEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' for experimentId: " +
+ taskTerminateEvent.getExperimentId() + "and taskId: " + taskTerminateEvent.getTaskId());
+ event = taskTerminateEvent;
+ gatewayId = null;
+ }
+ MessageContext messageContext = new MessageContext(event, message.getMessageType(), message.getMessageId(), gatewayId);
+ messageContext.setUpdatedTime(AiravataUtils.getTime(message.getUpdatedTime()));
+ handler.onMessage(messageContext);
+ } catch (TException e) {
+ String msg = "Failed to de-serialize the thrift message, from routing keys and queueName " + id;
+ log.warn(msg, e);
+ }
+ }
+ });
+ // save the name for deleting the queue
+ queueDetailsMap.put(id, new QueueDetails(queueName, keys));
+ return id;
+ } catch (Exception e) {
+ String msg = "could not open channel for exchange " + exchangeName;
+ log.error(msg);
+ throw new AiravataException(msg, e);
+ }
+ }
+
+ public void stopListen(final String id) throws AiravataException {
+ QueueDetails details = queueDetailsMap.get(id);
+ if (details != null) {
+ try {
+ for (String key : details.getRoutingKeys()) {
+ channel.queueUnbind(details.getQueueName(), exchangeName, key);
+ }
+ channel.queueDelete(details.getQueueName(), true, true);
+ } catch (IOException e) {
+ String msg = "could not un-bind queue: " + details.getQueueName() + " for exchange " + exchangeName;
+ log.debug(msg);
+ }
+ }
+ }
+
+ /**
+ * Private class for holding some information about the consumers registered
+ */
+ private class QueueDetails {
+ String queueName;
+
+ List<String> routingKeys;
+
+ private QueueDetails(String queueName, List<String> routingKeys) {
+ this.queueName = queueName;
+ this.routingKeys = routingKeys;
+ }
+
+ public String getQueueName() {
+ return queueName;
+ }
+
+ public List<String> getRoutingKeys() {
+ return routingKeys;
+ }
+ }
+
+ private String getId(List<String> routingKeys, String queueName) {
+ String id = "";
+ for (String key : routingKeys) {
+ id = id + "_" + key;
+ }
+ return id + "_" + queueName;
+ }
+
+ public void close() {
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (IOException ignore) {
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
index a4b4d1a..fe06ed7 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
@@ -45,7 +45,7 @@ public class RabbitMQStatusPublisher implements Publisher {
String exchangeName;
try {
brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
- exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_EXCHANGE_NAME);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_STATUS_EXCHANGE_NAME);
} catch (ApplicationSettingsException e) {
String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
log.error(message, e);
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
new file mode 100644
index 0000000..056dcac
--- /dev/null
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchConsumer.java
@@ -0,0 +1,239 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.messaging.core.impl;
+
+import com.rabbitmq.client.*;
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.AiravataUtils;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ThriftUtils;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.MessageHandler;
+import org.apache.airavata.messaging.core.MessagingConstants;
+import org.apache.airavata.model.messaging.event.*;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RabbitMQTaskLaunchConsumer {
+ private final static Logger logger = LoggerFactory.getLogger(RabbitMQTaskLaunchConsumer.class);
+ private static Logger log = LoggerFactory.getLogger(RabbitMQStatusConsumer.class);
+
+ private String taskLaunchExchangeName;
+ private String url;
+ private Connection connection;
+ private Channel channel;
+ private Map<String, QueueDetails> queueDetailsMap = new HashMap<String, QueueDetails>();
+
+ public RabbitMQTaskLaunchConsumer() throws AiravataException {
+ try {
+ url = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
+ taskLaunchExchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_TASK_LAUNCH_EXCHANGE_NAME);
+ createConnection();
+ } catch (ApplicationSettingsException e) {
+ String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
+ log.error(message, e);
+ throw new AiravataException(message, e);
+ }
+ }
+
+ public RabbitMQTaskLaunchConsumer(String brokerUrl, String exchangeName) throws AiravataException {
+ this.taskLaunchExchangeName = exchangeName;
+ this.url = brokerUrl;
+
+ createConnection();
+ }
+
+ private void createConnection() throws AiravataException {
+ try {
+ ConnectionFactory connectionFactory = new ConnectionFactory();
+ connectionFactory.setUri(url);
+ connection = connectionFactory.newConnection();
+ connection.addShutdownListener(new ShutdownListener() {
+ public void shutdownCompleted(ShutdownSignalException cause) {
+ }
+ });
+ log.info("connected to rabbitmq: " + connection + " for " + taskLaunchExchangeName);
+
+ channel = connection.createChannel();
+ channel.exchangeDeclare(taskLaunchExchangeName, "fanout");
+
+ } catch (Exception e) {
+ String msg = "could not open channel for exchange " + taskLaunchExchangeName;
+ log.error(msg);
+ throw new AiravataException(msg, e);
+ }
+ }
+
+ public String listen(final MessageHandler handler) throws AiravataException {
+ try {
+ Map<String, Object> props = handler.getProperties();
+ final Object routing = props.get(MessagingConstants.RABBIT_ROUTING_KEY);
+ if (routing == null) {
+ throw new IllegalArgumentException("The routing key must be present");
+ }
+
+ List<String> keys = new ArrayList<String>();
+ if (routing instanceof List) {
+ for (Object o : (List)routing) {
+ keys.add(o.toString());
+ }
+ } else if (routing instanceof String) {
+ keys.add((String) routing);
+ }
+
+ String queueName = (String) props.get(MessagingConstants.RABBIT_QUEUE);
+ String consumerTag = (String) props.get(MessagingConstants.RABBIT_CONSUMER_TAG);
+ if (queueName == null) {
+ if (!channel.isOpen()) {
+ channel = connection.createChannel();
+ channel.exchangeDeclare(taskLaunchExchangeName, "fanout");
+ }
+ queueName = channel.queueDeclare().getQueue();
+ } else {
+ channel.queueDeclare(queueName, true, false, false, null);
+ }
+
+ final String id = getId(keys, queueName);
+ if (queueDetailsMap.containsKey(id)) {
+ throw new IllegalStateException("This subscriber is already defined for this Consumer, " +
+ "cannot define the same subscriber twice");
+ }
+
+ if (consumerTag == null) {
+ consumerTag = "default";
+ }
+
+ // bind all the routing keys
+ for (String routingKey : keys) {
+ channel.queueBind(queueName, taskLaunchExchangeName, routingKey);
+ }
+
+ channel.basicConsume(queueName, true, consumerTag, new DefaultConsumer(channel) {
+ @Override
+ public void handleDelivery(String consumerTag,
+ Envelope envelope,
+ AMQP.BasicProperties properties,
+ byte[] body) {
+ Message message = new Message();
+
+ try {
+ ThriftUtils.createThriftFromBytes(body, message);
+ TBase event = null;
+ String gatewayId = null;
+ if(message.getMessageType().equals(MessageType.LAUNCHTASK)) {
+ TaskSubmitEvent taskSubmitEvent = new TaskSubmitEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), taskSubmitEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' for experimentId: " +
+ taskSubmitEvent.getExperimentId() + "and taskId: " + taskSubmitEvent.getTaskId());
+ event = taskSubmitEvent;
+ gatewayId = taskSubmitEvent.getGatewayId();
+ }else if(message.getMessageType().equals(MessageType.TERMINATETASK)) {
+ TaskTerminateEvent taskTerminateEvent = new TaskTerminateEvent();
+ ThriftUtils.createThriftFromBytes(message.getEvent(), taskTerminateEvent);
+ log.debug(" Message Received with message id '" + message.getMessageId()
+ + "' and with message type '" + message.getMessageType() + "' for experimentId: " +
+ taskTerminateEvent.getExperimentId() + "and taskId: " + taskTerminateEvent.getTaskId());
+ event = taskTerminateEvent;
+ gatewayId = null;
+ }
+ MessageContext messageContext = new MessageContext(event, message.getMessageType(), message.getMessageId(), gatewayId);
+ messageContext.setUpdatedTime(AiravataUtils.getTime(message.getUpdatedTime()));
+ handler.onMessage(messageContext);
+ } catch (TException e) {
+ String msg = "Failed to de-serialize the thrift message, from routing keys and queueName " + id;
+ log.warn(msg, e);
+ }
+ }
+ });
+ // save the name for deleting the queue
+ queueDetailsMap.put(id, new QueueDetails(queueName, keys));
+ return id;
+ } catch (Exception e) {
+ String msg = "could not open channel for exchange " + taskLaunchExchangeName;
+ log.error(msg);
+ throw new AiravataException(msg, e);
+ }
+ }
+
+ public void stopListen(final String id) throws AiravataException {
+ QueueDetails details = queueDetailsMap.get(id);
+ if (details != null) {
+ try {
+ for (String key : details.getRoutingKeys()) {
+ channel.queueUnbind(details.getQueueName(), taskLaunchExchangeName, key);
+ }
+ channel.queueDelete(details.getQueueName(), true, true);
+ } catch (IOException e) {
+ String msg = "could not un-bind queue: " + details.getQueueName() + " for exchange " + taskLaunchExchangeName;
+ log.debug(msg);
+ }
+ }
+ }
+
+ /**
+ * Private class for holding some information about the consumers registered
+ */
+ private class QueueDetails {
+ String queueName;
+
+ List<String> routingKeys;
+
+ private QueueDetails(String queueName, List<String> routingKeys) {
+ this.queueName = queueName;
+ this.routingKeys = routingKeys;
+ }
+
+ public String getQueueName() {
+ return queueName;
+ }
+
+ public List<String> getRoutingKeys() {
+ return routingKeys;
+ }
+ }
+
+ private String getId(List<String> routingKeys, String queueName) {
+ String id = "";
+ for (String key : routingKeys) {
+ id = id + "_" + key;
+ }
+ return id + "_" + queueName;
+ }
+
+ public void close() {
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (IOException ignore) {
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
----------------------------------------------------------------------
diff --git a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
index 8029a0c..fe58042 100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQTaskLaunchPublisher.java
@@ -44,7 +44,7 @@ public class RabbitMQTaskLaunchPublisher implements Publisher{
String exchangeName;
try {
brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
- exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_EXCHANGE_NAME);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_STATUS_EXCHANGE_NAME);
} catch (ApplicationSettingsException e) {
String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
log.error(message, e);
@@ -56,7 +56,7 @@ public class RabbitMQTaskLaunchPublisher implements Publisher{
public void publish(MessageContext msgCtx) throws AiravataException {
try {
- log.info("Publishing to lauch queue ...");
+ log.info("Publishing to launch queue ...");
byte[] body = ThriftUtils.serializeThriftObject(msgCtx.getEvent());
Message message = new Message();
message.setEvent(body);
@@ -65,13 +65,9 @@ public class RabbitMQTaskLaunchPublisher implements Publisher{
message.setUpdatedTime(msgCtx.getUpdatedTime().getTime());
String routingKey = null;
if (msgCtx.getType().equals(MessageType.LAUNCHTASK)){
- TaskSubmitEvent event = (TaskSubmitEvent) msgCtx.getEvent();
- routingKey = LAUNCH_TASK + "."+event.getExperimentId() + "." +
- event.getTaskId() + "." + event.getGatewayId();
+ routingKey = LAUNCH_TASK;
}else if(msgCtx.getType().equals(MessageType.TERMINATETASK)){
- TaskTerminateEvent event = (TaskTerminateEvent) msgCtx.getEvent();
- routingKey = TERMINATE_TASK + "."+event.getExperimentId() + "." +
- event.getTaskId();
+ routingKey = TERMINATE_TASK;
}
byte[] messageBody = ThriftUtils.serializeThriftObject(message);
rabbitMQProducer.send(messageBody, routingKey);
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java
index 0e0e425..97b85bc 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java
@@ -25,7 +25,6 @@ package org.apache.airavata.orchestrator.core.utils;
*
*/
public class OrchestratorConstants {
- private static final String SUBMITTER_PROPERTY = "job.submitter";
public static final String AIRAVATA_PROPERTIES = "airavata-server.properties";
public static final int hotUpdateInterval=1000;
public static final String JOB_SUBMITTER = "job.submitter";
http://git-wip-us.apache.org/repos/asf/airavata/blob/0149c1af/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/messaging/Monitor.java
----------------------------------------------------------------------
diff --git a/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/messaging/Monitor.java b/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/messaging/Monitor.java
index 896b248..6ee1111 100644
--- a/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/messaging/Monitor.java
+++ b/modules/xbaya-gui/src/main/java/org/apache/airavata/xbaya/messaging/Monitor.java
@@ -22,12 +22,11 @@
package org.apache.airavata.xbaya.messaging;
import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.common.utils.AiravataUtils;
import org.apache.airavata.messaging.core.Consumer;
import org.apache.airavata.messaging.core.MessageContext;
import org.apache.airavata.messaging.core.MessageHandler;
import org.apache.airavata.messaging.core.MessagingConstants;
-import org.apache.airavata.messaging.core.impl.RabbitMQConsumer;
+import org.apache.airavata.messaging.core.impl.RabbitMQStatusConsumer;
import org.apache.airavata.model.messaging.event.MessageType;
import org.apache.airavata.model.workspace.experiment.ExperimentState;
import org.apache.airavata.workflow.model.exceptions.WorkflowException;
@@ -101,7 +100,7 @@ public class Monitor extends EventProducer {
getEventDataRepository().triggerListenerForPreMonitorStart();
try {
// AiravataUtils.setExecutionAsServer();
- this.messageClient = new RabbitMQConsumer("amqp://localhost:5672", "airavata_rabbitmq_exchange");
+ this.messageClient = new RabbitMQStatusConsumer("amqp://localhost:5672", "airavata_rabbitmq_exchange");
} catch (AiravataException e) {
String msg = "Failed to start the consumer";
logger.error(msg, e);
[15/15] airavata git commit: changing default mode to passive
Posted by la...@apache.org.
changing default mode to passive
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/aa27ce10
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/aa27ce10
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/aa27ce10
Branch: refs/heads/master
Commit: aa27ce109a6b62dab19739051b40fbecc15a4389
Parents: 3eb9163
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Mon Mar 23 15:38:20 2015 -0400
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Mon Mar 23 15:38:20 2015 -0400
----------------------------------------------------------------------
.../apache/airavata/client/samples/CreateLaunchExperiment.java | 2 +-
.../server/src/main/resources/airavata-server.properties | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/aa27ce10/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 231da87..e65acb4 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -58,7 +58,7 @@ public class CreateLaunchExperiment {
private static final String DEFAULT_GATEWAY = "php_reference_gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_78c785c6-3748-4a93-8569-19fee56581bc";
+ private static String echoAppId = "Echo_5dd52cd4-f9a0-459f-9baf-f8e715e44548";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_aa083c86-4680-4002-b3ef-fad93c181926";
http://git-wip-us.apache.org/repos/asf/airavata/blob/aa27ce10/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index 2d8bc48..61ee3c5 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -137,7 +137,7 @@ myproxy.password=
myproxy.life=3600
# XSEDE Trusted certificates can be downloaded from https://software.xsede.org/security/xsede-certs.tar.gz
trusted.cert.location=/Users/lahirugunathilake/Downloads/certificates
-gfac.passive=false
+gfac.passive=true
# SSH PKI key pair or ssh password can be used SSH based authentication is used.
# if user specify both password authentication gets the higher preference
@@ -233,8 +233,8 @@ rabbitmq.exchange.name=airavata_rabbitmq_exchange
###########################################################################
#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
-#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
-job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
+job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
+#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
submitter.interval=10000
threadpool.size=10
[08/15] airavata git commit: merging with master
Posted by la...@apache.org.
merging with master
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/ffbb1b9f
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/ffbb1b9f
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/ffbb1b9f
Branch: refs/heads/master
Commit: ffbb1b9f6776cf90a3eb9e7d418bd5bed76a0ef4
Parents: 840e627 33d2e27
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Tue Feb 24 10:57:46 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Tue Feb 24 10:57:46 2015 -0500
----------------------------------------------------------------------
.../server/handler/AiravataServerHandler.java | 86 +-
.../java/org/apache/airavata/api/Airavata.java | 6323 ++++++++++++++++--
.../main/resources/lib/airavata/Airavata.cpp | 5246 +++++++++------
.../src/main/resources/lib/airavata/Airavata.h | 636 ++
.../lib/airavata/Airavata_server.skeleton.cpp | 20 +
.../resources/lib/Airavata/API/Airavata.php | 1270 +++-
.../client/samples/CreateLaunchExperiment.java | 60 +-
.../client/samples/RegisterSampleData.java | 2 +-
.../tools/RegisterSampleApplications.java | 13 +-
.../airavataAPI.thrift | 23 +
.../appcatalog/cpi/ComputeResource.java | 3 +
.../catalog/data/impl/ComputeResourceImpl.java | 30 +-
.../catalog/data/model/UnicoreDataMovement.java | 65 +
.../data/resources/AbstractResource.java | 7 +
.../resources/UnicoreDataMovementResource.java | 255 +
.../catalog/data/util/AppCatalogJPAUtils.java | 19 +-
.../data/util/AppCatalogResourceType.java | 1 +
.../data/util/AppCatalogThriftConversion.java | 20 +-
.../src/main/resources/META-INF/persistence.xml | 1 +
.../src/main/resources/appcatalog-derby.sql | 8 +
.../src/main/resources/appcatalog-mysql.sql | 9 +
.../src/test/resources/appcatalog-derby.sql | 8 +
.../main/resources/airavata-server.properties | 6 +
.../credential/store/client/TestSSLClient.java | 140 +
.../store/server/CredentialStoreServer.java | 21 +-
.../server/CredentialStoreServerHandler.java | 37 +-
.../airavata/credential/store/util/Utility.java | 14 +-
modules/distribution/server/pom.xml | 110 +-
modules/messaging/client/README | 15 +
modules/messaging/client/pom.xml | 103 +
.../messaging/client/RabbitMQListner.java | 230 +
.../core/impl/RabbitMQStatusPublisher.java | 12 +-
.../messaging/core/stats/CountWriterTask.java | 36 +
.../messaging/core/stats/LatencyWriterTask.java | 37 +
.../messaging/core/stats/StatCounter.java | 83 +
modules/messaging/pom.xml | 1 +
36 files changed, 12261 insertions(+), 2689 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/ffbb1b9f/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/ffbb1b9f/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --cc airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 1e9d983,9f4cd12..c4c303f
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@@ -60,10 -57,10 +57,10 @@@ public class CreateLaunchExperiment
private static final String DEFAULT_GATEWAY = "default.registry.gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_2e539083-665d-40fd-aaa2-4a751028326b";
+ private static String echoAppId = "Echo_a8fc8511-7b8e-431a-ad0f-de5eb1a9c576";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
- private static String amberAppId = "Amber_eda074ea-223d-49d7-a942-6c8742249f36";
+ private static String amberAppId = "Amber_42124128-628b-484c-829d-aff8b584eb00";
private static String gromacsAppId = "GROMACS_05622038-9edd-4cb1-824e-0b7cb993364b";
private static String espressoAppId = "ESPRESSO_10cc2820-5d0b-4c63-9546-8a8b595593c1";
private static String lammpsAppId = "LAMMPS_10893eb5-3840-438c-8446-d26c7ecb001f";
http://git-wip-us.apache.org/repos/asf/airavata/blob/ffbb1b9f/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --cc modules/configuration/server/src/main/resources/airavata-server.properties
index e309901,c493752..00191a7
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@@ -216,12 -215,9 +216,18 @@@ connection.name=xsed
#publisher
activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
publish.rabbitmq=false
++<<<<<<< HEAD
+status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
+task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
+rabbitmq.broker.url=amqp://localhost:5672
+rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
+
++=======
+ activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQPublisher
+ rabbitmq.broker.url=amqp://gw111.iu.xsede.org:5672
+ rabbitmq.exchange.name=airavata_rabbitmq_exchange
++>>>>>>> master
###########################################################################
# Orchestrator module Configuration
http://git-wip-us.apache.org/repos/asf/airavata/blob/ffbb1b9f/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
----------------------------------------------------------------------
diff --cc modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
index fe06ed7,0000000..70ed942
mode 100644,000000..100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
@@@ -1,99 -1,0 +1,103 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.messaging.core.impl;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ThriftUtils;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.MessagingConstants;
+import org.apache.airavata.messaging.core.Publisher;
++import org.apache.airavata.messaging.core.stats.StatCounter;
+import org.apache.airavata.model.messaging.event.*;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RabbitMQStatusPublisher implements Publisher {
+
+ private static Logger log = LoggerFactory.getLogger(RabbitMQStatusPublisher.class);
+
+ private RabbitMQProducer rabbitMQProducer;
+
++ StatCounter statCounter = StatCounter.getInstance();
+
+ public RabbitMQStatusPublisher() throws Exception {
+ String brokerUrl;
+ String exchangeName;
+ try {
+ brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_STATUS_EXCHANGE_NAME);
+ } catch (ApplicationSettingsException e) {
+ String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
+ log.error(message, e);
+ throw new AiravataException(message, e);
+ }
+ rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName);
+ rabbitMQProducer.open();
+ }
+
+ public void publish(MessageContext msgCtx) throws AiravataException {
+ try {
+ log.info("Publishing status to rabbitmq...");
+ byte[] body = ThriftUtils.serializeThriftObject(msgCtx.getEvent());
+ Message message = new Message();
+ message.setEvent(body);
+ message.setMessageId(msgCtx.getMessageId());
+ message.setMessageType(msgCtx.getType());
+ message.setUpdatedTime(msgCtx.getUpdatedTime().getTime());
++ String gatewayId = msgCtx.getGatewayId();
+ String routingKey = null;
+ if (msgCtx.getType().equals(MessageType.EXPERIMENT)){
+ ExperimentStatusChangeEvent event = (ExperimentStatusChangeEvent) msgCtx.getEvent();
- routingKey = event.getExperimentId();
++ routingKey = gatewayId + "." + event.getExperimentId();
+ } else if (msgCtx.getType().equals(MessageType.TASK)) {
+ TaskStatusChangeEvent event = (TaskStatusChangeEvent) msgCtx.getEvent();
- routingKey = event.getTaskIdentity().getExperimentId() + "." +
++ routingKey = gatewayId + "." + event.getTaskIdentity().getExperimentId() + "." +
+ event.getTaskIdentity().getWorkflowNodeId() + "." + event.getTaskIdentity().getTaskId();
+ }else if (msgCtx.getType().equals(MessageType.WORKFLOWNODE)){
+ WorkflowNodeStatusChangeEvent event = (WorkflowNodeStatusChangeEvent) msgCtx.getEvent();
+ WorkflowIdentifier workflowNodeIdentity = event.getWorkflowNodeIdentity();
- routingKey = workflowNodeIdentity.getExperimentId() + "." + workflowNodeIdentity.getWorkflowNodeId();
++ routingKey = gatewayId + "." + workflowNodeIdentity.getExperimentId() + "." + workflowNodeIdentity.getWorkflowNodeId();
+ }else if (msgCtx.getType().equals(MessageType.JOB)){
+ JobStatusChangeEvent event = (JobStatusChangeEvent)msgCtx.getEvent();
+ JobIdentifier identity = event.getJobIdentity();
- routingKey = identity.getExperimentId() + "." +
++ routingKey = gatewayId + "." + identity.getExperimentId() + "." +
+ identity.getWorkflowNodeId() + "." +
+ identity.getTaskId() + "." +
+ identity.getJobId();
+ }
+ byte[] messageBody = ThriftUtils.serializeThriftObject(message);
+ rabbitMQProducer.send(messageBody, routingKey);
++ statCounter.add(message);
+ } catch (TException e) {
+ String msg = "Error while deserializing the object";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ } catch (Exception e) {
+ String msg = "Error while sending to rabbitmq";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ }
+ }
+}
[10/15] airavata git commit: merging wth master
Posted by la...@apache.org.
merging wth master
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/93ed077e
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/93ed077e
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/93ed077e
Branch: refs/heads/master
Commit: 93ed077e87c3e29cd58301ead917fcf7e2025853
Parents: 1231c01 9a6eaaa
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Wed Feb 25 01:09:40 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Wed Feb 25 01:09:40 2015 -0500
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 192 +-
.../tools/RegisterSampleApplications.java | 31 +-
.../catalog/data/impl/ComputeResourceImpl.java | 15 +-
.../data/util/AppCatalogThriftConversion.java | 9 +-
modules/commons/utils/pom.xml | 4 +-
.../airavata/common/utils/WSConstants.java | 10 +-
.../apache/airavata/common/utils/WSDLUtil.java | 764 ++---
.../apache/airavata/common/utils/XMLUtil.java | 34 +-
.../credential-store/pom.xml | 5 +
modules/distribution/server/pom.xml | 64 +-
modules/gfac/gfac-bes/pom.xml | 5 +
modules/gfac/gfac-gsissh/pom.xml | 10 -
.../handler/GSISSHDirectorySetupHandler.java | 7 +-
modules/gfac/gfac-ssh/pom.xml | 10 -
.../ssh/handler/SSHDirectorySetupHandler.java | 7 +-
.../airavata/gfac/ssh/util/GFACSSHUtils.java | 45 +-
.../core/impl/RabbitMQStatusPublisher.java | 5 +-
modules/workflow-model/workflow-engine/pom.xml | 5 +-
.../engine/gfac/GFacRegistryClient.java | 186 +-
.../workflow/engine/gfac/SimpleWSClient.java | 166 +-
.../interpretor/SystemComponentInvoker.java | 86 +-
.../engine/interpretor/WorkflowInterpreter.java | 24 +-
.../engine/invoker/AsynchronousInvoker.java | 98 +-
.../workflow/engine/invoker/DynamicInvoker.java | 48 +-
.../workflow/engine/invoker/Invoker.java | 84 +-
.../workflow/engine/invoker/SimpleInvoker.java | 411 +--
.../workflow/engine/util/InterpreterUtil.java | 112 +-
.../workflow/engine/util/XBayaUtil.java | 94 +-
.../engine/workflow/proxy/WorkflowContext.java | 4 +-
.../workflow-model-component/pom.xml | 7 +-
.../workflow-model/workflow-model-core/pom.xml | 8 +-
.../component/system/SubWorkflowComponent.java | 4 +-
.../component/url/URLComponentRegistry.java | 12 +-
.../component/ws/WSComponentApplication.java | 2 +-
.../model/component/ws/WSComponentFactory.java | 8 +-
.../model/component/ws/WSComponentRegistry.java | 32 +-
.../model/component/ws/WorkflowComponent.java | 108 +-
.../airavata/workflow/model/gpel/DSCUtil.java | 140 +-
.../workflow/model/gpel/script/BPELScript.java | 1276 ++++----
.../model/gpel/script/WorkflowWSDL.java | 528 ++--
.../model/graph/system/StreamSourceNode.java | 7 +-
.../workflow/model/ode/ODEBPELTransformer.java | 1260 ++++----
.../airavata/workflow/model/ode/ODEClient.java | 200 +-
.../model/ode/ODEDeploymentDescriptor.java | 368 +--
.../workflow/model/ode/ODEWSDLTransformer.java | 872 +++---
.../workflow/model/ode/WSDLCleaner.java | 196 +-
.../workflow/model/wf/TridentTransformer.java | 50 +-
.../airavata/workflow/model/wf/Workflow.java | 542 ++--
modules/xbaya-gui/pom.xml | 5 +
.../xbaya/core/generators/BPELFiler.java | 38 +-
.../xbaya/core/generators/ODEScriptFiler.java | 68 +-
.../xbaya/invoker/factory/InvokerFactory.java | 16 +-
.../graph/dynamic/DynamicNodeWindow.java | 72 +-
tools/gsissh/pom.xml | 8 +-
.../java/com/jcraft/jsch/ExtendedSession.java | 2 -
.../illinois/ncsa/BCGSS/BCGSSContextImpl.java | 2894 +++++++++---------
.../illinois/ncsa/BCGSS/CircularByteBuffer.java | 1648 +++++-----
.../ncsa/BCGSS/GlobusTlsCipherFactory.java | 126 +-
.../illinois/ncsa/BCGSS/GlobusTlsClient.java | 494 +--
.../edu/illinois/ncsa/BCGSS/TlsHandlerUtil.java | 564 ++--
.../apache/airavata/gsi/ssh/GSSContextX509.java | 32 +-
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 163 +-
.../apache/airavata/gsi/ssh/util/SSHUtils.java | 4 +-
.../impl/DefaultSSHApiTestWithMyProxyAuth.java | 15 +-
.../gsi/ssh/impl/VanilaTestWithSSHAuth.java | 46 +-
65 files changed, 7232 insertions(+), 7118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/93ed077e/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --cc airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 78c2d71,a5bc81c..90b8e6d
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@@ -57,10 -57,10 +57,10 @@@ public class CreateLaunchExperiment
private static final String DEFAULT_GATEWAY = "default.registry.gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_2e539083-665d-40fd-aaa2-4a751028326b";
+ private static String echoAppId = "Echo_1365a7fd-eae1-4575-b447-99afb4d79c82";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
- private static String amberAppId = "Amber_42124128-628b-484c-829d-aff8b584eb00";
+ private static String amberAppId = "Amber_9e4f28b6-7a5d-4fe1-b07f-2053f8f0deb3";
private static String gromacsAppId = "GROMACS_05622038-9edd-4cb1-824e-0b7cb993364b";
private static String espressoAppId = "ESPRESSO_10cc2820-5d0b-4c63-9546-8a8b595593c1";
private static String lammpsAppId = "LAMMPS_10893eb5-3840-438c-8446-d26c7ecb001f";
@@@ -106,7 -104,7 +106,6 @@@
// final String expId = createExperimentForBR2Amber(airavataClient);
// final String expId = createExperimentWRFStampede(airavataClient);
// final String expId = createExperimentForStampedeAmber(airavataClient);
- // final String expId = createExperimentForTrestlesAmber(airavataClient);
- final String expId = createExperimentForTrestlesAmber(airavataClient);
// final String expId = createExperimentGROMACSStampede(airavataClient);
// final String expId = createExperimentESPRESSOStampede(airavataClient);
// final String expId = createExperimentLAMMPSStampede(airavataClient);
http://git-wip-us.apache.org/repos/asf/airavata/blob/93ed077e/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
----------------------------------------------------------------------
diff --cc modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
index 70ed942,0000000..ae5e29e
mode 100644,000000..100644
--- a/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
+++ b/modules/messaging/core/src/main/java/org/apache/airavata/messaging/core/impl/RabbitMQStatusPublisher.java
@@@ -1,103 -1,0 +1,102 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.messaging.core.impl;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ThriftUtils;
+import org.apache.airavata.messaging.core.MessageContext;
+import org.apache.airavata.messaging.core.MessagingConstants;
+import org.apache.airavata.messaging.core.Publisher;
- import org.apache.airavata.messaging.core.stats.StatCounter;
+import org.apache.airavata.model.messaging.event.*;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RabbitMQStatusPublisher implements Publisher {
+
+ private static Logger log = LoggerFactory.getLogger(RabbitMQStatusPublisher.class);
+
+ private RabbitMQProducer rabbitMQProducer;
+
- StatCounter statCounter = StatCounter.getInstance();
++// StatCounter statCounter = StatCounter.getInstance();
+
+ public RabbitMQStatusPublisher() throws Exception {
+ String brokerUrl;
+ String exchangeName;
+ try {
+ brokerUrl = ServerSettings.getSetting(MessagingConstants.RABBITMQ_BROKER_URL);
+ exchangeName = ServerSettings.getSetting(MessagingConstants.RABBITMQ_STATUS_EXCHANGE_NAME);
+ } catch (ApplicationSettingsException e) {
+ String message = "Failed to get read the required properties from airavata to initialize rabbitmq";
+ log.error(message, e);
+ throw new AiravataException(message, e);
+ }
+ rabbitMQProducer = new RabbitMQProducer(brokerUrl, exchangeName);
+ rabbitMQProducer.open();
+ }
+
+ public void publish(MessageContext msgCtx) throws AiravataException {
+ try {
+ log.info("Publishing status to rabbitmq...");
+ byte[] body = ThriftUtils.serializeThriftObject(msgCtx.getEvent());
+ Message message = new Message();
+ message.setEvent(body);
+ message.setMessageId(msgCtx.getMessageId());
+ message.setMessageType(msgCtx.getType());
+ message.setUpdatedTime(msgCtx.getUpdatedTime().getTime());
+ String gatewayId = msgCtx.getGatewayId();
+ String routingKey = null;
+ if (msgCtx.getType().equals(MessageType.EXPERIMENT)){
+ ExperimentStatusChangeEvent event = (ExperimentStatusChangeEvent) msgCtx.getEvent();
+ routingKey = gatewayId + "." + event.getExperimentId();
+ } else if (msgCtx.getType().equals(MessageType.TASK)) {
+ TaskStatusChangeEvent event = (TaskStatusChangeEvent) msgCtx.getEvent();
+ routingKey = gatewayId + "." + event.getTaskIdentity().getExperimentId() + "." +
+ event.getTaskIdentity().getWorkflowNodeId() + "." + event.getTaskIdentity().getTaskId();
+ }else if (msgCtx.getType().equals(MessageType.WORKFLOWNODE)){
+ WorkflowNodeStatusChangeEvent event = (WorkflowNodeStatusChangeEvent) msgCtx.getEvent();
+ WorkflowIdentifier workflowNodeIdentity = event.getWorkflowNodeIdentity();
+ routingKey = gatewayId + "." + workflowNodeIdentity.getExperimentId() + "." + workflowNodeIdentity.getWorkflowNodeId();
+ }else if (msgCtx.getType().equals(MessageType.JOB)){
+ JobStatusChangeEvent event = (JobStatusChangeEvent)msgCtx.getEvent();
+ JobIdentifier identity = event.getJobIdentity();
+ routingKey = gatewayId + "." + identity.getExperimentId() + "." +
+ identity.getWorkflowNodeId() + "." +
+ identity.getTaskId() + "." +
+ identity.getJobId();
+ }
+ byte[] messageBody = ThriftUtils.serializeThriftObject(message);
+ rabbitMQProducer.send(messageBody, routingKey);
- statCounter.add(message);
++// statCounter.add(message);
+ } catch (TException e) {
+ String msg = "Error while deserializing the object";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ } catch (Exception e) {
+ String msg = "Error while sending to rabbitmq";
+ log.error(msg, e);
+ throw new AiravataException(msg, e);
+ }
+ }
+}
[14/15] airavata git commit: Merge branch 'queue-gfac-rabbitmq'
Posted by la...@apache.org.
Merge branch 'queue-gfac-rabbitmq'
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/3eb91630
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/3eb91630
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/3eb91630
Branch: refs/heads/master
Commit: 3eb91630f9846f22ad84baad97fa6e1521b77ab9
Parents: c05d5e6 5310bb4
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Mon Mar 23 15:14:15 2015 -0400
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Mon Mar 23 15:14:15 2015 -0400
----------------------------------------------------------------------
.../airavata/api/server/AiravataAPIServer.java | 5 +-
.../server/handler/AiravataServerHandler.java | 2 +-
.../lib/airavata/messagingEvents_types.cpp | 202 +++++-
.../lib/airavata/messagingEvents_types.h | 106 ++-
.../Airavata/Model/Messaging/Event/Types.php | 228 +++++++
.../client/samples/CreateLaunchExperiment.java | 435 ++++++------
.../model/messaging/event/MessageType.java | 8 +-
.../model/messaging/event/TaskSubmitEvent.java | 684 +++++++++++++++++++
.../messaging/event/TaskTerminateEvent.java | 492 +++++++++++++
airavata-api/generate-thrift-files.sh | 22 +-
.../messagingEvents.thrift | 16 +-
.../airavata/common/utils/AiravataZKUtils.java | 27 +
.../apache/airavata/common/utils/Constants.java | 1 +
.../airavata/common/utils/ServerSettings.java | 24 +-
.../main/resources/airavata-server.properties | 15 +-
.../main/resources/airavata-server.properties | 10 +-
modules/gfac/airavata-gfac-service/pom.xml | 10 +
.../airavata/gfac/server/GfacServerHandler.java | 151 +++-
modules/gfac/gfac-core/pom.xml | 1 +
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 8 +-
.../core/monitor/GfacInternalStatusUpdator.java | 35 +-
.../airavata/gfac/core/monitor/MonitorID.java | 3 +-
.../airavata/gfac/core/utils/GFacUtils.java | 118 +++-
.../gfac/local/provider/impl/LocalProvider.java | 4 +-
.../handlers/GridPullMonitorHandler.java | 1 +
.../airavata/gfac/monitor/util/CommonUtils.java | 2 +-
.../messaging/client/RabbitMQListener.java | 4 +-
.../airavata/messaging/core/MessageContext.java | 17 +
.../messaging/core/MessagingConstants.java | 3 +-
.../messaging/core/PublisherFactory.java | 23 +-
.../airavata/messaging/core/TestClient.java | 5 +-
.../messaging/core/impl/RabbitMQConsumer.java | 258 -------
.../messaging/core/impl/RabbitMQProducer.java | 27 +-
.../messaging/core/impl/RabbitMQPublisher.java | 102 ---
.../core/impl/RabbitMQStatusConsumer.java | 274 ++++++++
.../core/impl/RabbitMQStatusPublisher.java | 102 +++
.../core/impl/RabbitMQTaskLaunchConsumer.java | 253 +++++++
.../core/impl/RabbitMQTaskLaunchPublisher.java | 85 +++
.../server/OrchestratorServerHandler.java | 95 +--
.../util/OrchestratorRecoveryHandler.java | 3 +-
modules/orchestrator/orchestrator-core/pom.xml | 4 +-
.../core/context/OrchestratorContext.java | 11 +
.../core/impl/GFACPassiveJobSubmitter.java | 220 ++++++
.../core/impl/GFACRPCJobSubmitter.java | 212 ++++++
.../core/impl/GFACServiceJobSubmitter.java | 212 ------
.../core/utils/OrchestratorConstants.java | 1 -
.../workflow/engine/WorkflowEngineImpl.java | 4 +-
.../airavata/xbaya/messaging/Monitor.java | 5 +-
pom.xml | 7 +
49 files changed, 3610 insertions(+), 927 deletions(-)
----------------------------------------------------------------------