You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/05/21 19:57:00 UTC
[46/51] [abbrv] git commit: adding tests for validator functionality
in orchestrator - AIRAVATA-1200
adding tests for validator functionality in orchestrator - AIRAVATA-1200
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/3132bf02
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/3132bf02
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/3132bf02
Branch: refs/heads/temp
Commit: 3132bf02da0d242fbc472bde89301322d88da5a9
Parents: 96fed6e
Author: lahiru <la...@apache.org>
Authored: Tue May 6 14:55:47 2014 -0400
Committer: lahiru <la...@apache.org>
Committed: Tue May 6 14:55:47 2014 -0400
----------------------------------------------------------------------
.../client/src/main/resources/log4j.properties | 2 +-
.../orchestrator/core/ValidatorTest.java | 106 +++++++
.../orchestrator/core/util/TestValidator.java | 44 +++
.../test/resources/airavata-server.properties | 298 +++++++++++++++++++
4 files changed, 449 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/3132bf02/modules/configuration/client/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/client/src/main/resources/log4j.properties b/modules/configuration/client/src/main/resources/log4j.properties
index ee40d6f..1a5c34d 100644
--- a/modules/configuration/client/src/main/resources/log4j.properties
+++ b/modules/configuration/client/src/main/resources/log4j.properties
@@ -34,7 +34,7 @@ log4j.appender.CONSOLE.layout.ConversionPattern=[%p] %m%n
# LOGFILE is set to be a File appender using a PatternLayout.
log4j.appender.LOGFILE=org.apache.log4j.FileAppender
-log4j.appender.LOGFILE.File=../../bin/axis2.log
+log4j.appender.LOGFILE.File=../../bin/airavata.log
log4j.appender.LOGFILE.Append=true
log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.LOGFILE.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n
http://git-wip-us.apache.org/repos/asf/airavata/blob/3132bf02/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/ValidatorTest.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/ValidatorTest.java b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/ValidatorTest.java
new file mode 100644
index 0000000..c45e9f2
--- /dev/null
+++ b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/ValidatorTest.java
@@ -0,0 +1,106 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.orchestrator.core;
+
+import org.apache.airavata.client.AiravataAPIFactory;
+import org.apache.airavata.client.api.AiravataAPI;
+import org.apache.airavata.client.api.exception.AiravataAPIInvocationException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.AiravataUtils;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.model.util.ExperimentModelUtil;
+import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.orchestrator.cpi.Orchestrator;
+import org.apache.airavata.orchestrator.cpi.impl.SimpleOrchestratorImpl;
+import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
+import org.apache.airavata.registry.cpi.ParentDataType;
+import org.apache.airavata.registry.cpi.Registry;
+import org.apache.airavata.schemas.gfac.DataType;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ValidatorTest extends BaseOrchestratorTest {
+ private static final Logger log = LoggerFactory.getLogger(NewOrchestratorTest.class);
+
+ private Orchestrator orchestrator;
+ private List<TaskDetails> tasks;
+
+ @BeforeTest
+ public void setUp() throws Exception {
+ AiravataUtils.setExecutionAsServer();
+ super.setUp();
+ orchestrator = new SimpleOrchestratorImpl();
+ }
+
+
+
+ @Test
+ public void testValidator() throws Exception {
+ // creating host description
+ List<DataObjectType> exInputs = new ArrayList<DataObjectType>();
+ DataObjectType input = new DataObjectType();
+ input.setKey("echo_input");
+ input.setType(DataType.STRING.toString());
+ input.setValue("echo_output=Hello World");
+ exInputs.add(input);
+
+ List<DataObjectType> exOut = new ArrayList<DataObjectType>();
+ DataObjectType output = new DataObjectType();
+ output.setKey("echo_output");
+ output.setType(DataType.STRING.toString());
+ output.setValue("");
+ exOut.add(output);
+
+ Experiment simpleExperiment =
+ ExperimentModelUtil.createSimpleExperiment("default", "admin", "echoExperiment", "SimpleEcho0", "SimpleEcho0", exInputs);
+ simpleExperiment.setExperimentOutputs(exOut);
+
+ WorkflowNodeDetails test = ExperimentModelUtil.createWorkflowNode("test", null);
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("localhost", 1, 1, 1, "normal", 0, 0, 1, "sds128");
+ scheduling.setResourceHostId("localhost");
+ UserConfigurationData userConfigurationData = new UserConfigurationData();
+ userConfigurationData.setAiravataAutoSchedule(false);
+ userConfigurationData.setOverrideManualScheduledParams(false);
+ userConfigurationData.setComputationalResourceScheduling(scheduling);
+ simpleExperiment.setUserConfigurationData(userConfigurationData);
+
+ Registry defaultRegistry = RegistryFactory.getDefaultRegistry();
+ String experimentId = (String)defaultRegistry.add(ParentDataType.EXPERIMENT, simpleExperiment);
+
+ simpleExperiment.setExperimentID(experimentId);
+ tasks = orchestrator.createTasks(experimentId);
+
+ Assert.assertTrue(orchestrator.validateExperiment(simpleExperiment, test, tasks.get(0)));
+
+ simpleExperiment.setExperimentID(null);
+
+ Assert.assertFalse(orchestrator.validateExperiment(simpleExperiment,test,tasks.get(0)));
+ tasks.get(0).setTaskID(null);
+ Assert.assertFalse(orchestrator.validateExperiment(simpleExperiment,test,tasks.get(0)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/3132bf02/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/util/TestValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/util/TestValidator.java b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/util/TestValidator.java
new file mode 100644
index 0000000..b92538c
--- /dev/null
+++ b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/util/TestValidator.java
@@ -0,0 +1,44 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.orchestrator.core.util;
+
+import org.apache.airavata.model.workspace.experiment.Experiment;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
+import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
+import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestValidator implements JobMetadataValidator {
+ private final static Logger logger = LoggerFactory.getLogger(TestValidator.class);
+
+ public boolean validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails taskID) throws OrchestratorException {
+ if (experiment.getProjectID() == null) {
+ logger.error("Project ID is not set");
+ return false;
+ } else if (experiment.getExperimentID() == null) {
+ logger.error("This experiment is wrong, no experimentID set");
+ return false;
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/3132bf02/modules/orchestrator/orchestrator-core/src/test/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/test/resources/airavata-server.properties b/modules/orchestrator/orchestrator-core/src/test/resources/airavata-server.properties
new file mode 100644
index 0000000..a882e14
--- /dev/null
+++ b/modules/orchestrator/orchestrator-core/src/test/resources/airavata-server.properties
@@ -0,0 +1,298 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+###########################################################################
+#
+# This properties file provides configuration for all Airavata Services:
+# Registry, Workflow Interpreter, GFac, Message Broker, Message Box
+#
+###########################################################################
+
+###########################################################################
+# Airavata Embedded-Tomcat Server Configuration
+###########################################################################
+# By default, tomcat runs on port 80. If the port is changed, it will be
+# have to be specified with port properties. This will be picked up by
+# the registry service to register the service end-points.
+
+port=8080
+
+# Axis2 server automatically picks up IP address from axis configuration,
+# but some DHCP enables machines do not report correct ip addresses,
+# in which case, the IP address can be manually specified.
+
+#ip=192.2.33.12
+
+
+#This property will enable https and stops http, during the url registration, https urls will be stored and
+# http urls will not get registered, because currently airavata supports only one url for each service
+#enable.https=false
+#system properties used by services to register service URLs
+system.gateway=default
+system.user=admin
+system.password=admin
+airavata.server.url=http://localhost:8080/airavata/services/registry
+
+###########################################################################
+# Airavata Registry Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+registry.jdbc.url=jdbc:derby://localhost:1527/persistent_data;create=true;user=airavata;password=airavata
+# MySql database configuration
+#registry.jdbc.driver=com.mysql.jdbc.Driver
+#registry.jdbc.url=jdbc:mysql://localhost:3306/persistent_data
+registry.jdbc.user=airavata
+registry.jdbc.password=airavata
+start.derby.server.mode=true
+validationQuery=SELECT 1 from CONFIGURATION
+jpa.connection.properties=MaxActive=10,MaxIdle=5,MinIdle=2,MaxWait=60000,testWhileIdle=true,testOnBorrow=true
+# Properties to setup registry service
+default.registry.user=admin
+default.registry.password=admin
+default.registry.password.hash.method=SHA
+default.registry.gateway=default
+#for rest [RegistryClient]
+#registry.jdbc.url=http://localhost:9080/airavata-services
+
+#user defined registry accessor classes
+#class.provenance.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.AiravataJPARegistry
+#class.configuration.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.AiravataJPARegistry
+#class.descriptor.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.AiravataJPARegistry
+#class.project.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.AiravataJPARegistry
+#class.user.workflow.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.AiravataJPARegistry
+#class.published.workflow.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.AiravataJPARegistry
+
+############################################################ ###############
+# Airavata Workflow Interpreter Configurations
+###########################################################################
+
+runInThread=true
+provenance=true
+provenanceWriterThreadPoolSize=20
+gfac.embedded=true
+
+#
+# Security Configuration used by Airavata Generic Factory Service
+# to interact with Computational Resources.
+#
+
+###########################################################################
+# Airavata GFac MyProxy GSI credentials to access Grid Resources.
+###########################################################################
+
+myproxy.server=myproxy.teragrid.org
+myproxy.username=ogce
+myproxy.password=
+myproxy.life=3600
+# XSEDE Trusted certificates can be downloaded from https://software.xsede.org/security/xsede-certs.tar.gz
+trusted.cert.location=/Users/lahirugunathilake/Downloads/certificates
+# SSH PKI key pair or ssh password can be used SSH based authentication is used.
+# if user specify both password authentication gets the higher preference
+
+################# ---------- For ssh key pair authentication ------------------- ################
+#public.ssh.key=/path to public key for ssh
+#ssh.username=username for ssh connection
+#private.ssh.key=/path to private key file for ssh
+#ssh.keypass=passphrase for the private key
+
+
+################# ---------- For ssh key pair authentication ------------------- ################
+#ssh.username=username for ssh connection
+#ssh.password=Password for ssh connection
+
+
+
+###########################################################################
+# Airavata Message Broker Basic Configurations.
+# the Advanced section provides detailed custom configuration properties.
+###########################################################################
+
+# Default derby database configuration
+broker.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
+broker.jdbc.url=jdbc:derby:wsmg;create=true;user=airavata;password=airavata
+
+# Note: This database will only be used if 'broker.storage.type=persistent'
+# is set in advanced properties below.
+# MySql database configuration
+#broker.jdbc.driver=com.mysql.jdbc.Driver
+#broker.jdbc.url=jdbc:mysql://localhost:3306/wsmg?user=airavata&password=airavata
+
+###########################################################################
+# Airavata Message Box Basic Configurations,
+# the Advanced section provides detailed custom configuration properties.
+###########################################################################
+
+# Default derby database
+msgBox.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
+msgBox.jdbc.url=jdbc:derby:wsmg;create=true;user=airavata;password=airavata
+
+# MySql database configuration
+#msgBox.jdbc.driver=com.mysql.jdbc.Driver
+#msgBox.jdbc.url=jdbc:mysql://localhost:3306/wsmg?user=airavata&password=airavata
+
+###########################################################################
+# Advance configuration to change service implementations
+###########################################################################
+
+#Update the server context root path if airavata server is deployed under a different context root other than axis2
+server.context-root=airavata-server
+
+#
+# Class which implemented HostScheduler interface. It will determine the which host to submit the request
+#
+host.scheduler=org.apache.airavata.gfac.core.scheduler.impl.SimpleHostScheduler
+
+#
+# Data Service Plugins classes
+#
+datachain.classes= org.apache.airavata.core.gfac.extension.data.RegistryDataService
+
+#
+# Pre execution Plugins classes. For example, GridFTP Input Staging
+#
+prechain.classes= org.apache.airavata.core.gfac.extension.pre.GridFtpInputStaging
+prechain.classes= org.apache.airavata.core.gfac.extension.pre.HttpInputStaging
+
+#
+# Post execution Plugins classes. For example, GridFTP Output Staging
+#
+postchain.classes= org.apache.airavata.core.gfac.extension.post.GridFtpOutputStaging
+postchain.classes= org.apache.airavata.core.gfac.extension.post.OutputRegister
+
+#
+# Advanced Message Broker Configurations
+#
+
+#socket time out in milliseconds for sending messages. (defaults is 20000)
+broker.socket.timeout=60000
+
+broker.storage.type=persistent
+#broker.storage.type=memory
+
+#specifies wether delivery component should be started or not.
+broker.start.delivery.thread=true
+
+#fixed thread pool based delivery
+#broker.delivery.method=pcrew
+
+#dynamic thread pool based delivery
+#broker.delivery.method=parallel
+
+#single thread delivery
+broker.delivery.method=serial
+
+#number of message delivery failures before a url become black listed (default is 2)
+#broker.msg.delivery.retries=2
+
+#time period (in seconds) a url will be kept blacklisted (default is 5 seconds)
+#consumer.expiration.time.gap=5
+
+#maximum number of messages to be send to a one consumer/url at time.
+#applicable if 'broker.delivery.method' is 'pcrew' . (default is 10)
+
+#sending.batch.size=10
+
+#size of the thread pool. only applicable if 'broker.delivery.method' is 'pcrew'. (default is 4)
+#sending.thread.pool.size=4
+
+#
+# Advanced Message Box Configurations
+#
+msgBox.usedatabase=true
+messagePreservationDays=2
+messagePreservationHours=0
+messagePreservationMinutes=0
+messagePerservationIntervalDays=0
+messagePerservationIntervalHours=1
+messagePerservationIntervalMinutes=0
+
+###---------------------------REGISTRY API IMPLEMENTATION---------------------------###
+
+class.registry.accessor=org.apache.airavata.persistance.registry.jpa.impl.AiravataJPARegistry
+#class.registry.accessor=org.apache.airavata.rest.client.RegistryClient
+
+###########################################################################
+# AMQP Notification Configuration
+###########################################################################
+amqp.notification.enable=1
+
+amqp.broker.host=localhost
+amqp.broker.port=5672
+amqp.broker.username=guest
+amqp.broker.password=guest
+
+amqp.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPSenderImpl
+amqp.topic.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPTopicSenderImpl
+amqp.broadcast.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPBroadcastSenderImpl
+
+###---------------------------Computational Middleware Configurations---------------------------###
+
+#enable.application.job.status.history=true
+#http://localhost:8080/axis2/services/RegistryService?wsdl
+registry.service.wsdl=http://localhost:${port}/${server.context-root}/services/RegistryService?wsdl
+
+# If false, disables two phase commit when submitting jobs
+TwoPhase=true
+
+
+###---------------------------Monitoring module Configurations---------------------------###
+#This will be the primary monitoring tool which runs in airavata, in future there will be multiple monitoring
+#mechanisms and one would be able to start a monitor
+monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor
+#,org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor
+#This is the amqp related configuration and this lists down the Rabbitmq host, this is an xsede specific configuration
+amqp.hosts=info1.dyn.teragrid.org,info2.dyn.teragrid.org
+proxy.file.path=/Users/lahirugunathilake/Downloads/x509up_u503876
+connection.name=xsede
+activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataExperimentStatusUpdator
+
+###---------------------------Orchestrator module Configurations---------------------------###
+job.submitter=org.apache.airavata.orchestrator.core.impl.EmbeddedGFACJobSubmitter
+job.validator=org.apache.airavata.orchestrator.core.util.TestValidator,org.apache.airavata.orchestrator.core.util.SecondValidator
+submitter.interval=10000
+threadpool.size=10
+start.submitter=true
+embedded.mode=true
+enable.validation=true
+orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
+
+###---------------------------API Server module Configurations---------------------------###
+apiserver=org.apache.airavata.api.server.AiravataAPIServer
+
+###---------------------------Airavata Server Configurations---------------------------###
+servers=apiserver,orchestrator
+#shutdown.trategy=NONE
+shutdown.trategy=SELF_TERMINATE
+# credential store specific parameters
+credential.store.keystore.url=../configuration/server/src/main/resources/airavata.jks
+credential.store.keystore.alias=airavata
+credential.store.keystore.password=airavata
+
+notifier.enabled=false
+#period in milliseconds
+notifier.duration=5000
+
+email.server=smtp.googlemail.com
+email.server.port=465
+email.user=airavata
+email.password=xxx
+email.ssl=true
+email.from=airavata@apache.org