You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2016/08/29 04:39:28 UTC
airavata git commit: First stage of kafka appender
Repository: airavata
Updated Branches:
refs/heads/lahiru/AIRAVATA-2065 415e9b701 -> 46b8502eb
First stage of kafka appender
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/46b8502e
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/46b8502e
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/46b8502e
Branch: refs/heads/lahiru/AIRAVATA-2065
Commit: 46b8502ebe3d2c76000db8d79329e58434810ef3
Parents: 415e9b7
Author: Lahiru Ginnaliya Gamathige <la...@apache.org>
Authored: Sun Aug 28 21:39:11 2016 -0700
Committer: Lahiru Ginnaliya Gamathige <la...@apache.org>
Committed: Sun Aug 28 21:39:11 2016 -0700
----------------------------------------------------------------------
distribution/pom.xml | 11 +-
modules/commons/pom.xml | 12 +-
.../airavata/common/logging/Exception.java | 66 +++++++++
.../airavata/common/logging/GsonTest.java | 20 +++
.../airavata/common/logging/LogEntry.java | 132 ++++++++++++++++++
.../airavata/common/logging/ServerId.java | 68 +++++++++
.../common/logging/kafka/KafkaAppender.java | 116 ++++++++++++++++
.../airavata/common/utils/AiravataZKUtils.java | 13 +-
.../common/utils/ApplicationSettings.java | 34 +++--
.../airavata/common/utils/AwsMetadata.java | 137 +++++++++++++++++++
.../airavata/common/utils/BuildConstant.java | 28 ++++
.../apache/airavata/common/utils/JSONUtil.java | 1 -
.../airavata/common/utils/ServerSettings.java | 36 +++++
.../main/resources/airavata-server.properties | 8 +-
.../gfac/bes/provider/impl/BESProvider.java | 7 +-
.../impl/JSDLGeneratorTestWithMyProxyAuth.java | 2 +-
.../apache/airavata/gfac/core/GFacUtils.java | 2 +-
.../gfac/impl/task/BESJobSubmissionTask.java | 7 +-
.../services/impl/BigRed2TestWithSSHAuth.java | 2 +-
.../impl/GSISSHProviderTestWithMyProxyAuth.java | 2 +-
.../impl/SSHProviderTestWithSSHAuth.java | 2 +-
modules/registry/registry-core/pom.xml | 27 ----
.../org/apache/airavata/server/ServerMain.java | 23 +++-
.../utils/PropertyReader.java | 3 +-
.../core/AbstractThriftDeserializer.java | 3 +-
pom.xml | 32 +++++
26 files changed, 728 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index d3124eb..e3d1b33 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -559,7 +559,16 @@
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka-clients.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
<!-- ======================== Sample =================== -->
<dependency>
<groupId>org.apache.airavata</groupId>
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/pom.xml
----------------------------------------------------------------------
diff --git a/modules/commons/pom.xml b/modules/commons/pom.xml
index a2136c6..f4231ce 100644
--- a/modules/commons/pom.xml
+++ b/modules/commons/pom.xml
@@ -115,7 +115,6 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
@@ -127,12 +126,21 @@
<artifactId>libthrift</artifactId>
<version>${thrift.version}</version>
</dependency>
-
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${google.gson.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka-clients.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
</dependencies>
<properties>
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java
new file mode 100644
index 0000000..cea0c95
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java
@@ -0,0 +1,66 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging;
+
+
+public class Exception {
+
+ private String message;
+
+ private String[] stackTrace;
+
+ private String className;
+
+ public Exception(String message, String[] stackTrace) {
+ this.message = message;
+ this.stackTrace = stackTrace;
+ }
+
+ public Exception(String message, String[] stackTrace, String className) {
+ this.message = message;
+ this.stackTrace = stackTrace;
+ this.className = className;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public String[] getStackTrace() {
+ return stackTrace;
+ }
+
+ public void setStackTrace(String[] stackTrace) {
+ this.stackTrace = stackTrace;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public void setClassName(String className) {
+ this.className = className;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java
new file mode 100644
index 0000000..b4e9cb2
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java
@@ -0,0 +1,20 @@
+///
+// Copyright (c) 2016. Highfive Technologies, Inc.
+///
+package org.apache.airavata.common.logging;
+
+import com.google.gson.Gson;
+
+import java.util.HashMap;
+
+public class GsonTest {
+ public static void main(String[] args) {
+ final String[] strings = {"roles"};
+ final Exception exception = new Exception("message", strings, "test");
+ final LogEntry logEntry = new LogEntry(new ServerId("serverId", "serverId", "serverId", strings), "serverId", "serverId", "serverId", "serverId", new HashMap<String, String>(), "thread");
+ final LogEntry logEntry1 = new LogEntry(new ServerId("serverId", "serverId", "serverId", strings), "serverId", "serverId", "serverId", "serverId", new HashMap<String, String>(), "thread", exception);
+
+ System.out.println(new Gson().toJson(logEntry));
+ System.out.println(new Gson().toJson(logEntry1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java
new file mode 100644
index 0000000..72fc4a0
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java
@@ -0,0 +1,132 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging;
+
+
+import java.lang.*;
+import java.util.Map;
+
+public class LogEntry {
+
+ private ServerId serverId;
+
+ private String message;
+
+ private String timestamp;
+
+ private String level;
+
+ private String loggerName;
+
+ private Map<String, String> mdc;
+
+ private String threadName;
+
+ private Exception exception;
+
+ public LogEntry(ServerId serverId, String message, String timestamp, String level, String loggerName, Map<String,
+ String> mdc, String threadName, Exception exception) {
+ this.serverId = serverId;
+ this.message = message;
+ this.timestamp = timestamp;
+ this.level = level;
+ this.loggerName = loggerName;
+ this.mdc = mdc;
+ this.threadName = threadName;
+ this.exception = exception;
+ }
+
+ public LogEntry(ServerId serverId, String message, String timestamp, String level, String loggerName, Map<String,
+ String> mdc, String threadName) {
+ this.serverId = serverId;
+ this.message = message;
+ this.timestamp = timestamp;
+ this.level = level;
+ this.loggerName = loggerName;
+ this.mdc = mdc;
+ this.threadName = threadName;
+ }
+
+
+ public ServerId getServerId() {
+ return serverId;
+ }
+
+ public void setServerId(ServerId serverId) {
+ this.serverId = serverId;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public String getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(String timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public String getLevel() {
+ return level;
+ }
+
+ public void setLevel(String level) {
+ this.level = level;
+ }
+
+ public String getLoggerName() {
+ return loggerName;
+ }
+
+ public void setLoggerName(String loggerName) {
+ this.loggerName = loggerName;
+ }
+
+ public Map<String, String> getMdc() {
+ return mdc;
+ }
+
+ public void setMdc(Map<String, String> mdc) {
+ this.mdc = mdc;
+ }
+
+ public String getThreadName() {
+ return threadName;
+ }
+
+ public void setThreadName(String threadName) {
+ this.threadName = threadName;
+ }
+
+ public Exception getException() {
+ return exception;
+ }
+
+ public void setException(Exception exception) {
+ this.exception = exception;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java
new file mode 100644
index 0000000..9611302
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging;
+
+public class ServerId {
+ private String serverId;
+ private String hostName;
+ private String version;
+
+ private String[] roles; // gfac, orchestrator, apiserver,
+
+ public ServerId(String serverId, String hostName, String version, String[] roles) {
+ this.serverId = serverId;
+ this.hostName = hostName;
+ this.version = version;
+ this.roles = roles;
+ }
+
+ public String getServerId() {
+ return serverId;
+ }
+
+ public void setServerId(String serverId) {
+ this.serverId = serverId;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public String[] getRoles() {
+ return roles;
+ }
+
+ public void setRoles(String[] roles) {
+ this.roles = roles;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java
new file mode 100644
index 0000000..a4a6a7b
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java
@@ -0,0 +1,116 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging.kafka;
+
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.IThrowableProxy;
+import ch.qos.logback.classic.spi.StackTraceElementProxy;
+import ch.qos.logback.core.UnsynchronizedAppenderBase;
+import com.google.gson.Gson;
+import org.apache.airavata.common.logging.Exception;
+import org.apache.airavata.common.logging.LogEntry;
+import org.apache.airavata.common.logging.ServerId;
+import org.apache.airavata.common.utils.AwsMetadata;
+import org.apache.airavata.common.utils.BuildConstant;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Properties;
+
+public class KafkaAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {
+ private final static Logger logger = LoggerFactory.getLogger(KafkaAppender.class);
+
+ private final Producer<String, String> producer;
+ private final String kafkaTopic;
+
+ private ServerId serverId = null;
+
+ public KafkaAppender(String kafkaTopicPrefix, String kafkaHost) {
+ Properties props = new Properties();
+ props.put("bootstrap.servers", kafkaHost);
+ props.put("acks", "0");
+ props.put("retries", 0);
+ props.put("batch.size", 16384);
+ props.put("linger.ms", 10000); // Send the batch every 5 seconds
+ props.put("buffer.memory", 33554432);
+ props.put("producer.type", "async");
+ props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ this.kafkaTopic = getKafkaTopicPrefix(kafkaTopicPrefix);
+
+ logger.info("Starting kafka producer: bootstrap-server:{}, topic : {}", kafkaHost, kafkaTopic);
+ this.producer = new KafkaProducer<>(props);
+ if(ServerSettings.isRunningOnAws()) {
+ final AwsMetadata awsMetadata = new AwsMetadata();
+ serverId = new ServerId(awsMetadata.getId(), awsMetadata.getHostname(),
+ BuildConstant.getGitDescribeVersion(), ServerSettings.getServerRoles());
+ } else {
+ serverId = new ServerId(ServerSettings.getIp(), ServerSettings.getIp(),
+ BuildConstant.getGitDescribeVersion(), ServerSettings.getServerRoles());
+ }
+ }
+
+ @Override
+ protected void append(ILoggingEvent event) {
+ //todo do more elegant streaming approach to publish logs
+ if (!event.getLevel().equals(Level.ALL) && // OFF AND ALL are not loggable levels
+ !event.getLevel().equals(Level.OFF)) {
+ final IThrowableProxy throwableProxy = event.getThrowableProxy();
+ final LogEntry entry = throwableProxy != null ?
+ new LogEntry(serverId, event.getMessage(), Instant.ofEpochMilli(event.getTimeStamp()).toString(),
+ event.getLevel().toString(), event.getLoggerName(), event.getMDCPropertyMap(),
+ event.getThreadName() != null ? event.getThreadName() : null,
+ new Exception(throwableProxy.getMessage(), toStringArray(throwableProxy.getStackTraceElementProxyArray())
+ , throwableProxy.getClassName()))
+ : new LogEntry(serverId, event.getMessage(), Instant.ofEpochMilli(event.getTimeStamp()).toString(),
+ event.getLevel().toString(), event.getLoggerName(), event.getMDCPropertyMap(),
+ event.getThreadName() != null ? event.getThreadName() : null);
+
+ producer.send(new ProducerRecord<>(kafkaTopic, new Gson().toJson(entry)));
+ }
+ }
+
+
+ private String[] toStringArray(StackTraceElementProxy[] stackTraceElement) {
+ return Arrays.stream(stackTraceElement).map(StackTraceElementProxy::getSTEAsString).toArray(String[]::new);
+ }
+
+ private String getKafkaTopicPrefix(String kafkaTopicPrefix) {
+ final StringBuilder stringBuffer = new StringBuilder("");
+ for (String role : ServerSettings.getServerRoles()) {
+ stringBuffer.append("_");
+ stringBuffer.append(role);
+ }
+ return kafkaTopicPrefix + stringBuffer.toString();
+ }
+
+ public void close() {
+ producer.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
index 5faf985..75f91fd 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
@@ -119,13 +119,14 @@ public class AiravataZKUtils implements Watcher {
public static void startEmbeddedZK(ServerCnxnFactory cnxnFactory) {
if (ServerSettings.isEmbeddedZK()) {
ServerConfig serverConfig = new ServerConfig();
- URL resource = AiravataZKUtils.class.getClassLoader().getResource("zoo.cfg");
- if (resource == null) {
- logger.error("There is no zoo.cfg file in the classpath... Failed to start Zookeeper Server");
- System.exit(1);
- }
+ URL resource = ApplicationSettings.loadFile("zoo.cfg");
try {
+ if (resource == null) {
+ logger.error("There is no zoo.cfg file in the classpath... Failed to start Zookeeper Server");
+ System.exit(1);
+ }
serverConfig.parse(resource.getPath());
+
} catch (QuorumPeerConfig.ConfigException e) {
logger.error("Error while starting embedded Zookeeper", e);
System.exit(2);
@@ -193,7 +194,5 @@ public class AiravataZKUtils implements Watcher {
} else {
return bytesToLong(curatorClient.getData().storingStatIn(exists).forPath(cancelDeliveryTagPath));
}
-
-
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
index 9ce0786..dc7944f 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
@@ -25,9 +25,11 @@ import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.regex.Matcher;
@@ -35,10 +37,12 @@ import java.util.regex.Pattern;
public class ApplicationSettings {
public static final String SERVER_PROPERTIES="airavata-server.properties";
-
- public static String ADDITIONAL_SETTINGS_FILES = "external.settings";
+ public static final String AIRAVATA_CONFIG_DIR = "airavata.config.dir";
+
+ public static String ADDITIONAL_SETTINGS_FILES = "external.settings";
protected Properties properties = new Properties();
+
private Exception propertyLoadException;
@@ -63,7 +67,6 @@ public class ApplicationSettings {
private void loadProperties() {
URL url = getPropertyFileURL();
try {
-
properties.load(url.openStream());
logger.info("Settings loaded from "+url.toString());
URL[] externalSettingsFileURLs = getExternalSettingsFileURLs();
@@ -77,7 +80,7 @@ public class ApplicationSettings {
}
protected URL getPropertyFileURL() {
- return ApplicationSettings.class.getClassLoader().getResource(SERVER_PROPERTIES);
+ return ApplicationSettings.loadFile(SERVER_PROPERTIES);
}
protected URL[] getExternalSettingsFileURLs(){
@@ -86,7 +89,7 @@ public class ApplicationSettings {
String externalSettingsFileNames = getSettingImpl(ADDITIONAL_SETTINGS_FILES);
String[] externalSettingFiles = externalSettingsFileNames.split(",");
for (String externalSettingFile : externalSettingFiles) {
- URL externalSettingFileURL = ApplicationSettings.class.getClassLoader().getResource(externalSettingFile);
+ URL externalSettingFileURL = ApplicationSettings.loadFile(externalSettingFile);
if (externalSettingFileURL==null){
logger.warn("Could not file external settings file "+externalSettingFile);
}else{
@@ -313,11 +316,11 @@ public class ApplicationSettings {
* Static methods which will be used by the users
*/
- public static String getSetting(String key) throws ApplicationSettingsException{
+ public static String getSetting(String key) throws ApplicationSettingsException {
return getInstance().getSettingImpl(key);
}
-
- public static String getSetting(String key, String defaultValue){
+
+ public static String getSetting(String key, String defaultValue) {
return getInstance().getSettingImpl(key,defaultValue);
}
@@ -426,4 +429,19 @@ public class ApplicationSettings {
public static ShutdownStrategy getShutdownStrategy() throws Exception{
return getInstance().getShutdownStrategyImpl();
}
+
+ public static URL loadFile(String fileName) {
+ final URL resource = ApplicationSettings.class.getClassLoader().getResource(fileName);
+ if(resource == null) {
+ if(System.getProperty(AIRAVATA_CONFIG_DIR) != null) {
+ final String airavataConfigDir = System.getProperty(AIRAVATA_CONFIG_DIR);
+ try {
+ return new File(airavataConfigDir + File.separator + fileName).toURI().toURL();
+ } catch (MalformedURLException e) {
+ logger.error("Error parsing the file from airavata.config.dir", airavataConfigDir);
+ }
+ }
+ }
+ return resource;
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java
new file mode 100644
index 0000000..f9b4a65
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java
@@ -0,0 +1,137 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.utils;
+
+
+import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
+import com.google.gson.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class AwsMetadata {
+ private static final Logger log = LoggerFactory.getLogger(AwsMetadata.class);
+ private static final String METADATA_URI_BASE = "http://169.254.169.254/";
+ private static final String REGION_SUFFIX = "/latest/dynamic/instance-identity/document";
+ private static final String ZONE_SUFFIX = "/latest/meta-data/placement/availability-zone";
+ private static final String PUBLIC_IPV4_SUFFIX = "/latest/meta-data/public-ipv4";
+ private static final String PRIVATE_IPV4_SUFFIX = "latest/meta-data/local-ipv4";
+ private static final String HOSTNAME_SUFFIX = "/latest/meta-data/hostname";
+ private static final String ID_SUFFIX = "/latest/meta-data/instance-id";
+
+ private final URI baseUri;
+
+ private String id;
+ private String region;
+ private String hostname;
+ private String zone;
+ private InetAddress publicIp;
+ private InetAddress privateIp;
+
+ public AwsMetadata() {
+ try {
+ baseUri = new URI(METADATA_URI_BASE);
+ } catch (URISyntaxException e) {
+ Preconditions.checkState(false, "Unexpected URI Syntax Exception: {}", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getRegion() {
+ if (region == null) {
+ try {
+ String dynamicData = getMetadataAt(REGION_SUFFIX);
+ if (dynamicData != null) {
+ final JsonObject asJsonObject = new JsonParser().parse(dynamicData).getAsJsonObject();
+ region = asJsonObject.get("region").getAsString();
+ }
+ } catch (ClassCastException e) {
+ log.error("Unable to get region, expecting a JSON Object", e);
+ }
+ }
+ return region;
+ }
+
+ public String getZoneName() {
+ if (zone == null) {
+ zone = getMetadataAt(ZONE_SUFFIX);
+ }
+ return zone;
+ }
+
+ public String getId() {
+ if (id == null) {
+ id = getMetadataAt(ID_SUFFIX);
+ }
+
+ return id;
+ }
+
+ public String getHostname() {
+ if (hostname == null) {
+ hostname = getMetadataAt(HOSTNAME_SUFFIX);
+ }
+ return hostname;
+ }
+
+ public InetAddress getPublicIpAddress() {
+ if (publicIp == null) {
+ String ip = getMetadataAt(PUBLIC_IPV4_SUFFIX);
+ if (ip != null) {
+ publicIp = InetAddresses.forString(ip);
+ }
+ }
+ return publicIp;
+ }
+
+ public InetAddress getInternalIpAddress() {
+ if (privateIp == null) {
+ String ip = getMetadataAt(PRIVATE_IPV4_SUFFIX);
+ if (ip != null) {
+ privateIp = InetAddresses.forString(ip);
+ }
+ }
+ return privateIp;
+ }
+
+ private String getMetadataAt(String suffix) {
+ try {
+ URI resolved = baseUri.resolve(suffix);
+ StringBuilder builder = new StringBuilder();
+ String line = null;
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(resolved.toURL().openStream()))) {
+ while ((line = reader.readLine()) != null) {
+ builder.append(line);
+ }
+ return builder.toString();
+ }
+ } catch (Exception e) {
+ // ignore for now to make sure local servers don't go verbose
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java
new file mode 100644
index 0000000..5598974
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java
@@ -0,0 +1,28 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.utils;
+
+public class BuildConstant {
+ public static final String GIT_DESCRIBE_VERSION = "airavata-0.16-128-g415e9b7";
+ public static String getGitDescribeVersion() {
+ return GIT_DESCRIBE_VERSION;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
index 222e5a2..59d065d 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
@@ -28,7 +28,6 @@ import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
-import com.google.gson.stream.JsonReader;
import java.io.File;
import java.io.FileReader;
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index 985daad..a14cb9d 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -27,6 +27,8 @@ import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
public class ServerSettings extends ApplicationSettings {
@@ -107,6 +109,15 @@ public class ServerSettings extends ApplicationSettings {
private static final String EMAIL_BASED_MONITOR_STORE_PROTOCOL = "email.based.monitor.store.protocol";
private static final String ENABLE_EMAIL_BASED_MONITORING = "enable.email.based.monitoring";
+ private static final String IS_RUNNING_ON_AWS = "isRunningOnAws";
+ private static final String ENABLE_KAFKA_LOGGING = "enable.kafka.logging";
+ private static final String KAFKA_BROKER_LIST = "kafka.broker.list";
+ private static final String KAFKA_TOPIC_PREFIX = "kafka.topic.prefix";
+ private static final String SERVER_ROLES = "server.roles";
+
+ // todo until AIRAVATA-2066 is finished, keep server side list configurations here.
+ private static Map<String, String[]> listConfigurations = new HashMap<>();
+
private static boolean stopAllThreads = false;
private static boolean emailBaseNotificationEnable;
private static String outputLocation;
@@ -400,4 +411,29 @@ public class ServerSettings extends ApplicationSettings {
public static String getLocalDataLocation() {
return System.getProperty("java.io.tmpdir");
}
+
+ public static boolean isRunningOnAws() {
+ return Boolean.valueOf(getSetting(IS_RUNNING_ON_AWS, "false"));
+ }
+
+ public static String getKafkaBrokerList() {
+ return getSetting(KAFKA_BROKER_LIST, null);
+ }
+
+ public static String getKafkaTopicPrefix() {
+ return getSetting(KAFKA_TOPIC_PREFIX, "all");
+ }
+
+ public static boolean isEnabledKafkaLogging() {
+ return Boolean.valueOf(getSetting(ENABLE_KAFKA_LOGGING, "false"));
+ }
+
+ public static void setServerRoles(String[] roles) {
+ listConfigurations.put(SERVER_ROLES, roles);
+ }
+
+ public static String[] getServerRoles() {
+ return listConfigurations.get(SERVER_ROLES);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index 0dad713..d079a6f 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -283,4 +283,10 @@ authorization.policy=airavata-default-xacml-policy
#### authorization cache related configuration ####
authz.cache.enabled=true
authz.cache.manager.class=org.apache.airavata.api.server.security.authzcache.DefaultAuthzCacheManager
-in.memory.cache.size=1000
\ No newline at end of file
+in.memory.cache.size=1000
+
+# Kafka Logging related configuration
+isRunningOnAws= false
+kafka.broker.list= 54.84.144.132:9092
+kafka.topic.prefix= local
+enable.kafka.logging= true
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
index b287b8e..efd2c5f 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
@@ -144,10 +144,9 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
JobDetails jobDetails = new JobDetails();
FactoryClient factory = new FactoryClient(eprt, secProperties);
- log.info(String.format("Activity Submitting to %s ... \n",
- factoryUrl));
+ log.info("Activity Submitting to {} ... \n", factoryUrl);
CreateActivityResponseDocument response = factory.createActivity(cad);
- log.info(String.format("Activity Submitted to %s \n", factoryUrl));
+ log.info("Activity Submitted to {} \n", factoryUrl);
EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
@@ -353,7 +352,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
}
protected String formatStatusMessage(String activityUrl, String status) {
- return String.format("Activity %s is %s.\n", activityUrl, status);
+ return String.format("Activity {} is {}.\n", activityUrl, status);
}
protected String subStatusAsString(ActivityStatusType statusType) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
index cf9b82a..90f67a1 100644
--- a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
@@ -106,7 +106,7 @@ import static org.junit.Assert.assertTrue;
// }
//
// protected GFacConfiguration getGFACConfig() throws Exception{
-// URL resource = BESProviderTestWithMyProxyAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
// System.out.println(resource.getFile());
// GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null,null);
// return gFacConfiguration;
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index 0ed836f..545e122 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -801,7 +801,7 @@ public class GFacUtils {
public static File createJobFile(TaskContext taskContext, JobDescriptor jobDescriptor, JobManagerConfiguration jobManagerConfiguration) throws GFacException {
try {
TransformerFactory factory = TransformerFactory.newInstance();
- URL resource = GFacUtils.class.getClassLoader().getResource(jobManagerConfiguration.getJobDescriptionTemplateName());
+ URL resource = ApplicationSettings.loadFile(jobManagerConfiguration.getJobDescriptionTemplateName());
if (resource == null) {
String error = "System configuration file '" + jobManagerConfiguration.getJobDescriptionTemplateName()
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
index 6529395..1955225 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
@@ -151,10 +151,9 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
jobDetails.setProcessId(taskContext.getProcessId());
FactoryClient factory = new FactoryClient(eprt, secProperties);
- log.info(String.format("Activity Submitting to %s ... \n",
- factoryUrl));
+ log.info("Activity Submitting to {} ... \n", factoryUrl);
CreateActivityResponseDocument response = factory.createActivity(cad);
- log.info(String.format("Activity Submitted to %s \n", factoryUrl));
+ log.info("Activity Submitted to {} \n", factoryUrl);
EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
@@ -356,7 +355,7 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
}
protected String formatStatusMessage(String activityUrl, String status) {
- return String.format("Activity %s is %s.\n", activityUrl, status);
+ return String.format("Activity {} is {}.\n", activityUrl, status);
}
protected void waitUntilDone(EndpointReferenceType factoryEpr, EndpointReferenceType activityEpr, ProcessContext processContext, DefaultClientConfiguration secProperties) throws Exception {
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
index 4aecd57..fbad51c 100644
--- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
+++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
@@ -91,7 +91,7 @@
// System.out.println("Use -Dssh.username=xxx -Dssh.password=yyy -Dssh.keypass=zzz " +
// "-Dprivate.ssh.key -Dpublic.ssh.key -Dssh.working.directory ");
// }
-// URL resource = BigRed2TestWithSSHAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
// assert resource != null;
// System.out.println(resource.getFile());
// GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()), null);
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
index 4ca5684..9d352a4 100644
--- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
@@ -90,7 +90,7 @@
// "E.g :- mvn clean install -Dmyproxy.username=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<");
// throw new Exception("Need my proxy user name password to run tests.");
// }
-// URL resource = GSISSHProviderTestWithMyProxyAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
// assert resource != null;
// System.out.println(resource.getFile());
// GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()), null);
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
index 4aa0df1..bdbadda 100644
--- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
+++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
@@ -54,7 +54,7 @@
// @Before
// public void setUp() throws Exception {
//
-// URL resource = SSHProviderTestWithSSHAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
// GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null);
//// gFacConfiguration.s
// //have to set InFlwo Handlers and outFlowHandlers
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/registry/registry-core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/pom.xml b/modules/registry/registry-core/pom.xml
index 85986ae..60da666 100644
--- a/modules/registry/registry-core/pom.xml
+++ b/modules/registry/registry-core/pom.xml
@@ -107,33 +107,6 @@
<build>
<plugins>
- <plugin>
- <groupId>org.apache.openjpa</groupId>
- <artifactId>openjpa-maven-plugin</artifactId>
- <version>2.2.0</version>
- <configuration>
- <includes>**/entities/*.class</includes>
- <excludes>**/entities/XML*.class</excludes>
- <addDefaultConstructor>true</addDefaultConstructor>
- <enforcePropertyRestrictions>true</enforcePropertyRestrictions>
- </configuration>
- <executions>
- <execution>
- <id>enhancer</id>
- <phase>process-classes</phase>
- <goals>
- <goal>enhance</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.openjpa</groupId>
- <artifactId>openjpa</artifactId>
- <version>2.2.0</version>
- </dependency>
- </dependencies>
- </plugin>
<!--<plugin>-->
<!--<groupId>org.apache.maven.plugins</groupId>-->
<!--<artifactId>maven-antrun-plugin</artifactId>-->
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
----------------------------------------------------------------------
diff --git a/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java b/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
index 99387de..bcde1e9 100644
--- a/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
+++ b/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
@@ -20,13 +20,16 @@
*/
package org.apache.airavata.server;
+import ch.qos.logback.classic.LoggerContext;
import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.logging.kafka.KafkaAppender;
import org.apache.airavata.common.utils.*;
import org.apache.airavata.common.utils.ApplicationSettings.ShutdownStrategy;
import org.apache.airavata.common.utils.IServer.ServerStatus;
import org.apache.airavata.common.utils.StringUtil.CommandLineParameters;
import org.apache.commons.cli.ParseException;
import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.slf4j.ILoggerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -155,14 +158,29 @@ public class ServerMain {
// }
public static void main(String args[]) throws ParseException, IOException {
- CommandLineParameters commandLineParameters = StringUtil.getCommandLineParser(args);
+ ServerSettings.mergeSettingsCommandLineArgs(args);
+ ServerSettings.setServerRoles(ApplicationSettings.getSetting(SERVERS_KEY, "all").split(","));
+
+ if (ServerSettings.isEnabledKafkaLogging()) {
+ final ILoggerFactory iLoggerFactory = LoggerFactory.getILoggerFactory();
+ if (iLoggerFactory instanceof LoggerContext) {
+ final KafkaAppender kafkaAppender = new KafkaAppender(ServerSettings.getKafkaBrokerList(),
+ ServerSettings.getKafkaTopicPrefix());
+ kafkaAppender.setContext((LoggerContext) iLoggerFactory);
+ kafkaAppender.setName("kafka-appender");
+ kafkaAppender.clearAllFilters();
+ kafkaAppender.start();
+ ((LoggerContext) iLoggerFactory).getLogger(Logger.ROOT_LOGGER_NAME).addAppender(kafkaAppender);
+ }
+ }
+
+ CommandLineParameters commandLineParameters = StringUtil.getCommandLineParser(args);
if (commandLineParameters.getArguments().contains(STOP_COMMAND_STR)){
performServerStopRequest(commandLineParameters);
}else{
AiravataZKUtils.startEmbeddedZK(cnxnFactory);
performServerStart(args);
}
-
}
@@ -173,7 +191,6 @@ public class ServerMain {
for (String string : args) {
logger.info("Server Arguments: " + string);
}
- ServerSettings.mergeSettingsCommandLineArgs(args);
String serverNames;
try {
serverNames = ApplicationSettings.getSetting(SERVERS_KEY);
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
----------------------------------------------------------------------
diff --git a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
index f7b8e1f..9df1745 100644
--- a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
+++ b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
@@ -21,6 +21,7 @@
package org.apache.airavata.testsuite.multitenantedairavata.utils;
+import org.apache.airavata.common.utils.ApplicationSettings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,7 +42,7 @@ public class PropertyReader {
}
protected void loadProperties() throws IOException {
- URL airavataURL = PropertyFileType.class.getClassLoader().getResource(TestFrameworkConstants.AIRAVATA_CLIENT_PROPERTIES);
+ URL airavataURL = ApplicationSettings.loadFile(TestFrameworkConstants.AIRAVATA_CLIENT_PROPERTIES);
if (airavataURL != null){
airavataClientProperties.load(airavataURL.openStream());
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
----------------------------------------------------------------------
diff --git a/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java b/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
index 83af52b..14a4b9c 100644
--- a/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
+++ b/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
@@ -91,8 +91,7 @@ public abstract class AbstractThriftDeserializer<E extends TFieldIdEnum, T exten
// Validate that the instance contains all required fields.
validate(instance);
} catch (final TException e) {
- log.error(String.format("Unable to deserialize JSON '%s' to type '%s'.",
- jp.getValueAsString(), instance.getClass().getName(), e));
+ log.error("Unable to deserialize JSON {} to type {}.", jp.getValueAsString(), instance.getClass().getName(), e);
ctxt.mappingException(e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6594026..00bc168 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,6 +101,8 @@
<snakeyaml.version>1.15</snakeyaml.version>
<maven.javadoc.failOnError>false</maven.javadoc.failOnError>
<maven.replacer.plugin.version>1.5.3</maven.replacer.plugin.version>
+ <kafka-clients.version>0.8.2.2</kafka-clients.version>
+ <logback.version>1.1.6</logback.version>
</properties>
<developers>
@@ -434,6 +436,16 @@
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka-clients.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -542,6 +554,26 @@
</excludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>com.lukegb.mojo</groupId>
+ <artifactId>gitdescribe-maven-plugin</artifactId>
+ <version>3.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>gitdescribe</goal>
+ </goals>
+ <id>git-describe</id>
+ <phase>initialize</phase>
+ <configuration>
+ <extraArguments>
+ <param>--tags</param>
+ </extraArguments>
+ <descriptionProperty>git-describe</descriptionProperty>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<activation>