You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/07/06 13:10:39 UTC
[46/50] [abbrv] ambari git commit: AMBARI-21399 Create property
descriptions for internal Log Feeder configs (mgergely)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
index 0fdc21f..a244a4e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
@@ -32,6 +32,7 @@ import java.util.Properties;
import org.apache.ambari.logfeeder.LogFeeder;
import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -46,6 +47,26 @@ import com.google.gson.reflect.TypeToken;
public class LogFeederUtil {
private static final Logger LOG = Logger.getLogger(LogFeederUtil.class);
+ public static final String LOGFEEDER_PROPERTIES_FILE = "logfeeder.properties";
+
+ @LogSearchPropertyDescription(
+ name = "cluster.name",
+ description = "The name of the cluster the Log Feeder program runs in.",
+ examples = {"cl1"},
+ sources = {LOGFEEDER_PROPERTIES_FILE}
+ )
+ private static final String CLUSTER_NAME_PROPERTY = "cluster.name";
+
+ private static final String DEFAULT_TMP_DIR = "/tmp/$username/logfeeder/";
+ @LogSearchPropertyDescription(
+ name = "logfeeder.tmp.dir",
+ description = "The tmp dir used for creating temporary files.",
+ examples = {"/tmp/"},
+ defaultValue = DEFAULT_TMP_DIR,
+ sources = {LOGFEEDER_PROPERTIES_FILE}
+ )
+ private static final String TMP_DIR_PROPERTY = "logfeeder.tmp.dir";
+
private final static String GSON_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";
private static Gson gson = new GsonBuilder().setDateFormat(GSON_DATE_FORMAT).create();
@@ -81,6 +102,10 @@ public class LogFeederUtil {
return props;
}
+ public static void loadProperties() throws Exception {
+ loadProperties(LOGFEEDER_PROPERTIES_FILE);
+ }
+
/**
* This method will read the properties from System, followed by propFile and finally from the map
*/
@@ -236,16 +261,20 @@ public class LogFeederUtil {
}
}
- private static String logfeederTempDir = null;
+ public static String getClusterName() {
+ return getStringProperty(CLUSTER_NAME_PROPERTY);
+ }
+
+ private static String logFeederTempDir = null;
- public synchronized static String getLogfeederTempDir() {
- if (logfeederTempDir == null) {
- String tempDirValue = getStringProperty("logfeeder.tmp.dir", "/tmp/$username/logfeeder/");
+ public synchronized static String getLogFeederTempDir() {
+ if (logFeederTempDir == null) {
+ String tempDirValue = getStringProperty(TMP_DIR_PROPERTY, DEFAULT_TMP_DIR);
HashMap<String, String> contextParam = new HashMap<String, String>();
String username = System.getProperty("user.name");
contextParam.put("username", username);
- logfeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam);
+ logFeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam);
}
- return logfeederTempDir;
+ return logFeederTempDir;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
deleted file mode 100644
index c975b99..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.util;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
-
-public class LogfeederHDFSUtil {
- private static final Logger LOG = Logger.getLogger(LogfeederHDFSUtil.class);
-
- private LogfeederHDFSUtil() {
- throw new UnsupportedOperationException();
- }
-
- public static boolean copyFromLocal(String sourceFilepath, String destFilePath, FileSystem fileSystem, boolean overwrite,
- boolean delSrc) {
- Path src = new Path(sourceFilepath);
- Path dst = new Path(destFilePath);
- boolean isCopied = false;
- try {
- LOG.info("copying localfile := " + sourceFilepath + " to hdfsPath := " + destFilePath);
- fileSystem.copyFromLocalFile(delSrc, overwrite, src, dst);
- isCopied = true;
- } catch (Exception e) {
- LOG.error("Error copying local file :" + sourceFilepath + " to hdfs location : " + destFilePath, e);
- }
- return isCopied;
- }
-
- public static FileSystem buildFileSystem(String hdfsHost, String hdfsPort) {
- try {
- Configuration configuration = buildHdfsConfiguration(hdfsHost, hdfsPort);
- FileSystem fs = FileSystem.get(configuration);
- return fs;
- } catch (Exception e) {
- LOG.error("Exception is buildFileSystem :", e);
- }
- return null;
- }
-
- private static Configuration buildHdfsConfiguration(String hdfsHost, String hdfsPort) {
- String url = "hdfs://" + hdfsHost + ":" + hdfsPort + "/";
- Configuration configuration = new Configuration();
- configuration.set("fs.default.name", url);
- return configuration;
- }
-
- public static void closeFileSystem(FileSystem fileSystem) {
- if (fileSystem != null) {
- try {
- fileSystem.close();
- } catch (IOException e) {
- LOG.error(e.getLocalizedMessage(), e.getCause());
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
index d963de3..573bb1c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.util;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
@@ -28,6 +29,8 @@ import org.apache.log4j.Logger;
import java.io.File;
import java.nio.charset.Charset;
+import static org.apache.ambari.logfeeder.util.LogFeederUtil.LOGFEEDER_PROPERTIES_FILE;
+
public class SSLUtil {
private static final Logger LOG = Logger.getLogger(SSLUtil.class);
@@ -42,6 +45,12 @@ public class SSLUtil {
private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt";
private static final String TRUSTSTORE_PASSWORD_FILE = "ts_pass.txt";
+ @LogSearchPropertyDescription(
+ name = "hadoop.security.credential.provider.path",
+ description = "The jceks file that provides passwords.",
+ examples = {"jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder.jceks"},
+ sources = {LOGFEEDER_PROPERTIES_FILE}
+ )
private static final String CREDENTIAL_STORE_PROVIDER_PATH = "hadoop.security.credential.provider.path";
private static final String LOGFEEDER_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys";
private static final String LOGFEEDER_STORE_DEFAULT_PASSWORD = "bigdata";
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index 8f75c3a..7abf177 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.TimeZone;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.common.LogfeederException;
+import org.apache.ambari.logfeeder.common.LogFeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl;
@@ -135,9 +135,9 @@ public class FilterJSONTest {
String inputStr="invalid json";
try{
filterJson.apply(inputStr,new InputMarker(null, null, 0));
- fail("Expected LogfeederException was not occured");
- }catch(LogfeederException logfeederException){
- assertEquals("Json parsing failed for inputstr = "+inputStr, logfeederException.getLocalizedMessage());
+ fail("Expected LogFeederException was not occured");
+ }catch(LogFeederException logFeederException){
+ assertEquals("Json parsing failed for inputstr = "+inputStr, logFeederException.getLocalizedMessage());
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
index 625e362..46fbc3b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
@@ -143,32 +143,6 @@ public class InputManagerTest {
}
@Test
- public void testInputManagr_waitOnAllInputs() throws Exception {
- Input input1 = strictMock(Input.class);
- Input input2 = strictMock(Input.class);
- Input input3 = strictMock(Input.class);
-
- Thread mockThread = strictMock(Thread.class);
-
- expect(input1.getThread()).andReturn(null);
- expect(input2.getThread()).andReturn(null);
- expect(input3.getThread()).andReturn(mockThread);
-
- mockThread.join(); expectLastCall();
-
- replay(input1, input2, input3);
-
- InputManager manager = new InputManager();
- manager.add("serviceName", input1);
- manager.add("serviceName", input2);
- manager.add("serviceName", input3);
-
- manager.waitOnAllInputs();
-
- verify(input1, input2, input3);
- }
-
- @Test
public void testInputManager_checkInAll() throws Exception {
Input input1 = strictMock(Input.class);
Input input2 = strictMock(Input.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
index 1461352..4b4067c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java
@@ -80,7 +80,6 @@ public class MetricsManagerTest {
assertEquals(metricList.size(), 1);
TimelineMetric metric = metricList.get(0);
- assertEquals(metric.getHostName(), "test_host_name");
assertEquals(metric.getAppId(), "logfeeder");
assertEquals(metric.getMetricName(), "metric1");
assertEquals(metric.getType(), "Long");
@@ -116,7 +115,6 @@ public class MetricsManagerTest {
assertEquals(metricList.size(), 1);
TimelineMetric metric = metricList.get(0);
- assertEquals(metric.getHostName(), "test_host_name");
assertEquals(metric.getAppId(), "logfeeder");
assertEquals(metric.getMetricName(), "metric1");
assertEquals(metric.getType(), "Long");
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
index 19027d1..5476c96 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
@@ -17,5 +17,4 @@ logfeeder.log.filter.enable=true
logfeeder.solr.config.interval=5
logfeeder.solr.zk_connect_string=some_connect_string
logfeeder.metrics.collector.hosts=some_collector_host
-node.hostname=test_host_name
logfeeder.include.default.level=FATAL,ERROR,WARN
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
index 978e91a..c34dce6 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
@@ -53,7 +53,7 @@ public class LogSearchConfigConfigurer implements Configurer {
while (true) {
try {
logSearchConfig = LogSearchConfigFactory.createLogSearchConfig(Component.SERVER, PropertiesHelper.getProperties(),
- LogSearchConfigZK.class);
+ null, LogSearchConfigZK.class);
logSearchConfigState.setLogSearchConfigAvailable(true);
break;
} catch (Exception e) {