You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/07/06 10:28:17 UTC
[1/2] ambari git commit: AMBARI-17492. Logsearch shows all the
services not only those which are available in the stack (Miklos Gergely via
oleewere)
Repository: ambari
Updated Branches:
refs/heads/trunk de5110ec4 -> c56844922
AMBARI-17492. Logsearch shows all the services not only those which are available in the stack (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c80ccd89
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c80ccd89
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c80ccd89
Branch: refs/heads/trunk
Commit: c80ccd89c9d3087b215e2a2f6d9e72ddd4b0990f
Parents: de5110e
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed Jul 6 12:13:16 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Jul 6 12:13:16 2016 +0200
----------------------------------------------------------------------
.../ambari-logsearch-assembly/pom.xml | 3 +-
.../ambari/server/agent/ExecutionCommand.java | 11 +
.../AmbariCustomCommandExecutionHelper.java | 2 +
.../AmbariManagementControllerImpl.java | 2 +
.../LOGSEARCH/0.5.0/package/scripts/params.py | 1 +
.../0.5.0/package/scripts/setup_logsearch.py | 6 +
.../templates/HadoopServiceConfig.json.j2 | 518 +++++++++++++++++++
.../stacks/2.4/LOGSEARCH/test_logsearch.py | 5 +
.../test/python/stacks/2.4/configs/default.json | 4 +
9 files changed, 551 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-logsearch/ambari-logsearch-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-assembly/pom.xml b/ambari-logsearch/ambari-logsearch-assembly/pom.xml
index 958232e..e0f6a09 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-assembly/pom.xml
@@ -139,6 +139,7 @@
<exclude>classes/log4j.xml</exclude>
<exclude>classes/logsearch.properties</exclude>
<exclude>classes/user_pass.json</exclude>
+ <exclude>classes/HadoopServiceConfig.json</exclude>
<exclude>solr_configsets/**</exclude>
</excludes>
</source>
@@ -391,7 +392,7 @@
<group>root</group>
</mapper>
<excludes>
- classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,solr_configsets/**
+ classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,classes/HadoopServiceConfig.json,solr_configsets/**
</excludes>
</data>
<data>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index f3f449d..9c262d9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -111,6 +111,9 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("localComponents")
private Set<String> localComponents = new HashSet<String>();
+ @SerializedName("availableServices")
+ private Set<String> availableServices = new HashSet<String>();
+
public String getCommandId() {
return commandId;
}
@@ -261,6 +264,14 @@ public class ExecutionCommand extends AgentCommand {
this.localComponents = localComponents;
}
+ public Set<String> getAvailableServices() {
+ return availableServices;
+ }
+
+ public void setAvailableServices(Set<String> availableServices) {
+ this.availableServices = availableServices;
+ }
+
public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() {
return configurationAttributes;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index fca66db..6750faf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -367,6 +367,8 @@ public class AmbariCustomCommandExecutionHelper {
execCmd.setForceRefreshConfigTagsBeforeExecution(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)));
}
+ execCmd.setAvailableServices(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()).keySet());
+
Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(CUSTOM_COMMAND, commandName);
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 5b4dae2..aaf69df 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2307,6 +2307,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
execCmd.setRoleParams(roleParams);
+
+ execCmd.setAvailableServices(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()).keySet());
if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) {
LOG.info("AmbariManagementControllerImpl.createHostAction: created ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, with cluster-env tags {}",
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index a89b3d6..165ac08 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -70,6 +70,7 @@ zookeeper_hosts_list.sort()
# get comma separated list of zookeeper hosts from clusterHostInfo
zookeeper_hosts = ",".join(zookeeper_hosts_list)
cluster_name = str(config['clusterName'])
+availableServices = config['availableServices']
# for now just pick first collector
if 'metrics_collector_hosts' in config['clusterHostInfo']:
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index bfab139..57d6e8b 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -56,6 +56,12 @@ def setup_logsearch():
group=params.user_group
)
+ File(format("{logsearch_server_conf}/HadoopServiceConfig.json"),
+ content=Template("HadoopServiceConfig.json.j2"),
+ owner=params.logsearch_user,
+ group=params.user_group
+ )
+
File(format("{logsearch_server_conf}/log4j.xml"),
content=InlineTemplate(params.logsearch_app_log4j_content),
owner=params.logsearch_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
new file mode 100644
index 0000000..64c81b5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
@@ -0,0 +1,518 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "service": {
+{% if "ACCUMULO" in availableServices %}
+ "accumulo": {
+ "label": "Accumulo",
+ "components": [
+ {
+ "name": "accumulo_gc"
+ },
+ {
+ "name": "accumulo_master"
+ },
+ {
+ "name": "accumulo_monitor"
+ },
+ {
+ "name": "accumulo_tracer"
+ },
+ {
+ "name": "accumulo_tserver"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "ATLAS" in availableServices %}
+ "atlas": {
+ "label": "Atlas",
+ "components": [
+ {
+ "name": "atlas_app"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+ "ambari": {
+ "label": "Ambari",
+ "components": [
+ {
+ "name": "ambari_agent"
+ },
+ {
+ "name": "ambari_server"
+ },
+ {
+ "name": "ambari_alerts"
+ },
+ {
+ "name": "ambari_audit"
+ },
+ {
+ "name": "ambari_config_changes"
+ },
+ {
+ "name": "ambari_eclipselink"
+ },
+ {
+ "name": "ambari_server_check_database"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% if "AMBARI_METRICS" in availableServices %}
+ "ams": {
+ "label": "AMS",
+ "components": [
+ {
+ "name": "ams_hbase_master"
+ },
+ {
+ "name": "ams_hbase_regionserver"
+ },
+ {
+ "name": "ams_collector"
+ },
+ {
+ "name": "ams_monitor"
+ },
+ {
+ "name": "ams_grafana"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "FALCON" in availableServices %}
+ "falcon": {
+ "label": "Falcon",
+ "components": [
+ {
+ "name": "falcon_app"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "FLUME" in availableServices %}
+ "flume": {
+ "label": "Flume",
+ "components": [
+ {
+ "name": "flume_handler"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "HBASE" in availableServices %}
+ "hbase": {
+ "label": "HBase",
+ "components": [
+ {
+ "name": "hbase_master"
+ },
+ {
+ "name": "hbase_regionserver"
+ },
+ {
+ "name": "hbase_phoenix_server"
+ }
+ ],
+ "dependencies": [
+ {
+ "service": "hdfs",
+ "components": [
+ "hdfs_namenode"
+ ]
+ }
+ ]
+ },
+{% endif %}
+{% if "HDFS" in availableServices %}
+ "hdfs": {
+ "label": "HDFS",
+ "components": [
+ {
+ "name": "hdfs_datanode"
+ },
+ {
+ "name": "hdfs_namenode"
+ },
+ {
+ "name": "hdfs_journalnode"
+ },
+ {
+ "name": "hdfs_secondarynamenode"
+ },
+ {
+ "name": "hdfs_zkfc"
+ },
+ {
+ "name": "hdfs_nfs3"
+ },
+ {
+ "name": "hdfs_audit",
+ "rowtype": "audit"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "HIVE" in availableServices %}
+ "hive": {
+ "label": "Hive",
+ "components": [
+ {
+ "name": "hive_hiveserver2"
+ },
+ {
+ "name": "hive_metastore"
+ },
+ {
+ "name": "webhcat_server"
+ }
+ ],
+ "dependencies": [
+ {
+ "service": "hdfs",
+ "components": [
+ "hdfs_namenode"
+ ]
+ }
+ ]
+ },
+{% endif %}
+{% if "KAFKA" in availableServices %}
+ "kafka": {
+ "label": "Kafka",
+ "components": [
+ {
+ "name": "kafka_controller"
+ },
+ {
+ "name": "kafka_request"
+ },
+ {
+ "name": "kafka_logcleaner"
+ },
+ {
+ "name": "kafka_server"
+ },
+ {
+ "name": "kafka_statechange"
+ }
+ ],
+ "dependencies": [
+ {
+ "service": "zookeeper",
+ "components": [
+ "zookeeper"
+ ]
+ }
+ ]
+ },
+{% endif %}
+{% if "KNOX" in availableServices %}
+ "knox": {
+ "label": "Knox",
+ "components": [
+ {
+ "name": "knox_gateway"
+ },
+ {
+ "name": "knox_cli"
+ },
+ {
+ "name": "knox_ldap"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "NIFI" in availableServices %}
+ "nifi": {
+ "label": "NiFi",
+ "components": [
+ {
+ "name": "nifi_app"
+ },
+ {
+ "name": "nifi_bootstrap"
+ },
+ {
+ "name": "nifi_setup"
+ },
+ {
+ "name": "nifi_user"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "MAPREDUCE2" in availableServices %}
+ "mapred": {
+ "label": "MapReduce",
+ "components": [
+ {
+ "name": "mapred_historyserver"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+ "logsearch": {
+ "label": "Logsearch",
+ "components": [
+ {
+ "name": "logsearch_app"
+ },
+ {
+ "name": "logsearch_feeder"
+ },
+ {
+ "name": "logsearch_perf"
+ },
+ {
+ "name": "logsearch_solr"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% if "RANGER" in availableServices %}
+ "ranger": {
+ "label": "Ranger",
+ "components": [
+ {
+ "name": "ranger_admin"
+ },
+ {
+ "name": "ranger_dbpatch"
+ },
+ {
+ "name": "ranger_kms"
+ },
+ {
+ "name": "ranger_usersync"
+ }
+ ],
+ "dependencies": [
+ {
+ "service": "hdfs",
+ "required": "optional",
+ "components": [
+ "hdfs_namenode"
+ ]
+ },
+ {
+ "service": "hbase",
+ "required": "optional",
+ "components": [
+ "hbase_master",
+ "hbase_regionserver"
+ ]
+ },
+ {
+ "service": "hive",
+ "required": "optional",
+ "components": [
+ "hive_hiveserver2"
+ ]
+ },
+ {
+ "service": "kafka",
+ "required": "optional",
+ "components": [
+ "kafka_ranger"
+ ]
+ },
+ {
+ "service": "knox",
+ "required": "optional",
+ "components": [
+ "knox_gateway"
+ ]
+ },
+ {
+ "service": "storm",
+ "required": "optional",
+ "components": [
+ "storm_supervisor"
+ ]
+ },
+ {
+ "service": "yarn",
+ "required": "optional",
+ "components": [
+ "yarn_resourcemanager"
+ ]
+ }
+ ]
+ },
+{% endif %}
+{% if "OOZIE" in availableServices %}
+ "oozie": {
+ "label": "Oozie",
+ "components": [
+ {
+ "name": "oozie_app"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "SMARTSENSE" in availableServices %}
+ "hst": {
+ "label": "SmartSense",
+ "components": [
+ {
+ "name": "hst_server"
+ },
+ {
+ "name": "hst_agent"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "SPARK" in availableServices %}
+ "spark": {
+ "label": "Spark",
+ "components": [
+ {
+ "name": "spark_jobhistory_server"
+ },
+ {
+ "name": "spark_thriftserver"
+ },
+ {
+ "name": "livy_server"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "SPARK2" in availableServices %}
+ "spark2": {
+ "label": "Spark 2",
+ "components": [
+ {
+ "name": "spark2_jobhistory_server"
+ },
+ {
+ "name": "spark2_thriftserver"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "STORM" in availableServices %}
+ "storm": {
+ "label": "Storm",
+ "components": [
+ {
+ "name": "storm_drpc"
+ },
+ {
+ "name": "storm_logviewer"
+ },
+ {
+ "name": "storm_nimbus"
+ },
+ {
+ "name": "storm_supervisor"
+ },
+ {
+ "name": "storm_ui"
+ },
+ {
+ "name": "storm_worker"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "YARN" in availableServices %}
+ "yarn": {
+ "label": "YARN",
+ "components": [
+ {
+ "name": "yarn_nodemanager"
+ },
+ {
+ "name": "yarn_resourcemanager"
+ },
+ {
+ "name": "yarn_timelineserver"
+ },
+ {
+ "name": "yarn_historyserver"
+ },
+ {
+ "name": "yarn_jobsummary"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "ZEPPELIN" in availableServices %}
+ "zeppelin": {
+ "label": "Zeppelin",
+ "components": [
+ {
+ "name": "zeppelin"
+ }
+ ],
+ "dependencies": [
+ ]
+ },
+{% endif %}
+{% if "ZOOKEEPER" in availableServices %}
+ "zookeeper": {
+ "label": "ZooKeeper",
+ "components": [
+ {
+ "name": "zookeeper"
+ }
+ ],
+ "dependencies": [
+ ]
+ }
+{% endif %}
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index 48e071d..cd45027 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -78,6 +78,11 @@ class TestLogSearch(RMFTestCase):
group='hadoop',
content = Template('logsearch.properties.j2')
)
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/HadoopServiceConfig.json',
+ owner = 'logsearch',
+ group='hadoop',
+ content = Template('HadoopServiceConfig.json.j2')
+ )
self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/log4j.xml',
owner = 'logsearch',
group='hadoop',
http://git-wip-us.apache.org/repos/asf/ambari/blob/c80ccd89/ambari-server/src/test/python/stacks/2.4/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json
index 233a733..462ee56 100644
--- a/ambari-server/src/test/python/stacks/2.4/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json
@@ -32,6 +32,10 @@
},
"taskId": 152,
"public_hostname": "c6401.ambari.apache.org",
+ "availableServices": [
+ "ACCUMULO",
+ "ATLAS"
+ ],
"configurations": {
"slider-client": {
"slider.yarn.queue": "default"
[2/2] ambari git commit: AMBARI-17561. Enable simulating logfeeder
inputs (Miklos Gergely via oleewere)
Posted by ol...@apache.org.
AMBARI-17561. Enable simulating logfeeder inputs (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c5684492
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c5684492
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c5684492
Branch: refs/heads/trunk
Commit: c56844922cb958c2879c292b0b57d2e5edf4a8a4
Parents: c80ccd8
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed Jul 6 12:14:21 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Jul 6 12:23:57 2016 +0200
----------------------------------------------------------------------
.../org/apache/ambari/logfeeder/LogFeeder.java | 21 ++-
.../ambari/logfeeder/input/InputSimulate.java | 158 +++++++++++++++++++
.../src/main/resources/alias_config.json | 105 ++++++------
3 files changed, 231 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c5684492/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
index 88a6737..8697f54 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
@@ -40,6 +40,7 @@ import org.apache.ambari.logfeeder.AliasUtil.ALIAS_PARAM;
import org.apache.ambari.logfeeder.AliasUtil.ALIAS_TYPE;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.input.InputSimulate;
import org.apache.ambari.logfeeder.logconfig.LogfeederScheduler;
import org.apache.ambari.logfeeder.output.Output;
import org.apache.ambari.logfeeder.util.FileUtil;
@@ -118,6 +119,9 @@ public class LogFeeder {
+ configFileName);
}
}
+
+ addSimulatedInputs();
+
mergeAllConfigs();
LogfeederScheduler.INSTANCE.start();
@@ -196,7 +200,22 @@ public class LogFeeder {
outputConfigList.addAll(mapList);
}
}
-
+ }
+
+ private void addSimulatedInputs() {
+ int simulatedInputNumber = LogFeederUtil.getIntProperty("logfeeder.simulate.input_number", 0);
+ if (simulatedInputNumber == 0)
+ return;
+
+ InputSimulate.loadTypeToFilePath(inputConfigList);
+ inputConfigList.clear();
+
+ for (int i = 0; i < simulatedInputNumber; i++) {
+ HashMap<String, Object> mapList = new HashMap<String, Object>();
+ mapList.put("source", "simulate");
+ mapList.put("rowtype", "service");
+ inputConfigList.add(mapList);
+ }
}
private void mergeAllConfigs() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/c5684492/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
new file mode 100644
index 0000000..72a0586
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.input;
+
+import java.net.Inet4Address;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.ambari.logfeeder.LogFeederUtil;
+import org.apache.ambari.logfeeder.filter.Filter;
+import org.apache.ambari.logfeeder.filter.JSONFilterCode;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.solr.common.util.Base64;
+
+public class InputSimulate extends Input {
+
+ private static final String LOG_MESSAGE_PREFIX = "Simulated log message for testing, line";
+
+ private static final String LOG_TEXT_PATTERN =
+ "{ logtime=\"%d\", level=\"%s\", log_message=\"<LOG_MESSAGE_PATTERN>\"}";
+
+ private static final Map<String, String> typeToFilePath = new HashMap<>();
+ public static void loadTypeToFilePath(List<Map<String, Object>> inputList) {
+ for (Map<String, Object> input : inputList) {
+ if (input.containsKey("type") && input.containsKey("path")) {
+ typeToFilePath.put((String)input.get("type"), (String)input.get("path"));
+ }
+ }
+ }
+
+ private static final Map<String, Integer> typeToLineNumber = new HashMap<>();
+
+ private final Random random = new Random(System.currentTimeMillis());
+
+ private final List<String> types;
+ private final String level;
+ private final String logText;
+ private final long sleepMillis;
+
+ public InputSimulate() throws Exception {
+ this.types = getSimulatedLogTypes();
+ this.level = LogFeederUtil.getStringProperty("logfeeder.simulate.log_level", "WARN");
+ this.logText = getLogText();
+ this.sleepMillis = LogFeederUtil.getIntProperty("logfeeder.simulate.sleep_milliseconds", 10000);
+
+ Filter filter = new JSONFilterCode();
+ filter.setInput(this);
+ setFirstFilter(filter);
+ }
+
+ private List<String> getSimulatedLogTypes() {
+ String logsToSimulate = LogFeederUtil.getStringProperty("logfeeder.simulate.log_ids");
+ if (logsToSimulate == null) {
+ return new ArrayList<>(typeToFilePath.keySet());
+ } else {
+ List<String> simulatedLogTypes = Arrays.asList(logsToSimulate.split(","));
+ simulatedLogTypes.retainAll(typeToFilePath.keySet());
+ return simulatedLogTypes;
+ }
+ }
+
+ private String getLogText() {
+ int logTextSize = LogFeederUtil.getIntProperty("logfeeder.simulate.log_message_size", 100);
+ int fillerSize = Math.max(logTextSize - LOG_MESSAGE_PREFIX.length() - 10, 0);
+ String filler = StringUtils.repeat("X", fillerSize);
+ String logMessagePattern = LOG_MESSAGE_PREFIX + " %08d " + filler;
+
+ return LOG_TEXT_PATTERN.replaceAll("<LOG_MESSAGE_PATTERN>", logMessagePattern);
+ }
+
+ @Override
+ public String getNameForThread() {
+ return "Simulated input";
+ }
+
+ @Override
+ public String getShortDescription() {
+ return "Simulated input";
+ }
+
+ @Override
+ void start() throws Exception {
+ if (types.isEmpty())
+ return;
+
+ getFirstFilter().setOutputMgr(outputMgr);
+ while (true) {
+ String type = imitateRandomLogFile();
+
+ InputMarker marker = getInputMarker(type);
+ String line = getLine(marker);
+
+ outputLine(line, marker);
+
+ try { Thread.sleep(sleepMillis); } catch(Exception e) {}
+ }
+ }
+
+ private String imitateRandomLogFile() {
+ int typePos = random.nextInt(types.size());
+ String type = types.get(typePos);
+ String filePath = typeToFilePath.get(type);
+
+ configs.put("type", type);
+ setFilePath(filePath);
+
+ return type;
+ }
+
+ private InputMarker getInputMarker(String type) throws Exception {
+ InputMarker marker = new InputMarker();
+ marker.input = this;
+ marker.lineNumber = getLineNumber(type);
+ marker.base64FileKey = getBase64FileKey();
+ return marker;
+ }
+
+ private static synchronized int getLineNumber(String type) {
+ if (!typeToLineNumber.containsKey(type)) {
+ typeToLineNumber.put(type, 0);
+ }
+ Integer lineNumber = typeToLineNumber.get(type) + 1;
+
+ typeToLineNumber.put(type, lineNumber);
+ return lineNumber;
+ }
+
+ private String getBase64FileKey() throws Exception {
+ String fileKey = Inet4Address.getLocalHost().getHostAddress() + "|" + filePath;
+ return Base64.byteArrayToBase64(fileKey.getBytes());
+ }
+
+ private String getLine(InputMarker marker) {
+ Date d = new Date();
+ return String.format(logText, d.getTime(), level, marker.lineNumber);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c5684492/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
index 978f581..bc221a0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json
@@ -1,54 +1,55 @@
{
- "input": {
- "file": {
- "klass": "org.apache.ambari.logfeeder.input.InputFile"
- },
- "s3_file": {
- "klass": "org.apache.ambari.logfeeder.input.InputS3File"
- }
-
- },
- "filter": {
- "json": {
- "klass": "org.apache.ambari.logfeeder.filter.JSONFilterCode"
- },
- "keyvalue": {
- "klass": "org.apache.ambari.logfeeder.filter.FilterKeyValue"
- },
- "grok": {
- "klass": "org.apache.ambari.logfeeder.filter.FilterGrok"
- }
- },
-
- "mapper": {
- "map_date": {
- "klass": "org.apache.ambari.logfeeder.mapper.MapperDate"
- },
- "map_fieldname": {
- "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldName"
- },
- "map_fieldvalue": {
- "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldValue"
- }
- },
- "output": {
- "solr": {
- "klass": "org.apache.ambari.logfeeder.output.OutputSolr"
- },
- "file": {
- "klass": "org.apache.ambari.logfeeder.output.OutputFile"
- },
- "kafka": {
- "klass": "org.apache.ambari.logfeeder.output.OutputKafka"
- },
- "dev_null": {
- "klass": "org.apache.ambari.logfeeder.output.OutputDevNull"
- },
- "s3_file": {
- "klass": "org.apache.ambari.logfeeder.output.OutputS3File"
- },
- "hdfs": {
- "klass": "org.apache.ambari.logfeeder.output.OutputHDFSFile"
- }
- }
+"input": {
+ "file": {
+ "klass": "org.apache.ambari.logfeeder.input.InputFile"
+ },
+ "s3_file": {
+ "klass": "org.apache.ambari.logfeeder.input.InputS3File"
+ },
+ "simulate": {
+ "klass": "org.apache.ambari.logfeeder.input.InputSimulate"
+ }
+ },
+ "filter": {
+ "json": {
+ "klass": "org.apache.ambari.logfeeder.filter.JSONFilterCode"
+ },
+ "keyvalue": {
+ "klass": "org.apache.ambari.logfeeder.filter.FilterKeyValue"
+ },
+ "grok": {
+ "klass": "org.apache.ambari.logfeeder.filter.FilterGrok"
+ }
+ },
+ "mapper": {
+ "map_date": {
+ "klass": "org.apache.ambari.logfeeder.mapper.MapperDate"
+ },
+ "map_fieldname": {
+ "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldName"
+ },
+ "map_fieldvalue": {
+ "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldValue"
+ }
+ },
+ "output": {
+ "solr": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputSolr"
+ },
+ "file": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputFile"
+ },
+ "kafka": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputKafka"
+ },
+ "dev_null": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputDevNull"
+ },
+ "s3_file": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputS3File"
+ },
+ "hdfs": {
+ "klass": "org.apache.ambari.logfeeder.output.OutputHDFSFile"
+ }
+ }
}
\ No newline at end of file