You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/04/09 12:15:55 UTC
[59/61] [abbrv] ambari git commit: AMBARI-15679. Stack definition for
Logsearch (oleewre)
AMBARI-15679. Stack definition for Logsearch (oleewre)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cb6a7b9b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cb6a7b9b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cb6a7b9b
Branch: refs/heads/branch-dev-logsearch
Commit: cb6a7b9bc145a5b01d51418c2fe335321a21d718
Parents: f729469
Author: oleewere <ol...@gmail.com>
Authored: Sat Apr 9 11:59:06 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Sat Apr 9 11:59:06 2016 +0200
----------------------------------------------------------------------
.../libraries/functions/package_conditions.py | 4 +
.../common-services/LOGSEARCH/0.5.0/alerts.json | 63 +
.../0.5.0/configuration/logfeeder-env.xml | 118 ++
.../0.5.0/configuration/logfeeder-log4j.xml | 80 +
.../0.5.0/configuration/logfeeder-site.xml | 87 +
.../logsearch-audit_logs-solrconfig.xml | 1929 ++++++++++++++++++
.../0.5.0/configuration/logsearch-env.xml | 143 ++
.../0.5.0/configuration/logsearch-log4j.xml | 110 +
.../logsearch-service_logs-solrconfig.xml | 1928 +++++++++++++++++
.../0.5.0/configuration/logsearch-site.xml | 80 +
.../0.5.0/configuration/logsearch-solr-env.xml | 193 ++
.../configuration/logsearch-solr-log4j.xml | 57 +
.../0.5.0/configuration/logsearch-solr-xml.xml | 41 +
.../LOGSEARCH/0.5.0/kerberos.json | 17 +
.../LOGSEARCH/0.5.0/metainfo.xml | 119 ++
.../0.5.0/package/scripts/logfeeder.py | 67 +
.../0.5.0/package/scripts/logsearch.py | 65 +
.../LOGSEARCH/0.5.0/package/scripts/params.py | 231 +++
.../0.5.0/package/scripts/setup_logfeeder.py | 67 +
.../0.5.0/package/scripts/setup_logsearch.py | 93 +
.../0.5.0/package/scripts/setup_solr.py | 67 +
.../LOGSEARCH/0.5.0/package/scripts/solr.py | 66 +
.../0.5.0/package/scripts/status_params.py | 36 +
.../package/templates/global.config.json.j2 | 28 +
.../0.5.0/package/templates/grok-patterns.j2 | 144 ++
.../templates/input.config-accumulo.json.j2 | 105 +
.../templates/input.config-ambari.json.j2 | 93 +
.../package/templates/input.config-ams.json.j2 | 92 +
.../templates/input.config-atlas.json.j2 | 55 +
.../templates/input.config-falcon.json.j2 | 55 +
.../templates/input.config-hbase.json.j2 | 62 +
.../package/templates/input.config-hdfs.json.j2 | 246 +++
.../package/templates/input.config-hive.json.j2 | 62 +
.../templates/input.config-kafka.json.j2 | 105 +
.../package/templates/input.config-knox.json.j2 | 68 +
.../templates/input.config-logsearch.json.j2 | 68 +
.../templates/input.config-oozie.json.j2 | 56 +
.../templates/input.config-ranger.json.j2 | 122 ++
.../templates/input.config-storm.json.j2 | 86 +
.../package/templates/input.config-yarn.json.j2 | 86 +
.../templates/input.config-zookeeper.json.j2 | 56 +
.../package/templates/logfeeder.properties.j2 | 18 +
.../package/templates/logsearch.properties.j2 | 40 +
.../package/templates/output.config.json.j2 | 97 +
.../0.5.0/package/templates/zoo.cfg.j2 | 31 +
.../LOGSEARCH/0.5.0/quicklinks/quicklinks.json | 28 +
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 3 +
.../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 3 +
.../stacks/HDP/2.3/role_command_order.json | 5 +-
.../HDP/2.3/services/LOGSEARCH/metainfo.xml | 26 +
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 3 +
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 3 +
.../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 3 +
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 152 ++
.../stacks/2.4/LOGSEARCH/test_logsearch.py | 159 ++
.../python/stacks/2.4/LOGSEARCH/test_solr.py | 155 ++
.../test/python/stacks/2.4/configs/default.json | 398 ++++
.../src/test/python/stacks/utils/RMFTestCase.py | 14 +-
.../HDP/2.2.0/upgrades/upgrade_test_checks.xml | 1 +
.../app/mappers/service_metrics_mapper.js | 3 +-
ambari-web/app/models/quick_links.js | 13 +
.../app/views/common/quick_view_link_view.js | 6 +-
.../test/views/common/quick_link_view_test.js | 5 +
63 files changed, 8412 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index df46ce0..e164f56 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -46,6 +46,10 @@ def should_install_ams_grafana():
config = Script.get_config()
return 'role' in config and config['role'] == "METRICS_GRAFANA"
+def should_install_logsearch_solr():
+ config = Script.get_config()
+ return 'role' in config and config['role'] != "LOGSEARCH_LOGFEEDER"
+
def should_install_mysql():
config = Script.get_config()
hive_database = config['configurations']['hive-env']['hive_database']
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json
new file mode 100644
index 0000000..32bbf67
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json
@@ -0,0 +1,63 @@
+{
+ "LOGSEARCH": {
+ "LOGSEARCH_SERVER": [
+ {
+ "name": "logsearch_ui",
+ "label": "LogSearch Web UI",
+ "description": "This host-level alert is triggered if the LogSearch UI is unreachable.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{logsearch-site/logsearch.ui.port}}",
+ "https": "{{logsearch-site/logsearch.ui.port}}",
+ "connection_timeout": 5.0,
+ "default_port": 61888
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f}s"
+ },
+ "warning":{
+ "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+ },
+ "critical": {
+ "text": "Connection failed to {1} ({3})"
+ }
+ }
+ }
+ }
+ ],
+ "LOGSEARCH_SOLR": [
+ {
+ "name": "logsearch_solr",
+ "label": "Solr Web UI",
+ "description": "This host-level alert is triggered if the Solr Cloud Instance is unreachable.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{logsearch-solr-env/logsearch_solr_port}}",
+ "https": "{{logsearch-solr-env/logsearch_solr_port}}",
+ "connection_timeout": 5.0,
+ "default_port": 8886
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f}s"
+ },
+ "warning":{
+ "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+ },
+ "critical": {
+ "text": "Connection failed to {1} ({3})"
+ }
+ }
+ }
+ }
+ ]
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
new file mode 100644
index 0000000..2722be8
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
@@ -0,0 +1,118 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+
+ <property>
+ <name>logfeeder_pid_dir</name>
+ <value>/var/run/ambari-logsearch-logfeeder</value>
+ <description>logfeeder Process ID Directory</description>
+ <display-name>Logfeeder pid dir</display-name>
+ </property>
+
+ <property>
+ <name>logfeeder_log_dir</name>
+ <value>/var/log/ambari-logsearch-logfeeder</value>
+ <description>Log dir for logfeeder</description>
+ <display-name>Logfeeder log dir</display-name>
+ </property>
+
+ <property>
+ <name>logfeeder_checkpoint_folder</name>
+ <value>/etc/ambari-logsearch-logfeeder/conf/checkpoints</value>
+ <description>Checkpoint folder for logfeeder</description>
+ <display-name>Logfeeder checkpoint dir</display-name>
+ </property>
+
+ <property>
+ <name>logfeeder_config_files</name>
+ <value>{{logfeeder_config_files}}</value>
+ <description>Comma separated config files in grok format</description>
+ <display-name>Logfeeder config files</display-name>
+ </property>
+
+ <property>
+ <name>logfeeder_user</name>
+ <value>logfeeder</value>
+ <property-type>USER</property-type>
+ <description>logfeeder user</description>
+ <display-name>Logsfeeder User</display-name>
+ </property>
+
+ <property>
+ <name>logfeeder_group</name>
+ <value>logfeeder</value>
+ <property-type>GROUP</property-type>
+ <description>logfeeder group</description>
+ <display-name>Logsfeeder Group</display-name>
+ </property>
+
+ <property>
+ <name>logfeeder_max_mem</name>
+ <value>512m</value>
+ <description>Max memory for LogFeeder</description>
+ <display-name>Logsfeeder max memory</display-name>
+ </property>
+
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for config.json file</description>
+ <value>#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export LOGFEEDER_PATH={{logfeeder_dir}}
+
+export LOGFEEDER_CONF_DIR={{logsearch_logfeeder_conf}}
+
+#Logfile e.g. /var/log/logfeeder.log
+export LOGFILE={{logfeeder_log}}
+
+#pid file e.g. /var/run/logfeeder.pid
+export PID_FILE={{logfeeder_pid_file}}
+
+export JAVA_HOME={{java64_home}}
+
+if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
+ export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}}
+fi
+ </value>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml
new file mode 100644
index 0000000..5a3331e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+
+ <!-- solrconfig.xml -->
+
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for solrconfig.xml file for service logs</description>
+ <value><?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender">
+ <param name="file" value="{{logfeeder_log_dir}}/logfeeder.log" />
+ <param name="append" value="true" />
+ <param name="maxFileSize" value="10MB" />
+ <param name="maxBackupIndex" value="10" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/>
+ </layout>
+ </appender>
+
+ <category name="org.apache.ambari.logfeeder" additivity="false">
+ <priority value="info" />
+ <appender-ref ref="rolling_file" />
+ </category>
+
+ <root>
+ <priority value="warn" />
+ <appender-ref ref="rolling_file" />
+ </root>
+</log4j:configuration>
+ </value>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml
new file mode 100644
index 0000000..40ee374
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>logfeeder.solr.service.logs.enable</name>
+ <value>true</value>
+ <display-name>Service Logs enabled</display-name>
+ <description>Enable service logs to Solr</description>
+ </property>
+
+ <property>
+ <name>logfeeder.solr.audit.logs.enable</name>
+ <value>true</value>
+ <display-name>Audit Logs enabled</display-name>
+ <description>Enable audit logs to Solr</description>
+ </property>
+
+ <property>
+ <name>logfeeder.kafka.service.logs.enable</name>
+ <value>false</value>
+ <display-name>Kafka Service Logs enabled</display-name>
+ <description>Enable service logs to Kafka</description>
+ </property>
+
+ <property>
+ <name>logfeeder.kafka.topic.service.logs</name>
+ <value>service_logs</value>
+ <display-name>Kafka Service Logs topic</display-name>
+ <description>Kafka topic for service logs</description>
+ </property>
+
+ <property>
+ <name>logfeeder.kafka.audit.logs.enable</name>
+ <value>false</value>
+ <display-name>Audit Logs enabled</display-name>
+ <description>Enable audit logs to Kafka</description>
+ </property>
+
+ <property>
+ <name>logfeeder.kafka.topic.audit.logs</name>
+ <value>audit_logs</value>
+ <display-name>Kafka Audit Logs topic</display-name>
+ <description>Kafka topic for audit logs</description>
+ </property>
+
+ <property>
+ <name>logfeeder.kafka.broker.list</name>
+ <value>NONE</value>
+ <display-name>Kafka broker list</display-name>
+ <description>List of brokers. e.g. broker1_host:port,broker2_host:port</description>
+ </property>
+
+ <property>
+ <name>logfeeder.kafka.security.protocol</name>
+ <value>NONE</value>
+ <display-name>Kafka security protocol</display-name>
+ <description>If security is enabled, then set this value. E.g. SASL_PLAINTEXT</description>
+ </property>
+
+ <property>
+ <name>logfeeder.kafka.kerberos.service.name</name>
+ <value>NONE</value>
+ <display-name>Kafka Kerberos service name</display-name>
+ <description>If security is enabled, then set this value. E.g. kafka</description>
+ </property>
+
+</configuration>