You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/04/09 12:15:50 UTC
[54/61] [abbrv] ambari git commit: AMBARI-15679. Stack definition for
Logsearch (oleewre)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2
new file mode 100644
index 0000000..ecce706
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2
@@ -0,0 +1,105 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"kafka_controller",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/controller.log"
+ },
+ {
+ "type":"kafka_request",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/kafka-request.log"
+ },
+ {
+ "type":"kafka_logcleaner",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/log-cleaner.log"
+ },
+ {
+ "type":"kafka_server",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/server.log"
+ },
+ {
+ "type":"kafka_statechange",
+ "rowtype":"service",
+ "path":"{{kafka_log_dir}}/state-change.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_controller",
+ "kafka_request",
+ "kafka_logcleaner"
+ ]
+
+ }
+
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed",
+ "conditions":{
+ "fields":{
+ "type":[
+ "kafka_server",
+ "kafka_statechange"
+ ]
+
+ }
+
+ },
+ "log4j_format":"[%d] %p %m (%c)%n",
+ "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+ "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2
new file mode 100644
index 0000000..7c8e635
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2
@@ -0,0 +1,68 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"knox_gateway",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/gateway.log"
+ },
+ {
+ "type":"knox_cli",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/knoxcli.log"
+ },
+ {
+ "type":"knox_ldap",
+ "rowtype":"service",
+ "path":"{{knox_log_dir}}/ldap.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "knox_gateway",
+ "knox_cli",
+ "knox_ldap"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
new file mode 100644
index 0000000..cf338da
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
@@ -0,0 +1,68 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"logsearch_app",
+ "rowtype":"service",
+ "path":"{{logsearch_log_dir}}/logsearch.log"
+ },
+ {
+ "type":"logsearch_feeder",
+ "rowtype":"service",
+ "path":"{{logfeeder_log_dir}}/logfeeder.log"
+ },
+ {
+ "type":"logsearch_perf",
+ "rowtype":"service",
+ "path":"{{logsearch_log_dir}}/logsearch-performance.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "logsearch_app",
+ "logsearch_feeder",
+ "logsearch_perf"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2
new file mode 100644
index 0000000..00cf789
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"oozie_app",
+ "rowtype":"service",
+ "path":"{{oozie_log_dir}}/oozie.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "oozie_app"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2
new file mode 100644
index 0000000..29a280a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2
@@ -0,0 +1,122 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"ranger_admin",
+ "rowtype":"service",
+ "path":"{{ranger_admin_log_dir}}/xa_portal.log"
+ },
+ {
+ "type":"ranger_dbpatch",
+ "is_enabled":"true",
+ "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log"
+ },
+ {
+ "type":"ranger_kms",
+ "rowtype":"service",
+ "path":"{{ranger_kms_log_dir}}/kms.log"
+ },
+ {
+ "type":"ranger_usersync",
+ "rowtype":"service",
+ "path":"{{ranger_usersync_log_dir}}/usersync.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_admin",
+ "ranger_dbpatch"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_kms"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "ranger_usersync"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n",
+ "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+ "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"dd MMM yyyy HH:mm:ss"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2
new file mode 100644
index 0000000..9995689
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2
@@ -0,0 +1,86 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"storm_drpc",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/drpc.log"
+ },
+ {
+ "type":"storm_logviewer",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/logviewer.log"
+ },
+ {
+ "type":"storm_nimbus",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/nimbus.log"
+ },
+ {
+ "type":"storm_supervisor",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/supervisor.log"
+ },
+ {
+ "type":"storm_ui",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/ui.log"
+ },
+ {
+ "type":"storm_worker",
+ "rowtype":"service",
+ "path":"{{storm_log_dir}}/*worker*.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_drpc",
+ "storm_logviewer",
+ "storm_nimbus",
+ "storm_supervisor",
+ "storm_ui",
+ "storm_worker"
+ ]
+
+ }
+
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2
new file mode 100644
index 0000000..8df7fb6
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2
@@ -0,0 +1,86 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"mapred_historyserver",
+ "rowtype":"service",
+ "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log"
+ },
+ {
+ "type":"yarn_nodemanager",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log"
+ },
+ {
+ "type":"yarn_resourcemanager",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log"
+ },
+ {
+ "type":"yarn_timelineserver",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log"
+ },
+ {
+ "type":"yarn_historyserver",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log"
+ },
+ {
+ "type":"yarn_jobsummary",
+ "rowtype":"service",
+ "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "mapred_historyserver",
+ "yarn_historyserver",
+ "yarn_jobsummary",
+ "yarn_nodemanager",
+ "yarn_resourcemanager",
+ "yarn_timelineserver"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2
new file mode 100644
index 0000000..b90cb6d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"zookeeper",
+ "rowtype":"service",
+ "path":"{{zk_log_dir}}/zookeeper*.out"
+ }
+
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "zookeeper"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
new file mode 100644
index 0000000..1edf16a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logfeeder.checkpoint.folder={{logfeeder_checkpoint_folder}}
+metrics.collector.hosts={{metrics_collector_hosts}}
+config.files={{logfeeder_config_files}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
new file mode 100644
index 0000000..1e183bd
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#solr.url=http://solr_host:{{logsearch_solr_port}}/solr
+solr.zkhosts={{zookeeper_quorum}}{{logsearch_solr_znode}}
+
+# Service Logs
+solr.core.logs={{logsearch_solr_collection_service_logs}}
+
+solr.service_logs.split_interval_mins={{service_logs_collection_splits_interval_mins}}
+solr.service_logs.shards={{logsearch_numshards}}
+solr.service_logs.replication_factor={{logsearch_repfactor}}
+
+solr.servicelogs.fields={{logsearch_service_logs_fields}}
+
+# Audit logs
+auditlog.solr.zkhosts={{solr_audit_logs_zk_quorum}}{{solr_audit_logs_zk_node}}
+auditlog.solr.core.logs={{logsearch_solr_collection_audit_logs}}
+auditlog.solr.url={{solr_audit_logs_url}}
+
+solr.audit_logs.split_interval_mins={{audit_logs_collection_splits_interval_mins}}
+solr.audit_logs.shards={{logsearch_numshards}}
+solr.audit_logs.replication_factor={{logsearch_repfactor}}
+
+# History logs
+solr.core.history=history
+solr.history.config_name=history
+solr.history.replication_factor={{logsearch_repfactor}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2
new file mode 100644
index 0000000..bd88ba7
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2
@@ -0,0 +1,97 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "output":[
+ {
+ "is_enabled":"{{solr_service_logs_enable}}",
+ "comment":"Output to solr for service logs",
+ "destination":"solr",
+ "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}",
+ "collection":"{{logsearch_solr_collection_service_logs}}",
+ "number_of_shards": "{{logsearch_numshards}}",
+ "splits_interval_mins": "{{service_logs_collection_splits_interval_mins}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "service"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "comment":"Output to solr for audit records",
+ "is_enabled":"{{solr_audit_logs_enable}}",
+ "destination":"solr",
+ "zk_hosts":"{{zookeeper_quorum}}{{logsearch_solr_znode}}",
+ "collection":"{{logsearch_solr_collection_audit_logs}}",
+ "number_of_shards": "{{logsearch_numshards}}",
+ "splits_interval_mins": "{{audit_logs_collection_splits_interval_mins}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "audit"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "is_enabled":"{{kafka_service_logs_enable}}",
+ "destination":"kafka",
+ "broker_list":"{{kafka_broker_list}}",
+ "topic":"{{kafka_topic_service_logs}}",
+ "kafka.security.protocol":"{{kafka_security_protocol}}",
+ "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "service"
+ ]
+
+ }
+
+ }
+
+ },
+ {
+ "is_enabled":"{{kafka_audit_logs_enable}}",
+ "destination":"kafka",
+ "broker_list":"{{kafka_broker_list}}",
+ "topic":"{{kafka_topic_audit_logs}}",
+ "kafka.security.protocol":"{{kafka_security_protocol}}",
+ "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
+ "conditions":{
+ "fields":{
+ "rowtype":[
+ "audit"
+ ]
+
+ }
+
+ }
+
+ }
+
+ ]
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2
new file mode 100644
index 0000000..6210f06
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/zoo.cfg.j2
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=5
+
+# the directory where the snapshot is stored.
+# dataDir=/opt/zookeeper/data
+# NOTE: Solr defaults the dataDir to $solrHome/zoo_data
+
+# the port at which the clients will connect
+# clientPort=2181
+# NOTE: Solr sets this based on zkRun / zkHost params
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json
new file mode 100644
index 0000000..16e1f4c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"http"
+ },
+ "links" : [
+ {
+ "name": "logsearch_ui",
+ "label": "Logsearch UI",
+ "url":"%@://%@:%@",
+ "requires_user_name": "false",
+ "port":{
+ "http_property": "logsearch.ui.port",
+ "http_default_port": "8888",
+ "https_property": "logsearch.ui.port",
+ "https_default_port": "8888",
+ "regex": "^(\\d+)$",
+ "site": "logsearch-site"
+ }
+ }
+ ]
+
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 77ff982..b0cff68 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -174,6 +174,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -210,6 +211,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -322,6 +324,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index 0ef4674..bf041de 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -180,6 +180,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -216,6 +217,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -328,6 +330,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index 9ab413e..d64714a 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -20,6 +20,9 @@
"KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"],
"KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"],
"NIMBUS-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"],
- "STORM_UI_SERVER-START" : ["NIMBUS-START", "NAMENODE-START"]
+ "STORM_UI_SERVER-START" : ["NIMBUS-START", "NAMENODE-START"],
+ "LOGSEARCH_SOLR-START" : ["ZOOKEEPER_SERVER-START"],
+ "LOGSEARCH_SERVER-START": ["LOGSEARCH_SOLR-START"],
+ "LOGSEARCH_LOGFEEDER-START": ["LOGSEARCH_SOLR-START", "LOGSEARCH_SERVER-START"]
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml
new file mode 100644
index 0000000..df697dc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/LOGSEARCH/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>LOGSEARCH</name>
+ <extends>common-services/LOGSEARCH/0.5.0</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index ed30846..6b74af0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -175,6 +175,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -213,6 +214,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -339,6 +341,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 4731631..9fb2bba 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -166,6 +166,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -203,6 +204,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -320,6 +322,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
index b53a090..e3bc7a3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
@@ -173,6 +173,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -211,6 +212,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
@@ -337,6 +339,7 @@
</priority>
<exclude>
<service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
</exclude>
</group>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
new file mode 100644
index 0000000..ab11ba1
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile
+from resource_management.core.exceptions import ComponentIsNotRunning
+
+class TestLogFeeder(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package"
+ STACK_VERSION = "2.4"
+
+ def configureResourcesCalled(self):
+ self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-logfeeder',
+ owner = 'logfeeder',
+ group = 'logfeeder',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-logfeeder',
+ owner = 'logfeeder',
+ group = 'logfeeder',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder',
+ owner = 'logfeeder',
+ group = 'logfeeder',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf',
+ owner = 'logfeeder',
+ group = 'logfeeder',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
+ owner = 'logfeeder',
+ group = 'logfeeder',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/var/log/ambari-logsearch-logfeeder/logfeeder.out',
+ owner = 'logfeeder',
+ group = 'logfeeder',
+ mode = 0644,
+ content = ''
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
+ owner = 'logfeeder',
+ content = Template('logfeeder.properties.j2')
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh',
+ owner = 'logfeeder',
+ content = InlineTemplate(self.getConfig()['configurations']['logfeeder-env']['content'])
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/log4j.xml',
+ owner = 'logfeeder',
+ content = InlineTemplate(self.getConfig()['configurations']['logfeeder-log4j']['content'])
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/grok-patterns',
+ owner = 'logfeeder',
+ content = Template('grok-patterns.j2'),
+ encoding = 'utf-8'
+ )
+
+ logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka',
+ 'knox', 'logsearch', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper']
+
+ logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
+
+ for file_name in logfeeder_config_file_names:
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name,
+ owner = 'logfeeder',
+ content = Template(file_name + ".j2")
+ )
+
+ def test_configure_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py",
+ classname = "LogFeeder",
+ command = "configure",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.configureResourcesCalled()
+ self.assertNoMoreResources()
+
+ def test_start_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py",
+ classname = "LogFeeder",
+ command = "start",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.configureResourcesCalled()
+ self.assertResourceCalled('Execute', '/usr/lib/ambari-logsearch-logfeeder/run.sh',
+ environment={'LOGFEEDER_INCLUDE': '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh'},
+ user = 'logfeeder'
+ )
+
+ def test_stop_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py",
+ classname = "LogFeeder",
+ command = "stop",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.assertResourceCalled('Execute', 'kill `cat /var/run/ambari-logsearch-logfeeder/logfeeder.pid`',
+ user = 'logfeeder',
+ only_if = 'test -f /var/run/ambari-logsearch-logfeeder/logfeeder.pid'
+ )
+ self.assertResourceCalled('File', '/var/run/ambari-logsearch-logfeeder/logfeeder.pid',
+ action = ['delete']
+ )
+
+# def test_status_default(self):
+# with self.assertRaises(ComponentIsNotRunning) as e:
+# self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py",
+# classname = "LogFeeder",
+# command = "status",
+# config_file = "default.json",
+# stack_version = self.STACK_VERSION,
+# target = RMFTestCase.TARGET_COMMON_SERVICES
+# )
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
new file mode 100644
index 0000000..585988d
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile
+from resource_management.core.exceptions import ComponentIsNotRunning
+from mock.mock import MagicMock, patch
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+
+class TestLogSearch(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package"
+ STACK_VERSION = "2.4"
+
+ def configureResourcesCalled(self):
+ self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-portal',
+ owner = 'logsearch',
+ group = 'logsearch',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-portal',
+ owner = 'logsearch',
+ group = 'logsearch',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-portal',
+ owner = 'logsearch',
+ group = 'logsearch',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/etc/ambari-logsearch-portal/conf',
+ owner = 'logsearch',
+ group = 'logsearch',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/var/log/ambari-logsearch-portal/logsearch.out',
+ owner = 'logsearch',
+ group = 'logsearch',
+ mode = 0644,
+ content = ''
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch.properties',
+ owner = 'logsearch',
+ content = Template('logsearch.properties.j2')
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/log4j.xml',
+ owner = 'logsearch',
+ content = InlineTemplate(self.getConfig()['configurations']['logsearch-log4j']['content'])
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch-env.sh',
+ content = InlineTemplate(self.getConfig()['configurations']['logsearch-env']['content']),
+ mode = 0755,
+ owner = "logsearch"
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf/solrconfig.xml',
+ owner = 'logsearch',
+ content = InlineTemplate(self.getConfig()['configurations']['logsearch-service_logs-solrconfig']['content'])
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/solrconfig.xml',
+ owner = 'logsearch',
+ content = InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content'])
+ )
+
+ self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_hadoop_logs_0.[0-9]* -confname hadoop_logs$',
+ only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/hadoop_logs$"
+ )
+ self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf -confname hadoop_logs$',
+ not_if = "^test -d /tmp/solr_config_hadoop_logs_0.[0-9]*$"
+ )
+ self.assertResourceCalled('Execute', '/usr/lib/ambari-logsearch-solr/bin/solr create -c history -d /etc/ambari-logsearch-portal/conf/solr_configsets/history/conf -s 10 -rf 1',
+ environment={'SOLR_INCLUDE': '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh'}
+ )
+ self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd downconfig -confdir /tmp/solr_config_audit_logs_0.[0-9]* -confname audit_logs$',
+ only_if = "^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd get /configs/audit_logs$"
+ )
+ self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:None/logsearch -cmd upconfig -confdir /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf -confname audit_logs$',
+ not_if = "^test -d /tmp/solr_config_audit_logs_0.[0-9]*$"
+ )
+ self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'),
+ sudo=True
+ )
+
+ def test_configure_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py",
+ classname = "LogSearch",
+ command = "configure",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.configureResourcesCalled()
+ self.assertNoMoreResources()
+
+ def test_start_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py",
+ classname = "LogSearch",
+ command = "start",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.configureResourcesCalled()
+ self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-portal/run.sh 61888",
+ environment = {'LOGSEARCH_INCLUDE': '/etc/ambari-logsearch-portal/conf/logsearch-env.sh'},
+ user = "logsearch"
+ )
+
+ def test_stop_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py",
+ classname = "LogSearch",
+ command = "stop",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.assertResourceCalled('Execute', 'kill `cat /var/run/ambari-logsearch-portal/logsearch.pid` >/dev/null 2>&1',
+ user = 'logsearch',
+ only_if = 'test -f /var/run/ambari-logsearch-portal/logsearch.pid'
+ )
+ self.assertResourceCalled('File', '/var/run/ambari-logsearch-portal/logsearch.pid',
+ action = ['delete']
+ )
+
+# def test_status_default(self):
+# with self.assertRaises(ComponentIsNotRunning) as e:
+# self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py",
+# classname = "LogSearch",
+# command = "status",
+# config_file = "default.json",
+# stack_version = self.STACK_VERSION,
+# target = RMFTestCase.TARGET_COMMON_SERVICES
+# )
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb6a7b9b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py
new file mode 100644
index 0000000..5a684e6
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+
+class TestSolr(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package"
+ STACK_VERSION = "2.4"
+
+ def configureResourcesCalled(self):
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr',
+ owner = 'solr',
+ group = 'solr',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr',
+ owner = 'solr',
+ group = 'solr',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/var/run/ambari-logsearch-solr',
+ owner = 'solr',
+ group = 'solr',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/etc/ambari-logsearch-solr/conf',
+ owner = 'solr',
+ group = 'solr',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/opt/logsearch_solr/data',
+ owner = 'solr',
+ group = 'solr',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('Directory', '/opt/logsearch_solr/data/resources',
+ owner = 'solr',
+ group = 'solr',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr/solr-install.log',
+ owner = 'solr',
+ group = 'solr',
+ mode = 0644,
+ content = ''
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh',
+ owner = 'solr',
+ mode = 0755,
+ content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-env']['content'])
+ )
+ self.assertResourceCalled('File', '/opt/logsearch_solr/data/solr.xml',
+ owner = 'solr',
+ content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-xml']['content'])
+ )
+ self.assertResourceCalled('File', '/etc/ambari-logsearch-solr/conf/log4j.properties',
+ owner = 'solr',
+ content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-log4j']['content'])
+ )
+ self.assertResourceCalled('File', '/opt/logsearch_solr/data/zoo.cfg',
+ owner = 'solr',
+ content = Template('zoo.cfg.j2')
+ )
+ self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org -cmd makepath /logsearch',
+ not_if = "export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org -cmd get /logsearch",
+ ignore_failures = True,
+ user = "solr"
+ )
+
+ def test_configure_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py",
+ classname = "Solr",
+ command = "configure",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.configureResourcesCalled()
+ self.assertNoMoreResources()
+
+ def test_start_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py",
+ classname = "Solr",
+ command = "start",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.configureResourcesCalled()
+ self.assertResourceCalled('Execute', "/usr/lib/ambari-logsearch-solr/bin/solr start -cloud -noprompt -s /opt/logsearch_solr/data >> /var/log/ambari-logsearch-solr/solr-install.log 2>&1",
+ environment = {'SOLR_INCLUDE': '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh'},
+ user = "solr"
+ )
+
+ def test_stop_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py",
+ classname = "Solr",
+ command = "stop",
+ config_file = "default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+
+ self.assertResourceCalled('Execute', '/usr/lib/ambari-logsearch-solr/bin/solr stop -all >> /var/log/ambari-logsearch-solr/solr-install.log',
+ environment = {'SOLR_INCLUDE': '/etc/ambari-logsearch-solr/conf/logsearch-solr-env.sh'},
+ user = "solr",
+ only_if = "test -f /var/run/ambari-logsearch-solr/solr-8886.pid"
+ )
+ self.assertResourceCalled('File', '/var/run/ambari-logsearch-solr/solr-8886.pid',
+ action = ['delete']
+ )
+
+# def test_status_default(self):
+# with self.assertRaises(ComponentIsNotRunning) as e:
+# self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/solr.py",
+# classname = "Solr",
+# command = "status",
+# config_file = "default.json",
+# stack_version = self.STACK_VERSION,
+# target = RMFTestCase.TARGET_COMMON_SERVICES
+# )