You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2017/06/28 00:24:42 UTC
[44/51] [partial] ambari git commit: AMBARI-21349. Create BigInsights
Stack Skeleton in Ambari 2.5 (alejandro)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py
new file mode 100755
index 0000000..d446baa
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from ambari_commons import OSCheck
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+hbase_pid_dir = config['configurations']['ams-hbase-env']['hbase_pid_dir']
+hbase_user = ams_user
+ams_collector_pid_dir = config['configurations']['ams-env']['metrics_collector_pid_dir']
+ams_monitor_pid_dir = config['configurations']['ams-env']['metrics_monitor_pid_dir']
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+ams_hbase_conf_dir = format("{hbase_conf_dir}")
+
+kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hostname = config['hostname']
+tmp_dir = Script.get_tmp_dir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2
new file mode 100755
index 0000000..c5fbc9b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2
@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{ams_user}} - nofile {{max_open_files_limit}}
+{{ams_user}} - nproc 65536
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2
new file mode 100755
index 0000000..f7f00eb
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2
@@ -0,0 +1,26 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{ams_collector_keytab_path}}"
+principal="{{ams_collector_jaas_princ}}";
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2
new file mode 100755
index 0000000..1929548
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2
@@ -0,0 +1,26 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+Server {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{ams_zookeeper_keytab}}"
+principal="{{ams_zookeeper_principal_name}}";
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2
new file mode 100755
index 0000000..a65ea88
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2
@@ -0,0 +1,63 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# syntax: [prefix].[source|sink|jmx].[instance].[options]
+# See package.html for org.apache.hadoop.metrics2 for details
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.period=30
+hbase.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+jvm.period=30
+jvm.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+rpc.period=30
+rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
+*.sink.timeline.slave.host.name={{hostname}}
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.sink.timeline.period={{metrics_collection_period}}
+hbase.sink.timeline.sendInterval={{metrics_report_interval}}000
+hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.serviceName-prefix=ams
+
+# Switch off metrics generation on a per region basis
+*.source.filter.class=org.apache.hadoop.metrics2.filter.GlobFilter
+hbase.*.source.filter.exclude=*Regions*
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2
new file mode 100755
index 0000000..38f9721
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2
@@ -0,0 +1,23 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=false
+useTicketCache=true;
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2
new file mode 100755
index 0000000..c29c674
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2
@@ -0,0 +1,39 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+grant '{{smoke_test_user}}', '{{smokeuser_permissions}}'
+exit
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2
new file mode 100755
index 0000000..a93c36c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2
@@ -0,0 +1,26 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{master_keytab_path}}"
+principal="{{master_jaas_princ}}";
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2
new file mode 100755
index 0000000..7097481
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2
@@ -0,0 +1,26 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{regionserver_keytab_path}}"
+principal="{{regionserver_jaas_princ}}";
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2
new file mode 100755
index 0000000..aa03d195
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2
@@ -0,0 +1,37 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{
+ "host_metric_groups": {
+ "all": {
+ "collect_every": "10",
+ "metrics": [
+ {
+ "name": "bytes_out",
+ "value_threshold": "128"
+ }
+ ]
+ }
+ },
+ "process_metric_groups": {
+ "": {
+ "collect_every": "15",
+ "metrics": []
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2
new file mode 100755
index 0000000..fc86a58
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2
@@ -0,0 +1,31 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+[default]
+debug_level = INFO
+metrics_server = {{metric_collector_host}}:{{metric_collector_port}}
+hostname = {{hostname}}
+enable_time_threshold = false
+enable_value_threshold = false
+
+[emitter]
+send_interval = {{metrics_report_interval}}
+
+[collector]
+collector_sleep_interval = 5
+max_queue_size = 5000
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2
new file mode 100755
index 0000000..81d060b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2
@@ -0,0 +1,20 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in rs_hosts %}{{host}}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2
new file mode 100755
index 0000000..2ee0efa
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2
@@ -0,0 +1,15 @@
+{
+ "metrics": [
+ {
+ "metricname": "AMBARI_METRICS.SmokeTest.FakeMetric",
+ "appid": "amssmoketestfake",
+ "hostname": "{{hostname}}",
+ "timestamp": {{current_time}},
+ "starttime": {{current_time}},
+ "metrics": {
+ "{{current_time}}": {{random1}},
+ "{{current_time + 1000}}": {{current_time}}
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json
new file mode 100755
index 0000000..4603d22
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json
@@ -0,0 +1,27 @@
+{
+ "FLUME": {
+ "service": [],
+ "FLUME_HANDLER": [
+ {
+ "name": "flume_agent_status",
+ "label": "Flume Agent Status",
+ "description": "This host-level alert is triggered if any of the expected flume agent processes are not available.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "SCRIPT",
+ "path": "FLUME/1.5.2.4.1/package/alerts/alert_flume_agent_status.py",
+ "parameters": [
+ {
+ "name": "run.directory",
+ "display_name": "Run Directory",
+ "value": "/var/run/flume",
+ "type": "STRING",
+ "description": "The directory where flume agent processes will place their PID files."
+ }
+ ]
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml
new file mode 100755
index 0000000..582e181
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+ <property>
+ <name>content</name>
+ <description>Flume agent configurations.
+ Specifying the configuration here applies to all hosts in this configuration-group. Please use host specific configuration-groups to provide different configurations on different hosts.
+ For configuring multiple agents on the same host, provide the combined agent configurations here. Each agent should have a different name.</description>
+ <value>
+# Flume agent config
+ </value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml
new file mode 100755
index 0000000..a448a66
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>flume_conf_dir</name>
+ <display-name>Flume Conf Dir</display-name>
+ <value>/etc/flume/conf</value>
+ <description>Location to save configuration files</description>
+ <value-attributes>
+ <type>directory</type>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+ <property>
+ <name>flume_log_dir</name>
+ <display-name>Flume Log Dir</display-name>
+ <value>/var/log/flume</value>
+ <description>Location to save log files</description>
+ <value-attributes>
+ <type>directory</type>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+ <property>
+ <name>flume_user</name>
+ <value>flume</value>
+ <property-type>USER</property-type>
+ <description>Flume User</description>
+ </property>
+
+ <!-- flume-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for flume-env.sh file</description>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced
+# during Flume startup.
+
+# Enviroment variables can be set here.
+
+export JAVA_HOME={{java_home}}
+
+# Give Flume more memory and pre-allocate, enable remote monitoring via JMX
+# export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote"
+
+# Note that the Flume conf directory is always included in the classpath.
+#FLUME_CLASSPATH=""
+
+export HIVE_HOME={{flume_hive_home}}
+export HCAT_HOME={{flume_hcat_home}}
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml
new file mode 100755
index 0000000..a19eff1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>FLUME</name>
+ <displayName>Flume</displayName>
+ <comment>A distributed service for collecting, aggregating, and moving large amounts of streaming data into HDFS</comment>
+ <version>1.5.2</version>
+ <components>
+ <component>
+ <name>FLUME_HANDLER</name>
+ <displayName>Flume</displayName>
+ <category>SLAVE</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <commandScript>
+ <script>scripts/flume_handler.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>flume</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/flume_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>flume-env</config-type>
+ <config-type>flume-conf</config-type>
+ </configuration-dependencies>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json
new file mode 100755
index 0000000..b0990ba
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json
@@ -0,0 +1,430 @@
+{
+ "FLUME_HANDLER": {
+ "Component": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "default": {
+ "metrics/flume/flume/CHANNEL/ChannelCapacity": {
+ "metric": "ChannelCapacity",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/StartTime": {
+ "metric": "StartTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeAttemptCount": {
+ "metric": "EventTakeAttemptCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount": {
+ "metric": "EventTakeSuccessCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutAttemptCount": {
+ "metric": "EventPutAttemptCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/StopTime": {
+ "metric": "StopTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelFillPercentage": {
+ "metric": "ChannelFillPercentage",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize": {
+ "metric": "ChannelSize",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount": {
+ "metric": "EventPutSuccessCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/ConnectionCreatedCount": {
+ "metric": "ConnectionCreatedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/BatchCompleteCount": {
+ "metric": "BatchCompleteCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/EventDrainSuccessCount": {
+ "metric": "EventDrainSuccessCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/StartTime": {
+ "metric": "StartTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/EventDrainAttemptCount": {
+ "metric": "EventDrainAttemptCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/ConnectionFailedCount": {
+ "metric": "ConnectionFailedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/BatchUnderflowCount": {
+ "metric": "BatchUnderflowCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/ConnectionClosedCount": {
+ "metric": "ConnectionClosedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/StopTime": {
+ "metric": "StopTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/BatchEmptyCount": {
+ "metric": "BatchEmptyCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendBatchReceivedCount": {
+ "metric": "AppendBatchReceivedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendAcceptedCount": {
+ "metric": "AppendAcceptedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/StartTime": {
+ "metric": "StartTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/OpenConnectionCount": {
+ "metric": "OpenConnectionCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendBatchAcceptedCount": {
+ "metric": "AppendBatchAcceptedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendReceivedCount": {
+ "metric": "AppendReceivedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/EventReceivedCount": {
+ "metric": "EventReceivedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/StopTime": {
+ "metric": "StopTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/EventAcceptedCount": {
+ "metric": "EventAcceptedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/min": {
+ "metric": "EventTakeSuccessCount._rate._min",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/max": {
+ "metric": "EventTakeSuccessCount._rate._max",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/avg": {
+ "metric": "EventTakeSuccessCount._rate._avg",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/sum": {
+ "metric": "EventTakeSuccessCount._rate._sum",
+ "pointInTime": false,
+ "temporal": true
+ },
+
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/avg": {
+ "metric": "EventPutSuccessCount._rate._avg",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/max": {
+ "metric": "EventPutSuccessCount._rate._max",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/min": {
+ "metric": "EventPutSuccessCount._rate._min",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/sum": {
+ "metric": "EventPutSuccessCount._rate._sum",
+ "pointInTime": false,
+ "temporal": true
+ },
+
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/avg": {
+ "metric": "ChannelSize._rate._avg",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/max": {
+ "metric": "ChannelSize._rate._max",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/min": {
+ "metric": "ChannelSize._rate._min",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/sum": {
+ "metric": "ChannelSize._rate._sum",
+ "pointInTime": false,
+ "temporal": true
+ }
+
+ }
+ }
+ }
+ ],
+ "HostComponent": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "default": {
+ "metrics/flume/flume/CHANNEL/ChannelCapacity": {
+ "metric": "ChannelCapacity",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/StartTime": {
+ "metric": "StartTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeAttemptCount": {
+ "metric": "EventTakeAttemptCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount": {
+ "metric": "EventTakeSuccessCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutAttemptCount": {
+ "metric": "EventPutAttemptCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/StopTime": {
+ "metric": "StopTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelFillPercentage": {
+ "metric": "ChannelFillPercentage",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize": {
+ "metric": "ChannelSize",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount": {
+ "metric": "EventPutSuccessCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/ConnectionCreatedCount": {
+ "metric": "ConnectionCreatedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/BatchCompleteCount": {
+ "metric": "BatchCompleteCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/EventDrainSuccessCount": {
+ "metric": "EventDrainSuccessCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/StartTime": {
+ "metric": "StartTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/EventDrainAttemptCount": {
+ "metric": "EventDrainAttemptCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/ConnectionFailedCount": {
+ "metric": "ConnectionFailedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/BatchUnderflowCount": {
+ "metric": "BatchUnderflowCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/ConnectionClosedCount": {
+ "metric": "ConnectionClosedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/StopTime": {
+ "metric": "StopTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SINK/BatchEmptyCount": {
+ "metric": "BatchEmptyCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendBatchReceivedCount": {
+ "metric": "AppendBatchReceivedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendAcceptedCount": {
+ "metric": "AppendAcceptedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/StartTime": {
+ "metric": "StartTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/OpenConnectionCount": {
+ "metric": "OpenConnectionCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendBatchAcceptedCount": {
+ "metric": "AppendBatchAcceptedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/AppendReceivedCount": {
+ "metric": "AppendReceivedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/EventReceivedCount": {
+ "metric": "EventReceivedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/StopTime": {
+ "metric": "StopTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/flume/flume/SOURCE/EventAcceptedCount": {
+ "metric": "EventAcceptedCount",
+ "pointInTime": true,
+ "temporal": true
+ },
+
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/avg": {
+ "metric": "EventTakeSuccessCount._rate._avg",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/max": {
+ "metric": "EventTakeSuccessCount._rate._max",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/min": {
+ "metric": "EventTakeSuccessCount._rate._min",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/sum": {
+ "metric": "EventTakeSuccessCount._rate._sum",
+ "pointInTime": false,
+ "temporal": true
+ },
+
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/avg": {
+ "metric": "EventPutSuccessCount._rate._avg",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/max": {
+ "metric": "EventPutSuccessCount._rate._max",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/min": {
+ "metric": "EventPutSuccessCount._rate._min",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/sum": {
+ "metric": "EventPutSuccessCount._rate._sum",
+ "pointInTime": false,
+ "temporal": true
+ },
+
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/avg": {
+ "metric": "ChannelSize._rate._avg",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/max": {
+ "metric": "ChannelSize._rate._max",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/min": {
+ "metric": "ChannelSize._rate._min",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/flume/flume/CHANNEL/ChannelSize/rate/sum": {
+ "metric": "ChannelSize._rate._sum",
+ "pointInTime": false,
+ "temporal": true
+ }
+
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py
new file mode 100755
index 0000000..df08bb1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import socket
+
+from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names
+from resource_management.libraries.functions.flume_agent_helper import get_flume_status
+
+RESULT_CODE_OK = 'OK'
+RESULT_CODE_CRITICAL = 'CRITICAL'
+RESULT_CODE_UNKNOWN = 'UNKNOWN'
+
+FLUME_CONF_DIR_KEY = '{{flume-env/flume_conf_dir}}'
+
+FLUME_RUN_DIR_KEY = "run.directory"
+FLUME_RUN_DIR_DEFAULT = '/var/run/flume'
+
+def get_tokens():
+ """
+ Returns a tuple of tokens in the format {{site/property}} that will be used
+ to build the dictionary passed into execute
+ """
+ return (FLUME_CONF_DIR_KEY,)
+
+
+def execute(configurations={}, parameters={}, host_name=None):
+ """
+ Returns a tuple containing the result code and a pre-formatted result label
+
+ Keyword arguments:
+ configurations (dictionary): a mapping of configuration key to value
+ parameters (dictionary): a mapping of script parameter key to value
+ host_name (string): the name of this host where the alert is running
+ """
+
+ if configurations is None:
+ return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.'])
+
+ flume_conf_directory = None
+ if FLUME_CONF_DIR_KEY in configurations:
+ flume_conf_directory = configurations[FLUME_CONF_DIR_KEY]
+
+ if flume_conf_directory is None:
+ return (RESULT_CODE_UNKNOWN, ['The Flume configuration directory is a required parameter.'])
+
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ # parse script arguments
+ flume_run_directory = FLUME_RUN_DIR_DEFAULT
+ if FLUME_RUN_DIR_KEY in parameters:
+ flume_run_directory = parameters[FLUME_RUN_DIR_KEY]
+
+ processes = get_flume_status(flume_conf_directory, flume_run_directory)
+ expected_agents = find_expected_agent_names(flume_conf_directory)
+
+ alert_label = ''
+ alert_state = RESULT_CODE_OK
+
+ if len(processes) == 0 and len(expected_agents) == 0:
+ alert_label = 'No agents defined on {0}'.format(host_name)
+ else:
+ ok = []
+ critical = []
+ text_arr = []
+
+ for process in processes:
+ if not process.has_key('status') or process['status'] == 'NOT_RUNNING':
+ critical.append(process['name'])
+ else:
+ ok.append(process['name'])
+
+ if len(critical) > 0:
+ text_arr.append("{0} {1} NOT running".format(", ".join(critical),
+ "is" if len(critical) == 1 else "are"))
+
+ if len(ok) > 0:
+ text_arr.append("{0} {1} running".format(", ".join(ok),
+ "is" if len(ok) == 1 else "are"))
+
+ plural = len(critical) > 1 or len(ok) > 1
+ alert_label = "Agent{0} {1} {2}".format(
+ "s" if plural else "",
+ " and ".join(text_arr),
+ "on " + host_name)
+
+ alert_state = RESULT_CODE_CRITICAL if len(critical) > 0 else RESULT_CODE_OK
+
+ return (alert_state, [alert_label])
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py
new file mode 100755
index 0000000..8edba7f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py
@@ -0,0 +1,228 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import glob
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+import os
+from resource_management import *
+from resource_management.libraries.functions.flume_agent_helper import is_flume_process_live
+from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names
+from resource_management.libraries.functions.flume_agent_helper import await_flume_process_termination
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def flume(action = None):
+ import params
+
+ if action == 'config':
+ # remove previously defined meta's
+ for n in find_expected_agent_names(params.flume_conf_dir):
+ File(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'),
+ action = "delete",
+ )
+
+ Directory(params.flume_run_dir,
+ )
+
+ Directory(params.flume_conf_dir,
+ create_parents=True,
+ owner=params.flume_user,
+ )
+ Directory(params.flume_log_dir, owner=params.flume_user)
+
+ flume_agents = {}
+ if params.flume_conf_content is not None:
+ flume_agents = build_flume_topology(params.flume_conf_content)
+
+ for agent in flume_agents.keys():
+ flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent)
+ flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf')
+ flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json')
+ flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties')
+ flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.sh')
+
+ Directory(flume_agent_conf_dir,
+ owner=params.flume_user,
+ )
+
+ PropertiesFile(flume_agent_conf_file,
+ properties=flume_agents[agent],
+ owner=params.flume_user,
+ mode = 0644)
+
+ File(flume_agent_log4j_file,
+ content=Template('log4j.properties.j2', agent_name = agent),
+ owner=params.flume_user,
+ mode = 0644)
+
+ File(flume_agent_meta_file,
+ content = json.dumps(ambari_meta(agent, flume_agents[agent])),
+ owner=params.flume_user,
+ mode = 0644)
+
+ File(flume_agent_env_file,
+ owner=params.flume_user,
+ content=InlineTemplate(params.flume_env_sh_template)
+ )
+
+ if params.has_metric_collector:
+ File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"),
+ owner=params.flume_user,
+ content=Template("flume-metrics2.properties.j2")
+ )
+
+ elif action == 'start':
+ # desired state for service should be STARTED
+ if len(params.flume_command_targets) == 0:
+ _set_desired_state('STARTED')
+
+ # It is important to run this command as a background process.
+
+ flume_base = as_user(format("{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/{{4}}.out 2>&1"), params.flume_user, env={'JAVA_HOME': params.java_home}) + " &"
+
+ for agent in cmd_target_names():
+ flume_agent_conf_dir = params.flume_conf_dir + os.sep + agent
+ flume_agent_conf_file = flume_agent_conf_dir + os.sep + "flume.conf"
+ flume_agent_pid_file = params.flume_run_dir + os.sep + agent + ".pid"
+
+ if not os.path.isfile(flume_agent_conf_file):
+ continue
+
+ if not is_flume_process_live(flume_agent_pid_file):
+ # TODO someday make the ganglia ports configurable
+ extra_args = ''
+ if params.ganglia_server_host is not None:
+ extra_args = '-Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts={0}:{1}'
+ extra_args = extra_args.format(params.ganglia_server_host, '8655')
+ if params.has_metric_collector:
+ extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \
+ '-Dflume.monitoring.node={0}:{1}'
+ extra_args = extra_args.format(params.metric_collector_host, params.metric_collector_port)
+
+ flume_cmd = flume_base.format(agent, flume_agent_conf_dir,
+ flume_agent_conf_file, extra_args, agent)
+
+ Execute(flume_cmd,
+ wait_for_finish=False,
+ environment={'JAVA_HOME': params.java_home}
+ )
+ # sometimes startup spawns a couple of threads - so only the first line may count
+ pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}.*{agent}.*'))) + \
+ " | " + as_sudo(('tee', flume_agent_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0"
+ Execute(pid_cmd,
+ logoutput=True,
+ tries=20,
+ try_sleep=10)
+
+ pass
+ elif action == 'stop':
+ # desired state for service should be INSTALLED
+ if len(params.flume_command_targets) == 0:
+ _set_desired_state('INSTALLED')
+
+ pid_files = glob.glob(params.flume_run_dir + os.sep + "*.pid")
+
+ if 0 == len(pid_files):
+ return
+
+ agent_names = cmd_target_names()
+
+
+ for agent in agent_names:
+ pid_file = format("{flume_run_dir}/{agent}.pid")
+
+ if is_flume_process_live(pid_file):
+ pid = shell.checked_call(("cat", pid_file), sudo=True)[1].strip()
+ Execute(("kill", "-15", pid), sudo=True) # kill command has to be a tuple
+
+ if not await_flume_process_termination(pid_file):
+ raise Fail("Can't stop flume agent: {0}".format(agent))
+
+ File(pid_file, action = 'delete')
+
+
+def ambari_meta(agent_name, agent_conf):
+ res = {}
+
+ sources = agent_conf[agent_name + '.sources'].split(' ')
+ res['sources_count'] = len(sources)
+
+ sinks = agent_conf[agent_name + '.sinks'].split(' ')
+ res['sinks_count'] = len(sinks)
+
+ channels = agent_conf[agent_name + '.channels'].split(' ')
+ res['channels_count'] = len(channels)
+
+ return res
+
+
+# define a map of dictionaries, where the key is agent name
+# and the dictionary is the name/value pair
+def build_flume_topology(content):
+
+ result = {}
+ agent_names = []
+
+ for line in content.split('\n'):
+ rline = line.strip()
+ if 0 != len(rline) and not rline.startswith('#'):
+ pair = rline.split('=')
+ lhs = pair[0].strip()
+ rhs = pair[1].strip()
+
+ part0 = lhs.split('.')[0]
+
+ if lhs.endswith(".sources"):
+ agent_names.append(part0)
+
+ if not result.has_key(part0):
+ result[part0] = {}
+
+ result[part0][lhs] = rhs
+
+ # trim out non-agents
+ for k in result.keys():
+ if not k in agent_names:
+ del result[k]
+
+ return result
+
+
+def cmd_target_names():
+ import params
+
+ if len(params.flume_command_targets) > 0:
+ return params.flume_command_targets
+ else:
+ return find_expected_agent_names(params.flume_conf_dir)
+
+
+def _set_desired_state(state):
+ import params
+ File(params.ambari_state_file,
+ content = state,
+ )
+
+def get_desired_state():
+ import params
+ from resource_management.core import sudo
+ if os.path.exists(params.ambari_state_file):
+ return sudo.read_file(params.ambari_state_file)
+ else:
+ return 'INSTALLED'
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py
new file mode 100755
index 0000000..64cd1c1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py
@@ -0,0 +1,40 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class FlumeServiceCheck(Script):
+
+ @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ if params.security_enabled:
+ principal_replaced = params.http_principal.replace("_HOST", params.hostname)
+ Execute(format("{kinit_path_local} -kt {http_keytab} {principal_replaced}"),
+ user=params.smoke_user)
+
+ Execute(format('env JAVA_HOME={java_home} {flume_bin} version'),
+ logoutput=True,
+ tries = 3,
+ try_sleep = 20)
+
+if __name__ == "__main__":
+ FlumeServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py
new file mode 100755
index 0000000..d0c6e3b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py
@@ -0,0 +1,145 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import flume_upgrade
+
+from flume import flume
+from flume import get_desired_state
+
+from resource_management import *
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names
+from resource_management.libraries.functions.flume_agent_helper import get_flume_status
+from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions import Direction
+
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class FlumeHandler(Script):
+
+ def get_component_name(self):
+ return "flume-server"
+
+ def install(self, env):
+ import params
+ self.install_packages(env)
+ env.set_params(params)
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ flume(action='start')
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ flume(action='stop')
+
+ if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE:
+ flume_upgrade.post_stop_backup()
+ #if rolling_restart:
+ # flume_upgrade.post_stop_backup()
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ flume(action='config')
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ processes = get_flume_status(params.flume_conf_dir, params.flume_run_dir)
+ expected_agents = find_expected_agent_names(params.flume_conf_dir)
+
+ json = {}
+ json['processes'] = processes
+ json['alerts'] = []
+
+ alert = {}
+ alert['name'] = 'flume_agent'
+ alert['label'] = 'Flume Agent process'
+
+ if len(processes) == 0 and len(expected_agents) == 0:
+ alert['state'] = 'OK'
+
+ if not params.hostname is None:
+ alert['text'] = 'No agents defined on ' + params.hostname
+ else:
+ alert['text'] = 'No agents defined'
+
+ else:
+ crit = []
+ ok = []
+
+ for proc in processes:
+ if not proc.has_key('status') or proc['status'] == 'NOT_RUNNING':
+ crit.append(proc['name'])
+ else:
+ ok.append(proc['name'])
+
+ text_arr = []
+
+ if len(crit) > 0:
+ text_arr.append("{0} {1} NOT running".format(", ".join(crit),
+ "is" if len(crit) == 1 else "are"))
+
+ if len(ok) > 0:
+ text_arr.append("{0} {1} running".format(", ".join(ok),
+ "is" if len(ok) == 1 else "are"))
+
+ plural = len(crit) > 1 or len(ok) > 1
+ alert['text'] = "Agent{0} {1} {2}".format(
+ "s" if plural else "",
+ " and ".join(text_arr),
+ "" if params.hostname is None else "on " + str(params.hostname))
+
+ alert['state'] = 'CRITICAL' if len(crit) > 0 else 'OK'
+
+ json['alerts'].append(alert)
+ self.put_structured_out(json)
+
+ # only throw an exception if there are agents defined and there is a
+ # problem with the processes; if there are no agents defined, then
+ # the service should report STARTED (green) ONLY if the desired state is started. otherwise, INSTALLED (red)
+ if len(expected_agents) > 0:
+ for proc in processes:
+ if not proc.has_key('status') or proc['status'] == 'NOT_RUNNING':
+ raise ComponentIsNotRunning()
+ elif len(expected_agents) == 0 and 'INSTALLED' == get_desired_state():
+ raise ComponentIsNotRunning()
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ # this function should not execute if the version can't be determined or
+ # is not at least IOP 4.0.0.0
+ if not params.version or compare_versions(format_stack_version(params.version), '4.0.0.0') < 0:
+ return
+
+ Logger.info("Executing Flume Stack Upgrade pre-restart")
+ conf_select.select(params.stack_name, "flume", params.version)
+ stack_select.select("flume-server", params.version)
+ if params.upgrade_direction == Direction.UPGRADE:
+ flume_upgrade.pre_start_restore()
+
+if __name__ == "__main__":
+ FlumeHandler().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py
new file mode 100755
index 0000000..181668b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py
@@ -0,0 +1,94 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+import shutil
+import tarfile
+import tempfile
+
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions import tar_archive
+
+BACKUP_TEMP_DIR = "flume-upgrade-backup"
+BACKUP_CONF_DIR_ARCHIVE = "flume-conf-backup.tar"
+
+def post_stop_backup():
+ """
+ Backs up the flume config, config dir, file/spillable channels as part of the
+ upgrade process.
+ :return:
+ """
+ Logger.info('Backing up Flume data and configuration before upgrade...')
+ directoryMappings = _get_directory_mappings()
+
+ absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
+ if not os.path.isdir(absolute_backup_dir):
+ os.makedirs(absolute_backup_dir)
+
+ for directory in directoryMappings:
+ if not os.path.isdir(directory):
+ raise Fail("Unable to backup missing directory {0}".format(directory))
+
+ archive = os.path.join(absolute_backup_dir, directoryMappings[directory])
+ Logger.info('Compressing {0} to {1}'.format(directory, archive))
+
+ if os.path.exists(archive):
+ os.remove(archive)
+
+ # backup the directory, following symlinks instead of including them
+ tar_archive.archive_directory_dereference(archive, directory)
+
+
+def pre_start_restore():
+ """
+ Restores the flume config, config dir, file/spillable channels to their proper locations
+ after an upgrade has completed.
+ :return:
+ """
+ Logger.info('Restoring Flume data and configuration after upgrade...')
+ directoryMappings = _get_directory_mappings()
+
+ for directory in directoryMappings:
+ archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
+ directoryMappings[directory])
+
+ if os.path.isfile(archive):
+ Logger.info('Extracting {0} to {1}'.format(archive, directory))
+ tarball = None
+ try:
+ tarball = tarfile.open(archive, "r")
+ tarball.extractall(directory)
+ finally:
+ if tarball:
+ tarball.close()
+
+ # cleanup
+ if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)):
+ shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
+
+def _get_directory_mappings():
+ """
+ Gets a dictionary of directory to archive name that represents the
+ directories that need to be backed up and their output tarball archive targets
+ :return: the dictionary of directory to tarball mappings
+ """
+ import params
+
+ return { params.flume_conf_dir : BACKUP_CONF_DIR_ARCHIVE}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py
new file mode 100755
index 0000000..f9addb9
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py
@@ -0,0 +1,101 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.libraries.functions.constants import Direction
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.script.script import Script
+
+from params_linux import *
+
+config = Script.get_config()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE)
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+user_group = config['configurations']['cluster-env']['user_group']
+proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group']
+
+security_enabled = False
+
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+iop_stack_version = format_stack_version(stack_version_unformatted)
+
+# hadoop default parameters
+flume_bin = '/usr/iop/current/flume-server/bin/flume-ng'
+
+# hadoop parameters for 2.2+
+flume_hive_home = '/usr/iop/current/hive-metastore'
+flume_hcat_home = '/usr/iop/current/hive-webhcat'
+
+# flume agent runtime for management
+flume_log_dir = config['configurations']['flume-env']['flume_log_dir']
+
+# flume_run_dir is a new parameter added since Ambari 2.1 in IOP 4.1
+# The default value must be the same as the value of "flume_run_dir" in params.py of Ambari 1.7 in IOP 4.0
+# See ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py
+FLUME_RUN_DIR_DEFAULT = '/var/run/flume'
+if (('flume-env' in config['configurations']) and ('flume_run_dir' in config['configurations']['flume-env'])):
+ flume_run_dir = config['configurations']['flume-env']['flume_run_dir']
+else:
+ flume_run_dir = FLUME_RUN_DIR_DEFAULT
+
+java_home = config['hostLevelParams']['java_home']
+ambari_state_file = format("{flume_run_dir}/ambari-state.txt")
+
+if 'flume-env' in config['configurations'] and 'flume_user' in config['configurations']['flume-env']:
+ flume_user = config['configurations']['flume-env']['flume_user']
+
+if (('flume-conf' in config['configurations']) and('content' in config['configurations']['flume-conf'])):
+ flume_conf_content = config['configurations']['flume-conf']['content']
+else:
+ flume_conf_content = None
+
+if (('flume-log4j' in config['configurations']) and ('content' in config['configurations']['flume-log4j'])):
+ flume_log4j_content = config['configurations']['flume-log4j']['content']
+else:
+ flume_log4j_content = None
+
+targets = default('/commandParams/flume_handler', None)
+flume_command_targets = [] if targets is None else targets.split(',')
+
+flume_env_sh_template = config['configurations']['flume-env']['content']
+
+ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', [])
+ganglia_server_host = None
+if 0 != len(ganglia_server_hosts):
+ ganglia_server_host = ganglia_server_hosts[0]
+
+hostname = None
+if config.has_key('hostname'):
+ hostname = config['hostname']
+
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
+has_metric_collector = not len(ams_collector_hosts) == 0
+if has_metric_collector:
+ metric_collector_host = ams_collector_hosts[0]
+ metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
+ if metric_collector_port and metric_collector_port.find(':') != -1:
+ metric_collector_port = metric_collector_port.split(':')[1]
+ pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py
new file mode 100755
index 0000000..7ecbe27
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py
@@ -0,0 +1,30 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.libraries.functions import format
+from resource_management.libraries.script.script import Script
+
+# server configurations
+config = Script.get_config()
+
+flume_conf_dir = '/usr/iop/current/flume-server/conf'
+
+flume_user = 'flume'
+flume_group = 'flume'
+if 'flume-env' in config['configurations'] and 'flume_user' in config['configurations']['flume-env']:
+ flume_user = config['configurations']['flume-env']['flume_user']
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2
new file mode 100755
index 0000000..bc18043
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2
@@ -0,0 +1,26 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+collector={{metric_collector_host}}
+port={{metric_collector_port}}
+collectionFrequency=60000
+maxRowCacheSize=10000
+sendInterval=59000
+
+# Metric names having type COUNTER
+counters=EventTakeSuccessCount,EventPutSuccessCount,EventTakeAttemptCount,EventPutAttemptCount
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2
new file mode 100755
index 0000000..4dee67f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# flume.conf: Add your flume configuration here and start flume
+# Note if you are using the Windows srvice or Unix service
+# provided by the HDP distribution, they will assume the
+# agent's name in this file to be 'a1'
+#
+{{flume_agent_conf_content}}