You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/10/02 13:41:40 UTC
[1/2] ambari git commit: AMBARI-13290. Set kafka userid ulimit open
files to 32k for kafka broker through Ambari (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 deb5a06a4 -> 989d34c14
refs/heads/trunk f0949259f -> 9286b1c02
AMBARI-13290. Set kafka userid ulimit open files to 32k for kafka broker through Ambari (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9286b1c0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9286b1c0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9286b1c0
Branch: refs/heads/trunk
Commit: 9286b1c0206221ffd4fe0774c1b151da12ac8cc3
Parents: f094925
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Oct 2 14:41:30 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Oct 2 14:41:30 2015 +0300
----------------------------------------------------------------------
.../KAFKA/0.8.1.2.2/configuration/kafka-env.xml | 10 ++++++
.../KAFKA/0.8.1.2.2/package/scripts/kafka.py | 13 ++++++++
.../KAFKA/0.8.1.2.2/package/scripts/params.py | 4 +++
.../0.8.1.2.2/package/templates/kafka.conf.j2 | 35 ++++++++++++++++++++
4 files changed, 62 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
index ecc0782..d515e73 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
@@ -42,6 +42,16 @@
<overridable>false</overridable>
</value-attributes>
</property>
+ <property>
+ <name>kafka_user_nofile_limit</name>
+ <value>32000</value>
+ <description>Max open files limit setting for KAFKA user.</description>
+ </property>
+ <property>
+ <name>kafka_user_nproc_limit</name>
+ <value>16000</value>
+ <description>Max number of processes limit setting for KAFKA user.</description>
+ </property>
<!-- kafka-env.sh -->
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
index 11492a7..4d28c41 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
@@ -94,6 +94,19 @@ def kafka():
TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"),
owner=params.kafka_user)
+ # On some OS this folder could be not exists, so we will create it before pushing there files
+ Directory(params.limits_conf_dir,
+ recursive=True,
+ owner='root',
+ group='root'
+ )
+
+ File(os.path.join(params.limits_conf_dir, 'kafka.conf'),
+ owner='root',
+ group='root',
+ mode=0644,
+ content=Template("kafka.conf.j2")
+ )
setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
index 26ea3e2..0a55504 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
@@ -43,6 +43,10 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
kafka_home = '/usr/lib/kafka/'
kafka_bin = kafka_home+'/bin/kafka'
conf_dir = "/etc/kafka/conf"
+limits_conf_dir = "/etc/security/limits.d"
+
+kafka_user_nofile_limit = config['configurations']['kafka-env']['kafka_user_nofile_limit']
+kafka_user_nproc_limit = config['configurations']['kafka-env']['kafka_user_nproc_limit']
# parameters for 2.2+
if Script.is_hdp_stack_greater_or_equal("2.2"):
http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2
new file mode 100644
index 0000000..9e18e1d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2
@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{kafka_user}} - nofile {{kafka_user_nofile_limit}}
+{{kafka_user}} - nproc {{kafka_user_nproc_limit}}
[2/2] ambari git commit: AMBARI-13290. Set kafka userid ulimit open
files to 32k for kafka broker through Ambari (aonishuk)
Posted by ao...@apache.org.
AMBARI-13290. Set kafka userid ulimit open files to 32k for kafka broker through Ambari (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/989d34c1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/989d34c1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/989d34c1
Branch: refs/heads/branch-2.1
Commit: 989d34c14dc8a73d5d36eba29b3e0db910446806
Parents: deb5a06
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Oct 2 14:41:33 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Oct 2 14:41:33 2015 +0300
----------------------------------------------------------------------
.../KAFKA/0.8.1.2.2/configuration/kafka-env.xml | 10 ++++++
.../KAFKA/0.8.1.2.2/package/scripts/kafka.py | 13 ++++++++
.../KAFKA/0.8.1.2.2/package/scripts/params.py | 4 +++
.../0.8.1.2.2/package/templates/kafka.conf.j2 | 35 ++++++++++++++++++++
4 files changed, 62 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/989d34c1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
index ecc0782..d515e73 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml
@@ -42,6 +42,16 @@
<overridable>false</overridable>
</value-attributes>
</property>
+ <property>
+ <name>kafka_user_nofile_limit</name>
+ <value>32000</value>
+ <description>Max open files limit setting for KAFKA user.</description>
+ </property>
+ <property>
+ <name>kafka_user_nproc_limit</name>
+ <value>16000</value>
+ <description>Max number of processes limit setting for KAFKA user.</description>
+ </property>
<!-- kafka-env.sh -->
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/989d34c1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
index 11492a7..4d28c41 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
@@ -94,6 +94,19 @@ def kafka():
TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"),
owner=params.kafka_user)
+ # On some OS this folder could be not exists, so we will create it before pushing there files
+ Directory(params.limits_conf_dir,
+ recursive=True,
+ owner='root',
+ group='root'
+ )
+
+ File(os.path.join(params.limits_conf_dir, 'kafka.conf'),
+ owner='root',
+ group='root',
+ mode=0644,
+ content=Template("kafka.conf.j2")
+ )
setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/989d34c1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
index 26ea3e2..0a55504 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
@@ -43,6 +43,10 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
kafka_home = '/usr/lib/kafka/'
kafka_bin = kafka_home+'/bin/kafka'
conf_dir = "/etc/kafka/conf"
+limits_conf_dir = "/etc/security/limits.d"
+
+kafka_user_nofile_limit = config['configurations']['kafka-env']['kafka_user_nofile_limit']
+kafka_user_nproc_limit = config['configurations']['kafka-env']['kafka_user_nproc_limit']
# parameters for 2.2+
if Script.is_hdp_stack_greater_or_equal("2.2"):
http://git-wip-us.apache.org/repos/asf/ambari/blob/989d34c1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2
new file mode 100644
index 0000000..9e18e1d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2
@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{kafka_user}} - nofile {{kafka_user_nofile_limit}}
+{{kafka_user}} - nproc {{kafka_user_nproc_limit}}