You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/02/07 19:50:26 UTC
metron git commit: METRON-1447 Heap Size Not Set Correctly by MPack
for ES 5.x (nickwallen) closes apache/metron#927
Repository: metron
Updated Branches:
refs/heads/master 38b7d5e33 -> 31f3aa8cb
METRON-1447 Heap Size Not Set Correctly by MPack for ES 5.x (nickwallen) closes apache/metron#927
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/31f3aa8c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/31f3aa8c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/31f3aa8c
Branch: refs/heads/master
Commit: 31f3aa8cb0f4c3821c264e375aa14eda207adcb0
Parents: 38b7d5e
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Feb 7 14:50:06 2018 -0500
Committer: nickallen <ni...@apache.org>
Committed: Wed Feb 7 14:50:06 2018 -0500
----------------------------------------------------------------------
.../5.6.2/configuration/elastic-jvm-options.xml | 144 +++++++++++++++++++
.../5.6.2/configuration/elastic-sysconfig.xml | 12 +-
.../ELASTICSEARCH/5.6.2/metainfo.xml | 1 +
.../5.6.2/package/scripts/elastic_commands.py | 13 ++
.../5.6.2/package/scripts/params.py | 3 +
5 files changed, 163 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
new file mode 100644
index 0000000..5c6aaca
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>heap_size</name>
+ <value>512m</value>
+ <description>JVM heap size</description>
+ </property>
+ <property>
+ <name>content</name>
+ <description>The jinja template for the Elasticsearch JVM options file.</description>
+ <value>
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{heap_size}}
+-Xmx{{heap_size}}
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM (remove on 32-bit client JVMs)
+-server
+
+# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Dlog4j.skipJansi=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
index cb069b8..ea6ca38 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
@@ -40,11 +40,6 @@
<description>Elasticsearch Configuration Directory</description>
</property>
<property>
- <name>heap_size</name>
- <value>512m</value>
- <description>Heap size</description>
- </property>
- <property>
<name>max_open_files</name>
<value>65536</value>
<description>Maximum number of open files</description>
@@ -92,11 +87,8 @@ PID_DIR={{pid_dir}}
# JAVA_HOME must be provided here for OS that use systemd service launch
JAVA_HOME={{java64_home}}
-# Additional Java OPTS
-ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \
--XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \
--XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8 \
--Xms{{heap_size}} -Xmx{{heap_size}}"
+# Additional Java options - now preferential to use 'jvm.options' file instead
+ES_JAVA_OPTS=""
# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html
MAX_LOCKED_MEMORY=unlimited
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
index 0943eec..47abb45 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
@@ -83,6 +83,7 @@
<config-type>elastic-site</config-type>
<config-type>elastic-sysconfig</config-type>
<config-type>elastic-systemd</config-type>
+ <config-type>elastic-jvm-options</config-type>
</configuration-dependencies>
<restartRequiredAfterChange>true</restartRequiredAfterChange>
<quickLinksConfigurations>
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
index afbaff2..618d10a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
@@ -190,6 +190,17 @@ def create_elastic_pam_limits(params):
owner="root",
group="root")
+def create_elastic_jvm_options(params):
+ """
+ Defines the jvm.options file used to specify JVM options.
+ """
+ path = "{0}/jvm.options".format(params.conf_dir)
+ Logger.info("Creating Elasticsearch JVM Options; file={0}".format(path))
+ File(path,
+ content=InlineTemplate(params.jvm_options_template),
+ owner=params.elastic_user,
+ group=params.elastic_group)
+
def get_data_directories(params):
"""
Returns the directories to use for storing Elasticsearch data.
@@ -225,6 +236,7 @@ def configure_master():
create_elastic_site(params, "elasticsearch.master.yaml.j2")
create_elastic_config(params)
create_elastic_pam_limits(params)
+ create_elastic_jvm_options(params)
if is_systemd_running():
configure_systemd(params)
@@ -249,5 +261,6 @@ def configure_slave():
create_elastic_site(params, "elasticsearch.slave.yaml.j2")
create_elastic_config(params)
create_elastic_pam_limits(params)
+ create_elastic_jvm_options(params)
if is_systemd_running():
configure_systemd(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
index 0399c60..24f2306 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
@@ -103,3 +103,6 @@ systemd_parent_dir = '/etc/systemd/system/'
systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/'
systemd_override_file = systemd_elasticsearch_dir + 'override.conf'
systemd_override_template = config['configurations']['elastic-systemd']['content']
+
+heap_size = config['configurations']['elastic-jvm-options']['heap_size']
+jvm_options_template = config['configurations']['elastic-jvm-options']['content']