You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ni...@apache.org on 2017/10/04 19:51:25 UTC
ambari git commit: AMBARI-22125. Better defaults for druid memory
configs to avoid OutOfMemoryErrors on small machines
Repository: ambari
Updated Branches:
refs/heads/trunk d86f76497 -> 1da77356b
AMBARI-22125. Better defaults for druid memory configs to avoid OutOfMemoryErrors on small machines
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1da77356
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1da77356
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1da77356
Branch: refs/heads/trunk
Commit: 1da77356b0de98df51f48bedc087daf01aedbd7c
Parents: d86f764
Author: Nishant <ni...@gmail.com>
Authored: Thu Oct 5 01:21:00 2017 +0530
Committer: Nishant <ni...@gmail.com>
Committed: Thu Oct 5 01:21:00 2017 +0530
----------------------------------------------------------------------
.../DRUID/0.9.2/configuration/druid-broker.xml | 6 +
.../0.9.2/configuration/druid-historical.xml | 6 +
.../stacks/HDP/2.6/services/stack_advisor.py | 28 ++-
.../stacks/2.6/common/test_stack_advisor.py | 190 ++++++++++++++++++-
4 files changed, 219 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml
index 4f05da0..6146ca3 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-broker.xml
@@ -69,6 +69,12 @@
<on-ambari-upgrade add="false"/>
</property>
<property>
+ <name>druid.processing.numMergeBuffers</name>
+ <value>2</value>
+ <description>The number of direct memory buffers available for merging query results. The buffers are sized by druid.processing.buffer.sizeBytes.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>druid.broker.cache.useCache</name>
<value>true</value>
<description>Enable the cache on the broker.</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
index 9b65404..5ff30ce 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
@@ -39,6 +39,12 @@
<on-ambari-upgrade add="false"/>
</property>
<property>
+ <name>druid.processing.numMergeBuffers</name>
+ <value>2</value>
+ <description>The number of direct memory buffers available for merging query results. The buffers are sized by druid.processing.buffer.sizeBytes.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>druid.processing.buffer.sizeBytes</name>
<value>1073741824</value>
<value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 723ff4e..0d2925e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -16,9 +16,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
-import math
-
import json
+import math
import re
from resource_management.libraries.functions import format
@@ -144,7 +143,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
# JVM Configs go to env properties
putEnvProperty = self.putProperty(configurations, "druid-env", services)
- # processing thread pool Config
+ # processing thread pool and memory configs
for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']:
component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
@@ -154,8 +153,31 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
processingThreads = 1
if totalAvailableCpu > 1:
processingThreads = totalAvailableCpu - 1
+ numMergeBuffers = max(2, processingThreads/4)
putComponentProperty('druid.processing.numThreads', processingThreads)
putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30)
+ putComponentProperty('druid.processing.numMergeBuffers', numMergeBuffers)
+ totalAvailableMemInMb = self.getMinMemory(component_hosts) / 1024
+ maxAvailableBufferSizeInMb = totalAvailableMemInMb/(processingThreads + numMergeBuffers)
+ putComponentProperty('druid.processing.buffer.sizeBytes', self.getDruidProcessingBufferSizeInMb(totalAvailableMemInMb) * 1024 * 1024)
+
+
+ # returns the recommended druid processing buffer size in Mb.
+ # the recommended buffer size is kept lower then the max available memory to have enough free memory to load druid data.
+ # for low memory nodes, the actual allocated buffer size is small to keep some free memory for memory mapping of segments
+ # If user installs all druid processes on a single node, memory available for loading segments will be further decreased.
+ def getDruidProcessingBufferSizeInMb(self, maxAvailableBufferSizeInMb):
+ if maxAvailableBufferSizeInMb <= 256:
+ return min(64, maxAvailableBufferSizeInMb)
+ elif maxAvailableBufferSizeInMb <= 1024:
+ return 128
+ elif maxAvailableBufferSizeInMb <= 2048:
+ return 256
+ elif maxAvailableBufferSizeInMb <= 6144:
+ return 512
+ # High Memory nodes below
+ else :
+ return 1024
def recommendSupersetConfigurations(self, configurations, clusterData, services, hosts):
# superset is in list of services to be installed
http://git-wip-us.apache.org/repos/asf/ambari/blob/1da77356/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
index f8483c1..ec44b3d 100644
--- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -188,9 +188,15 @@ class TestHDP26StackAdvisor(TestCase):
self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations,
{'druid-historical': {
- 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '3',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '1073741824'}},
'druid-broker': {
- 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '3',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '1073741824'}},
'druid-common': {'properties': {'druid.extensions.loadList': '["mysql-metadata-storage"]',
'druid.metadata.storage.connector.port': '3306',
'druid.metadata.storage.connector.connectURI': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true',
@@ -767,9 +773,15 @@ class TestHDP26StackAdvisor(TestCase):
self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations,
{'druid-historical': {
- 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '3',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '1073741824'}},
'druid-broker': {
- 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '3',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '1073741824'}},
'druid-common': {'properties': {'druid.extensions.loadList': '["postgresql-metadata-storage"]',
'druid.metadata.storage.connector.port': '5432',
'druid.metadata.storage.connector.connectURI': 'jdbc:postgresql://c6401.ambari.apache.org:5432/druid',
@@ -875,9 +887,15 @@ class TestHDP26StackAdvisor(TestCase):
self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations,
{'druid-historical': {
- 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '3',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '1073741824'}},
'druid-broker': {
- 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '3',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '1073741824'}},
'druid-common': {'properties': {'druid.extensions.loadList': '[]',
'druid.metadata.storage.connector.port': '1527',
'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
@@ -892,6 +910,7 @@ class TestHDP26StackAdvisor(TestCase):
)
+
def test_recommendDruidConfigurations_property_existence_check(self):
# Test for https://issues.apache.org/jira/browse/AMBARI-19144
hosts = {
@@ -1069,9 +1088,15 @@ class TestHDP26StackAdvisor(TestCase):
self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations,
{'druid-historical': {
- 'properties': {'druid.processing.numThreads': '2', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '2',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '536870912'}},
'druid-broker': {
- 'properties': {'druid.processing.numThreads': '1', 'druid.server.http.numThreads': '40'}},
+ 'properties': {'druid.processing.numThreads': '1',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '268435456'}},
'druid-common': {'properties': {'druid.extensions.loadList': '[]',
'druid.metadata.storage.connector.port': '1527',
'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
@@ -1086,6 +1111,155 @@ class TestHDP26StackAdvisor(TestCase):
'druid.broker.jvm.heap.memory': {'maximum': '1877'}}}}
)
+ def test_recommendDruidConfigurations_low_mem_hosts(self):
+ hosts = {
+ "items": [
+ {
+ "href": "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts": {
+ "cpu_count": 8,
+ "total_mem": 102400,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ }
+ }, {
+ "href": "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts": {
+ "cpu_count": 4,
+ "total_mem": 204800,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6402.ambari.apache.org",
+ "host_name": "c6402.ambari.apache.org"
+ }
+ },
+ {
+ "href": "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 409600,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6403.ambari.apache.org",
+ "host_name": "c6403.ambari.apache.org"
+ }
+ }
+ ]
+ }
+
+ services = {
+ "Versions": {
+ "parent_stack_version": "2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.6",
+ "stack_hierarchy": {
+ "stack_name": "HDP",
+ "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+ }
+ },
+ "services": [{
+ "StackServices": {
+ "service_name": "DRUID",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_COORDINATOR",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_OVERLORD",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_BROKER",
+ "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_HISTORICAL",
+ "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_MIDDLEMANAGER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ }
+ ]
+ }
+ ],
+ "configurations": {
+ "druid-common": {
+ "properties": {
+ "database_name": "druid",
+ "metastore_hostname": "c6401.ambari.apache.org",
+ "druid.metadata.storage.type": "derby",
+ "druid.extensions.loadList": "[\"mysql-metadata-storage\"]",
+ "druid.extensions.pullList": "[]"
+ }
+ }
+ }
+ }
+
+ clusterData = {
+ }
+
+ configurations = {
+ }
+
+ self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations,
+ {'druid-historical': {
+ 'properties': {'druid.processing.numThreads': '5',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '67108864'}},
+ 'druid-broker': {
+ 'properties': {'druid.processing.numThreads': '3',
+ 'druid.server.http.numThreads': '40',
+ 'druid.processing.numMergeBuffers': '2',
+ 'druid.processing.buffer.sizeBytes': '67108864'}},
+ 'druid-common': {'properties': {'druid.extensions.loadList': '[]',
+ 'druid.metadata.storage.connector.port': '1527',
+ 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
+ 'druid.zk.service.host': ''
+ }},
+ 'druid-env': {'properties': {},
+ 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '1024'},
+ 'druid.overlord.jvm.heap.memory': {'maximum': '1024'},
+ 'druid.middlemanager.jvm.heap.memory': {
+ 'maximum': '1024'},
+ 'druid.historical.jvm.heap.memory': {'maximum': '1024'},
+ 'druid.broker.jvm.heap.memory': {'maximum': '1024'}}}}
+ )
+
def test_recommendAtlasConfigurations(self):
configurations = {