You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/10/24 15:04:02 UTC
[13/51] [abbrv] metron git commit: METRON-1508 In Ubuntu14 Dev
Indexing Fails to Write to Elasticsearch (nickwallen) closes
apache/metron#1185
METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08fe1cff
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08fe1cff
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08fe1cff
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08fe1cff1430c71205d62e375cbacd6906e4fc4e
Parents: 0d34e77
Author: nickwallen <ni...@nickallen.org>
Authored: Fri Sep 21 15:31:24 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Fri Sep 21 15:31:24 2018 -0400
----------------------------------------------------------------------
.../ansible/roles/ambari_config/vars/single_node_vm.yml | 4 ++--
.../development/centos6/ansible/inventory/group_vars/all | 1 +
.../development/ubuntu14/ansible/inventory/group_vars/all | 1 +
.../ELASTICSEARCH/5.6.2/configuration/elastic-site.xml | 8 ++++----
4 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 962726c..2ec956a 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -92,7 +92,7 @@ configurations:
# Storm expects ambari metrics to be available in 2.6. We do *not* install ambari metrics in full-dev, so we need to revert to the old consumer
storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]'
topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]'
- # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties
+ # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties
nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER'
supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER'
worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER"
@@ -117,7 +117,7 @@ configurations:
required_configurations:
- metron-env:
storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
- es_hosts: "{{ groups.search | join(',') }}"
+ es_hosts: "{{ elasticsearch_hosts }}"
zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
solr_zookeeper_url: "{{ groups.search[0] }}:9983"
- metron-rest-env:
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/centos6/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
index dcb7b42..550eb8f 100644
--- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
threat_intel_bulk_load: False
+elasticsearch_hosts: "{{ groups.search | join(',') }}"
# data directories - only required to override defaults
zookeeper_data_dir: "/data1/hadoop/zookeeper"
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
index ef9f705..1338d7c 100644
--- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
threat_intel_bulk_load: False
+elasticsearch_hosts: "localhost"
# data directories - only required to override defaults
zookeeper_data_dir: "/data1/hadoop/zookeeper"
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
index 34df1e4..440c66d 100755
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
@@ -111,7 +111,7 @@
<value>0</value>
<description>expected_data_nodes</description>
</property>
- <!-- Index -->
+ <!-- Index -->
<property>
<name>index_merge_scheduler_max_thread_count</name>
<value>5</value>
@@ -171,7 +171,7 @@
<name>cluster_routing_allocation_disk_threshold_enabled</name>
<value>true</value>
<description>Property used when multiple drives are used to understand if thresholding is active</description>
- </property>
+ </property>
<property>
<name>cluster_routing_allocation_disk_watermark_low</name>
<value>.97</value>
@@ -185,7 +185,7 @@
<property>
<name>network_host</name>
<value>[ _local_, _site_ ]</value>
- <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
+ <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation.</description>
</property>
<property>
<name>network_publish_host</name>
@@ -193,6 +193,6 @@
<value-attributes>
<empty-value-valid>true</empty-value-valid>
</value-attributes>
- <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
+ <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation.</description>
</property>
</configuration>