You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/02/25 15:32:01 UTC
incubator-metron git commit: METRON-54: Allow Configurable Data Dirs
(dlyle65535 via cestella) closes apache/incubator-metron#30
Repository: incubator-metron
Updated Branches:
refs/heads/master 5fc17e63c -> a1464cc91
METRON-54: Allow Configurable Data Dirs (dlyle65535 via cestella) closes apache/incubator-metron#30
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/a1464cc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/a1464cc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/a1464cc9
Branch: refs/heads/master
Commit: a1464cc91bf203e6f21b059de3c779de47e793b8
Parents: 5fc17e6
Author: dlyle65535 <dl...@gmail.com>
Authored: Thu Feb 25 09:31:52 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Feb 25 09:31:52 2016 -0500
----------------------------------------------------------------------
.../inventory/metron_example/group_vars/all | 18 +++++++++++++++
.../inventory/multinode-vagrant/group_vars/all | 16 ++++++++++++++
.../inventory/singlenode-vagrant/group_vars/all | 19 ++++++++++++++++
.../vars/multi_vagrant_cluster.yml | 17 +++++++++++++++
.../roles/ambari_config/vars/single_node_vm.yml | 17 +++++++++++++++
.../roles/ambari_config/vars/small_cluster.yml | 23 ++++++++++++++++++++
deployment/roles/elasticsearch/tasks/main.yml | 13 ++++++++++-
deployment/roles/pcap_replay/files/pcap-replay | 2 +-
deployment/roles/pcap_replay/tasks/service.yml | 7 +++++-
9 files changed, 129 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index e72888c..b8cf9dc 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -56,5 +56,23 @@ pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
+
+#PCAP Replay
pcap_replay: True
+pcap_replay_interface: eth1
+
+#data directories - only required to override defaults
+#zookeeper_data_dir: "/newdir/hadoop/zookeeper"
+#namenode_checkpoint_dir: "/newdir/hadoop/hdfs/namesecondary"
+#namenode_name_dir: "/newdir/hadoop/hdfs/namenode"
+#datanode_data_dir: "/newdir/hadoop/hdfs/data"
+#journalnode_edits_dir: "/newdir/hadoop/hdfs/journalnode"
+#nodemanager_local_dirs: "/newdir/hadoop/yarn/local"
+#timeline_ldb_store_path: "/newdir/hadoop/yarn/timeline"
+#timeline_ldb_state_path: "/newdir/hadoop/yarn/timeline"
+#nodemanager_log_dirs: "/newdir/hadoop/yarn/log"
+#jhs_recovery_store_ldb_path: "/newdir/hadoop/mapreduce/jhs"
+#storm_local_dir: "/newdir/hadoop/storm"
+#kafka_log_dirs: "/newdir/kafka-log"
+#elasticsearch_data_dir: "/newdir1/elasticsearch"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/inventory/multinode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/multinode-vagrant/group_vars/all b/deployment/inventory/multinode-vagrant/group_vars/all
index 2f7b10f..bb41e89 100644
--- a/deployment/inventory/multinode-vagrant/group_vars/all
+++ b/deployment/inventory/multinode-vagrant/group_vars/all
@@ -53,3 +53,19 @@ pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
+
+#data directories
+zookeeper_data_dir: "/newdir/hadoop/zookeeper"
+namenode_checkpoint_dir: "/newdir/hadoop/hdfs/namesecondary"
+namenode_name_dir: "/newdir/hadoop/hdfs/namenode"
+datanode_data_dir: "/newdir/hadoop/hdfs/data"
+journalnode_edits_dir: "/newdir/hadoop/hdfs/journalnode"
+nodemanager_local_dirs: "/newdir/hadoop/yarn/local"
+timeline_ldb_store_path: "/newdir/hadoop/yarn/timeline"
+timeline_ldb_state_path: "/newdir/hadoop/yarn/timeline"
+nodemanager_log_dirs: "/newdir/hadoop/yarn/log"
+jhs_recovery_store_ldb_path: "/newdir/hadoop/mapreduce/jhs"
+storm_local_dir: "/newdir/hadoop/storm"
+kafka_log_dirs: "/newdir/kafka-log"
+elasticsearch_data_dir: "/newdir1/elasticsearch"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index efef4e5..1e08a6a 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -53,4 +53,23 @@ pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
+
+#PCAP Replay
pcap_replay: True
+pcap_replay_interface: eth1
+
+#data directories - only required to override defaults
+zookeeper_data_dir: "/data1/hadoop/zookeeper"
+namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
+namenode_name_dir: "/data1/hadoop/hdfs/namenode"
+datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
+journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
+nodemanager_local_dirs: "/data1/hadoop/yarn/local"
+timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
+timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
+nodemanager_log_dirs: "/data1/hadoop/yarn/log"
+jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
+storm_local_dir: "/data1/hadoop/storm"
+kafka_log_dirs: "/data1/kafka-log"
+elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml b/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
index 1e2e1e8..574268f 100644
--- a/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
+++ b/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
@@ -39,6 +39,13 @@ cluster_name: "metron"
blueprint_name: "metron_blueprint"
configurations:
+ - zoo.cfg:
+ dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
+ - hdfs-site:
+ dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
+ dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
+ dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
+ dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
- hadoop-env:
namenode_heapsize: 1024
dtnode_heapsize: 1024
@@ -54,10 +61,20 @@ configurations:
- yarn-site:
yarn.nodemanager.resource.memory-mb: 1024
yarn.scheduler.maximum-allocation-mb: 1024
+ yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
+ yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
+ yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
+ yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
+
+ - mapred-site:
+ mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
- storm-site:
supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
+ storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
- kafka-env:
content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
+ - kafka-broker:
+ log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
blueprint:
stack_name: HDP
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/single_node_vm.yml b/deployment/roles/ambari_config/vars/single_node_vm.yml
index abae6a4..d0d3b78 100644
--- a/deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -35,25 +35,42 @@ cluster_name: "metron_cluster"
blueprint_name: "metron_blueprint"
configurations:
+ - zoo.cfg:
+ dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
- hadoop-env:
namenode_heapsize: 1024
dtnode_heapsize: 1024
- hbase-env:
hbase_regionserver_heapsize: 1024
hbase_master_heapsize: 1024
+ - hdfs-site:
+ dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
+ dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
+ dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
+ dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
- yarn-env:
nodemanager_heapsize: 512
yarn_heapsize: 512
apptimelineserver_heapsize : 512
- mapred-env:
jobhistory_heapsize: 256
+ - mapred-site:
+ mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
- yarn-site:
yarn.nodemanager.resource.memory-mb: 1024
yarn.scheduler.maximum-allocation-mb: 1024
+ yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
+ yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
+ yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
+ yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
- storm-site:
supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
+ storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
- kafka-env:
content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
+ - kafka-broker:
+ log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
+
blueprint:
stack_name: HDP
stack_version: 2.3
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/small_cluster.yml b/deployment/roles/ambari_config/vars/small_cluster.yml
index 592c2dc..ff985dc 100644
--- a/deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/deployment/roles/ambari_config/vars/small_cluster.yml
@@ -39,8 +39,31 @@ cluster_name: "metron"
blueprint_name: "metron_blueprint"
configurations:
+ - zoo.cfg:
+ dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
+ - hadoop-env:
+ namenode_heapsize: 1024
+ dtnode_heapsize: 1024
+ - hbase-env:
+ hbase_regionserver_heapsize: 1024
+ hbase_master_heapsize: 1024
+ - hdfs-site:
+ dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
+ dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
+ dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
+ dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
+ - mapred-site:
+ mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
+ - yarn-site:
+ yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
+ yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
+ yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
+ yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
- storm-site:
supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
+ storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
+ - kafka-broker:
+ log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
blueprint:
stack_name: HDP
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/tasks/main.yml b/deployment/roles/elasticsearch/tasks/main.yml
index 9a08c60..555666b 100644
--- a/deployment/roles/elasticsearch/tasks/main.yml
+++ b/deployment/roles/elasticsearch/tasks/main.yml
@@ -15,7 +15,6 @@
# limitations under the License.
#
---
-
- name: Install libselinux-python and java
yum: name={{item}}
with_items:
@@ -36,6 +35,17 @@
- name: Install Elasticsearch.
yum: pkg=elasticsearch state=installed
+- name: Create Data Directories
+ file:
+ path: "{{ item }}"
+ state: directory
+ mode: 0755
+ owner: elasticsearch
+ group: elasticsearch
+ when: elasticsearch_data_dir is defined
+ with_items:
+ - '{{ elasticsearch_data_dir.split(",") }}'
+
- name: Configure Elasticsearch.
lineinfile: >
dest=/etc/elasticsearch/elasticsearch.yml
@@ -48,6 +58,7 @@
elasticsearch_network_interface | default("eth0") }}:ipv4_' }
- { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
+ - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir | default("/var/lib/elasticsearch")}}' }
notify: restart elasticsearch
- name: Start Elasticsearch.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/roles/pcap_replay/files/pcap-replay
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/files/pcap-replay b/deployment/roles/pcap_replay/files/pcap-replay
index 700236d..87259b5 100644
--- a/deployment/roles/pcap_replay/files/pcap-replay
+++ b/deployment/roles/pcap_replay/files/pcap-replay
@@ -23,7 +23,7 @@
DAEMON_PATH="/opt/pcap-replay"
PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null`
-IFACE="eth1"
+IFACE="{{ pcap_replay_interface | default("eth0") }}"
DAEMON=/usr/local/bin/tcpreplay
DAEMONOPTS="--intf1=$IFACE --loop=0 $PCAPIN"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a1464cc9/deployment/roles/pcap_replay/tasks/service.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/tasks/service.yml b/deployment/roles/pcap_replay/tasks/service.yml
index 351bbe0..ce52f50 100644
--- a/deployment/roles/pcap_replay/tasks/service.yml
+++ b/deployment/roles/pcap_replay/tasks/service.yml
@@ -19,7 +19,12 @@
file: path={{ pcap_replay_path }} state=directory mode=0755
- name: Install init.d service script
- copy: src=pcap-replay dest=/etc/init.d/pcap-replay
+ template:
+ src: "../roles/pcap_replay/files/pcap-replay"
+ dest: "/etc/init.d/pcap-replay"
+ mode: 0755
+ owner: "root"
+ group: "root"
- name: Set permissions on service script
file: path=/etc/init.d/pcap-replay mode=0755