You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ly...@apache.org on 2016/09/29 14:19:15 UTC
incubator-metron git commit: METRON-466 Full Dev Platform build
sometimes fails closes apache/incubator-metron#279
Repository: incubator-metron
Updated Branches:
refs/heads/master 1c6de7fe5 -> dcda53825
METRON-466 Full Dev Platform build sometimes fails closes apache/incubator-metron#279
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/dcda5382
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/dcda5382
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/dcda5382
Branch: refs/heads/master
Commit: dcda538252728d87f36bf140145f04d14a763cd3
Parents: 1c6de7f
Author: dlyle65535 <dl...@gmail.com>
Authored: Thu Sep 29 10:18:24 2016 -0400
Committer: David Lyle <dl...@gmail.com>
Committed: Thu Sep 29 10:18:24 2016 -0400
----------------------------------------------------------------------
.../extra_modules/ambari_cluster_state.py | 5 +-
.../inventory/full-dev-platform/group_vars/all | 3 +-
.../inventory/quick-dev-platform/group_vars/all | 95 ++++++++++++++++++++
.../inventory/quick-dev-platform/hosts | 56 ++++++++++++
.../vagrant/full-dev-platform/run.sh | 2 +-
.../vagrant/quick-dev-platform/Vagrantfile | 2 +-
.../vagrant/quick-dev-platform/run.sh | 2 +-
7 files changed, 160 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dcda5382/metron-deployment/extra_modules/ambari_cluster_state.py
----------------------------------------------------------------------
diff --git a/metron-deployment/extra_modules/ambari_cluster_state.py b/metron-deployment/extra_modules/ambari_cluster_state.py
index f1d86d5..a61d0f1 100644
--- a/metron-deployment/extra_modules/ambari_cluster_state.py
+++ b/metron-deployment/extra_modules/ambari_cluster_state.py
@@ -181,7 +181,10 @@ def main():
request = set_cluster_state(ambari_url, username, password, cluster_name, state)
if wait_for_complete:
- request_id = json.loads(request.content)['Requests']['id']
+ try:
+ request_id = json.loads(request.content)['Requests']['id']
+ except ValueError:
+ module.exit_json(changed=True, results=request.content)
status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
if status != 'COMPLETED':
module.fail_json(msg="Request failed with status {0}".format(status))
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dcda5382/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 2194069..609f388 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -38,6 +38,7 @@ ambari_user: admin
ambari_password: admin
cluster_type: single_node_vm
ambari_server_mem: 512
+java_home: /usr/jdk64/jdk1.8.0_77
# hbase
pcap_hbase_table: pcap
@@ -78,7 +79,7 @@ sensor_test_mode: True
install_pycapa: False
install_bro: True
install_snort: True
-install_yaf: True
+install_yaf: False
install_pcap_replay: True
sniff_interface: eth1
pcap_replay_interface: "{{ sniff_interface }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dcda5382/metron-deployment/inventory/quick-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/quick-dev-platform/group_vars/all b/metron-deployment/inventory/quick-dev-platform/group_vars/all
new file mode 100644
index 0000000..e5a0cb8
--- /dev/null
+++ b/metron-deployment/inventory/quick-dev-platform/group_vars/all
@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# which services should be started?
+services_to_start:
+ - mysql
+ - elasticsearch
+ - pcap-service
+ - kibana
+ - snort
+ - snort-logs
+ - bro
+ - pcap-replay
+ - bro-parser
+ - snort-parser
+ - enrichment
+ - indexing
+
+# ambari
+ambari_host: "{{ groups.ambari_master[0] }}"
+hdp_host_group: "{{ groups.ambari_slave }}"
+ambari_port: 8080
+ambari_user: admin
+ambari_password: admin
+cluster_type: single_node_vm
+ambari_server_mem: 512
+
+# hbase
+pcap_hbase_table: pcap
+tracker_hbase_table: access_tracker
+threatintel_hbase_table: threatintel
+enrichment_hbase_table: enrichment
+
+# metron
+metron_version: 0.2.0BETA
+metron_directory: /usr/metron/{{ metron_version }}
+bro_version: "2.4.1"
+fixbuf_version: "1.7.1"
+yaf_version: "2.8.0"
+daq_version: "2.0.6-1"
+pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
+pycapa_home: "/opt/pycapa"
+snort_version: "2.9.8.0-1"
+snort_alert_csv_path: "/var/log/snort/alert.csv"
+threat_intel_bulk_load: False
+
+# data directories - only required to override defaults
+zookeeper_data_dir: "/data1/hadoop/zookeeper"
+namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
+namenode_name_dir: "/data1/hadoop/hdfs/namenode"
+datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
+journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
+nodemanager_local_dirs: "/data1/hadoop/yarn/local"
+timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
+timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
+nodemanager_log_dirs: "/data1/hadoop/yarn/log"
+jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
+storm_local_dir: "/data1/hadoop/storm"
+kafka_log_dirs: "/data1/kafka-log"
+elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
+
+# sensors
+sensor_test_mode: True
+install_pycapa: False
+install_bro: True
+install_snort: True
+install_yaf: False
+install_pcap_replay: True
+sniff_interface: eth1
+pcap_replay_interface: "{{ sniff_interface }}"
+pcapservice_port: 8081
+
+# search
+install_elasticsearch: True
+install_solr: False
+solr_collection_name: Metron
+solr_number_shards: 1
+solr_replication_factor: 1
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth1
+elasticsearch_web_port: 9200
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dcda5382/metron-deployment/inventory/quick-dev-platform/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/quick-dev-platform/hosts b/metron-deployment/inventory/quick-dev-platform/hosts
new file mode 100644
index 0000000..7dacc20
--- /dev/null
+++ b/metron-deployment/inventory/quick-dev-platform/hosts
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[ambari_master]
+node1
+
+[ambari_slave]
+node1
+
+[metron_hbase_tables]
+node1
+
+[metron_kafka_topics]
+node1
+
+[enrichment]
+node1
+
+[search]
+node1
+
+[web]
+node1
+
+[sensors]
+node1
+
+[pcap_server]
+node1
+
+[mysql]
+node1
+
+[metron:children]
+enrichment
+search
+web
+sensors
+mysql
+metron_kafka_topics
+metron_hbase_tables
+pcap_server
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dcda5382/metron-deployment/vagrant/full-dev-platform/run.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/run.sh b/metron-deployment/vagrant/full-dev-platform/run.sh
index b1dea7e..c2e510b 100755
--- a/metron-deployment/vagrant/full-dev-platform/run.sh
+++ b/metron-deployment/vagrant/full-dev-platform/run.sh
@@ -17,4 +17,4 @@
# limitations under the License.
#
-vagrant --ansible-skip-tags="solr" up
+vagrant --ansible-skip-tags="solr,yaf" up
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dcda5382/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
index 1509252..285c216 100644
--- a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
+++ b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
@@ -87,7 +87,7 @@ Vagrant.configure(2) do |config|
ansible.sudo = true
ansible.tags = ansibleTags.split(",") if ansibleTags != ''
ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != ''
- ansible.inventory_path = "../../inventory/full-dev-platform"
+ ansible.inventory_path = "../../inventory/quick-dev-platform"
end
end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dcda5382/metron-deployment/vagrant/quick-dev-platform/run.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/quick-dev-platform/run.sh b/metron-deployment/vagrant/quick-dev-platform/run.sh
index 56e7d1a..1f31507 100755
--- a/metron-deployment/vagrant/quick-dev-platform/run.sh
+++ b/metron-deployment/vagrant/quick-dev-platform/run.sh
@@ -19,5 +19,5 @@
vagrant \
--ansible-tags="hdp-deploy,metron" \
- --ansible-skip-tags="solr" \
+ --ansible-skip-tags="solr,yaf" \
up