You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/06/14 19:07:40 UTC
incubator-metron git commit: METRON-211 Enable creation of Kafka
topics for custom parsers (nickwallen) closes apache/incubator-metron#144
Repository: incubator-metron
Updated Branches:
refs/heads/master 925469b85 -> bd3138832
METRON-211 Enable creation of Kafka topics for custom parsers (nickwallen) closes apache/incubator-metron#144
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/bd313883
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/bd313883
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/bd313883
Branch: refs/heads/master
Commit: bd3138832315543b5dd9ee897747b3341ac1ab44
Parents: 925469b
Author: nickwallen <ni...@nickallen.org>
Authored: Tue Jun 14 15:07:08 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Tue Jun 14 15:07:08 2016 -0400
----------------------------------------------------------------------
metron-deployment/README.md | 6 +--
metron-deployment/amazon-ec2/playbook.yml | 2 +-
.../inventory/devimage-vagrant/hosts | 8 +++-
.../inventory/full-dev-platform/hosts | 8 +++-
.../inventory/metron_example/hosts | 10 +++--
.../packer-build/ansible/playbook.yml | 3 +-
.../packer-build/metron-centos-6.7.json | 2 +-
metron-deployment/playbooks/metron_install.yml | 24 ++++++++----
.../roles/hadoop_setup/defaults/main.yml | 25 ------------
.../roles/hadoop_setup/meta/main.yml | 20 ----------
.../roles/hadoop_setup/tasks/main.yml | 41 --------------------
.../roles/hadoop_setup/vars/main.yml | 18 ---------
.../roles/metron-hbase/defaults/main.yml | 22 +++++++++++
.../roles/metron-hbase/meta/main.yml | 19 +++++++++
.../roles/metron-hbase/tasks/main.yml | 24 ++++++++++++
.../roles/metron-kafka/defaults/main.yml | 24 ++++++++++++
.../roles/metron-kafka/meta/main.yml | 19 +++++++++
.../roles/metron-kafka/tasks/main.yml | 30 ++++++++++++++
.../roles/metron_hbase_tables/defaults/main.yml | 22 +++++++++++
.../roles/metron_hbase_tables/meta/main.yml | 19 +++++++++
.../roles/metron_hbase_tables/tasks/main.yml | 24 ++++++++++++
.../roles/metron_kafka_topics/defaults/main.yml | 24 ++++++++++++
.../roles/metron_kafka_topics/meta/main.yml | 19 +++++++++
.../roles/metron_kafka_topics/tasks/main.yml | 30 ++++++++++++++
.../roles/metron_streaming/defaults/main.yml | 1 -
.../roles/metron_streaming/tasks/main.yml | 14 +++----
.../roles/metron_streaming/tasks/topologies.yml | 4 ++
27 files changed, 326 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/README.md b/metron-deployment/README.md
index edf8a37..e7e988d 100644
--- a/metron-deployment/README.md
+++ b/metron-deployment/README.md
@@ -61,10 +61,10 @@ Metron services will be installed on by updating these groups:
- web - host where the Metron UI and underlying services will run
- sensors - host where network data will be collected and published to Kafka
-The Metron topologies depend on Kafka topics and HBase tables being created beforehand. Declare a host that has Kafka
- and HBase clients installed by updating this group:
+The Metron topologies depend on Kafka topics and HBase tables being created beforehand. Declare a host that has Kafka and HBase clients installed by updating these groups:
-- hadoop_client
+- metron_kafka_topics
+- metron_hbase_tables
If only installing Metron, these groups can be ignored:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml
index cef6a2c..2ae6767 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -28,7 +28,7 @@
- include: tasks/create-open-outbound-security-group.yml
- include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,metron,ec2
- include: tasks/create-hosts.yml host_count=4 host_type=ambari_slave,ec2
- - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,hadoop_client,metron,ec2,pcap_server
+ - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,metron_kafka_topics,metron_hbase_tables,metron,ec2,pcap_server
- include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2
- include: tasks/create-hosts.yml host_count=2 host_type=search,metron,ec2
- include: tasks/create-hosts.yml host_count=1 host_type=web,mysql,metron,ec2
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/inventory/devimage-vagrant/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/devimage-vagrant/hosts b/metron-deployment/inventory/devimage-vagrant/hosts
index 528094b..36ec7fa 100644
--- a/metron-deployment/inventory/devimage-vagrant/hosts
+++ b/metron-deployment/inventory/devimage-vagrant/hosts
@@ -21,7 +21,10 @@ node1
[ambari_slave]
node1
-[hadoop_client]
+[metron_hbase_tables]
+node1
+
+[metron_kafka_topics]
node1
[enrichment]
@@ -48,5 +51,6 @@ search
web
sensors
mysql
-hadoop_client
+metron_kafka_topics
+metron_hbase_tables
pcap_server
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/inventory/full-dev-platform/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/hosts b/metron-deployment/inventory/full-dev-platform/hosts
index 93c53c2..7dacc20 100644
--- a/metron-deployment/inventory/full-dev-platform/hosts
+++ b/metron-deployment/inventory/full-dev-platform/hosts
@@ -21,7 +21,10 @@ node1
[ambari_slave]
node1
-[hadoop_client]
+[metron_hbase_tables]
+node1
+
+[metron_kafka_topics]
node1
[enrichment]
@@ -48,5 +51,6 @@ search
web
sensors
mysql
-hadoop_client
+metron_kafka_topics
+metron_hbase_tables
pcap_server
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/inventory/metron_example/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/hosts b/metron-deployment/inventory/metron_example/hosts
index d190434..d2d540f 100644
--- a/metron-deployment/inventory/metron_example/hosts
+++ b/metron-deployment/inventory/metron_example/hosts
@@ -28,8 +28,10 @@ node6
node7
node8
-#last ambari_slave
-[hadoop_client]
+[metron_hbase_tables]
+node9
+
+[metron_kafka_topics]
node9
[pcap_server]
@@ -62,5 +64,5 @@ search
web
sensors
mysql
-hadoop_client
-
+metron_kafka_topics
+metron_hbase_tables
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/packer-build/ansible/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/ansible/playbook.yml b/metron-deployment/packer-build/ansible/playbook.yml
index 72c3743..8f922ba 100644
--- a/metron-deployment/packer-build/ansible/playbook.yml
+++ b/metron-deployment/packer-build/ansible/playbook.yml
@@ -30,6 +30,7 @@
become: true
roles:
- role: ambari_gather_facts
- - role: hadoop_setup
+ - role: metron_hbase_tables
+ - role: metron_kafka_topics
tags:
- packer-cleanup
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/packer-build/metron-centos-6.7.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/metron-centos-6.7.json b/metron-deployment/packer-build/metron-centos-6.7.json
index 7bbc8bb..e6de665 100644
--- a/metron-deployment/packer-build/metron-centos-6.7.json
+++ b/metron-deployment/packer-build/metron-centos-6.7.json
@@ -89,7 +89,7 @@
"sftp_command": "/usr/libexec/openssh/sftp-server",
"user" : "vagrant",
"host_alias" : "node1",
- "groups" : ["packer","ambari_master","ambari_slave","hadoop_client","enrichment","search","web","sensors","mysql","pcap_server","metron"]
+ "groups" : ["packer","ambari_master","ambari_slave","metron_kafka_topics","metron_hbase_tables","enrichment","search","web","sensors","mysql","pcap_server","metron"]
},
{
"type": "shell",
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index 4284234..7435b97 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -39,14 +39,6 @@
tags:
- metron-prereqs
-- hosts: hadoop_client
- become: true
- roles:
- - role: hadoop_setup
- tags:
- - metron-prereqs
- - hadoop-setup
-
#
# search
#
@@ -86,6 +78,22 @@
#
# parsers, enrichment, and indexing topologies
#
+- hosts: metron_hbase_tables
+ become: true
+ roles:
+ - role: ambari_gather_facts
+ - role: metron_hbase_tables
+ tags:
+ - metron-hbase-tables
+
+- hosts: metron_kafka_topics
+ become: true
+ roles:
+ - role: ambari_gather_facts
+ - role: metron_kafka_topics
+ tags:
+ - metron-kafka-topics
+
- hosts: enrichment
become: true
roles:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/hadoop_setup/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/defaults/main.yml b/metron-deployment/roles/hadoop_setup/defaults/main.yml
deleted file mode 100644
index 99a55f6..0000000
--- a/metron-deployment/roles/hadoop_setup/defaults/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-num_partitions: 1
-retention_in_gb: 10
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: yaf
-snort_topic: snort
-enrichments_topic: enrichments
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/hadoop_setup/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/meta/main.yml b/metron-deployment/roles/hadoop_setup/meta/main.yml
deleted file mode 100644
index 8f0bf2b..0000000
--- a/metron-deployment/roles/hadoop_setup/meta/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - ambari_gather_facts
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/hadoop_setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/tasks/main.yml b/metron-deployment/roles/hadoop_setup/tasks/main.yml
deleted file mode 100644
index cc54641..0000000
--- a/metron-deployment/roles/hadoop_setup/tasks/main.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#must run on hadoop host
-- name: Create HBase tables
- shell: echo "create '{{ item }}','t'" | hbase shell -n
- with_items:
- - "{{ pcap_hbase_table }}"
- - "{{ tracker_hbase_table }}"
- - "{{ threatintel_hbase_table }}"
- - "{{ enrichment_hbase_table }}"
- register: hbase_result
- failed_when: hbase_result.rc != 0 and ("Table already exists" not in hbase_result.stdout)
- changed_when: ("Table already exists" not in hbase_result.stdout)
-
-- name: Create Kafka topics
- shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions {{ num_partitions }} --replication-factor 1 --config retention.bytes={{ retention_in_gb * 1024 * 1024 * 1024}}"
- with_items:
- - "{{ pycapa_topic }}"
- - "{{ bro_topic }}"
- - "{{ yaf_topic }}"
- - "{{ snort_topic }}"
- - "{{ enrichments_topic }}"
- register: kafka_result
- failed_when: kafka_result.rc != 0 and ("already exists" not in kafka_result.stderr)
- changed_when: ("already exists" not in kafka_result.stderr)
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/hadoop_setup/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/vars/main.yml b/metron-deployment/roles/hadoop_setup/vars/main.yml
deleted file mode 100644
index 9747044..0000000
--- a/metron-deployment/roles/hadoop_setup/vars/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-kafka_home: /usr/hdp/current/kafka-broker/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron-hbase/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-hbase/defaults/main.yml b/metron-deployment/roles/metron-hbase/defaults/main.yml
new file mode 100644
index 0000000..9d40d1b
--- /dev/null
+++ b/metron-deployment/roles/metron-hbase/defaults/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hbase_tables_to_create:
+ - pcap
+ - access_tracker
+ - threatintel
+ - enrichment
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron-hbase/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-hbase/meta/main.yml b/metron-deployment/roles/metron-hbase/meta/main.yml
new file mode 100644
index 0000000..c3d807b
--- /dev/null
+++ b/metron-deployment/roles/metron-hbase/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron-hbase/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-hbase/tasks/main.yml b/metron-deployment/roles/metron-hbase/tasks/main.yml
new file mode 100644
index 0000000..cc86c09
--- /dev/null
+++ b/metron-deployment/roles/metron-hbase/tasks/main.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# must run on hadoop host
+- name: Create the HBase tables required for Metron
+ shell: echo "create '{{ item }}','t'" | hbase shell -n
+ with_items: "{{ hbase_tables_to_create }}"
+ register: out
+ failed_when: out.rc != 0 and ("Table already exists" not in out.stdout)
+ changed_when: ("Table already exists" not in out.stdout)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron-kafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-kafka/defaults/main.yml b/metron-deployment/roles/metron-kafka/defaults/main.yml
new file mode 100644
index 0000000..4e83a67
--- /dev/null
+++ b/metron-deployment/roles/metron-kafka/defaults/main.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+kafka_home: /usr/hdp/current/kafka-broker/
+topics_to_create:
+ - { topic: "pcap", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "bro", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "yaf", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "snort", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "enrichments", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron-kafka/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-kafka/meta/main.yml b/metron-deployment/roles/metron-kafka/meta/main.yml
new file mode 100644
index 0000000..c3d807b
--- /dev/null
+++ b/metron-deployment/roles/metron-kafka/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron-kafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-kafka/tasks/main.yml b/metron-deployment/roles/metron-kafka/tasks/main.yml
new file mode 100644
index 0000000..8d3cb0a
--- /dev/null
+++ b/metron-deployment/roles/metron-kafka/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create the Kafka topics required for Metron
+ shell: >
+ {{ kafka_home }}/bin/kafka-topics.sh \
+ --zookeeper {{ zookeeper_url }} \
+ --create \
+ --topic {{ item.topic }} \
+ --partitions {{ item.num_partitions }} \
+ --replication-factor {{ item.replication_factor }} \
+ --config retention.bytes={{ item.retention_gb * 1024 * 1024 * 1024 }}
+ with_items: "{{ topics_to_create }}"
+ register: out
+ failed_when: out.rc != 0 and ("already exists" not in out.stderr)
+ changed_when: ("already exists" not in out.stderr)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_hbase_tables/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_hbase_tables/defaults/main.yml b/metron-deployment/roles/metron_hbase_tables/defaults/main.yml
new file mode 100644
index 0000000..9d40d1b
--- /dev/null
+++ b/metron-deployment/roles/metron_hbase_tables/defaults/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hbase_tables_to_create:
+ - pcap
+ - access_tracker
+ - threatintel
+ - enrichment
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_hbase_tables/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_hbase_tables/meta/main.yml b/metron-deployment/roles/metron_hbase_tables/meta/main.yml
new file mode 100644
index 0000000..c3d807b
--- /dev/null
+++ b/metron-deployment/roles/metron_hbase_tables/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_hbase_tables/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_hbase_tables/tasks/main.yml b/metron-deployment/roles/metron_hbase_tables/tasks/main.yml
new file mode 100644
index 0000000..cc86c09
--- /dev/null
+++ b/metron-deployment/roles/metron_hbase_tables/tasks/main.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# must run on hadoop host
+- name: Create the HBase tables required for Metron
+ shell: echo "create '{{ item }}','t'" | hbase shell -n
+ with_items: "{{ hbase_tables_to_create }}"
+ register: out
+ failed_when: out.rc != 0 and ("Table already exists" not in out.stdout)
+ changed_when: ("Table already exists" not in out.stdout)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_kafka_topics/defaults/main.yml b/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
new file mode 100644
index 0000000..4e83a67
--- /dev/null
+++ b/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+kafka_home: /usr/hdp/current/kafka-broker/
+topics_to_create:
+ - { topic: "pcap", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "bro", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "yaf", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "snort", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "enrichments", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_kafka_topics/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_kafka_topics/meta/main.yml b/metron-deployment/roles/metron_kafka_topics/meta/main.yml
new file mode 100644
index 0000000..c3d807b
--- /dev/null
+++ b/metron-deployment/roles/metron_kafka_topics/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_kafka_topics/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_kafka_topics/tasks/main.yml b/metron-deployment/roles/metron_kafka_topics/tasks/main.yml
new file mode 100644
index 0000000..8d3cb0a
--- /dev/null
+++ b/metron-deployment/roles/metron_kafka_topics/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create the Kafka topics required for Metron
+ shell: >
+ {{ kafka_home }}/bin/kafka-topics.sh \
+ --zookeeper {{ zookeeper_url }} \
+ --create \
+ --topic {{ item.topic }} \
+ --partitions {{ item.num_partitions }} \
+ --replication-factor {{ item.replication_factor }} \
+ --config retention.bytes={{ item.retention_gb * 1024 * 1024 * 1024 }}
+ with_items: "{{ topics_to_create }}"
+ register: out
+ failed_when: out.rc != 0 and ("already exists" not in out.stderr)
+ changed_when: ("already exists" not in out.stderr)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/defaults/main.yml b/metron-deployment/roles/metron_streaming/defaults/main.yml
index 1af0939..cbec601 100644
--- a/metron-deployment/roles/metron_streaming/defaults/main.yml
+++ b/metron-deployment/roles/metron_streaming/defaults/main.yml
@@ -36,7 +36,6 @@ metron_solr_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-solr/t
metron_elasticsearch_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-elasticsearch/target/{{ metron_elasticsearch_bundle_name }}"
metron_parsers_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-parsers/target/{{ metron_parsers_bundle_name }}"
-
config_path: "{{ metron_directory }}/config"
zookeeper_config_path: "{{ config_path }}/zookeeper"
zookeeper_global_config_path: "{{ zookeeper_config_path }}/global.json"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml
index c960dba..0945e3d 100644
--- a/metron-deployment/roles/metron_streaming/tasks/main.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/main.yml
@@ -15,12 +15,12 @@
# limitations under the License.
#
---
-- name: Create Metron streaming directories
- file: path="{{ metron_directory }}/{{ item.name }}" state=directory mode=0755
+- name: Create Metron directories
+ file: path={{ metron_directory }}/{{ item }} state=directory mode=0755
with_items:
- - { name: 'lib'}
- - { name: 'bin'}
- - { name: 'config'}
+ - lib
+ - bin
+ - config
- name: Create Source Config Directory
file:
@@ -30,10 +30,6 @@
- include: copy_bundles.yml
-- name: Get Default mysql passowrd
- include_vars: "../roles/mysql_server/defaults/main.yml"
- when: mysql_root_password is undefined
-
- include: hdfs_filesystem.yml
run_once: true
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/bd313883/metron-deployment/roles/metron_streaming/tasks/topologies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/topologies.yml b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
index 17bcfbb..2010eae 100644
--- a/metron-deployment/roles/metron_streaming/tasks/topologies.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
@@ -15,6 +15,10 @@
# limitations under the License.
#
---
+- name: Get Default mysql passowrd
+ include_vars: "../roles/mysql_server/defaults/main.yml"
+ when: mysql_root_password is undefined
+
- name: Configure Metron Pcap Topology
lineinfile: >
dest={{ metron_pcap_properties_config_path }}