You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2016/04/26 16:46:36 UTC
[48/51] [partial] incubator-metron git commit: METRON-113 Project
Reorganization (merrimanr) closes apache/incubator-metron#88
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/java_jdk/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/java_jdk/tasks/main.yml b/deployment/roles/java_jdk/tasks/main.yml
deleted file mode 100644
index 999b9c1..0000000
--- a/deployment/roles/java_jdk/tasks/main.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Check for java at "{{ java_home }}"
- stat: path="{{ java_home }}"
- register: jdk_dir
-
-- name: Alternatives link for java
- alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }}
- with_items:
- - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
- - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
- when: jdk_dir.stat.exists
-
-- name: Install openjdk
- yum: name={{item}}
- with_items:
- - java-1.8.0-openjdk
- - java-1.8.0-openjdk-devel
- when: not jdk_dir.stat.exists
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/kafka-broker/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/defaults/main.yml b/deployment/roles/kafka-broker/defaults/main.yml
deleted file mode 100644
index 351d125..0000000
--- a/deployment/roles/kafka-broker/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/kafka-broker/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/meta/main.yml b/deployment/roles/kafka-broker/meta/main.yml
deleted file mode 100644
index 9587e79..0000000
--- a/deployment/roles/kafka-broker/meta/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-dependencies:
- - libselinux-python
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/kafka-broker/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/tasks/main.yml b/deployment/roles/kafka-broker/tasks/main.yml
deleted file mode 100644
index db05cb0..0000000
--- a/deployment/roles/kafka-broker/tasks/main.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Retrieve HDP repository definition
- get_url:
- url: "{{ hdp_repo_def }}"
- dest: /etc/yum.repos.d/hdp.repo
- mode: 0644
-
-- name: Install kafka
- yum: name={{item}}
- with_items:
- - java-1.8.0-openjdk
- - kafka
- - zookeeper-server
-
-- name: Create pid directories
- file: path={{ item }} state=directory mode=0755
- with_items:
- - /var/run/zookeeper
- - /var/run/kafka
-
-- name: Start zookeeper
- shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start
-
-- name: Start kafka
- shell: /usr/hdp/current/kafka-broker/bin/kafka start
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/kafka-broker/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/vars/main.yml b/deployment/roles/kafka-broker/vars/main.yml
deleted file mode 100644
index 351d125..0000000
--- a/deployment/roles/kafka-broker/vars/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/kafka-client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-client/tasks/main.yml b/deployment/roles/kafka-client/tasks/main.yml
deleted file mode 100644
index 1674225..0000000
--- a/deployment/roles/kafka-client/tasks/main.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: Retrieve HDP repository definition
- get_url:
- url: "{{ hdp_repo_def }}"
- dest: /etc/yum.repos.d/hdp.repo
- mode: 0644
-
- - name: Install kafka
- yum:
- name: kafka
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/defaults/main.yml b/deployment/roles/librdkafka/defaults/main.yml
deleted file mode 100644
index d920883..0000000
--- a/deployment/roles/librdkafka/defaults/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-librdkafka_version: 0.8.6
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz
-librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/librdkafka/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/dependencies.yml b/deployment/roles/librdkafka/tasks/dependencies.yml
deleted file mode 100644
index 431e861..0000000
--- a/deployment/roles/librdkafka/tasks/dependencies.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install prerequisites
- yum: name={{ item }}
- with_items:
- - cmake
- - make
- - gcc
- - gcc-c++
- - flex
- - bison
- - libpcap
- - libpcap-devel
- - openssl-devel
- - python-devel
- - swig
- - zlib-devel
- - perl
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/librdkafka/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/librdkafka.yml b/deployment/roles/librdkafka/tasks/librdkafka.yml
deleted file mode 100644
index 652d319..0000000
--- a/deployment/roles/librdkafka/tasks/librdkafka.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download librdkafka
- get_url:
- url: "{{ librdkafka_url }}"
- dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
-
-- name: Extract librdkafka tarball
- unarchive:
- src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/librdkafka-{{ librdkafka_version }}"
-
-- name: Compile and install librdkafka
- shell: "{{ item }}"
- args:
- chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
- creates: "{{ librdkafka_home }}/lib/librdkafka.so"
- with_items:
- - rm -rf build/
- - "./configure --prefix={{ librdkafka_home }}"
- - make
- - make install
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/librdkafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/main.yml b/deployment/roles/librdkafka/tasks/main.yml
deleted file mode 100644
index 2144d7f..0000000
--- a/deployment/roles/librdkafka/tasks/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: librdkafka.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/libselinux-python/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/libselinux-python/tasks/main.yml b/deployment/roles/libselinux-python/tasks/main.yml
deleted file mode 100644
index 78f5a27..0000000
--- a/deployment/roles/libselinux-python/tasks/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install libselinux-python
- yum:
- name: libselinux-python
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/defaults/main.yml b/deployment/roles/metron_common/defaults/main.yml
deleted file mode 100644
index 50aaefd..0000000
--- a/deployment/roles/metron_common/defaults/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
-metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_common/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/meta/main.yml b/deployment/roles/metron_common/meta/main.yml
deleted file mode 100644
index 4db50aa..0000000
--- a/deployment/roles/metron_common/meta/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - yum-update
- - epel
- - ntp
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/tasks/main.yml b/deployment/roles/metron_common/tasks/main.yml
deleted file mode 100644
index 64e6ab9..0000000
--- a/deployment/roles/metron_common/tasks/main.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Check OS Version
- fail: msg="Metron deployment supports CentOS 6 only."
- when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6")
-
-- name: Check for Metron jar path
- stat: path={{ metron_jar_path }}
- register: metron_jars
-
-- name: Verify Metron jars exist
- fail: msg="Unable to locate staged Metron jars at {{ metron_jar_path }}. Did you run 'mvn package'?"
- when: metron_jars.stat.exists == True
-
-- name: Ensure iptables is stopped and is not running at boot time.
- ignore_errors: yes
- service:
- name: iptables
- state: stopped
- enabled: no
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/defaults/main.yml b/deployment/roles/metron_pcapservice/defaults/main.yml
deleted file mode 100644
index 0dbca48..0000000
--- a/deployment/roles/metron_pcapservice/defaults/main.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_version: 0.1BETA
-metron_directory: /usr/metron/{{ metron_version }}
-pcapservice_jar_name: Metron-Pcap_Service-{{ metron_version }}-jar-with-dependencies.jar
-pcapservice_jar_src: "{{ playbook_dir }}/../../metron-streaming/Metron-Pcap_Service/target/{{ pcapservice_jar_name }}"
-pcapservice_jar_dst: "{{ metron_directory }}/lib/{{ pcapservice_jar_name }}"
-pcapservice_port: 8081
-hbase_config_path: "/etc/hbase/conf"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_pcapservice/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/meta/main.yml b/deployment/roles/metron_pcapservice/meta/main.yml
deleted file mode 100644
index ddf6aa9..0000000
--- a/deployment/roles/metron_pcapservice/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - java_jdk
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_pcapservice/tasks/config-hbase.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/config-hbase.yml b/deployment/roles/metron_pcapservice/tasks/config-hbase.yml
deleted file mode 100644
index b77c1ec..0000000
--- a/deployment/roles/metron_pcapservice/tasks/config-hbase.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: Fetch hbase-site.xml
- fetch: src=/etc/hbase/conf/hbase-site.xml dest=/tmp/hbase/conf/hbase-site.xml flat=yes
- delegate_to: "{{ groups.ambari_slave[0] }}"
-
- - name: Create hbase conf directory
- file: path=/etc/hbase/conf state=directory mode=0755
-
- - name: Copy hbase-site.xml
- copy: src=/tmp/hbase/conf/hbase-site.xml dest=/etc/hbase/conf/hbase-site.xml mode=0644
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_pcapservice/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/main.yml b/deployment/roles/metron_pcapservice/tasks/main.yml
deleted file mode 100644
index ee9cac2..0000000
--- a/deployment/roles/metron_pcapservice/tasks/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Check for hbase-site
- stat: path=/etc/hbase/conf/hbase-site.xml
- register: hbase_site
-
-- include: config-hbase.yml
- when: not hbase_site.stat.exists
-
-- include: pcapservice.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_pcapservice/tasks/pcapservice.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/pcapservice.yml b/deployment/roles/metron_pcapservice/tasks/pcapservice.yml
deleted file mode 100644
index 651f7fb..0000000
--- a/deployment/roles/metron_pcapservice/tasks/pcapservice.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Metron streaming directories
- file: path={{ metron_directory }}/{{ item.name }} state=directory mode=0755
- with_items:
- - { name: 'lib'}
- - { name: 'config'}
-
-- name: Copy Metron pcapservice jar
- copy:
- src: "{{ pcapservice_jar_src }}"
- dest: "{{ pcapservice_jar_dst }}"
-
-- name: Add hbase-site.xml to pcapservice jar
- shell: "jar -uf {{ pcapservice_jar_dst }} hbase-site.xml"
- args:
- chdir: "{{ hbase_config_path }}"
-
-- name: Install service script
- template: src=pcapservice dest=/etc/init.d/pcapservice mode=0755
-
-- name: Start pcapservice
- service: name=pcapservice state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_pcapservice/templates/pcapservice
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/templates/pcapservice b/deployment/roles/metron_pcapservice/templates/pcapservice
deleted file mode 100644
index a3ad92b..0000000
--- a/deployment/roles/metron_pcapservice/templates/pcapservice
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# metron pcap service
-# chkconfig: 345 20 80
-# description: Metron PCAP Service Daemon
-# processname: pcapservice
-#
-NAME=pcapservice
-DESC="Metron pcap service"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-LOGFILE="/var/log/metron_pcapservice.log"
-EXTRA_ARGS="${@:2}"
-DAEMON_PATH="/"
-DAEMON="/usr/bin/java"
-DAEMONOPTS="-cp {{ pcapservice_jar_dst }} org.apache.metron.pcapservice.rest.PcapService -port {{ pcapservice_port }}"
-
-case "$1" in
- start)
- printf "%-50s" "Starting $NAME..."
-
- # kick-off the daemon
- cd $DAEMON_PATH
- PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!`
- if [ -z $PID ]; then
- printf "%s\n" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%s\n" "Ok"
- fi
- ;;
-
- status)
- printf "%-50s" "Checking $NAME..."
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- echo "Running"
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- stop)
- printf "%-50s" "Stopping $NAME"
- PID=`cat $PIDFILE`
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- kill -HUP $PID
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- restart)
- $0 stop
- $0 start
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart}"
- exit 1
-esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
deleted file mode 100644
index f0f605d..0000000
--- a/deployment/roles/metron_streaming/defaults/main.yml
+++ /dev/null
@@ -1,75 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
-metron_directory: /usr/metron/{{ metron_version }}
-metron_solr_jar_name: Metron-Solr-{{ metron_version }}.jar
-metron_elasticsearch_jar_name: Metron-Elasticsearch-{{ metron_version }}.jar
-
-metron_dataloads_name: Metron-DataLoads-{{ metron_version }}-archive.tar.gz
-metron_dataloads_path: "{{ playbook_dir }}/../../metron-streaming/Metron-DataLoads/target/{{ metron_dataloads_name }}"
-metron_topologies_bundle_name: Metron-Topologies-{{ metron_version }}-archive.tar.gz
-metron_solr_bundle_name: Metron-Solr-{{ metron_version }}-archive.tar.gz
-metron_elasticsearch_bundle_name: Metron-Elasticsearch-{{ metron_version }}-archive.tar.gz
-metron_solr_bundle_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Solr/target/{{ metron_solr_bundle_name }}"
-metron_elasticsearch_bundle_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Elasticsearch/target/{{ metron_elasticsearch_bundle_name }}"
-metron_topologies_bundle_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_topologies_bundle_name }}"
-
-config_path: "{{ metron_directory }}/config"
-zookeeper_config_path: "{{ config_path }}/zookeeper"
-zookeeper_global_config_path: "{{ zookeeper_config_path }}/global.json"
-metron_solr_properties_config_path: "{{ metron_directory }}/config/etc/env/solr.properties"
-metron_elasticsearch_properties_config_path: "{{ metron_directory }}/config/etc/env/elasticsearch.properties"
-hbase_config_path: "/etc/hbase/conf"
-hdfs_config_path: "/etc/hadoop/conf"
-
-threat_intel_bulk_load: True
-threat_intel_bin: "{{ metron_directory }}/bin/threatintel_bulk_load.sh"
-threat_intel_work_dir: /tmp/ti_bulk
-threat_intel_csv_filename: "threat_ip.csv"
-threat_intel_csv_filepath: "../roles/metron_streaming/templates/{{ threat_intel_csv_filename }}"
-
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: yaf
-snort_topic: snort
-enrichments_topic: enrichments
-storm_topologies:
- - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
- - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
- - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
- - "{{ metron_directory }}/config/topologies/pcap/remote.yaml"
- - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
-
-hdfs_retention_days: 30
-hdfs_bro_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/enrichment/indexed/bro_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/bro-purge/cron-hdfs-bro-purge.log 2>&1"
-hdfs_yaf_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/enrichment/indexed/yaf_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-hdfs-yaf-purge.log 2>&1"
-hdfs_snort_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/enrichment/indexed/snort_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-hdfs-yaf-purge.log 2>&1"
-
-elasticsearch_config_path: /etc/elasticsearch
-elasticsearch_cluster_name: metron
-elasticsearch_transport_port: 9300
-
-es_retention_days: 30
-es_bro_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p bro_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/bro-purge/cron-es-bro-purge.log 2>&1"
-es_yaf_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p yaf_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-es-yaf-purge.log 2>&1"
-es_snort_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p yaf_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/snort-purge/cron-es-snort-purge.log 2>&1"
-
-metron_hdfs_output_dir: "/apps/metron"
-metron_hdfs_rotation_policy: org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
-metron_hdfs_rotation_policy_count: 1
-metron_hdfs_rotation_policy_units: DAYS
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/files/config/sensors/bro.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/bro.json b/deployment/roles/metron_streaming/files/config/sensors/bro.json
deleted file mode 100644
index 2b534b4..0000000
--- a/deployment/roles/metron_streaming/files/config/sensors/bro.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "index": "bro",
- "batchSize": 5,
- "enrichmentFieldMap":
- {
- "geo": ["ip_dst_addr", "ip_src_addr"],
- "host": ["host"]
- },
- "threatIntelFieldMap":
- {
- "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
- },
- "fieldToThreatIntelTypeMap":
- {
- "ip_src_addr" : ["malicious_ip"],
- "ip_dst_addr" : ["malicious_ip"]
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/files/config/sensors/pcap.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/pcap.json b/deployment/roles/metron_streaming/files/config/sensors/pcap.json
deleted file mode 100644
index 7792165..0000000
--- a/deployment/roles/metron_streaming/files/config/sensors/pcap.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "index": "pcap",
- "batchSize": 5,
- "enrichmentFieldMap":
- {
- "geo": ["ip_src_addr", "ip_dst_addr"],
- "host": ["ip_src_addr", "ip_dst_addr"]
- },
- "threatIntelFieldMap":
- {
- "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"]
- },
- "fieldToThreatIntelTypeMap":
- {
- "ip_dst_addr" : [ "malicious_ip" ]
- ,"ip_src_addr" : [ "malicious_ip" ]
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/files/config/sensors/snort.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/snort.json b/deployment/roles/metron_streaming/files/config/sensors/snort.json
deleted file mode 100644
index c5b6dcc..0000000
--- a/deployment/roles/metron_streaming/files/config/sensors/snort.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "index": "snort",
- "batchSize": 1,
- "enrichmentFieldMap":
- {
- "geo": ["ip_dst_addr", "ip_src_addr"],
- "host": ["host"]
- },
- "threatIntelFieldMap":
- {
- "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
- },
- "fieldToThreatIntelTypeMap":
- {
- "ip_src_addr" : ["malicious_ip"],
- "ip_dst_addr" : ["malicious_ip"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/files/config/sensors/yaf.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/yaf.json b/deployment/roles/metron_streaming/files/config/sensors/yaf.json
deleted file mode 100644
index 2b46c9a..0000000
--- a/deployment/roles/metron_streaming/files/config/sensors/yaf.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "index": "yaf",
- "batchSize": 5,
- "enrichmentFieldMap":
- {
- "geo": ["ip_dst_addr", "ip_src_addr"],
- "host": ["host"]
- },
- "threatIntelFieldMap":
- {
- "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
- },
- "fieldToThreatIntelTypeMap":
- {
- "ip_src_addr" : ["malicious_ip"],
- "ip_dst_addr" : ["malicious_ip"]
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/files/extractor.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/extractor.json b/deployment/roles/metron_streaming/files/extractor.json
deleted file mode 100644
index 545202a..0000000
--- a/deployment/roles/metron_streaming/files/extractor.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "config": {
- "columns": {
- "ip": 0
- },
- "indicator_column": "ip",
- "type" : "malicious_ip",
- "separator": ","
- },
- "extractor": "CSV"
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/yaf_index.template b/deployment/roles/metron_streaming/files/yaf_index.template
deleted file mode 100644
index c8c1702..0000000
--- a/deployment/roles/metron_streaming/files/yaf_index.template
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "template" : "yaf_index*",
- "mappings" : {
- "yaf_doc" : {
- "properties": {
- "message": {
- "properties": {
- "@timestamp":{"type":"date","format":"dateOptionalTime"},
- "end-time":{"type":"string"},
- "duration":{"type":"string"},
- "rtt":{"type":"string"},
- "proto":{"type":"string"},
- "sip":{"type":"string"},
- "sp":{"type":"string"},
- "dip":{"type":"string"},
- "dp":{"type":"string"},
- "iflags":{"type":"string"},
- "uflags":{"type":"string"},
- "riflags":{"type":"string"},
- "ruflags":{"type":"string"},
- "isn":{"type":"string"},
- "risn":{"type":"string"},
- "tag":{"type":"string"},
- "rtag":{"type":"string"},
- "pkt":{"type":"string"},
- "oct":{"type":"string"},
- "rpkt":{"type":"string"},
- "roct":{"type":"string"},
- "app":{"type":"string"},
- "end-reason":{"type":"string"}
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/handlers/main.yml b/deployment/roles/metron_streaming/handlers/main.yml
deleted file mode 100644
index 5aee8b4..0000000
--- a/deployment/roles/metron_streaming/handlers/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Load Config
- shell: "{{ metron_directory }}/bin/zk_load_configs.sh -p {{ zookeeper_config_path }} -z {{ zookeeper_url }} && touch {{ zookeeper_config_path }}/configured"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/meta/main.yml b/deployment/roles/metron_streaming/meta/main.yml
deleted file mode 100644
index 6820b2c..0000000
--- a/deployment/roles/metron_streaming/meta/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - ambari_gather_facts
- - java_jdk
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/es_purge.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/es_purge.yml b/deployment/roles/metron_streaming/tasks/es_purge.yml
deleted file mode 100644
index 22616ca..0000000
--- a/deployment/roles/metron_streaming/tasks/es_purge.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Empty Log Files for ES Purge
- file:
- path: "{{ item }}"
- state: touch
- owner: hdfs
- group: hdfs
- mode: 0644
- with_items:
- - /var/log/bro-purge/cron-es-bro-purge.log
- - /var/log/yaf-purge/cron-es-yaf-purge.log
- - /var/log/snort-purge/cron-es-snort-purge.log
-
-
-- name: Purge Elasticsearch Indices every 30 days.
- cron:
- name: "{{ item.name }}"
- job: "{{ item.job }}"
- special_time: daily
- user: hdfs
- with_items:
- - { name: "bro_es_purge", job: "{{ es_bro_purge_cronjob }}" }
- - { name: "yaf_es_purge", job: "{{ es_yaf_purge_cronjob }}" }
- - { name: "snort_es_purge", job: "{{ es_snort_purge_cronjob }}" }
-
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/grok_upload.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/grok_upload.yml b/deployment/roles/metron_streaming/tasks/grok_upload.yml
deleted file mode 100644
index ef560f0..0000000
--- a/deployment/roles/metron_streaming/tasks/grok_upload.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create HDFS directory for grok patterns
- command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns
- become: yes
- become_user: hdfs
-
-- name: Assign hfds user as owner of {{ metron_hdfs_output_dir }}/patterns HDFS directory
- command: hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns
- become: yes
- become_user: hdfs
-
-- name: Assign permissions of HDFS {{ metron_hdfs_output_dir }}/patterns directory
- command: hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns
- become: yes
- become_user: hdfs
-
-- name: Upload Grok Patterns to hdfs://{{ metron_hdfs_output_dir }}
- command: hdfs dfs -put -f {{ metron_directory }}/config/patterns {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml b/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
deleted file mode 100644
index 252e671..0000000
--- a/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create root user HDFS directory
- command: hdfs dfs -mkdir -p /user/root
- become: yes
- become_user: hdfs
-
-- name: Assign root as owner of /user/root HDFS directory
- command: hdfs dfs -chown root:root /user/root
- become: yes
- become_user: hdfs
-
-- name: Create Metron HDFS output directory
- command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
-
-- name: Assign hdfs as owner of HDFS output directory
- command: hdfs dfs -chown hdfs:hadoop {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
-
-- name: Assign permissions of HDFS output directory
- command: hdfs dfs -chmod 775 {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/hdfs_purge.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/hdfs_purge.yml b/deployment/roles/metron_streaming/tasks/hdfs_purge.yml
deleted file mode 100644
index 33442e4..0000000
--- a/deployment/roles/metron_streaming/tasks/hdfs_purge.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Log Directories for HDFS Purge
- file:
- path: "{{ item }}"
- state: directory
- mode: 0755
- owner: hdfs
- group: hdfs
- with_items:
- - /var/log/bro-purge
- - /var/log/yaf-purge
- - /var/log/snort-purge
-
-- name: Create Empty Log Files for HDFS Purge
- file:
- path: "{{ item }}"
- state: touch
- owner: hdfs
- group: hdfs
- mode: 0644
- with_items:
- - /var/log/bro-purge/cron-hdfs-bro-purge.log
- - /var/log/yaf-purge/cron-hdfs-yaf-purge.log
- - /var/log/snort-purge/cron-hdfs-snort-purge.log
-
-- name: Purge HDFS Sensor Data every 30 days.
- cron:
- name: "{{ item.name }}"
- job: "{{ item.job }}"
- special_time: daily
- user: hdfs
- with_items:
- - { name: "bro_hdfs_purge", job: "{{ hdfs_bro_purge_cronjob }}" }
- - { name: "yaf_hdfs_purge", job: "{{ hdfs_yaf_purge_cronjob }}" }
- - { name: "snort_hdfs_purge", job: "{{ hdfs_snort_purge_cronjob }}" }
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
deleted file mode 100644
index e076645..0000000
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ /dev/null
@@ -1,136 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Metron streaming directories
- file: path="{{ metron_directory }}/{{ item.name }}" state=directory mode=0755
- with_items:
- - { name: 'lib'}
- - { name: 'bin'}
- - { name: 'config'}
-
-
-- name: Copy Metron Solr bundle
- copy:
- src: "{{ metron_solr_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron Elasticsearch bundle
- copy:
- src: "{{ metron_elasticsearch_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron Topologies bundle
- copy:
- src: "{{ metron_topologies_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron DataLoads bundle
- copy:
- src: "{{ metron_dataloads_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Unbundle Metron bundles
- shell: cd {{ metron_directory }} && tar xzvf Metron-Solr*.tar.gz && tar xzvf Metron-Elasticsearch*.tar.gz && tar xzvf Metron-Topologies*.tar.gz && tar xzvf Metron-DataLoads*.tar.gz && rm *.tar.gz
-
-- name: Add *-site.xml files to topology jars
- shell: cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }}
- with_items:
- - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" }
- - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hbase-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "core-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hdfs-site.xml" }
-
-- name: Get Default mysql passowrd
- include_vars: "../roles/mysql_server/defaults/main.yml"
- when: mysql_root_password is undefined
-
-- include: hdfs_filesystem.yml
- run_once: true
-
-- include: grok_upload.yml
- run_once: true
-
-- name: Configure Metron Solr topologies
- lineinfile: >
- dest={{ metron_solr_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
- - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
- - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
- - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
- - { regexp: "bolt.hbase.table.name=", line: "bolt.hbase.table.name={{ pcap_hbase_table }}" }
- - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
- - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
- - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
- - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
- - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
- - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
- - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
- - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
- - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
- - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
- - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
-
-- name: Configure Metron Elasticsearch topologies
- lineinfile: >
- dest={{ metron_elasticsearch_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
- - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
- - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
- - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
- - { regexp: "bolt.hbase.table.name=", line: "bolt.hbase.table.name={{ pcap_hbase_table }}" }
- - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
- - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
- - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
- - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
- - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
- - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
- - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
- - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
- - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
- - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
- - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
-
-- include: source_config.yml
- run_once: true
-
-- include: threat_intel.yml
- run_once: true
- when: threat_intel_bulk_load == True
-
-- include: metron_topology.yml
-
-- include: hdfs_purge.yml
-
-- include: es_purge.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/metron_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/metron_topology.yml b/deployment/roles/metron_streaming/tasks/metron_topology.yml
deleted file mode 100644
index f8bf539..0000000
--- a/deployment/roles/metron_streaming/tasks/metron_topology.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Submit Solr Metron topologies
- command: storm jar {{ metron_directory }}/lib/{{ metron_solr_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_solr_properties_config_path }} --remote {{ item }}
- with_items:
- - "{{ storm_topologies }}"
- when: install_solr | default(False) == True
-
-- name: Submit Elasticsearch Metron topologies
- command: storm jar {{ metron_directory }}/lib/{{ metron_elasticsearch_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_elasticsearch_properties_config_path }} --remote {{ item }}
- with_items:
- - "{{ storm_topologies }}"
- when: install_elasticsearch | default(False) == True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/source_config.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/source_config.yml b/deployment/roles/metron_streaming/tasks/source_config.yml
deleted file mode 100644
index 959056c..0000000
--- a/deployment/roles/metron_streaming/tasks/source_config.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Source Config Directory
- file:
- path: "{{ zookeeper_config_path }}"
- state: directory
-
-- name: Copy Elasticsearch Global Config File
- template:
- src: "templates/config/elasticsearch.global.json"
- dest: "{{ zookeeper_global_config_path }}"
- mode: 0644
- when: install_elasticsearch | default(False) == True
-
-- name: Copy Solr Global Config File
- template:
- src: "../roles/metron_streaming/templates/config/solr.global.json"
- dest: "{{ zookeeper_global_config_path }}"
- mode: 0644
- when: install_solr | default(False) == True
-
-- name: Copy Sensor Config Files
- copy:
- src: "{{ item }}"
- dest: "{{ zookeeper_config_path }}"
- mode: 0644
- with_items:
- - ../roles/metron_streaming/files/config/
- notify: Load Config
-
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/tasks/threat_intel.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/threat_intel.yml b/deployment/roles/metron_streaming/tasks/threat_intel.yml
deleted file mode 100644
index 6dd8144..0000000
--- a/deployment/roles/metron_streaming/tasks/threat_intel.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Create Bulk load working Directory
- file:
- path: "{{ threat_intel_work_dir }}"
- state: directory
-
-- name: Copy extractor.json to {{ inventory_hostname }}
- copy:
- src: ../roles/metron_streaming/files/extractor.json
- dest: "{{ threat_intel_work_dir }}"
- mode: 0644
-
-- name: Copy Bulk Load CSV File
- template:
- src: "{{ threat_intel_csv_filepath }}"
- dest: "{{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }}"
- mode: 0644
-
-- name: Copy Bulk Load CSV File to HDFS
- command: "hdfs dfs -put {{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }} ."
-
-- name: Run Threat Intel Bulk Load
- shell: "{{ threat_intel_bin }} -f t --table {{threatintel_hbase_table}} -e {{ threat_intel_work_dir }}/extractor.json -i /user/root && touch {{ threat_intel_work_dir }}/loaded"
- args:
- creates: "{{ threat_intel_work_dir }}/loaded"
-
-- name: Clean up HDFS File
- command: "hdfs dfs -rm {{ threat_intel_csv_filename }}"
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json b/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
deleted file mode 100644
index 8177102..0000000
--- a/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "es.clustername": "{{ elasticsearch_cluster_name }}",
- "es.ip": "{{ groups.search[0] }}",
- "es.port": "{{ elasticsearch_transport_port }}",
- "es.date.format": "yyyy.MM.dd.HH"
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/templates/config/solr.global.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/templates/config/solr.global.json b/deployment/roles/metron_streaming/templates/config/solr.global.json
deleted file mode 100644
index 5cb7a4d..0000000
--- a/deployment/roles/metron_streaming/templates/config/solr.global.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "solr.zookeeper": "{{ zookeeper_url }}",
- "solr.collection": "{{ solr_collection_name }}",
- "solr.numShards": {{ solr_number_shards }},
- "solr.replicationFactor": {{ solr_replication_factor }}
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_streaming/templates/threat_ip.csv
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/templates/threat_ip.csv b/deployment/roles/metron_streaming/templates/threat_ip.csv
deleted file mode 100644
index 3ac38f3..0000000
--- a/deployment/roles/metron_streaming/templates/threat_ip.csv
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#Add single column of ip address to alert
-#Public lists are available on the internet
-# example:
-23.113.113.105
-24.107.205.249
-24.108.62.255
-24.224.153.71
-27.4.1.212
-27.131.149.102
-31.24.30.31
-31.131.251.33
-31.186.99.250
-31.192.209.119
-31.192.209.150
-31.200.244.17
-37.34.52.185
-37.58.112.101
-37.99.146.27
-37.128.132.96
-37.140.195.177
-37.140.199.100
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_ui/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/defaults/main.yml b/deployment/roles/metron_ui/defaults/main.yml
deleted file mode 100644
index 23aed40..0000000
--- a/deployment/roles/metron_ui/defaults/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_version: 0.1BETA
-metron_directory: /usr/metron/{{ metron_version }}
-metron_ui_directory: "{{ metron_directory }}/metron-ui"
-metron_temp_archive: /tmp/metron-ui.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_ui/tasks/copy-source.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/copy-source.yml b/deployment/roles/metron_ui/tasks/copy-source.yml
deleted file mode 100644
index 703b7f9..0000000
--- a/deployment/roles/metron_ui/tasks/copy-source.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Verify {{ metron_ui_directory }} exists
- file:
- path: "{{ metron_ui_directory }}"
- state: directory
- mode: 0755
- owner: root
- group: root
-
-- name: Archive metron-ui on localhost
- shell: tar --exclude='./node_modules' -czf {{ metron_temp_archive }} .
- args:
- chdir: "{{ playbook_dir }}/../../metron-ui"
- creates: "{{ metron_temp_archive }}"
- warn: false #Warns to use unarchive - unarchive does not archive
- become: false
- delegate_to: localhost
- run_once: true
-
-- name: Extract metron-ui tarball
- unarchive:
- src: "{{ metron_temp_archive }}"
- dest: "{{ metron_ui_directory }}"
- creates: "{{ metron_ui_directory}}/config"
-
-- name: Delete {{ metron_temp_archive }}
- local_action: file path="{{ metron_temp_archive }}" state=absent
- become: false
- run_once: true
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/main.yml b/deployment/roles/metron_ui/tasks/main.yml
deleted file mode 100644
index fd3422b..0000000
--- a/deployment/roles/metron_ui/tasks/main.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install Metron UI dependencies
- yum:
- pkg: "{{ item }}"
- state: installed
- with_items:
- - libpcap-devel
- - wireshark
- - nodejs
- - npm
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- include: copy-source.yml
-
-- name: Configure Metron UI
- lineinfile:
- dest="{{ metron_ui_directory }}/config.json"
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- state=present
- with_items:
- - { regexp: '"elasticsearch":', line: '"elasticsearch": { "url": "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}" },' }
- - { regexp: '"pcap":', line: ' "pcap": { "url": "http://{{ groups.web[0] }}:{{ pcapservice_port }}/pcapGetter","mock": false }' }
-
-- name: Install Node dependencies
- npm:
- name: pm2
- path: "{{ metron_ui_directory }}"
- global: true
-
-- name: Install Metron UI
- npm:
- path: "{{ metron_ui_directory }}"
- production: no
-
-- name: Start Metron UI
- shell: "pm2 start {{ metron_ui_directory }}/lib/metron-ui.js --name metron"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/mysql_client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_client/tasks/main.yml b/deployment/roles/mysql_client/tasks/main.yml
deleted file mode 100644
index 8c54c23..0000000
--- a/deployment/roles/mysql_client/tasks/main.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Get Default mysql passowrd
- include_vars: "../roles/mysql_server/defaults/main.yml"
- when: mysql_root_password is undefined
-
-- name: Allow remote login to mysql
- template:
- src: "../roles/mysql_client/templates/db_config.sql"
- dest: "/tmp/{{ansible_fqdn}}.sql"
- delegate_to: "{{ groups.mysql[0] }}"
-
-- name: Import DB_Config
- mysql_db:
- name: "all"
- state: "import"
- target: "/tmp/{{ansible_fqdn}}.sql"
- ignore_errors: True
- delegate_to: "{{ groups.mysql[0] }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/mysql_client/templates/db_config.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_client/templates/db_config.sql b/deployment/roles/mysql_client/templates/db_config.sql
deleted file mode 100644
index c407a13..0000000
--- a/deployment/roles/mysql_client/templates/db_config.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-CREATE USER 'root'@'{{ ansible_fqdn }}' IDENTIFIED BY '{{ mysql_root_password }}';
-SET PASSWORD FOR 'root'@'{{ ansible_fqdn }}' = PASSWORD('{{ mysql_root_password }}');
-GRANT ALL PRIVILEGES ON *.* to 'root'@'{{ ansible_fqdn }}' WITH GRANT OPTION;
-FLUSH PRIVILEGES;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/mysql_server/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/defaults/main.yml b/deployment/roles/mysql_server/defaults/main.yml
deleted file mode 100644
index 0acbd17..0000000
--- a/deployment/roles/mysql_server/defaults/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-mysql_rpm_version: mysql57-community-release-el6-7.noarch
-mysql_yum_repo_url: https://dev.mysql.com/get/{{ mysql_rpm_version }}.rpm
-mysql_root_password: P@ssw0rd
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/mysql_server/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/files/geoip_ddl.sql b/deployment/roles/mysql_server/files/geoip_ddl.sql
deleted file mode 100644
index 02616c6..0000000
--- a/deployment/roles/mysql_server/files/geoip_ddl.sql
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-CREATE DATABASE IF NOT EXISTS GEO;
-
-USE GEO;
-
-DROP TABLE IF EXISTS `blocks`;
-CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
-int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) )
-ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
-
-DROP TABLE IF EXISTS `location`;
-CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
- default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
-NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
- (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
-
-load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
-load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
-
-
-DELIMITER $$
-DROP FUNCTION IF EXISTS `IPTOLOCID` $$
-CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
- BEGIN
- DECLARE ipn INTEGER UNSIGNED;
- DECLARE locID_var INTEGER;
- IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
- END IF;
- SET ipn = INET_ATON(ip);
- SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
- RETURN locID_var;
- END
-$$
-DELIMITER ;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/deployment/roles/mysql_server/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/handlers/main.yml b/deployment/roles/mysql_server/handlers/main.yml
deleted file mode 100644
index 112c5ca..0000000
--- a/deployment/roles/mysql_server/handlers/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: restart elasticsearch
- service: name=elasticsearch state=restarted