You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2018/01/30 14:54:00 UTC
[18/31] metron git commit: METRON-1370 Create Full Dev Equivalent for
Ubuntu (nickwallen via cestella) closes apache/incubator-metron#903
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst
new file mode 100644
index 0000000..42a2ca7
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed AFTER installation of the 'metron-config' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+# install the service script
+cp -f /usr/metron/${FULL_VERSION}/bin/metron-management-ui /etc/init.d/
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm
new file mode 100644
index 0000000..f9f4bcd
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed AFTER removal of the 'metron-management' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+rm -f /etc/init.d/metron-management-ui
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst
new file mode 100644
index 0000000..3ddedc9
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed BEFORE installation of the 'metron-config' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+if [ -f "/etc/init.d/metron-management-ui"]; then
+ # if service already exists, stop it before upgrading
+ /etc/init.d/metron-management-ui stop
+fi
+
+exit 0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm
new file mode 100644
index 0000000..5b2ad74
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed BEFORE removal of the 'metron-management' package.
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+# if service exists, stop it
+if [ -f "/etc/init.d/metron-management-ui"]; then
+ /etc/init.d/metron-management-ui stop
+fi
+
+exit 0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog b/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog
new file mode 100755
index 0000000..cfbf6af
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this file is added to ALL metron packages
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+$PACKAGE ($FULL_VERSION) $DISTRIBUTION; urgency=medium
+
+ * Initial release. (Closes: METRON-1351)
+
+ -- Apache Metron <de...@metron.apache.org> Wed, 13 Dec 2017 21:19:45 +0000
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/control
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/control b/metron-deployment/packaging/docker/deb-docker/debian/metron/control
new file mode 100755
index 0000000..d7611f9
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/control
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this file is added to ALL metron packages
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+Source: metron
+Section: misc
+Priority: extra
+Maintainer: Apache Metron <de...@metron.apache.org>
+Homepage: https://metron.apache.org/
+Package: $PACKAGE
+Architecture: all
+Version: $FULL_VERSION
+Depends:
+Description: Apache Metron
+ Apache Metron provides a scalable advanced security analytics framework.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright b/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright
new file mode 100755
index 0000000..d7f20c1
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this file is added to ALL metron packages
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: $PACKAGE
+Upstream-Contact: Apache Metron <de...@metron.apache.org>
+
+Files: *
+License: ASL-2
+Copyright:
+Apache Metron
+Copyright 2015-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software developed by Cisco Systems (http://www.cisco.com)
+Copyright (c) 2014 Cisco Systems.
+
+This product includes software developed by Chef Software (https://www.chef.io)
+Copyright (c) 2012-2015, Chef Software, Inc.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml
index 5770e40..7fbe47b 100644
--- a/metron-deployment/packaging/docker/deb-docker/pom.xml
+++ b/metron-deployment/packaging/docker/deb-docker/pom.xml
@@ -49,6 +49,12 @@
<include>**/*</include>
</includes>
</fileset>
+ <fileset>
+ <directory>.npm</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ </fileset>
</filesets>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 0b7218d..4ed2edd 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -85,6 +85,12 @@
<directory>SRPMS</directory>
<followSymlinks>false</followSymlinks>
</fileset>
+ <fileset>
+ <directory>.npm</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ </fileset>
</filesets>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/ambari_install.yml b/metron-deployment/playbooks/ambari_install.yml
deleted file mode 100644
index c0cfef3..0000000
--- a/metron-deployment/playbooks/ambari_install.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: ec2
- become: true
- tasks:
- - include_vars: ../amazon-ec2/conf/defaults.yml
- tags:
- - ec2
-
-- hosts: packer
- become: true
- tasks:
- - include_vars: ../inventory/full-dev-platform/group_vars/all
- tags:
- - packer
-
-- hosts: ambari_*
- become: true
- roles:
- - role: ambari_common
- tags:
- - ambari-prereqs
- - hdp-install
-
-- hosts: ambari_master
- become: true
- roles:
- - role: ambari_master
- tags:
- - ambari-server
- - hdp-install
-
-- hosts: ambari_slave
- become: true
- roles:
- - role: ambari_slave
- tags:
- - ambari-agent
- - hdp-install
-
-- hosts: metron
- become: true
- roles:
- - role: metron-rpms
- tags:
- - metron-deploy
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/docker_probe_install.yml b/metron-deployment/playbooks/docker_probe_install.yml
deleted file mode 100644
index 7be779a..0000000
--- a/metron-deployment/playbooks/docker_probe_install.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# sensors
-#
-- hosts: localhost
- tasks:
- - name: add container to inventory
- add_host:
- name: amb-server
- ansible_connection: docker
- groups: sensors
- changed_when: false
- tags: add-host
-
-- hosts: sensors
- vars:
- metron_version: 0.4.3
- metron_directory: /usr/metron/{{ metron_version }}
- bro_version: "2.5.2"
- fixbuf_version: "1.7.1"
- yaf_version: "2.8.0"
- daq_version: "2.0.6-1"
- pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
- pycapa_home: "/opt/pycapa"
- snort_version: "2.9.8.0-1"
- snort_alert_csv_path: "/var/log/snort/alert.csv"
- threat_intel_bulk_load: False
- sensor_test_mode: True
- install_pycapa: False
- install_bro: True
- install_snort: True
- install_yaf: True
- install_pcap_replay: True
- sniff_interface: eth0
- pcap_replay_interface: "{{ sniff_interface }}"
- pcapservice_port: 8081
- kafka_broker_url: amb4.service.consul:6667
- connection: docker
- roles:
- - { role: bro, tags: ['bro'] }
- - { role: snort, tags: ['snort'] }
- - { role: yaf, tags: ['yaf'] }
- - { role: pcap_replay, tags: ['pcap-replay'] }
- - { role: sensor-test-mode, tags: ['sensor-test-mode'] }
- tags:
- - sensors
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_build.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_build.yml b/metron-deployment/playbooks/metron_build.yml
deleted file mode 100644
index 83eb352..0000000
--- a/metron-deployment/playbooks/metron_build.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: all
- become: false
- roles:
- - role: metron-builder
- tags:
- - build
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_full_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_full_install.yml b/metron-deployment/playbooks/metron_full_install.yml
deleted file mode 100644
index 670f5c8..0000000
--- a/metron-deployment/playbooks/metron_full_install.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: all
- pre_tasks:
- - name: Verify Ansible Version
- fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version is {{ ansible_version }}"
- when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=')"
-
-- include: metron_build.yml
- tags:
- - build
-
-- include: ambari_install.yml
- tags:
- - ambari
-
-- include: metron_install.yml
- tags:
- - metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
deleted file mode 100644
index bed615e..0000000
--- a/metron-deployment/playbooks/metron_install.yml
+++ /dev/null
@@ -1,102 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: ec2
- become: true
- tasks:
- - include_vars: ../amazon-ec2/conf/defaults.yml
- tags:
- - ec2
-
-- hosts: packer
- become: true
- tasks:
- - include_vars: ../inventory/full-dev-platform/group_vars/all
- tags:
- - packer
-
-#
-# start installation of components in Ambari
-#
-- hosts: ambari_master
- become: true
- roles:
- - role: ambari_config
- tags:
- - hdp-install
- - hdp-deploy
-
-- hosts: ambari_master
- become: true
- roles:
- - role: load_web_templates
- tags:
- - load_templates
-
-- hosts: pcap_server
- become: true
- roles:
- - role: metron_pcapservice
- tags:
- - pcap-service
-
-#
-# sensors
-#
-- hosts: sensors
- become: true
- roles:
- - { role: ambari_gather_facts, tags: ['always'] }
- - { role: tap_interface, tags: ['tap'] }
- - { role: pycapa, tags: ['pycapa'] }
- - { role: bro, tags: ['bro'] }
- - { role: snort, tags: ['snort'] }
- - { role: yaf, tags: ['yaf'] }
- - { role: pcap_replay, tags: ['pcap-replay'] }
- - { role: sensor-test-mode, tags: ['sensor-test-mode'] }
- tags:
- - sensors
-
-#
-# sensor stubs - rather than deploying the sensors, deploy ligher weight
-# stubs that mimic the behavior of the real sensors
-#
-- hosts: sensors
- become: true
- roles:
- - { role: ambari_gather_facts }
- - { role: sensor-stubs }
- tags:
- - sensor-stubs
-
-#
-# monitor and start metron services with monit
-#
-- hosts: monit
- become: true
- roles:
- - { role: ambari_gather_facts, tags: ['always'] }
- - { role: monit, tags: ['monit'] }
- - { role: monit-start, tags: ['start'] }
-
-#
-# deployment report
-#
-- hosts: monit
- become: false
- roles:
- - { role: deployment-report, tags: ['report'] }
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/README.md b/metron-deployment/roles/README.md
deleted file mode 100644
index 1cb5878..0000000
--- a/metron-deployment/roles/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Ansible Roles
-
-- Monit
-- OpenTaxii
-- Pcap Replay
-- Sensor Stubs
-- Sensor Test Mode
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/defaults/main.yml b/metron-deployment/roles/ambari_common/defaults/main.yml
deleted file mode 100644
index 2c9389c..0000000
--- a/metron-deployment/roles/ambari_common/defaults/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-hadoop_logrotate_frequency: daily
-hadoop_logrotate_retention: 30
-rhel_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/meta/main.yml b/metron-deployment/roles/ambari_common/meta/main.yml
deleted file mode 100644
index d7e46d6..0000000
--- a/metron-deployment/roles/ambari_common/meta/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - libselinux-python
- - epel
- - ntp
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/tasks/main.yml b/metron-deployment/roles/ambari_common/tasks/main.yml
deleted file mode 100644
index 011c05f..0000000
--- a/metron-deployment/roles/ambari_common/tasks/main.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Check OS Version
- fail: msg="Ambari HDP deployment supports CentOS 6 only."
- when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6")
-
-- name: Ensure iptables is stopped and is not running at boot time.
- ignore_errors: yes
- service: name=iptables state=stopped enabled=no
-
-#
-# ambari uses socket.getfqdn() to find the hostname. with 'localhost.localdomain'
-# in '/etc/hosts' this function will report the hostname as 'localhost.localdomain'
-# rather than 'node1' as would be expected. other functions like socket.gethostname()
-# will always return 'node1' as expected. ambari needs to see 'node1' to be able to
-# communicate between the master and agents.
-
-- name: Remove ipv4 'localhost.localdomain' from /etc/hosts
- lineinfile: dest=/etc/hosts state=absent regexp="^127.0.0.1(.*)localdomain(.*)$"
-
-- name: Remove ipv6 'localhost.localdomain' from /etc/hosts
- lineinfile: dest=/etc/hosts state=absent regexp="^::1(.*)localdomain(.*)$"
-
-- name: Add localhost to /etc/hosts
- lineinfile: dest=/etc/hosts line="127.0.0.1 localhost"
-
-- name: Download Ambari repo
- get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
-
-- name: Create Logrotate Script for Hadoop Services
- template:
- src: "metron-hadoop-logrotate.yml"
- dest: "/etc/logrotate.d/metron-ambari"
- mode: 0644
-
-- name: Install Nodejs repository
- shell: curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
- args:
- warn: false
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml b/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
deleted file mode 100644
index d95c10b..0000000
--- a/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
+++ /dev/null
@@ -1,149 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#Hadoop HDFS Logs
-/var/log/hadoop/hdfs/*.log* {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hadoop/hdfs/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hadoop/hdfs/*.audit {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Hadoop Yarn Logs
-/var/log/hadoop/yarn/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Hadoop Mapreduce Logs
-/var/log/hadoop/mapreduce/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Storm Logs
-/var/log/storm/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/storm/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Kafka Logs
-/var/log/kafka/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/kafka/*.err {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#HBase Logs
-/var/log/hbase/*.log* {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hbase/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hbase/*.audit {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Zookeeper Logs
-/var/log/zookeeper/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/zookeeper/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/defaults/main.yml b/metron-deployment/roles/ambari_config/defaults/main.yml
deleted file mode 100644
index e0de145..0000000
--- a/metron-deployment/roles/ambari_config/defaults/main.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-zookeeper_data_dir: /hadoop/zookeeper
-namenode_checkpoint_dir: /hadoop/hdfs/namesecondary
-namenode_name_dir: /hadoop/hdfs/namenode
-datanode_data_dir: /hadoop/hdfs/data
-journalnode_edits_dir: /hadoop/hdfs/journalnode
-jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs
-nodemanager_local_dirs: /hadoop/yarn/local
-timeline_ldb_store_path: /hadoop/yarn/timeline
-timeline_ldb_state_path: /hadoop/yarn/timeline
-nodemanager_log_dirs: /hadoop/yarn/log
-storm_local_dir: /hadoop/storm
-kafka_log_dirs: /kafka-log
-cluster_type: small_cluster
-nodemanager_mem_mb : 4096
-mapred_map_java_opts : -Xmx1024m
-mapred_reduce_java_opts : -Xmx1024m
-mapred_map_mem_mb : 1229
-mapred_reduce_mem_mb : 1229
-topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf'
-hdp_stack: "2.5"
-elasticsearch_network_interface: _site_
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/meta/main.yml b/metron-deployment/roles/ambari_config/meta/main.yml
deleted file mode 100644
index 8f65a28..0000000
--- a/metron-deployment/roles/ambari_config/meta/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - epel
- - python-pip
- - httplib2
- - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/tasks/main.yml b/metron-deployment/roles/ambari_config/tasks/main.yml
deleted file mode 100644
index 6588d4a..0000000
--- a/metron-deployment/roles/ambari_config/tasks/main.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include_vars: "{{ cluster_type }}.yml"
-
-- name: Install python-requests
- yum:
- name: python-requests
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name : Wait for Ambari to start; http://{{ ambari_host }}:{{ ambari_port }}
- wait_for :
- host: "{{ ambari_host }}"
- port: "{{ ambari_port }}"
- timeout: 600
-
-- name: Deploy cluster with Ambari; http://{{ ambari_host }}:{{ ambari_port }}
- ambari_cluster_state:
- host: "{{ ambari_host }}"
- port: "{{ ambari_port }}"
- username: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- cluster_name: "{{ cluster_name }}"
- cluster_state: present
- blueprint_name: "{{ blueprint_name }}"
- configurations: "{{ configurations }}"
- wait_for_complete: True
- blueprint_var: "{{ blueprint }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
deleted file mode 100644
index 6a60902..0000000
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ /dev/null
@@ -1,135 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-# vars file for single_node_vm blueprint
-
-hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
-hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, HDFS_CLIENT, NODEMANAGER, YARN_CLIENT, MAPREDUCE2_CLIENT]
-spark_master: [SPARK_JOBHISTORYSERVER]
-spark_slave: [SPARK_CLIENT]
-storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
-storm_slave: [SUPERVISOR]
-kafka_broker: [KAFKA_BROKER]
-zookeeper_master: [ZOOKEEPER_SERVER]
-zookeeper_slave: [ZOOKEEPER_CLIENT]
-hbase_master: [HBASE_MASTER, HBASE_CLIENT]
-hbase_slave: [HBASE_REGIONSERVER]
-es_master: [ES_MASTER]
-kibana_master: [KIBANA_MASTER]
-metron_indexing: [METRON_INDEXING]
-metron_profiler: [METRON_PROFILER]
-metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
-metron_parsers : [METRON_PARSERS]
-metron_rest: [METRON_REST]
-metron_management_ui: [METRON_MANAGEMENT_UI]
-metron_alerts_ui: [METRON_ALERTS_UI]
-
-metron_components: >
- {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
- union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
- union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }}
-
-cluster_name: "metron_cluster"
-blueprint_name: "metron_blueprint"
-
-configurations:
- - zoo.cfg:
- dataDir: '{{ zookeeper_data_dir }}'
- - hadoop-env:
- hadoop_heapsize: 1024
- namenode_heapsize: 2048
- dtnode_heapsize: 512
- namenode_opt_permsize: 128m
- - hbase-env:
- hbase_regionserver_heapsize: 512
- hbase_master_heapsize: 512
- hbase_regionserver_xmn_max: 512
- - hdfs-site:
- dfs.replication: 1
- dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir }}'
- dfs.namenode.name.dir: '{{ namenode_name_dir }}'
- dfs.datanode.data.dir: '{{ datanode_data_dir }}'
- dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}'
- - yarn-env:
- nodemanager_heapsize: 512
- yarn_heapsize: 512
- apptimelineserver_heapsize : 512
- resourcemanager_heapsize: 1024
- - mapred-env:
- jobhistory_heapsize: 256
- - mapred-site:
- mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}'
- mapreduce.map.java.opts : '{{ mapred_map_java_opts }}'
- mapreduce.reduce.java.opts : '{{ mapred_reduce_java_opts }}'
- mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
- mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
- - yarn-site:
- yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}'
- yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}'
- yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}'
- yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}'
- yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
- - storm-site:
- supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
- storm.local.dir: '{{ storm_local_dir }}'
- topology.classpath: '{{ topology_classpath }}'
- - kafka-env:
- content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
- - kafka-broker:
- log.dirs: '{{ kafka_log_dirs }}'
- delete.topic.enable: "true"
- - metron-rest-env:
- metron_spring_profiles_active: "dev"
- - metron-parsers-env:
- parsers: "bro,snort"
- - elastic-site:
- index_number_of_shards: 1
- index_number_of_replicas: 0
- zen_discovery_ping_unicast_hosts: "[ {{ groups.search | join(', ') }} ]"
- gateway_recover_after_data_nodes: 1
- network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"
- masters_also_are_datanodes: "1"
-
-required_configurations:
- - metron-env:
- storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
- es_hosts: "{{ groups.search | join(',') }}"
- zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
- - metron-rest-env:
- metron_jdbc_driver: "org.h2.Driver"
- metron_jdbc_url: "jdbc:h2:file:~/metrondb"
- metron_jdbc_username: "root"
- metron_jdbc_password: "root"
- metron_jdbc_platform: "h2"
- - kibana-env:
- kibana_pid_dir: /var/run/kibana
- kibana_es_url: http://{{ groups.search[0] }}:9200
- kibana_log_dir: /var/log/kibana
- kibana_server_port: 5000
- kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9"
- kibana_server_host: 0.0.0.0
-
-blueprint:
- stack_name: HDP
- stack_version: "{{ hdp_stack }}"
- required_configurations: "{{ required_configurations }}"
- groups:
- - name : host_group_1
- cardinality: 1
- configurations: []
- components: "{{ metron_components }}"
- hosts: "{{ hdp_host_group }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
deleted file mode 100644
index 4ec8458..0000000
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ /dev/null
@@ -1,149 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
-app_timeline_server: [APP_TIMELINE_SERVER]
-hadoop_slave: [DATANODE, NODEMANAGER]
-spark_master: [SPARK_JOBHISTORYSERVER]
-storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
-storm_slave: [SUPERVISOR]
-kafka_broker: [KAFKA_BROKER]
-zookeeper_master: [ZOOKEEPER_SERVER]
-hbase_master: [HBASE_MASTER]
-hbase_slave: [HBASE_REGIONSERVER]
-hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
-es_master: [ES_MASTER]
-es_slave: [ES_SLAVE]
-kibana_master: [KIBANA_MASTER]
-metron_indexing: [METRON_INDEXING]
-metron_profiler: [METRON_PROFILER]
-metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
-metron_parsers : [METRON_PARSERS]
-metron_rest: [METRON_REST]
-metron_management_ui: [METRON_MANAGEMENT_UI]
-metron_alerts_ui: [METRON_ALERTS_UI]
-
-master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}"
-master_1_host:
- - "{{groups.ambari_slave[0]}}"
-master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}"
-master_2_host:
- - "{{groups.ambari_slave[1]}}"
-metron_components: >
- {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
- union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
-metron_host:
- - "{{ groups.metron[0] }}"
-web_components: "{{ kibana_master | union(es_master) }}"
-web_host:
- - "{{ groups.web[0] }}"
-slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}"
-
-cluster_name: "metron"
-blueprint_name: "metron_blueprint"
-
-configurations:
- - zoo.cfg:
- dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
- - hadoop-env:
- namenode_heapsize: 2048
- dtnode_heapsize: 1024
- - hbase-env:
- hbase_regionserver_heapsize: 1024
- hbase_master_heapsize: 1024
- - hdfs-site:
- dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
- dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
- dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
- dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
- - mapred-site:
- mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
- mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
- mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
- - yarn-site:
- yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
- yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
- yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
- yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
- yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
- - storm-site:
- supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
- storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
- topology.classpath: '{{ topology_classpath }}'
- - kafka-broker:
- log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
- - metron-rest-env:
- metron_spring_profiles_active: "dev"
- - metron-env:
- parsers: "bro,snort,yaf"
- - elastic-site:
- index_number_of_shards: 2
- index_number_of_replicas: 1
- zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]"
- gateway_recover_after_data_nodes: 1
- network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"
-
-required_configurations:
- - metron-env:
- storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744"
- es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}"
- zeppelin_server_url: "{{ groups.zeppelin[0] }}"
- - metron-rest-env:
- metron_jdbc_driver: "org.h2.Driver"
- metron_jdbc_url: "jdbc:h2:file:~/metrondb"
- metron_jdbc_username: "root"
- metron_jdbc_password: "root"
- metron_jdbc_platform: "h2"
- - kibana-env:
- kibana_pid_dir: /var/run/kibana
- kibana_es_url: http://{{ groups.web[0] }}:9200
- kibana_log_dir: /var/log/kibana
- kibana_server_port: 5000
- kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9"
- kibana_server_host: 0.0.0.0
-
-blueprint:
- stack_name: HDP
- stack_version: "{{ hdp_stack }}"
- required_configurations: "{{ required_configurations }}"
- groups:
- - name : master_1
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ master_1_components }}"
- hosts: "{{ master_1_host }}"
- - name : master_2
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ master_2_components }}"
- hosts: "{{ master_2_host }}"
- - name : metron
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ metron_components }}"
- hosts: "{{ metron_host }}"
- - name : web
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ web_components }}"
- hosts: "{{ web_host }}"
- - name: slaves
- cardinality: 1+
- configuration: [] # configuration not yet implemented
- components: "{{ slave_components }}"
- hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_gather_facts/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_gather_facts/meta/main.yml b/metron-deployment/roles/ambari_gather_facts/meta/main.yml
deleted file mode 100644
index 61197e3..0000000
--- a/metron-deployment/roles/ambari_gather_facts/meta/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - epel
- - python-pip
- - httplib2
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_gather_facts/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/roles/ambari_gather_facts/tasks/main.yml
deleted file mode 100644
index 2b37eec..0000000
--- a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml
+++ /dev/null
@@ -1,234 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# cluster_name
-#
-- name: "Ask Ambari: cluster_name"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: cluster_name_response
- when: cluster_name is undefined
-
-- set_fact:
- cluster_name: "{{ (cluster_name_response.content | from_json)['items'][0].Clusters.cluster_name }}"
- when: cluster_name is undefined
-
-#
-# namenode_host
-#
-- name: "Ask Ambari: namenode_host"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: namenode_hosts_response
- when: namenode_host is undefined
-
-- set_fact:
- namenode_host: "{{ (namenode_hosts_response.content | from_json).host_components[0].HostRoles.host_name }}"
- when: namenode_host is undefined
-
-#
-# core_site_tag
-#
-- name: "Ask Ambari: core_site_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: core_site_tag_response
- when: core_site_tag is undefined
-
-- set_fact:
- core_site_tag: "{{ (core_site_tag_response.content | from_json).HostRoles.actual_configs['core-site'].default }}"
- when: core_site_tag is undefined
-
-#
-# hdfs_url
-#
-- name: "Ask Ambari: hdfs_url"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: core_site_response
- when: hdfs_url is undefined
-
-- set_fact:
- hdfs_url: "{{ (core_site_response.content | from_json)['items'][0].properties['fs.defaultFS'] }}"
- when: hdfs_url is undefined
-
-#
-# kafka_broker_hosts
-#
-- name: "Ask Ambari: kafka_broker_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: kafka_broker_hosts_response
- when: kafka_broker_hosts is undefined
-
-- set_fact:
- kafka_broker_hosts: "{{ (kafka_broker_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: kafka_broker_hosts is undefined
-
-#
-# kafka_broker_tag
-#
-- name: "Ask Ambari: kafka_broker_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: kafka_broker_tag_response
- when: kafka_broker_tag is undefined
-
-- set_fact:
- kafka_broker_tag: "{{ (kafka_broker_tag_response.content | from_json).HostRoles.actual_configs['kafka-broker'].default }}"
- when: kafka_broker_tag is undefined
-
-#
-# kafka_broker_port
-#
-- name: "Ask Ambari: kafka_broker_port"
- shell: >
- curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H "X-Requested-By: ambari" "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}" | python -c 'import sys, json; print json.load(sys.stdin)["items"][0]["properties"]["listeners"]'
- args:
- warn: false
- register: kafka_broker_port_response
- when: (kafka_broker_url is undefined) or (kafka_broker_port is undefined)
-
-- set_fact:
- kafka_broker_port: "{{ kafka_broker_port_response.stdout_lines[0] | replace('PLAINTEXT://localhost:', '')}}"
- when: kafka_broker_port is undefined
-
-- set_fact:
- kafka_broker_url: "{% for host in kafka_broker_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ kafka_broker_port }}{% endfor %}"
- when: kafka_broker_url is undefined
-
-#
-# zookeeper_hosts
-#
-- name: "Ask Ambari: zookeeper_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/ZOOKEEPER/components/ZOOKEEPER_SERVER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: zookeeper_hosts_response
- when: zookeeper_hosts is undefined
-
-- set_fact:
- zookeeper_hosts: "{{ (zookeeper_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: zookeeper_hosts is undefined
-
-#
-# zookeeper_tag
-#
-- name: "Ask Ambari: zookeeper_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: zookeeper_tag_response
- when: zookeeper_tag is undefined
-
-- set_fact:
- zookeeper_tag: "{{ (zookeeper_tag_response.content | from_json).HostRoles.actual_configs['zoo.cfg'].default }}"
- when: zookeeper_tag is undefined
-
-#
-# zookeeper_url, zookeeper_port
-#
-- name: "Ask Ambari: zookeeper_url, zookeeper_port"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: zookeeper_properties_response
- when: zookeeper_url is undefined or zookeeper_port is undefined
-
-- set_fact:
- zookeeper_port: "{{ (zookeeper_properties_response.content | from_json)['items'][0].properties['clientPort'] }}"
- when: zookeeper_port is undefined
-
-- set_fact:
- zookeeper_url: "{% for host in zookeeper_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ zookeeper_port }}{% endfor %}"
- when: zookeeper_url is undefined
-
-- name: "Ask Ambari: metron_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: metron_hosts_response
- when: metron_hosts is undefined
-
-- set_fact:
- metron_hosts: "{{ (metron_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: metron_hosts is undefined
-
-- name: "Ask Ambari: kibana_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: kibana_hosts_response
- when: kibana_hosts is undefined
-
-- set_fact:
- kibana_hosts: "{{ (kibana_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: kibana_hosts is undefined
-
-
-#
-# debug output
-#
-- name: debug
- debug:
- msg: "zookeeper_port = {{ zookeeper_port }},
- zookeeper_hosts = {{ zookeeper_hosts }},
- zookeeper_url = {{ zookeeper_url }},
- kafka_broker_port = {{ kafka_broker_port }},
- kafka_broker_hosts = {{ kafka_broker_hosts }},
- kafka_broker_url = {{ kafka_broker_url }},
- metron_hosts = {{ metron_hosts }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_master/defaults/main.yml b/metron-deployment/roles/ambari_master/defaults/main.yml
deleted file mode 100644
index bc6c08c..0000000
--- a/metron-deployment/roles/ambari_master/defaults/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-ambari_server_mem: 2048
-ambari_mpack_version: 0.4.3.0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml
deleted file mode 100644
index bc8e837..0000000
--- a/metron-deployment/roles/ambari_master/tasks/main.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-# tasks file for ambari_master
-- name: Install ambari server
- yum:
- name: ambari-server
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Set Ambari Server Max Memory
- replace:
- dest: /var/lib/ambari-server/ambari-env.sh
- regexp: "\ -Xmx2048m\ "
- replace: " -Xmx{{ ambari_server_mem }}m "
- backup: no
-
-- name: Setup ambari server
- shell: ambari-server setup -s && touch /etc/ambari-server/configured creates=/etc/ambari-server/configured
- register: ambari_server_setup
- failed_when: "(ambari_server_setup.stderr is defined and ambari_server_setup.stderr != '') or 'FATAL' in ambari_server_setup.stdout"
-
-- name: Copy MPack to Ambari Host
- copy:
- src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-{{ ambari_mpack_version }}.tar.gz"
- dest: /tmp
-
-- name: Install MPack on Ambari Host
- shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-0.4.3.0.tar.gz
- args:
- creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services
-
-- name: start ambari server
- service:
- name: ambari-server
- state: restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/defaults/main.yml b/metron-deployment/roles/ambari_slave/defaults/main.yml
deleted file mode 100644
index f3cb4c5..0000000
--- a/metron-deployment/roles/ambari_slave/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-ambari_installation_user: "root"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/files/hostname.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/files/hostname.sh b/metron-deployment/roles/ambari_slave/files/hostname.sh
deleted file mode 100644
index cc8c1cd..0000000
--- a/metron-deployment/roles/ambari_slave/files/hostname.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-echo {{ inventory_hostname }}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/meta/main.yml b/metron-deployment/roles/ambari_slave/meta/main.yml
deleted file mode 100644
index ddf6aa9..0000000
--- a/metron-deployment/roles/ambari_slave/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/tasks/main.yml b/metron-deployment/roles/ambari_slave/tasks/main.yml
deleted file mode 100644
index ca81a7a..0000000
--- a/metron-deployment/roles/ambari_slave/tasks/main.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: ../roles/metron-rpms/tasks/create_directory.yml
-- include: ../roles/metron-rpms/tasks/create_repo.yml
-
-- name: Install ambari-agent
- yum:
- name: ambari-agent
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Create ambari-agent hostname script
- template:
- src: "../roles/ambari_slave/files/hostname.sh"
- dest: "/var/lib/ambari-agent/hostname.sh"
- mode: 0744
- owner: "{{ ambari_installation_user }}"
- group: "{{ ambari_installation_user }}"
-
-- name: Configure ambari-server hostname in ambari-agent configuration
- lineinfile:
- dest: /etc/ambari-agent/conf/ambari-agent.ini
- regexp: "{{ item.regexp }}"
- line: "{{ item.line }}"
- insertafter: "{{ item.insertafter }}"
- backup: yes
- with_items:
- - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' }
- - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'}
-
-- name: Ensure ambari-agent is running
- service:
- name: ambari-agent
- state: restarted
- enabled: yes
-
-- name : Wait for agent to register
- command : sleep 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/meta/main.yml b/metron-deployment/roles/bro/meta/main.yml
deleted file mode 100644
index df226e7..0000000
--- a/metron-deployment/roles/bro/meta/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - libselinux-python
- - build-tools
- - kafka-client
- - librdkafka
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/bro.yml b/metron-deployment/roles/bro/tasks/bro.yml
deleted file mode 100644
index 222ef0e..0000000
--- a/metron-deployment/roles/bro/tasks/bro.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download bro
- get_url:
- url: "https://www.bro.org/downloads/bro-{{ bro_version }}.tar.gz"
- dest: "/tmp/bro-{{ bro_version }}.tar.gz"
-
-- name: Extract bro tarball
- unarchive:
- src: "/tmp/bro-{{ bro_version }}.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/bro-{{ bro_version }}"
-
-- name: Compile and Install bro
- shell: "{{ item }}"
- environment:
- CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
- CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
- args:
- chdir: "/tmp/bro-{{ bro_version }}"
- creates: "{{ bro_home }}/bin/bro"
- with_items:
- - "./configure --prefix={{ bro_home }}"
- - make
- - make install
-
-- name: Configure bro
- lineinfile:
- dest: "{{ bro_home }}/etc/node.cfg"
- regexp: '^interface=.*$'
- line: 'interface={{ sniff_interface }}'
-
-- name: Install bro configuration
- shell: "{{ bro_home }}/bin/broctl install"
-
-- name: Bro Cronjob
- cron:
- name: Bro Cron
- minute: "{{ bro_crontab_minutes }}"
- job: "{{ item }}"
- with_items:
- - "{{ bro_crontab_job }}"
- - "{{ bro_clean_job }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/dependencies.yml b/metron-deployment/roles/bro/tasks/dependencies.yml
deleted file mode 100644
index 3cd3bae..0000000
--- a/metron-deployment/roles/bro/tasks/dependencies.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install yum repositories
- yum: name={{ item }} update_cache=yes
- with_items:
- - centos-release-scl
-
-- name: Install prerequisites
- yum: name={{ item }}
- with_items:
- - cmake
- - make
- - gcc
- - gcc-c++
- - flex
- - bison
- - libpcap
- - libpcap-devel
- - openssl-devel
- - python-devel
- - swig
- - zlib-devel
- - perl
- - crontabs
- - net-tools
- - devtoolset-4-gcc
- - devtoolset-4-gcc-c++
- - python27
- - rh-git29
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/librdkafka.yml b/metron-deployment/roles/bro/tasks/librdkafka.yml
deleted file mode 100644
index 652d319..0000000
--- a/metron-deployment/roles/bro/tasks/librdkafka.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download librdkafka
- get_url:
- url: "{{ librdkafka_url }}"
- dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
-
-- name: Extract librdkafka tarball
- unarchive:
- src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/librdkafka-{{ librdkafka_version }}"
-
-- name: Compile and install librdkafka
- shell: "{{ item }}"
- args:
- chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
- creates: "{{ librdkafka_home }}/lib/librdkafka.so"
- with_items:
- - rm -rf build/
- - "./configure --prefix={{ librdkafka_home }}"
- - make
- - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/main.yml b/metron-deployment/roles/bro/tasks/main.yml
deleted file mode 100644
index 440e4b5..0000000
--- a/metron-deployment/roles/bro/tasks/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: librdkafka.yml
-- include: bro.yml
-- include: metron-bro-plugin-kafka.yml
-- include: nic.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
deleted file mode 100644
index 7043387..0000000
--- a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install bro-pkg
- environment:
- LD_LIBRARY_PATH: "{{ python27_lib }}"
- command: "{{ python27_bin }}/pip install bro-pkg"
-
-- name: Configure bro-pkg
- environment:
- PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
- LD_LIBRARY_PATH: "{{ python27_lib }}"
- command: "{{ python27_bin }}/bro-pkg autoconfig"
-
-- name: Install the metron-bro-plugin-kafka package
- environment:
- PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
- LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
- CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
- CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
- command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}"
-
-- name: Configure bro-kafka plugin
- lineinfile:
- dest: "{{ bro_home }}/share/bro/site/local.bro"
- line: "{{ item }}"
- with_items:
- - "@load packages"
- - "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);"
- - "redef Kafka::topic_name = \"{{ bro_topic }}\";"
- - "redef Kafka::tag_json = T;"
- - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");"
-
-- name: Deploy bro configuration changes
- shell: "{{ bro_bin }}/broctl deploy"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/nic.yml b/metron-deployment/roles/bro/tasks/nic.yml
deleted file mode 100644
index 6053618..0000000
--- a/metron-deployment/roles/bro/tasks/nic.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"