You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/08/06 16:10:02 UTC
[1/4] metron git commit: METRON-1604 : Add RHEL 7 power pc to OS
family for the HCP management pack repo info closes
apache/incubator-metron#1052
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1554-pcap-query-panel 09a7f1899 -> 897266598
METRON-1604 : Add RHEL 7 power pc to OS family for the HCP management pack repo info closes apache/incubator-metron#1052
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4e1dd5d8
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4e1dd5d8
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4e1dd5d8
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 4e1dd5d8bb38eec54be3dba6c9de2b95fba60448
Parents: ccdbeff
Author: MohanDV <mo...@gmail.com>
Authored: Fri Jul 27 09:40:44 2018 -0400
Committer: cstella <ce...@gmail.com>
Committed: Fri Jul 27 09:40:44 2018 -0400
----------------------------------------------------------------------
.../addon-services/METRON/CURRENT/repos/repoinfo.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/4e1dd5d8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
index 9b1888c..aec012d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
@@ -35,4 +35,11 @@
<reponame>METRON</reponame>
</repo>
</os>
+ <os family="redhat-ppc7">
+ <repo>
+ <baseurl>file:///localrepo</baseurl>
+ <repoid>METRON-${metron.version}</repoid>
+ <reponame>METRON</reponame>
+ </repo>
+ </os>
</reposinfo>
[4/4] metron git commit: Merge branch 'master' into
feature/METRON-1554-pcap-query-panel
Posted by rm...@apache.org.
Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/89726659
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/89726659
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/89726659
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 897266598b7984884d6cf8a6a699fe902e792483
Parents: 09a7f18 1b2bdff
Author: merrimanr <me...@gmail.com>
Authored: Mon Aug 6 11:08:48 2018 -0500
Committer: merrimanr <me...@gmail.com>
Committed: Mon Aug 6 11:08:48 2018 -0500
----------------------------------------------------------------------
.../METRON/CURRENT/repos/repoinfo.xml | 7 ++
.../packaging/docker/ansible-docker/Dockerfile | 118 ++++++++++++-------
.../packaging/docker/deb-docker/Dockerfile | 23 ++--
.../packaging/docker/rpm-docker/Dockerfile | 46 ++++++--
metron-sensors/pycapa/README.md | 47 +++++---
5 files changed, 156 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
[3/4] metron git commit: METRON-1701 Update General notes on the
installation of Pycapa on Kerberized cluster (MohanDV via nickwallen) closes
apache/metron#1136
Posted by rm...@apache.org.
METRON-1701 Update General notes on the installation of Pycapa on Kerberized cluster (MohanDV via nickwallen) closes apache/metron#1136
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b2bdff5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b2bdff5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b2bdff5
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 1b2bdff53106694a9223ecc97f7ce9d0a78b4b09
Parents: a1e25a4
Author: MohanDV <mo...@gmail.com>
Authored: Wed Aug 1 12:13:19 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Wed Aug 1 12:13:19 2018 -0400
----------------------------------------------------------------------
metron-sensors/pycapa/README.md | 47 ++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/1b2bdff5/metron-sensors/pycapa/README.md
----------------------------------------------------------------------
diff --git a/metron-sensors/pycapa/README.md b/metron-sensors/pycapa/README.md
index 7e688a8..9f01d33 100644
--- a/metron-sensors/pycapa/README.md
+++ b/metron-sensors/pycapa/README.md
@@ -34,9 +34,9 @@ Pycapa performs network packet capture, both off-the-wire and from a Kafka topic
Installation
============
-General notes on the installation of Pycapa.
+General notes on the installation of Pycapa.
* Python 2.7 is required.
-* The following package dependencies are required and can be installed automatically with `pip`.
+* The following package dependencies are required and can be installed automatically with `pip`. The requirements are installed as part of step 4
* [confluent-kafka-python](https://github.com/confluentinc/confluent-kafka-python)
* [pcapy](https://github.com/CoreSecurity/pcapy)
* These instructions can be used directly on CentOS 7+.
@@ -54,8 +54,8 @@ General notes on the installation of Pycapa.
```
export PREFIX=/usr
- wget https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz -O - | tar -xz
- cd librdkafka-0.9.4/
+ wget https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz -O - | tar -xz
+ cd librdkafka-0.11.5/
./configure --prefix=$PREFIX
make
make install
@@ -231,18 +231,24 @@ Capturing on 'Standard input'
### Kerberos
-The probe can be used in a Kerberized environment. Follow these additional steps to use Pycapa with Kerberos. The following assumptions have been made. These may need altered to fit your environment.
+The probe can be used in a Kerberized environment. The Python client README (https://github.com/confluentinc/confluent-kafka-python) has an important note for Kerberos case that the pre-built Linux wheels do NOT contain SASL Kerberos support. You will need to use the non-binary wheel to install confluent-kafka-python and build/install librdkafka separately. Follow these additional steps to use Pycapa with Kerberos. The following assumptions have been made. These may need altered to fit your environment.
* The Kafka broker is at `kafka1:6667`
* Zookeeper is at `zookeeper1:2181`
* The Kafka security protocol is `SASL_PLAINTEXT`
* The keytab used is located at `/etc/security/keytabs/metron.headless.keytab`
* The service principal is `metron@EXAMPLE.COM`
+
+
+1. If it is not, ensure that you have `libsasl` or `libsasl2` installed. On CentOS, this can be installed with the following command.
+ ```
+ yum install -y cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi
+ ```
1. Build Librdkafka with SASL support (` --enable-sasl`) and install at your chosen $PREFIX.
```
- wget https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz -O - | tar -xz
- cd librdkafka-0.9.4/
+ wget https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz -O - | tar -xz
+ cd librdkafka-0.11.5/
./configure --prefix=$PREFIX --enable-sasl
make
make install
@@ -250,14 +256,19 @@ The probe can be used in a Kerberized environment. Follow these additional step
1. Validate Librdkafka does indeed support SASL. Run the following command and ensure that `sasl` is returned as a built-in feature.
```
- $ examples/rdkafka_example -X builtin.features
- builtin.features = gzip,snappy,ssl,sasl,regex
- ```
-
- If it is not, ensure that you have `libsasl` or `libsasl2` installed. On CentOS, this can be installed with the following command.
- ```
- yum install -y cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi
+ $ examples/rdkafka_example -X builtin.features
+ builtin.features = gzip,snappy,ssl,sasl,regex,lz4,sasl_gssapi,sasl_plain,sasl_scram,plugins
```
+1. The source install of confluent-kafka.
+
+ If you have already installed, remove the binary wheel python client first, repeat until it says no longer installed
+ ```
+ pip uninstall -y confluent-kafka
+ ```
+
+ ```
+ pip install --no-binary :all: confluent-kafka
+ ```
1. Grant access to your Kafka topic. In this example the topic is simply named `pcap`.
```
@@ -279,8 +290,8 @@ The probe can be used in a Kerberized environment. Follow these additional step
* `security.protocol`
* `sasl.kerberos.keytab`
* `sasl.kerberos.principal`
-
- ```
+
+ ```
$ pycapa --producer \
--interface eth0 \
--kafka-broker kafka1:6667 \
@@ -292,8 +303,8 @@ The probe can be used in a Kerberized environment. Follow these additional step
INFO:root:Starting packet capture
INFO:root:Waiting for '1' message(s) to flush
INFO:root:'10' packet(s) in, '10' packet(s) out
- ```
-
+ ```
+
FAQs
====
[2/4] metron git commit: METRON-1650 Packaging docker containers are
too large (jameslamb via merrimanr) closes apache/metron#1091
Posted by rm...@apache.org.
METRON-1650 Packaging docker containers are too large (jameslamb via merrimanr) closes apache/metron#1091
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a1e25a4b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a1e25a4b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a1e25a4b
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: a1e25a4bf618e544b2062bf7f7f25bec0029956a
Parents: 4e1dd5d
Author: jameslamb <ja...@gmail.com>
Authored: Fri Jul 27 15:06:40 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Fri Jul 27 15:06:40 2018 -0500
----------------------------------------------------------------------
.../packaging/docker/ansible-docker/Dockerfile | 118 ++++++++++++-------
.../packaging/docker/deb-docker/Dockerfile | 23 ++--
.../packaging/docker/rpm-docker/Dockerfile | 46 ++++++--
3 files changed, 120 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a1e25a4b/metron-deployment/packaging/docker/ansible-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/ansible-docker/Dockerfile b/metron-deployment/packaging/docker/ansible-docker/Dockerfile
index ebf002a..df3aeb5 100644
--- a/metron-deployment/packaging/docker/ansible-docker/Dockerfile
+++ b/metron-deployment/packaging/docker/ansible-docker/Dockerfile
@@ -17,49 +17,81 @@
FROM centos:centos6.9
MAINTAINER Apache Metron
-RUN yum install -y tar
-RUN yum install -y wget
-# base development tools required
-RUN yum groupinstall -y "Development tools"
-# newer cpp 11 support required for building node modules
-RUN yum install -y centos-release-scl
-RUN yum install -y devtoolset-4-gcc-c++ devtoolset-4-gcc
-RUN yum install -y zlib-dev openssl-devel sqlite-devel bzip2-devel libffi-devel
-# install python 2.7.11 but do not make it the default
-RUN wget https://www.python.org/ftp/python/2.7.11/Python-2.7.11.tgz -O /usr/src/Python-2.7.11.tgz
-WORKDIR /usr/src
-RUN tar xvf Python-2.7.11.tgz
-WORKDIR /usr/src/Python-2.7.11
-RUN ./configure
-RUN make altinstall
-WORKDIR /usr/src
-RUN wget --no-check-certificate https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz -O /usr/src/setuptools-11.3.tar.gz
-RUN tar xvf setuptools-11.3.tar.gz
-WORKDIR /usr/src/setuptools-11.3
-RUN python2.7 setup.py install
-RUN easy_install-2.7 pip
-# install ansible and set the configuration var
-RUN pip2.7 install ansible==2.0.0.2
-RUN pip2.7 install boto
COPY ansible.cfg /root/
ENV ANSIBLE_CONFIG /root/ansible.cfg
-# java
-RUN yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel
-RUN yum install -y which
-RUN yum install -y nss
-WORKDIR /usr/src
-# setup maven
-RUN wget http://apache.cs.utah.edu/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-RUN tar xzvf apache-maven-3.3.9-bin.tar.gz
-RUN mv apache-maven-3.3.9 /opt/maven
-RUN ln -s /opt/maven/bin/mvn /usr/bin/mvn
-# install rpm tools required to build rpms
-RUN yum -y install asciidoc rpm-build rpm2cpio tar unzip xmlto zip rpmlint make && yum clean all
-# create a .bashrc for root, enabling the cpp 11 toolset
-RUN touch /root/.bashrc \
- && cat '/opt/rh/devtoolset-4/enable' >> /root/.bashrc
-# install node so that the node dependencies can be packaged into the RPMs
-RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
-RUN yum -y install nodejs
-WORKDIR /root
+RUN yum install -y \
+ tar \
+ wget \
+ # base development tools required \
+ && yum groupinstall -y \
+ "Development tools" \
+ # Install Software Collections repo (needs to be done in separate command) \
+ && yum install -y \
+ centos-release-scl \
+ # newer cpp 11 support required for building node modules \
+ && yum install -y \
+ devtoolset-4-gcc-c++ \
+ devtoolset-4-gcc \
+ zlib-dev \
+ openssl-devel \
+ sqlite-devel \
+ bzip2-devel \
+ libffi-devel \
+ # install python 2.7.11 but do not make it the default \
+ && wget https://www.python.org/ftp/python/2.7.11/Python-2.7.11.tgz -O /usr/src/Python-2.7.11.tgz \
+ && cd /usr/src \
+ && tar xvf Python-2.7.11.tgz \
+ && rm -rf Python-2.7.11.tgz \
+ && cd /usr/src/Python-2.7.11 \
+ && ./configure \
+ && make altinstall \
+ && cd /usr/src \
+ && wget --no-check-certificate https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz -O /usr/src/setuptools-11.3.tar.gz \
+ && tar xvf setuptools-11.3.tar.gz \
+ && rm setuptools-11.3.tar.gz \
+ && cd /usr/src/setuptools-11.3 \
+ && python2.7 setup.py install \
+ && easy_install-2.7 pip \
+ # install ansible and set the configuration var \
+ && pip2.7 install \
+ ansible==2.0.0.2 \
+ boto \
+ # java \
+ && yum install -y \
+ java-1.8.0-openjdk \
+ java-1.8.0-openjdk-devel \
+ which \
+ nss \
+ && cd /usr/src \
+ # setup maven \
+ && wget http://apache.cs.utah.edu/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
+ && tar xzvf apache-maven-3.3.9-bin.tar.gz \
+ && rm apache-maven-3.3.9-bin.tar.gz \
+ && mv apache-maven-3.3.9 /opt/maven \
+ && ln -s /opt/maven/bin/mvn /usr/bin/mvn \
+ # install rpm tools required to build rpms \
+ && yum install -y \
+ asciidoc \
+ rpm-build \
+ rpm2cpio \
+ tar \
+ unzip \
+ xmlto \
+ zip \
+ rpmlint \
+ make \
+ # create a .bashrc for root, enabling the cpp 11 toolset \
+ && touch /root/.bashrc \
+ && echo '/opt/rh/devtoolset-4/enable' >> /root/.bashrc \
+ # install node so that the node dependencies can be packaged into the RPMs \
+ && curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - \
+ && yum install -y \
+ nodejs \
+ # Remove packages just needed for builds \
+ && yum remove -y \
+ wget \
+ # Clean up yum caches \
+ && yum clean all
+
+WORKDIR /root
http://git-wip-us.apache.org/repos/asf/metron/blob/a1e25a4b/metron-deployment/packaging/docker/deb-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/Dockerfile b/metron-deployment/packaging/docker/deb-docker/Dockerfile
index 44203c6..5169efc 100644
--- a/metron-deployment/packaging/docker/deb-docker/Dockerfile
+++ b/metron-deployment/packaging/docker/deb-docker/Dockerfile
@@ -18,16 +18,17 @@
FROM ubuntu:trusty
# install prerequisites
-RUN apt-get update && apt-get install -y \
- build-essential \
- openjdk-7-jdk \
- wget \
- curl \
- dpkg-dev \
- gettext
-
-# install nodejs so that the node dependencies can be packaged into the DEBs
-RUN curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
-RUN apt-get install -y nodejs
+RUN apt-get update \
+ && apt-get install -y \
+ build-essential \
+ openjdk-7-jdk \
+ curl \
+ dpkg-dev \
+ gettext \
+ # install nodejs so that the node dependencies can be packaged into the DEBs \
+ && curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - \
+ && apt-get install -y \
+ nodejs \
+ && apt-get clean
WORKDIR /root
http://git-wip-us.apache.org/repos/asf/metron/blob/a1e25a4b/metron-deployment/packaging/docker/rpm-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/Dockerfile b/metron-deployment/packaging/docker/rpm-docker/Dockerfile
index a2dae8e..f151678 100644
--- a/metron-deployment/packaging/docker/rpm-docker/Dockerfile
+++ b/metron-deployment/packaging/docker/rpm-docker/Dockerfile
@@ -17,17 +17,37 @@
FROM centos:centos6
-RUN yum install -y tar
-RUN yum install -y wget
-RUN yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel
-WORKDIR /usr/src
-RUN wget http://apache.cs.utah.edu/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz
-RUN tar xzvf apache-maven-3.2.5-bin.tar.gz
-RUN mv apache-maven-3.2.5 /opt/maven
-RUN ln -s /opt/maven/bin/mvn /usr/bin/mvn
-RUN yum -y install asciidoc rpm-build rpm2cpio tar unzip xmlto zip rpmlint && yum clean all
-WORKDIR /root
+RUN yum install -y \
+ tar \
+ wget \
+ java-1.8.0-openjdk \
+ java-1.8.0-openjdk-devel \
+ && cd /usr/src \
+ && wget http://apache.cs.utah.edu/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz \
+ && tar xzvf apache-maven-3.2.5-bin.tar.gz \
+ && rm apache-maven-3.2.5-bin.tar.gz \
+ && mv apache-maven-3.2.5 /opt/maven \
+ && ln -s /opt/maven/bin/mvn /usr/bin/mvn \
+ && yum install -y \
+ asciidoc \
+ rpm-build \
+ rpm2cpio \
+ tar \
+ unzip \
+ xmlto \
+ zip \
+ rpmlint \
+ # install node so that the node dependencies can be packaged into the RPMs \
+ && cd /root \
+ && curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - \
+ && yum install -y \
+ gcc-c++ \
+ make \
+ nodejs \
+ # Remove packages just needed for builds \
+ && yum remove -y \
+ wget \
+ # Clean up yum caches \
+ && yum clean all
-# install node so that the node dependencies can be packaged into the RPMs
-RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
-RUN yum -y install gcc-c++ make nodejs
+WORKDIR /root