You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ky...@apache.org on 2017/03/29 00:40:09 UTC
incubator-metron git commit: METRON-700 Add hadoop container to
metron-docker (kylerichardson) closes apache/incubator-metron#472
Repository: incubator-metron
Updated Branches:
refs/heads/master 88d72339f -> c13ee8265
METRON-700 Add hadoop container to metron-docker (kylerichardson) closes apache/incubator-metron#472
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/c13ee826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/c13ee826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/c13ee826
Branch: refs/heads/master
Commit: c13ee82656830e131f929e443930465c475ff4eb
Parents: 88d7233
Author: kylerichardson <ky...@gmail.com>
Authored: Tue Mar 28 20:38:37 2017 -0400
Committer: Kyle Richardson <ky...@apache.org>
Committed: Tue Mar 28 20:38:37 2017 -0400
----------------------------------------------------------------------
metron-docker/.gitignore | 4 +--
metron-docker/README.md | 2 ++
metron-docker/compose/docker-compose.yml | 12 ++++++-
metron-docker/compose/hadoop/Dockerfile | 34 ++++++++++++++++++
metron-docker/compose/hadoop/conf/core-site.xml | 24 +++++++++++++
metron-docker/compose/hadoop/conf/hdfs-site.xml | 32 +++++++++++++++++
.../compose/hadoop/docker-entrypoint.sh | 38 ++++++++++++++++++++
metron-docker/compose/kafkazk/Dockerfile | 10 ++----
metron-docker/compose/kafkazk/bin/init-zk.sh | 1 +
metron-docker/compose/storm/Dockerfile | 6 ++--
metron-docker/pom.xml | 24 +++++++++++--
11 files changed, 169 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/.gitignore
----------------------------------------------------------------------
diff --git a/metron-docker/.gitignore b/metron-docker/.gitignore
index 2001ab7..05cbde5 100644
--- a/metron-docker/.gitignore
+++ b/metron-docker/.gitignore
@@ -1,8 +1,6 @@
/compose/.env
/compose/elasticsearch/es_templates
-/compose/kafkazk/common
-/compose/kafkazk/parser
-/compose/kafkazk/enrichment
+/compose/kafkazk/packages/
/compose/hbase/data-management
/compose/storm/elasticsearch
/compose/storm/enrichment
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/README.md
----------------------------------------------------------------------
diff --git a/metron-docker/README.md b/metron-docker/README.md
index eb0138b..32892f8 100644
--- a/metron-docker/README.md
+++ b/metron-docker/README.md
@@ -13,6 +13,7 @@ Metron Docker includes these images that have been customized for Metron:
- Storm (with all topologies deployed)
- Elasticsearch
- Kibana
+ - HDFS
Setup
-----
@@ -80,6 +81,7 @@ Then, assuming a host ip of `192.168.99.100`, the UIs and APIs are available at:
* HBase - http://192.168.99.100:16010/
* Elasticsearch - http://192.168.99.100:9200/_plugin/head/
* Kibana - http://192.168.99.100:5601/
+* HDFS (Namenode) - http://192.168.99.100:50070/
The Storm logs can be useful when troubleshooting topologies. They can be found on the Storm container in `/usr/share/apache-storm/logs`.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/docker-compose.yml
----------------------------------------------------------------------
diff --git a/metron-docker/compose/docker-compose.yml b/metron-docker/compose/docker-compose.yml
index 8ef2d85..efc4227 100644
--- a/metron-docker/compose/docker-compose.yml
+++ b/metron-docker/compose/docker-compose.yml
@@ -37,6 +37,14 @@ services:
- "/opt/hbase-1.1.6/conf"
depends_on:
- kafkazk
+ hadoop:
+ build:
+ context: ./hadoop
+ ports:
+ - "50070:50070"
+ volumes:
+ - "/opt/hadoop-2.7.3/etc/hadoop"
+ command: tail -f /dev/null
storm:
build:
context: ./storm
@@ -48,13 +56,15 @@ services:
- "8081:8081"
environment:
ZOOKEEPER_ADDR: kafkazk
- CONFIG_TOPOLOGY_CLASSPATH: /opt/hbase-1.1.6/conf
+ CONFIG_TOPOLOGY_CLASSPATH: "/opt/hbase-1.1.6/conf:/opt/hadoop-2.7.3/etc/hadoop"
volumes_from:
- hbase
+ - hadoop
depends_on:
- kafkazk
- hbase
- elasticsearch
+ - hadoop
command: --daemon nimbus supervisor ui logviewer
elasticsearch:
build:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/Dockerfile b/metron-docker/compose/hadoop/Dockerfile
new file mode 100644
index 0000000..caddbd9
--- /dev/null
+++ b/metron-docker/compose/hadoop/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM openjdk:7
+
+ENV HADOOP_PREFIX=/opt/hadoop-2.7.3
+ENV HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
+ENV HADOOP_IDENT_STRING=root
+
+RUN curl -sL http://archive.apache.org/dist/hadoop/core/hadoop-2.7.3/hadoop-2.7.3.tar.gz | tar -xzC /opt
+
+COPY ./conf/* $HADOOP_CONF_DIR/
+RUN mkdir -p /hadoop/dfs/name \
+ && mkdir -p /hadoop/dfs/data \
+ && $HADOOP_PREFIX/bin/hdfs namenode -format
+
+WORKDIR $HADOOP_PREFIX
+EXPOSE 9000 50010 50020 50070 50075
+
+COPY ./docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/conf/core-site.xml
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/conf/core-site.xml b/metron-docker/compose/hadoop/conf/core-site.xml
new file mode 100644
index 0000000..f7a9465
--- /dev/null
+++ b/metron-docker/compose/hadoop/conf/core-site.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://hadoop:9000</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/conf/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/conf/hdfs-site.xml b/metron-docker/compose/hadoop/conf/hdfs-site.xml
new file mode 100644
index 0000000..c75a464
--- /dev/null
+++ b/metron-docker/compose/hadoop/conf/hdfs-site.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/hadoop/dfs/name</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/hadoop/dfs/data</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/docker-entrypoint.sh
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/docker-entrypoint.sh b/metron-docker/compose/hadoop/docker-entrypoint.sh
new file mode 100755
index 0000000..97b9809
--- /dev/null
+++ b/metron-docker/compose/hadoop/docker-entrypoint.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# exit immediately on error
+set -e
+
+# start namenode
+$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_PREFIX/etc/hadoop --script hdfs start namenode
+
+# start datanode
+$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_PREFIX/etc/hadoop --script hdfs start datanode
+
+# create metron base directory
+$HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /apps/metron
+
+# create directory for geo database
+$HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /apps/metron/geo/default
+
+# download geo database to hdfs
+curl http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | $HADOOP_PREFIX/bin/hdfs dfs -put - /apps/metron/geo/default/GeoLite2-City.mmdb.gz
+
+# pass through CMD as PID 1
+exec "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/kafkazk/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-docker/compose/kafkazk/Dockerfile b/metron-docker/compose/kafkazk/Dockerfile
index 401b7fd..77b6b7e 100644
--- a/metron-docker/compose/kafkazk/Dockerfile
+++ b/metron-docker/compose/kafkazk/Dockerfile
@@ -34,13 +34,9 @@ RUN yum install -y java-1.8.0-openjdk lsof
RUN mkdir -p $METRON_HOME
ADD ./bin /opt/kafka_2.11-0.10.0.0/bin
RUN chmod 755 /opt/kafka_2.11-0.10.0.0/bin/wait-for-it.sh
-ADD ./common /common
-ADD ./parser /parser
-ADD ./enrichment /enrichment
-ADD ./data /data
-RUN tar -xzf /common/metron-common-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
-RUN tar -xzf /parser/metron-parsers-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
-RUN tar -xzf /enrichment/metron-enrichment-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
+COPY ./data /data/
+COPY ./packages/* /packages/
+RUN find /packages -type f -name '*.tar.gz' -exec tar -xzf {} -C /usr/metron/$METRON_VERSION/ \;
ADD ./conf /$METRON_HOME/config/zookeeper
EXPOSE 2181 9092
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/kafkazk/bin/init-zk.sh
----------------------------------------------------------------------
diff --git a/metron-docker/compose/kafkazk/bin/init-zk.sh b/metron-docker/compose/kafkazk/bin/init-zk.sh
index 6e2dd3f..4974683 100755
--- a/metron-docker/compose/kafkazk/bin/init-zk.sh
+++ b/metron-docker/compose/kafkazk/bin/init-zk.sh
@@ -19,4 +19,5 @@ echo "create /metron metron" | ./bin/zookeeper-shell.sh localhost:2181
echo "create /metron/topology topology" | ./bin/zookeeper-shell.sh localhost:2181
echo "create /metron/topology/parsers parsers" | ./bin/zookeeper-shell.sh localhost:2181
echo "create /metron/topology/enrichments enrichments" | ./bin/zookeeper-shell.sh localhost:2181
+echo "create /metron/topology/indexing indexing" | ./bin/zookeeper-shell.sh localhost:2181
$METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m PUSH -i $METRON_HOME/config/zookeeper
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/storm/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-docker/compose/storm/Dockerfile b/metron-docker/compose/storm/Dockerfile
index 4d80479..78aeba4 100644
--- a/metron-docker/compose/storm/Dockerfile
+++ b/metron-docker/compose/storm/Dockerfile
@@ -20,7 +20,6 @@ ARG METRON_VERSION
ENV METRON_VERSION $METRON_VERSION
ENV METRON_HOME /usr/metron/$METRON_VERSION/
-ENV GEO_DIR /apps/metron/geo/default/
ADD ./bin $METRON_HOME/bin
ADD ./parser /parser
@@ -28,7 +27,6 @@ ADD ./enrichment /enrichment
ADD ./indexing /indexing
ADD ./elasticsearch /elasticsearch
RUN mkdir -p $METRON_HOME
-RUN mkdir -p $GEO_DIR
RUN tar -xzf /parser/metron-parsers-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
RUN tar -xzf /enrichment/metron-enrichment-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
@@ -49,10 +47,10 @@ RUN tar -xzf /elasticsearch/metron-elasticsearch-$METRON_VERSION-archive.tar.gz
RUN sed -i -e "s/kafka.zk=.*:/kafka.zk=kafkazk:/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
RUN sed -i -e "s/kafka.broker=.*/kafka.broker=kafkazk:9092/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
RUN sed -i -e "s/es.ip=.*/es.ip=metron-elasticsearch/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
-RUN sed -i -e "s/bolt.hdfs.file.system.url=.*:8020/bolt.hdfs.file.system.url=file\:\/\/\//g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
+RUN sed -i -e "s/bolt.hdfs.file.system.url=.*/bolt.hdfs.file.system.url=hdfs:\/\/hadoop:9000/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
+RUN sed -i -e "s/index.hdfs.output=.*/index.hdfs.output=\/apps\/metron\/indexing\/indexed/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
EXPOSE 8080 8000
EXPOSE 8081 8081
WORKDIR $METRON_HOME
-RUN cd $GEO_DIR && wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-docker/pom.xml b/metron-docker/pom.xml
index 1351cdf..0f4b1f8 100644
--- a/metron-docker/pom.xml
+++ b/metron-docker/pom.xml
@@ -41,7 +41,7 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${project.basedir}/compose/kafkazk/common</outputDirectory>
+ <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
<resources>
<resource>
<directory>../metron-platform/metron-common/target/</directory>
@@ -59,7 +59,7 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${project.basedir}/compose/kafkazk/parser</outputDirectory>
+ <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
<resources>
<resource>
<directory>../metron-platform/metron-parsers/target/</directory>
@@ -77,7 +77,7 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${project.basedir}/compose/kafkazk/enrichment</outputDirectory>
+ <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
<resources>
<resource>
<directory>../metron-platform/metron-enrichment/target/</directory>
@@ -89,6 +89,24 @@
</configuration>
</execution>
<execution>
+ <id>copy-indexing-to-kafkazk</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
+ <resources>
+ <resource>
+ <directory>../metron-platform/metron-indexing/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
<id>copy-templates-to-elasticsearch</id>
<phase>prepare-package</phase>
<goals>