You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/10/01 23:34:48 UTC
[ambari-logsearch] branch cloudbreak created (now c35688a)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a change to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git.
at c35688a Add Log Searach container support
This branch includes the following new commits:
new c35688a Add Log Searach container support
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ambari-logsearch] 01/01: Add Log Searach container support
Posted by ol...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit c35688a3bef89037c799540a4ca6854ccac9d389
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Tue Oct 2 01:34:24 2018 +0200
Add Log Searach container support
---
Makefile | 38 +++++++++++++
ambari-logsearch-it/pom.xml | 4 +-
ambari-logsearch-server/build.xml | 6 ++
ambari-logsearch-server/pom.xml | 19 +++++++
.../containers/docker-logsearch-portal/Dockerfile | 49 ++++++++++++++++
.../docker-logsearch-portal/bin/entrypoint.sh | 32 +++++++++++
.../containers/docker-logsearch-portal/bin/init.sh | 13 +++++
.../docker-logsearch-portal/conf/log4j2.yml | 50 ++++++++++++++++
.../docker-logsearch-portal/conf/logsearch-env.sh | 13 +++++
.../conf/logsearch.properties | 66 ++++++++++++++++++++++
pom.xml | 7 +++
11 files changed, 295 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..a23e441
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GIT_REV_SHORT = $(shell git rev-parse --short HEAD)
+MAVEN_BINARY ?= mvn
+
+ifeq ("$(USE_GIT_BUILD_TAG)", "true")
+ LOGSEARCH_BUILD_DOCKER_TAG = build-$(GIT_REV_SHORT)
+else
+ LOGSEARCH_BUILD_DOCKER_TAG = "latest"
+endif
+
+package:
+ $(MAVEN_BINARY) clean package
+
+test:
+ $(MAVEN_BINARY) clean test
+
+rpm:
+ $(MAVEN_BINARY) clean package -Dbuild-rpm -DskipTests
+
+deb:
+ $(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests
+
+docker-build:
+ $(MAVEN_BINARY) clean package docker:build -DskipTests -Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
+
+docker-push:
+ $(MAVEN_BINARY) clean package docker:build docker:push -DskipTests -Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
\ No newline at end of file
diff --git a/ambari-logsearch-it/pom.xml b/ambari-logsearch-it/pom.xml
index cf8970e..a4e160f 100644
--- a/ambari-logsearch-it/pom.xml
+++ b/ambari-logsearch-it/pom.xml
@@ -38,7 +38,7 @@
<jackson-jaxrs.version>2.9.4</jackson-jaxrs.version>
<failsafe-plugin.version>2.20</failsafe-plugin.version>
<forkCount>1</forkCount>
- <docker.host>localhost</docker.host>
+ <logsearch.docker.host>localhost</logsearch.docker.host>
<backend.stories.location>NONE</backend.stories.location>
<ui.stories.location>NONE</ui.stories.location>
</properties>
@@ -283,7 +283,7 @@
</includes>
<systemPropertyVariables>
<log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration>
- <docker.host>${docker.host}</docker.host>
+ <docker.host>${logsearch.docker.host}</docker.host>
<backend.stories.location>${backend.stories.location}</backend.stories.location>
</systemPropertyVariables>
</configuration>
diff --git a/ambari-logsearch-server/build.xml b/ambari-logsearch-server/build.xml
index b509a15..88ccd6b 100644
--- a/ambari-logsearch-server/build.xml
+++ b/ambari-logsearch-server/build.xml
@@ -66,5 +66,11 @@
<exclude name="**/*.sh"/>
</tarfileset>
</tar>
+ <copy todir="target" >
+ <fileset dir="../jenkins/containers" includes="**"/>
+ </copy>
+ <copy todir="target/docker-logsearch-portal/logsearch-portal" >
+ <fileset dir="target/package" includes="**"/>
+ </copy>
</target>
</project>
diff --git a/ambari-logsearch-server/pom.xml b/ambari-logsearch-server/pom.xml
index 39a174e..5649361 100755
--- a/ambari-logsearch-server/pom.xml
+++ b/ambari-logsearch-server/pom.xml
@@ -41,6 +41,25 @@
<jjwt.version>0.6.0</jjwt.version>
<javax-servlet.version>4.0.1</javax-servlet.version>
</properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>io.fabric8</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <configuration>
+ <verbose>true</verbose>
+ <images>
+ <image>
+ <name>apache/ambari-logsearch-portal:${logsearch.docker.tag}</name>
+ <build>
+ <dockerFileDir>${project.build.directory}/docker-logsearch-portal</dockerFileDir>
+ </build>
+ </image>
+ </images>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
<profiles>
<profile>
<id>dev</id>
diff --git a/jenkins/containers/docker-logsearch-portal/Dockerfile b/jenkins/containers/docker-logsearch-portal/Dockerfile
new file mode 100644
index 0000000..03ded63
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-portal/Dockerfile
@@ -0,0 +1,49 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM openjdk:11-jre
+MAINTAINER dev@ambari.apache.org
+
+ADD bin/entrypoint.sh /entrypoint.sh
+ADD bin/init.sh /init.sh
+ADD logsearch-portal /usr/lib/ambari-logsearch-portal
+
+ADD conf/log4j2.yml /usr/lib/ambari-logsearch-portal/conf/log4j2.yml
+ADD conf/logsearch.properties /usr/lib/ambari-logsearch-portal/conf/logsearch.properties
+ADD conf/logsearch-env.sh /usr/lib/ambari-logsearch-portal/conf/logsearch-env.sh
+
+ENV JAVA_HOME="/usr/java/default"
+ENV LOGSEARCH_USER="logsearch"
+ENV LOGSEARCH_GROUP="logsearch"
+ENV LOGSEARCH_UID="61888"
+ENV LOGSEARCH_GID="61888"
+ENV LOGSEARCH_INIT_FILE /init.sh
+ENV LOGSEARCH_PORT="61888"
+ENV LOGSEARCH_PATH="/usr/lib/ambari-logsearch-portal"
+ENV LOGSEARCH_CONF_DIR="/usr/lib/ambari-logsearch-portal/conf"
+ENV LOGFILE="/var/log/ambari-logsearch-portal/logsearch.log"
+ENV PID_FILE="/var/run/ambari-logsearch-portal/logsearch.pid"
+
+ENV LOGSEARCH_DEBUG=false
+ENV LOGSEARCH_DEBUG_PORT=5005
+ENV LOGSEARCH_SSL="false"
+
+RUN groupadd -r --gid $LOGSEARCH_GID $LOGSEARCH_GROUP && useradd -r --uid $LOGSEARCH_UID --gid $LOGSEARCH_GID $LOGSEARCH_USER
+RUN chown -R $LOGSEARCH_USER:$LOGSEARCH_GROUP /usr/lib/ambari-logsearch-portal
+RUN find /usr/lib/ambari-logsearch-portal -type d -exec chmod 755 {} +
+RUN find /usr/lib/ambari-logsearch-portal -type f -exec chmod 644 {} +
+RUN chmod 755 $LOGSEARCH_PATH/bin/logsearch.sh
+RUN chmod 755 $LOGSEARCH_PATH/conf/logsearch-env.sh
+RUN chmod 755 /entrypoint.sh
+RUN chmod 755 /init.sh
+
+WORKDIR /usr/lib/ambari-logsearch-portal
+CMD ["/entrypoint.sh"]
\ No newline at end of file
diff --git a/jenkins/containers/docker-logsearch-portal/bin/entrypoint.sh b/jenkins/containers/docker-logsearch-portal/bin/entrypoint.sh
new file mode 100644
index 0000000..c8baba4
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-portal/bin/entrypoint.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ZK_CONNECT_STRING=${ZK_CONNECT_STRING:-"localhost:9983"}
+
+function set_custom_zookeeper_address() {
+ local file_to_update=${1:?"usage: <filename_to_update>"}
+ local zk_connect_string="$ZK_CONNECT_STRING"
+ if [ "$zk_connect_string" != "localhost:9983" ] ; then
+ sed -i "s|localhost:9983|$zk_connect_string|g" $file_to_update
+ fi
+}
+
+function start() {
+ set_custom_zookeeper_address /usr/lib/ambari-logsearch-portal/conf/logsearch.properties
+ /usr/lib/ambari-logsearch-portal/bin/logsearch.sh start -f
+}
+
+if [[ -f "$LOGSEARCH_INIT_FILE" ]]; then
+ $LOGSEARCH_INIT_FILE
+fi
+
+start ${@}
diff --git a/jenkins/containers/docker-logsearch-portal/bin/init.sh b/jenkins/containers/docker-logsearch-portal/bin/init.sh
new file mode 100644
index 0000000..72ce056
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-portal/bin/init.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+echo "No init file specified."
\ No newline at end of file
diff --git a/jenkins/containers/docker-logsearch-portal/conf/log4j2.yml b/jenkins/containers/docker-logsearch-portal/conf/log4j2.yml
new file mode 100644
index 0000000..3e4fa63
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-portal/conf/log4j2.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+Configutation:
+ name: LogSearchConfig
+ packages: org.apache.ambari.logsearch.layout
+
+ Properties:
+ Property:
+ name: log-path
+ value: "/var/log/ambari-logsearch-portal"
+
+ Appenders:
+
+ Console:
+ name: Console_Appender
+ target: SYSTEM_OUT
+ PatternLayout:
+ pattern: "%d [%t] %-5p %C{6} (%F:%L) - %m%n"
+
+ RollingFile:
+ - name: RollingFile_Appender
+ fileName: ${log-path}/logsearch.log
+ filePattern: "logsearch-app.log.%d{yyyy-MM-dd-hh-mm}.gz"
+ PatternLayout:
+ pattern: "%d [%t] %-5p %C{6} (%F:%L) - %m%n"
+ Policies:
+ SizeBasedTriggeringPolicy:
+ size: 10 MB
+ DefaultRollOverStrategy:
+ max: 10
+
+ Loggers:
+
+ Root:
+ level: info
+ AppenderRef:
+ - ref: Console_Appender
+ - ref: RollingFile_Appender
\ No newline at end of file
diff --git a/jenkins/containers/docker-logsearch-portal/conf/logsearch-env.sh b/jenkins/containers/docker-logsearch-portal/conf/logsearch-env.sh
new file mode 100644
index 0000000..7e0290f
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-portal/conf/logsearch-env.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+LOGSEARCH_JAVA_MEM=${LOGSEARCH_JAVA_MEM:-"-Xmx1024m"}
\ No newline at end of file
diff --git a/jenkins/containers/docker-logsearch-portal/conf/logsearch.properties b/jenkins/containers/docker-logsearch-portal/conf/logsearch.properties
new file mode 100644
index 0000000..59d2077
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-portal/conf/logsearch.properties
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logsearch.solr.zk_connect_string=localhost:9983
+logsearch.config.zk_connect_string=localhost:9983
+
+logsearch.config.api.enabled=false
+logsearch.config.api.filter.zk.enabled=true
+
+# Service Logs
+logsearch.solr.collection.service.logs=service_logs
+logsearch.solr.service.logs.config.name=hadoop_logs
+
+logsearch.collection.service.logs.numshards=2
+logsearch.collection.service.logs.replication.factor=2
+
+# Audit logs
+logsearch.solr.audit.logs.zk_connect_string=localhost:9983
+logsearch.solr.collection.audit.logs=audit_logs
+logsearch.solr.audit.logs.url=
+
+logsearch.audit.logs.split.interval.mins=15
+logsearch.collection.audit.logs.numshards=2
+logsearch.collection.audit.logs.replication.factor=2
+
+logsearch.solr.config_set.folder=/usr/lib/ambari-logsearch-portal/conf/solr_configsets
+logsearch.solr.audit.logs.config_set.folder=/usr/lib/ambari-logsearch-portal/conf/solr_configsets
+
+# History logs
+logsearch.solr.collection.history=history
+logsearch.solr.history.config.name=history
+logsearch.collection.history.replication.factor=1
+
+# Metrics
+logsearch.solr.metrics.collector.hosts=
+logsearch.solr.jmx.port=18886
+
+# Logfeeder Settings
+
+logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO
+
+# logsearch-admin.json
+logsearch.auth.file.enable=true
+logsearch.login.credentials.file=user_pass.json
+
+logsearch.auth.ldap.enable=false
+logsearch.auth.simple.enable=false
+logsearch.auth.external_auth.enable=false
+
+logsearch.auth.redirect.forward=true
+
+logsearch.protocol=http
+
+logsearch.session.timeout=30
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 0f8b0e9..d6ecde5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,6 +92,7 @@
<skipSurefireTests>false</skipSurefireTests>
<compiler.version>3.8.0</compiler.version>
<ambari-metrics.version>2.7.0.0.0</ambari-metrics.version>
+ <logsearch.docker.tag>latest</logsearch.docker.tag>
</properties>
<licenses>
@@ -148,6 +149,11 @@
<pluginManagement>
<plugins>
<plugin>
+ <groupId>io.fabric8</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <version>0.27.1</version>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.4.1</version>
@@ -339,6 +345,7 @@
<exclude>**/docker/knox/**</exclude>
<exclude>**/node_modules/**</exclude>
<exclude>**/dist/**</exclude>
+ <exclude>.repository/**</exclude>
</excludes>
</configuration>
<executions>