You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2021/07/19 10:27:47 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #828] Manage
Spark-Block-Cleaner log files
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 7ee92ce [KYUUBI #828] Manage Spark-Block-Cleaner log files
7ee92ce is described below
commit 7ee92ce5c1831fa20e62c1a703fd0564416f75cd
Author: Binjie Yang <22...@qq.com>
AuthorDate: Mon Jul 19 18:27:35 2021 +0800
[KYUUBI #828] Manage Spark-Block-Cleaner log files
### _Why are the changes needed?_
Spark-Block-Cleaner log files are not governed by itself.
Add log4j-defaults.properties to help input both console and file.
And use rollingFileApeender to split big logfile to small logFiles.
Create a log file under /logs/spark-block-cleaner-log/ so that you can manage the log yourself with the ability to clear the folder leading to `spark-`.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [X] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request
Closes #831 from zwangsheng/feature/block-cleaner.
Closes #828
83dc639e [ulysses] nit
5458f84b [Binjie Yang] make init log4j static
f0d65dd7 [Binjie Yang] import format
48514496 [Binjie Yang] optimize properties
52713f1e [Binjie Yang] assign log4j.properties
a8723f1a [Binjie Yang] fix
4d44a033 [Binjie Yang] use rollingfileappender
Lead-authored-by: Binjie Yang <22...@qq.com>
Co-authored-by: ulysses <ul...@gmail.com>
Signed-off-by: ulysses-you <ul...@gmail.com>
---
build/dist | 4 ++--
.../kubernetes/docker/entrypoint.sh | 3 +--
tools/spark-block-cleaner/pom.xml | 5 +++++
.../main/resources/log4j-block-cleaner.properties} | 21 +++++++++++++++------
.../kyuubi/tools/KubernetesSparkBlockCleaner.scala | 5 +++++
5 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/build/dist b/build/dist
index 89a8732..645d792 100755
--- a/build/dist
+++ b/build/dist
@@ -195,9 +195,9 @@ cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engin
# Copy kyuubi tools
if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" ]]; then
- mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes/docker"
+ mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes"
mkdir -p "$DISTDIR/tools/spark-block-cleaner/jars"
- cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/kubernetes/"
+ cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/"
cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" "$DISTDIR/tools/spark-block-cleaner/jars/"
fi
diff --git a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh b/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh
index 63a6e7c..953a803 100644
--- a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh
+++ b/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh
@@ -20,5 +20,4 @@
# shellcheck disable=SC2046
exec /usr/bin/tini -s -- java -cp "${CLASS_PATH}:${CLEANER_CLASSPATH}" \
- org.apache.kyuubi.tools.KubernetesSparkBlockCleaner \
- | tee /log/cleanerLog/cleaner$(date "+%Y%m%d%H%M%S").out
+ org.apache.kyuubi.tools.KubernetesSparkBlockCleaner
diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml
index 7db8bda..b03d851 100644
--- a/tools/spark-block-cleaner/pom.xml
+++ b/tools/spark-block-cleaner/pom.xml
@@ -55,5 +55,10 @@
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
</build>
</project>
diff --git a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh b/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties
similarity index 51%
copy from tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh
copy to tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties
index 63a6e7c..2649bc4 100644
--- a/tools/spark-block-cleaner/kubernetes/docker/entrypoint.sh
+++ b/tools/spark-block-cleaner/src/main/resources/log4j-block-cleaner.properties
@@ -1,4 +1,3 @@
-#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,9 +15,19 @@
# limitations under the License.
#
-# entrypoint for spark-block-cleaner
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console, logFile
-# shellcheck disable=SC2046
-exec /usr/bin/tini -s -- java -cp "${CLASS_PATH}:${CLEANER_CLASSPATH}" \
- org.apache.kyuubi.tools.KubernetesSparkBlockCleaner \
- | tee /log/cleanerLog/cleaner$(date "+%Y%m%d%H%M%S").out
+### console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.out
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %d{yyyy} %p %c{2}: %m%n
+
+### logFile
+log4j.appender.logFile=org.apache.log4j.RollingFileAppender
+log4j.appender.logFile.File=/logs/spark-block-cleaner-log/cleaner-log.out
+log4j.appender.logFile.MaxFileSize=20MB
+log4j.appender.logFile.MaxBackupIndex=5
+log4j.appender.logFile.layout=org.apache.log4j.PatternLayout
+log4j.appender.logFile.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n
diff --git a/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala b/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala
index 04b8787..7e67dd4 100644
--- a/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala
+++ b/tools/spark-block-cleaner/src/main/scala/org/apache/kyuubi/tools/KubernetesSparkBlockCleaner.scala
@@ -21,6 +21,8 @@ import java.io.File
import java.nio.file.{Files, Paths}
import java.util.concurrent.{CountDownLatch, Executors}
+import org.apache.log4j.PropertyConfigurator
+
import org.apache.kyuubi.Logging
/*
@@ -45,6 +47,9 @@ object KubernetesSparkBlockCleaner extends Logging {
private val envMap = System.getenv()
+ PropertyConfigurator.configure(
+ Thread.currentThread().getContextClassLoader.getResource("log4j-block-cleaner.properties"))
+
private val freeSpaceThreshold = envMap.getOrDefault(FREE_SPACE_THRESHOLD_KEY,
"60").toInt
private val fileExpiredTime = envMap.getOrDefault(FILE_EXPIRED_TIME_KEY,