You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by zi...@apache.org on 2022/03/02 05:37:28 UTC
[gobblin] branch master updated: add config to set log level for any class (#3473)
This is an automated email from the ASF dual-hosted git repository.
zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new b24ba4f add config to set log level for any class (#3473)
b24ba4f is described below
commit b24ba4f6feda09aff1953f36e99f75b00b62e61a
Author: Arjun Singh Bora <ab...@linkedin.com>
AuthorDate: Tue Mar 1 21:37:22 2022 -0800
add config to set log level for any class (#3473)
---
.../org/apache/gobblin/azkaban/AzkabanJobLauncher.java | 5 +++++
.../org/apache/gobblin/runtime/AbstractJobLauncher.java | 2 --
.../java/org/apache/gobblin/runtime/TaskExecutor.java | 2 ++
.../gobblin/util/logs/Log4jConfigurationHelper.java | 17 +++++++++++++++++
4 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanJobLauncher.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanJobLauncher.java
index 43543d8..c26d3ef 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanJobLauncher.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanJobLauncher.java
@@ -74,6 +74,7 @@ import org.apache.gobblin.util.HadoopUtils;
import org.apache.gobblin.util.PropertiesUtils;
import org.apache.gobblin.util.TimeRangeChecker;
import org.apache.gobblin.util.hadoop.TokenUtils;
+import org.apache.gobblin.util.logs.Log4jConfigurationHelper;
import static org.apache.gobblin.runtime.AbstractJobLauncher.resolveGobblinJobTemplateIfNecessary;
import static org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
@@ -101,6 +102,8 @@ public class AzkabanJobLauncher extends AbstractJob implements ApplicationLaunch
private static final Logger LOG = Logger.getLogger(AzkabanJobLauncher.class);
public static final String GOBBLIN_LOG_LEVEL_KEY = "gobblin.log.levelOverride";
+ public static final String LOG_LEVEL_OVERRIDE_MAP = "log.levelOverride.map";
+
public static final String GOBBLIN_CUSTOM_JOB_LISTENERS = "gobblin.custom.job.listeners";
private static final String HADOOP_FS_DEFAULT_NAME = "fs.default.name";
@@ -139,6 +142,8 @@ public class AzkabanJobLauncher extends AbstractJob implements ApplicationLaunch
Logger.getLogger("org.apache.gobblin").setLevel(logLevel);
}
+ Log4jConfigurationHelper.setLogLevel(PropertiesUtils.getPropAsList(props, Log4jConfigurationHelper.LOG_LEVEL_OVERRIDE_MAP, ""));
+
this.props = new Properties();
this.props.putAll(props);
diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java
index aa40bce..42ce5c5 100644
--- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java
+++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java
@@ -1037,8 +1037,6 @@ public abstract class AbstractJobLauncher implements JobLauncher {
}
}
-
-
private static String getJobIdPrefix(String jobId) {
return jobId.substring(0, jobId.lastIndexOf(Id.Job.SEPARATOR) + 1);
}
diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/TaskExecutor.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/TaskExecutor.java
index 755d972..1fa8808 100644
--- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/TaskExecutor.java
+++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/TaskExecutor.java
@@ -54,6 +54,7 @@ import org.apache.gobblin.configuration.ConfigurationKeys;
import org.apache.gobblin.metrics.GobblinMetrics;
import org.apache.gobblin.runtime.fork.Fork;
import org.apache.gobblin.util.ExecutorsUtils;
+import org.apache.gobblin.util.logs.Log4jConfigurationHelper;
import lombok.Getter;
@@ -214,6 +215,7 @@ public class TaskExecutor extends AbstractIdleService {
ConfigurationKeys.DEFAULT_QUEUED_TASK_TIME_MAX_AGE),
conf.getInt(ConfigurationKeys.METRIC_TIMER_WINDOW_SIZE_IN_MINUTES,
ConfigurationKeys.DEFAULT_METRIC_TIMER_WINDOW_SIZE_IN_MINUTES));
+ Log4jConfigurationHelper.setLogLevel(conf.getTrimmedStringCollection(Log4jConfigurationHelper.LOG_LEVEL_OVERRIDE_MAP));
}
@Override
diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/logs/Log4jConfigurationHelper.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/logs/Log4jConfigurationHelper.java
index e1559fd..b790b5e 100644
--- a/gobblin-utility/src/main/java/org/apache/gobblin/util/logs/Log4jConfigurationHelper.java
+++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/logs/Log4jConfigurationHelper.java
@@ -20,10 +20,13 @@ package org.apache.gobblin.util.logs;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Collection;
import java.util.Map.Entry;
import java.util.Properties;
+import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import com.google.common.io.Closer;
@@ -35,6 +38,8 @@ import com.google.common.io.Closer;
* @author Yinan Li
*/
public class Log4jConfigurationHelper {
+ private static final Logger LOG = Logger.getLogger(Log4jConfigurationHelper.class);
+ public static final String LOG_LEVEL_OVERRIDE_MAP = "log.levelOverride.map";
/**
* Update the log4j configuration.
@@ -67,4 +72,16 @@ public class Log4jConfigurationHelper {
closer.close();
}
}
+
+ // change log level of any class using this config
+ // e.g. log.levelOverride.map=org.apache.gobblin=debug,org.apache.kafka=trace
+ public static void setLogLevel(Collection<String> logClassesAndLevelString) {
+ for (String logClassAndLevelString : logClassesAndLevelString) {
+ String[] logClassAndLevel = logClassAndLevelString.split("=");
+ if (logClassAndLevel.length != 2) {
+ LOG.warn("Invalid value for config " + LOG_LEVEL_OVERRIDE_MAP);
+ }
+ Logger.getLogger(logClassAndLevel[0]).setLevel(Level.toLevel(logClassAndLevel[1]));
+ }
+ }
}