You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2020/02/27 17:06:11 UTC
[hadoop] branch trunk updated: HDFS-15124. Crashing bugs in
NameNode when using a valid configuration for . Contributed by Ctest.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new cd2c6b1 HDFS-15124. Crashing bugs in NameNode when using a valid configuration for . Contributed by Ctest.
cd2c6b1 is described below
commit cd2c6b1aac470991b9b90339ce2721ba179e7c48
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Thu Feb 27 22:19:35 2020 +0530
HDFS-15124. Crashing bugs in NameNode when using a valid configuration for . Contributed by Ctest.
---
.../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 10 +++++++++-
.../hadoop/hdfs/server/namenode/top/TopAuditLogger.java | 15 +++++++++++++++
.../hadoop/hdfs/server/namenode/TestFSNamesystem.java | 8 ++++++++
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8c5286c..ce44652 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1110,6 +1110,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
Collection<String> alClasses =
conf.getTrimmedStringCollection(DFS_NAMENODE_AUDIT_LOGGERS_KEY);
List<AuditLogger> auditLoggers = Lists.newArrayList();
+ boolean topAuditLoggerAdded = false;
if (alClasses != null && !alClasses.isEmpty()) {
for (String className : alClasses) {
try {
@@ -1118,9 +1119,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
logger = new FSNamesystemAuditLogger();
} else {
logger = (AuditLogger) Class.forName(className).newInstance();
+ if (TopAuditLogger.class.getName().equals(
+ logger.getClass().getName())) {
+ topAuditLoggerAdded = true;
+ }
}
logger.initialize(conf);
auditLoggers.add(logger);
+ } catch (InstantiationException e) {
+ LOG.error("{} instantiation failed.", className, e);
+ throw new RuntimeException(e);
} catch (RuntimeException re) {
throw re;
} catch (Exception e) {
@@ -1137,7 +1145,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// Add audit logger to calculate top users
- if (topConf.isEnabled) {
+ if (topConf.isEnabled && !topAuditLoggerAdded) {
topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs);
if (DefaultMetricsSystem.instance().getSource(
TOPMETRICS_METRICS_SOURCE_NAME) == null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
index 49c9153..feba0e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
@@ -23,12 +23,16 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.namenode.AuditLogger;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics;
+import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
+
/**
* An {@link AuditLogger} that sends logged data directly to the metrics
* systems. It is used when the top service is used directly by the name node
@@ -39,6 +43,17 @@ public class TopAuditLogger implements AuditLogger {
private final TopMetrics topMetrics;
+ public TopAuditLogger() {
+ Configuration conf = new HdfsConfiguration();
+ TopConf topConf = new TopConf(conf);
+ this.topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs);
+ if (DefaultMetricsSystem.instance().getSource(
+ TOPMETRICS_METRICS_SOURCE_NAME) == null) {
+ DefaultMetricsSystem.instance().register(TOPMETRICS_METRICS_SOURCE_NAME,
+ "Top N operations by user", topMetrics);
+ }
+ }
+
public TopAuditLogger(TopMetrics topMetrics) {
Preconditions.checkNotNull(topMetrics, "Cannot init with a null " +
"TopMetrics");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
index cb18bcf..f2f4244 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
@@ -303,6 +303,14 @@ public class TestFSNamesystem {
.or(instanceOf(TopAuditLogger.class))
.or(instanceOf(DummyAuditLogger.class)));
}
+
+ // Test Configuring TopAuditLogger.
+ conf.set(DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY,
+ "org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger");
+ fsn = new FSNamesystem(conf, fsImage);
+ auditLoggers = fsn.getAuditLoggers();
+ assertEquals(1, auditLoggers.size());
+ assertThat(auditLoggers.get(0), instanceOf(TopAuditLogger.class));
}
static class DummyAuditLogger implements AuditLogger {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org