You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ec...@apache.org on 2015/09/15 21:11:50 UTC
[05/50] [abbrv] hadoop git commit: HDFS-8929. Add a metric to expose
the timestamp of the last journal (Contributed by surendra singh lilhore)
HDFS-8929. Add a metric to expose the timestamp of the last journal (Contributed by surendra singh lilhore)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94cf7ab9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94cf7ab9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94cf7ab9
Branch: refs/heads/HADOOP-11890
Commit: 94cf7ab9d28a885181afeb2c181dfe857d158254
Parents: a153b96
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Sep 9 11:03:57 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Sep 9 11:03:57 2015 +0530
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 1 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/qjournal/server/Journal.java | 10 +++++++++-
.../hadoop/hdfs/qjournal/server/JournalMetrics.java | 7 ++++++-
.../hadoop/hdfs/qjournal/server/TestJournalNode.java | 9 +++++++++
5 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94cf7ab9/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 8722968..efe1f2f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -275,6 +275,7 @@ The server-side metrics for a journal from the JournalNode's perspective. Each m
| `CurrentLagTxns` | The number of transactions that this JournalNode is lagging |
| `LastWrittenTxId` | The highest transaction id stored on this JournalNode |
| `LastPromisedEpoch` | The last epoch number which this node has promised not to accept any lower epoch, or 0 if no promises have been made |
+| `LastJournalTimestamp` | The timestamp of last successfully written transaction |
datanode
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94cf7ab9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cfedb0a..14666dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -921,6 +921,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8862. BlockManager#excessReplicateMap should use a HashMap. (yliu)
+ HDFS-8929. Add a metric to expose the timestamp of the last journal
+ (surendra singh lilhore via vinayakumarb)
+
BUG FIXES
HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94cf7ab9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
index 813f267..b94cd8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StopWatch;
+import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
@@ -132,6 +133,8 @@ public class Journal implements Closeable {
private final JournalMetrics metrics;
+ private long lastJournalTimestamp = 0;
+
/**
* Time threshold for sync calls, beyond which a warning should be logged to the console.
*/
@@ -253,7 +256,11 @@ public class Journal implements Closeable {
synchronized long getCommittedTxnIdForTests() throws IOException {
return committedTxnId.get();
}
-
+
+ synchronized long getLastJournalTimestamp() {
+ return lastJournalTimestamp;
+ }
+
synchronized long getCurrentLagTxns() throws IOException {
long committed = committedTxnId.get();
if (committed == 0) {
@@ -411,6 +418,7 @@ public class Journal implements Closeable {
updateHighestWrittenTxId(lastTxnId);
nextTxId = lastTxnId + 1;
+ lastJournalTimestamp = Time.now();
}
public void heartbeat(RequestInfo reqInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94cf7ab9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java
index 40c0bff..cffe2c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java
@@ -109,7 +109,12 @@ class JournalMetrics {
return -1L;
}
}
-
+
+ @Metric("The timestamp of last successfully written transaction")
+ public long getLastJournalTimestamp() {
+ return journal.getLastJournalTimestamp();
+ }
+
void addSync(long us) {
for (MutableQuantiles q : syncsQuantiles) {
q.add(us);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94cf7ab9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
index 2115671..9dd6846 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
@@ -106,7 +106,9 @@ public class TestJournalNode {
MetricsAsserts.assertCounter("BatchesWritten", 0L, metrics);
MetricsAsserts.assertCounter("BatchesWrittenWhileLagging", 0L, metrics);
MetricsAsserts.assertGauge("CurrentLagTxns", 0L, metrics);
+ MetricsAsserts.assertGauge("LastJournalTimestamp", 0L, metrics);
+ long beginTimestamp = System.currentTimeMillis();
IPCLoggerChannel ch = new IPCLoggerChannel(
conf, FAKE_NSINFO, journalId, jn.getBoundIpcAddress());
ch.newEpoch(1).get();
@@ -119,6 +121,10 @@ public class TestJournalNode {
MetricsAsserts.assertCounter("BatchesWritten", 1L, metrics);
MetricsAsserts.assertCounter("BatchesWrittenWhileLagging", 0L, metrics);
MetricsAsserts.assertGauge("CurrentLagTxns", 0L, metrics);
+ long lastJournalTimestamp = MetricsAsserts.getLongGauge(
+ "LastJournalTimestamp", metrics);
+ assertTrue(lastJournalTimestamp > beginTimestamp);
+ beginTimestamp = lastJournalTimestamp;
ch.setCommittedTxId(100L);
ch.sendEdits(1L, 2, 1, "goodbye".getBytes(Charsets.UTF_8)).get();
@@ -128,6 +134,9 @@ public class TestJournalNode {
MetricsAsserts.assertCounter("BatchesWritten", 2L, metrics);
MetricsAsserts.assertCounter("BatchesWrittenWhileLagging", 1L, metrics);
MetricsAsserts.assertGauge("CurrentLagTxns", 98L, metrics);
+ lastJournalTimestamp = MetricsAsserts.getLongGauge(
+ "LastJournalTimestamp", metrics);
+ assertTrue(lastJournalTimestamp > beginTimestamp);
}