You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/05/31 22:42:46 UTC
[11/50] [abbrv] hadoop git commit: HDFS-11877.
FileJournalManager#getLogFile should ignore in progress edit logs during JN
sync. Contributed by Hanisha Koneru.
HDFS-11877. FileJournalManager#getLogFile should ignore in progress edit logs during JN sync. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e83ed5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e83ed5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e83ed5e
Branch: refs/heads/YARN-1011
Commit: 0e83ed5e7372c801c9fee01df91b6b56de467ab1
Parents: 1c8dd6d
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed May 24 16:09:00 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed May 24 16:09:00 2017 -0700
----------------------------------------------------------------------
.../hdfs/qjournal/client/IPCLoggerChannel.java | 2 +-
.../qjournal/server/GetJournalEditServlet.java | 16 +++++++++++++---
.../hdfs/qjournal/server/JournalNodeSyncer.java | 2 +-
.../hdfs/server/namenode/FileJournalManager.java | 18 +++++++++++++++---
4 files changed, 30 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index cbfe5b8..6cd892c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -276,7 +276,7 @@ public class IPCLoggerChannel implements AsyncLogger {
try {
String path = GetJournalEditServlet.buildPath(
- journalId, segmentTxId, nsInfo);
+ journalId, segmentTxId, nsInfo, true);
return new URL(httpServerURL, path);
} catch (MalformedURLException e) {
// should never get here.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
index 2335ea5..e96fd4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
@@ -72,6 +72,7 @@ public class GetJournalEditServlet extends HttpServlet {
static final String STORAGEINFO_PARAM = "storageInfo";
static final String JOURNAL_ID_PARAM = "jid";
static final String SEGMENT_TXID_PARAM = "segmentTxId";
+ static final String IN_PROGRESS_OK = "inProgressOk";
protected boolean isValidRequestor(HttpServletRequest request, Configuration conf)
throws IOException {
@@ -186,6 +187,14 @@ public class GetJournalEditServlet extends HttpServlet {
final Configuration conf = (Configuration) getServletContext()
.getAttribute(JspHelper.CURRENT_CONF);
final String journalId = request.getParameter(JOURNAL_ID_PARAM);
+ final String inProgressOkStr = request.getParameter(IN_PROGRESS_OK);
+ final boolean inProgressOk;
+ if (inProgressOkStr != null &&
+ inProgressOkStr.equalsIgnoreCase("false")) {
+ inProgressOk = false;
+ } else {
+ inProgressOk = true;
+ }
QuorumJournalManager.checkJournalId(journalId);
final JNStorage storage = JournalNodeHttpServer
.getJournalFromContext(context, journalId).getStorage();
@@ -210,8 +219,7 @@ public class GetJournalEditServlet extends HttpServlet {
// Synchronize on the FJM so that the file doesn't get finalized
// out from underneath us while we're in the process of opening
// it up.
- EditLogFile elf = fjm.getLogFile(
- segmentTxId);
+ EditLogFile elf = fjm.getLogFile(segmentTxId, inProgressOk);
if (elf == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,
"No edit log found starting at txid " + segmentTxId);
@@ -239,7 +247,7 @@ public class GetJournalEditServlet extends HttpServlet {
}
public static String buildPath(String journalId, long segmentTxId,
- NamespaceInfo nsInfo) {
+ NamespaceInfo nsInfo, boolean inProgressOk) {
StringBuilder path = new StringBuilder("/getJournal?");
try {
path.append(JOURNAL_ID_PARAM).append("=")
@@ -248,6 +256,8 @@ public class GetJournalEditServlet extends HttpServlet {
.append(segmentTxId);
path.append("&" + STORAGEINFO_PARAM).append("=")
.append(URLEncoder.encode(nsInfo.toColonSeparatedString(), "UTF-8"));
+ path.append("&" + IN_PROGRESS_OK).append("=")
+ .append(inProgressOk);
} catch (UnsupportedEncodingException e) {
// Never get here -- everyone supports UTF-8
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
index 788c5de..73defc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
@@ -296,7 +296,7 @@ public class JournalNodeSyncer {
}
String urlPath = GetJournalEditServlet.buildPath(jid, missingLog
- .getStartTxId(), nsInfo);
+ .getStartTxId(), nsInfo, false);
url = new URL(remoteJNproxy.httpServerUrl, urlPath);
success = downloadMissingLogSegment(url, missingLog);
} catch (MalformedURLException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
index 5d62825..c71c09a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
@@ -451,16 +451,28 @@ public class FileJournalManager implements JournalManager {
}
public EditLogFile getLogFile(long startTxId) throws IOException {
- return getLogFile(sd.getCurrentDir(), startTxId);
+ return getLogFile(sd.getCurrentDir(), startTxId, true);
}
-
+
+ public EditLogFile getLogFile(long startTxId, boolean inProgressOk)
+ throws IOException {
+ return getLogFile(sd.getCurrentDir(), startTxId, inProgressOk);
+ }
+
public static EditLogFile getLogFile(File dir, long startTxId)
throws IOException {
+ return getLogFile(dir, startTxId, true);
+ }
+
+ public static EditLogFile getLogFile(File dir, long startTxId,
+ boolean inProgressOk) throws IOException {
List<EditLogFile> files = matchEditLogs(dir);
List<EditLogFile> ret = Lists.newLinkedList();
for (EditLogFile elf : files) {
if (elf.getFirstTxId() == startTxId) {
- ret.add(elf);
+ if (inProgressOk || !elf.isInProgress()) {
+ ret.add(elf);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org