You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/10/04 19:15:16 UTC
[hadoop] branch branch-3.1 updated (9fdb849 -> 166d38c)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a change to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 9fdb849 HDFS-14245. [SBN read] Enable ObserverReadProxyProvider to work with non-ClientProtocol proxy types. Contributed by Erik Krogen.
new d91c687 HDFS-14497. Write lock held by metasave impact following RPC processing. Contributed by He Xiaoqiao.
new c61c114 HDFS-14497. Addendum: Write lock held by metasave impact following RPC processing.
new 166d38c HDFS-14890. Fixed namenode and journalnode startup on Windows. Contributed by Siddharth Wagle
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hdfs/server/blockmanagement/BlockManager.java | 2 +-
.../apache/hadoop/hdfs/server/common/Storage.java | 11 ++--
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 28 ++++++----
.../hadoop/hdfs/server/namenode/TestMetaSave.java | 60 ++++++++++++++++++++++
4 files changed, 88 insertions(+), 13 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HDFS-14497. Write lock held by metasave impact
following RPC processing. Contributed by He Xiaoqiao.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d91c68729c18a2f853beb66b506ef9dff3526a38
Author: He Xiaoqiao <he...@apache.org>
AuthorDate: Thu May 30 13:27:48 2019 -0700
HDFS-14497. Write lock held by metasave impact following RPC processing. Contributed by He Xiaoqiao.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
(cherry picked from commit 33c62f8f4e94442825fe286c2b18518925d980e6)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(cherry picked from commit 80392e94b6dca16229fc35426d107184be68c908)
---
.../hdfs/server/blockmanagement/BlockManager.java | 2 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 28 ++++++----
.../hadoop/hdfs/server/namenode/TestMetaSave.java | 60 ++++++++++++++++++++++
3 files changed, 80 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 4e450e2..321f44e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -726,7 +726,7 @@ public class BlockManager implements BlockStatsMXBean {
/** Dump meta data to out. */
public void metaSave(PrintWriter out) {
- assert namesystem.hasWriteLock(); // TODO: block manager read lock and NS write lock
+ assert namesystem.hasReadLock(); // TODO: block manager read lock and NS write lock
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
datanodeManager.fetchDatanodes(live, dead, false);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7ebc8da..108cfbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -122,6 +122,7 @@ import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -588,6 +589,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private String nameNodeHostName = null;
/**
+ * HDFS-14497: Concurrency control when many metaSave request to write
+ * meta to same out stream after switch to read lock.
+ */
+ private Object metaSaveLock = new Object();
+
+ /**
* Notify that loading of this FSDirectory is complete, and
* it is imageLoaded for use
*/
@@ -1757,23 +1764,26 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String operationName = "metaSave";
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.READ);
- writeLock();
+ readLock();
try {
checkOperation(OperationCategory.READ);
- File file = new File(System.getProperty("hadoop.log.dir"), filename);
- PrintWriter out = new PrintWriter(new BufferedWriter(
- new OutputStreamWriter(new FileOutputStream(file), Charsets.UTF_8)));
- metaSave(out);
- out.flush();
- out.close();
+ synchronized(metaSaveLock) {
+ File file = new File(System.getProperty("hadoop.log.dir"), filename);
+ PrintWriter out = new PrintWriter(new BufferedWriter(
+ new OutputStreamWriter(Files.newOutputStream(file.toPath()),
+ Charsets.UTF_8)));
+ metaSave(out);
+ out.flush();
+ out.close();
+ }
} finally {
- writeUnlock(operationName);
+ readUnlock(operationName);
}
logAuditEvent(true, operationName, null);
}
private void metaSave(PrintWriter out) {
- assert hasWriteLock();
+ assert hasReadLock();
long totalInodes = this.dir.totalInodes();
long totalBlocks = this.getBlocksTotal();
out.println(totalInodes + " files and directories, " + totalBlocks
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
index 8cc1433..d4748f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
@@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.util.ArrayList;
import java.util.concurrent.TimeoutException;
import com.google.common.base.Supplier;
@@ -215,6 +216,65 @@ public class TestMetaSave {
}
}
+ class MetaSaveThread extends Thread {
+ NamenodeProtocols nnRpc;
+ String filename;
+ public MetaSaveThread(NamenodeProtocols nnRpc, String filename) {
+ this.nnRpc = nnRpc;
+ this.filename = filename;
+ }
+
+ @Override
+ public void run() {
+ try {
+ nnRpc.metaSave(filename);
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ /**
+ * Tests that metasave concurrent output file (not append).
+ */
+ @Test
+ public void testConcurrentMetaSave() throws Exception {
+ ArrayList<MetaSaveThread> threads = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ threads.add(new MetaSaveThread(nnRpc, "metaSaveConcurrent.out.txt"));
+ }
+ for (int i = 0; i < 10; i++) {
+ threads.get(i).start();
+ }
+ for (int i = 0; i < 10; i++) {
+ threads.get(i).join();
+ }
+ // Read output file.
+ FileInputStream fis = null;
+ InputStreamReader isr = null;
+ BufferedReader rdr = null;
+ try {
+ fis = new FileInputStream(getLogFile("metaSaveConcurrent.out.txt"));
+ isr = new InputStreamReader(fis);
+ rdr = new BufferedReader(isr);
+
+ // Validate that file was overwritten (not appended) by checking for
+ // presence of only one "Live Datanodes" line.
+ boolean foundLiveDatanodesLine = false;
+ String line = rdr.readLine();
+ while (line != null) {
+ if (line.startsWith("Live Datanodes")) {
+ if (foundLiveDatanodesLine) {
+ fail("multiple Live Datanodes lines, output file not overwritten");
+ }
+ foundLiveDatanodesLine = true;
+ }
+ line = rdr.readLine();
+ }
+ } finally {
+ IOUtils.cleanup(null, rdr, isr, fis);
+ }
+ }
+
@After
public void tearDown() throws IOException {
if (fileSys != null)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HDFS-14890. Fixed namenode and journalnode startup
on Windows. Contributed by Siddharth Wagle
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 166d38ceaacf76b099d4d296ce17c810d8a3840e
Author: Eric Yang <ey...@apache.org>
AuthorDate: Fri Oct 4 13:13:10 2019 -0400
HDFS-14890. Fixed namenode and journalnode startup on Windows.
Contributed by Siddharth Wagle
(cherry picked from commit aa24add8f0e9812d1f787efb3c40155b0fdeed9c)
(cherry picked from commit 8bb2b00d38978859b22b892034eb3f559b820942)
---
.../java/org/apache/hadoop/hdfs/server/common/Storage.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 2ba943a..e7da44e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -447,9 +447,14 @@ public abstract class Storage extends StorageInfo {
throw new IOException("Cannot create directory " + curDir);
}
if (permission != null) {
- Set<PosixFilePermission> permissions =
- PosixFilePermissions.fromString(permission.toString());
- Files.setPosixFilePermissions(curDir.toPath(), permissions);
+ try {
+ Set<PosixFilePermission> permissions =
+ PosixFilePermissions.fromString(permission.toString());
+ Files.setPosixFilePermissions(curDir.toPath(), permissions);
+ } catch (UnsupportedOperationException uoe) {
+ // Default to FileUtil for non posix file systems
+ FileUtil.setPermission(curDir, permission);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: HDFS-14497. Addendum: Write lock held by metasave
impact following RPC processing.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit c61c114a3db021d1d86f290ce8fb6185582ed986
Author: He Xiaoqiao <he...@apache.org>
AuthorDate: Tue Aug 27 15:26:21 2019 -0700
HDFS-14497. Addendum: Write lock held by metasave impact following RPC processing.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
(cherry picked from commit dde9399b37bffb77da17c025f0b9b673d7088bc6)
(cherry picked from commit e29ae7db1258f08339cf0f53968fce6f98ada3ac)
---
.../main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 108cfbc..a091d19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -592,7 +592,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* HDFS-14497: Concurrency control when many metaSave request to write
* meta to same out stream after switch to read lock.
*/
- private Object metaSaveLock = new Object();
+ private final Object metaSaveLock = new Object();
/**
* Notify that loading of this FSDirectory is complete, and
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org