You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xi...@apache.org on 2017/12/06 04:48:19 UTC
hadoop git commit: HDFS-12872. EC Checksum broken when
BlockAccessToken is enabled.
Repository: hadoop
Updated Branches:
refs/heads/trunk 05c347fe5 -> 56b1ff80d
HDFS-12872. EC Checksum broken when BlockAccessToken is enabled.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56b1ff80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56b1ff80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56b1ff80
Branch: refs/heads/trunk
Commit: 56b1ff80dd9fbcde8d21a604eff0babb3a16418f
Parents: 05c347f
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Dec 5 20:48:02 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Dec 5 20:48:16 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java | 3 ++-
.../hadoop/hdfs/server/blockmanagement/BlockManager.java | 9 ++++-----
.../test/java/org/apache/hadoop/hdfs/TestFileChecksum.java | 9 ++++++++-
3 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56b1ff80/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
index 4dbaa01..e7d5b3a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
@@ -192,6 +192,7 @@ public class LocatedBlocks {
+ "\n underConstruction=" + underConstruction
+ "\n blocks=" + blocks
+ "\n lastLocatedBlock=" + lastLocatedBlock
- + "\n isLastBlockComplete=" + isLastBlockComplete + "}";
+ + "\n isLastBlockComplete=" + isLastBlockComplete
+ + "\n ecPolicy=" + ecPolicy + "}";
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56b1ff80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 1cdb159..ae04aac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1378,12 +1378,11 @@ public class BlockManager implements BlockStatsMXBean {
b.getStorageIDs());
}
sb.setBlockTokens(blockTokens);
- } else {
- b.setBlockToken(blockTokenSecretManager.generateToken(
- NameNode.getRemoteUser().getShortUserName(),
- b.getBlock(), EnumSet.of(mode), b.getStorageTypes(),
- b.getStorageIDs()));
}
+ b.setBlockToken(blockTokenSecretManager.generateToken(
+ NameNode.getRemoteUser().getShortUserName(),
+ b.getBlock(), EnumSet.of(mode), b.getStorageTypes(),
+ b.getStorageIDs()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56b1ff80/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
index 9d6687c..d201ce1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -33,8 +34,12 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.slf4j.event.Level;
import java.io.IOException;
+import java.util.Random;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY;
/**
* This test serves a prototype to demo the idea proposed so far. It creates two
@@ -77,6 +82,7 @@ public class TestFileChecksum {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
+ conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
Path ecPath = new Path(ecDir);
cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault());
@@ -89,6 +95,7 @@ public class TestFileChecksum {
bytesPerCRC = conf.getInt(
HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,
HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
+ GenericTestUtils.setLogLevel(FileChecksumHelper.LOG, Level.DEBUG);
}
@After
@@ -518,7 +525,7 @@ public class TestFileChecksum {
LocatedBlock locatedBlock = locatedBlocks.get(0);
DatanodeInfo[] datanodes = locatedBlock.getLocations();
- DatanodeInfo chosenDn = datanodes[0];
+ DatanodeInfo chosenDn = datanodes[new Random().nextInt(datanodes.length)];
int idx = 0;
for (DataNode dn : cluster.getDataNodes()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org