You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2019/10/01 00:26:58 UTC
[hadoop] branch branch-3.1 updated (db5b835 -> 8fc4e40)
This is an automated email from the ASF dual-hosted git repository.
shv pushed a change to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from db5b835 HADOOP-15014. Addendum: KMS should log the IP address of the clients. Contributed by Zsombor Gegesy.
new bece1a1 Revert "HDFS-14305. Fix serial number calculation in BlockTokenSecretManager to avoid token key ID overlap between NameNodes. Contributed by He Xiaoqiao."
new 8fc4e40 HDFS-14305. Fix serial number calculation in BlockTokenSecretManager to avoid token key ID overlap between NameNodes. Contributed by Konstantin V Shvachko.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../token/block/BlockTokenSecretManager.java | 31 ++++++++-----------
.../hdfs/security/token/block/TestBlockToken.java | 23 ++++++++++++++
.../ha/TestFailoverWithBlockTokensEnabled.java | 36 ++--------------------
3 files changed, 39 insertions(+), 51 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: Revert "HDFS-14305. Fix serial number calculation
in BlockTokenSecretManager to avoid token key ID overlap between NameNodes.
Contributed by He Xiaoqiao."
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit bece1a1f70a66891d32c0fe44a1eec9a6f7998ea
Author: Konstantin V Shvachko <sh...@apache.org>
AuthorDate: Sun Sep 29 13:11:31 2019 -0700
Revert "HDFS-14305. Fix serial number calculation in BlockTokenSecretManager to avoid token key ID overlap between NameNodes. Contributed by He Xiaoqiao."
This reverts commit 0feba4396f6e96c332743a39f965de7995b67bde.
---
.../token/block/BlockTokenSecretManager.java | 21 +++++----------
.../ha/TestFailoverWithBlockTokensEnabled.java | 31 +---------------------
2 files changed, 7 insertions(+), 45 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
index 20ad2bb..a541976 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
@@ -64,17 +64,6 @@ public class BlockTokenSecretManager extends
public static final Token<BlockTokenIdentifier> DUMMY_TOKEN = new Token<BlockTokenIdentifier>();
- /**
- * In order to prevent serial No. of different NameNode from overlapping,
- * Using 6 bits (identify 64=2^6 namenodes, and presuppose that no scenario
- * where deploy more than 64 namenodes (include ANN, SBN, Observers, etc.)
- * in one namespace) to identify index of NameNode, and the remainder 26 bits
- * auto-incr to change the serial No.
- */
- @VisibleForTesting
- public static final int NUM_VALID_BITS = 26;
- private static final int LOW_MASK = (1 << NUM_VALID_BITS) - 1;
-
private final boolean isMaster;
/**
@@ -91,8 +80,8 @@ public class BlockTokenSecretManager extends
private String blockPoolId;
private final String encryptionAlgorithm;
- private final int nnIndex;
-
+ private final int intRange;
+ private final int nnRangeStart;
private final boolean useProto;
private final boolean shouldWrapQOP;
@@ -151,7 +140,8 @@ public class BlockTokenSecretManager extends
private BlockTokenSecretManager(boolean isMaster, long keyUpdateInterval,
long tokenLifetime, String blockPoolId, String encryptionAlgorithm,
int nnIndex, int numNNs, boolean useProto, boolean shouldWrapQOP) {
- this.nnIndex = nnIndex;
+ this.intRange = Integer.MAX_VALUE / numNNs;
+ this.nnRangeStart = intRange * nnIndex;
this.isMaster = isMaster;
this.keyUpdateInterval = keyUpdateInterval;
this.tokenLifetime = tokenLifetime;
@@ -166,7 +156,8 @@ public class BlockTokenSecretManager extends
@VisibleForTesting
public synchronized void setSerialNo(int serialNo) {
- this.serialNo = (serialNo & LOW_MASK) | (nnIndex << NUM_VALID_BITS);
+ // we mod the serial number by the range and then add that times the index
+ this.serialNo = (serialNo % intRange) + (nnRangeStart);
}
public void setBlockPoolId(String blockPoolId) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java
index 850b961..43ab69d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java
@@ -116,36 +116,7 @@ public class TestFailoverWithBlockTokensEnabled {
}
}
}
-
- @Test
- public void testSerialNumberMaskMatchIndex() {
- BlockTokenSecretManager btsm1 = cluster.getNamesystem(0).getBlockManager()
- .getBlockTokenSecretManager();
- BlockTokenSecretManager btsm2 = cluster.getNamesystem(1).getBlockManager()
- .getBlockTokenSecretManager();
- BlockTokenSecretManager btsm3 = cluster.getNamesystem(2).getBlockManager()
- .getBlockTokenSecretManager();
- int[] testSet = {0, Integer.MAX_VALUE, Integer.MIN_VALUE,
- Integer.MAX_VALUE / 2, Integer.MIN_VALUE / 2,
- Integer.MAX_VALUE / 3, Integer.MIN_VALUE / 3};
- for (int i = 0; i < testSet.length; i++) {
- setAndCheckHighBitsSerialNumber(testSet[i], btsm1, 0);
- setAndCheckHighBitsSerialNumber(testSet[i], btsm2, 1);
- setAndCheckHighBitsSerialNumber(testSet[i], btsm3, 2);
- }
- }
-
- /**
- * Check mask of serial number if equal to index of NameNode.
- */
- private void setAndCheckHighBitsSerialNumber(int serialNumber,
- BlockTokenSecretManager btsm, int nnIndex) {
- btsm.setSerialNo(serialNumber);
- int serialNo = btsm.getSerialNoForTesting();
- int index = serialNo >> BlockTokenSecretManager.NUM_VALID_BITS;
- assertEquals(index, nnIndex);
- }
-
+
@Test
public void ensureInvalidBlockTokensAreRejected() throws IOException,
URISyntaxException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-14305. Fix serial number calculation in
BlockTokenSecretManager to avoid token key ID overlap between NameNodes.
Contributed by Konstantin V Shvachko.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8fc4e406ad52649753f463cee02631f1c0ebb23b
Author: Konstantin V Shvachko <sh...@apache.org>
AuthorDate: Mon Sep 30 16:48:10 2019 -0700
HDFS-14305. Fix serial number calculation in BlockTokenSecretManager to avoid token key ID overlap between NameNodes. Contributed by Konstantin V Shvachko.
---
.../token/block/BlockTokenSecretManager.java | 12 +++++++----
.../hdfs/security/token/block/TestBlockToken.java | 23 ++++++++++++++++++++++
.../ha/TestFailoverWithBlockTokensEnabled.java | 5 ++---
3 files changed, 33 insertions(+), 7 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
index a541976..1c5c19b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
@@ -121,8 +121,6 @@ public class BlockTokenSecretManager extends
encryptionAlgorithm, nnIndex, numNNs, useProto, shouldWrapQOP);
Preconditions.checkArgument(nnIndex >= 0);
Preconditions.checkArgument(numNNs > 0);
- setSerialNo(new SecureRandom().nextInt());
- generateKeys();
}
/**
@@ -151,13 +149,19 @@ public class BlockTokenSecretManager extends
this.useProto = useProto;
this.shouldWrapQOP = shouldWrapQOP;
this.timer = new Timer();
+ setSerialNo(new SecureRandom().nextInt(Integer.MAX_VALUE));
+ LOG.info("Block token key range: [" +
+ nnRangeStart + ", " + (nnRangeStart + intRange) + ")");
generateKeys();
}
@VisibleForTesting
- public synchronized void setSerialNo(int serialNo) {
+ public synchronized void setSerialNo(int nextNo) {
// we mod the serial number by the range and then add that times the index
- this.serialNo = (serialNo % intRange) + (nnRangeStart);
+ this.serialNo = (nextNo % intRange) + (nnRangeStart);
+ assert serialNo >= nnRangeStart && serialNo < (nnRangeStart + intRange) :
+ "serialNo " + serialNo + " is not in the designated range: [" +
+ nnRangeStart + ", " + (nnRangeStart + intRange) + ")";
}
public void setBlockPoolId(String blockPoolId) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
index c16b471..6f62042 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
@@ -814,4 +814,27 @@ public class TestBlockToken {
testBadStorageIDCheckAccess(true);
}
+ /**
+ * Verify that block token serialNo is always within the range designated to
+ * to the NameNode.
+ */
+ @Test
+ public void testBlockTokenRanges() throws IOException {
+ final int interval = 1024;
+ final int numNNs = Integer.MAX_VALUE / interval;
+ for(int nnIdx = 0; nnIdx < 64; nnIdx++) {
+ BlockTokenSecretManager sm = new BlockTokenSecretManager(
+ blockKeyUpdateInterval, blockTokenLifetime, nnIdx, numNNs,
+ "fake-pool", null, false);
+ int rangeStart = nnIdx * interval;
+ for(int i = 0; i < interval * 3; i++) {
+ int serialNo = sm.getSerialNoForTesting();
+ assertTrue(
+ "serialNo " + serialNo + " is not in the designated range: [" +
+ rangeStart + ", " + (rangeStart + interval) + ")",
+ serialNo >= rangeStart && serialNo < (rangeStart + interval));
+ sm.updateKeys();
+ }
+ }
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java
index 43ab69d..ff90121 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailoverWithBlockTokensEnabled.java
@@ -92,11 +92,10 @@ public class TestFailoverWithBlockTokensEnabled {
setAndCheckSerialNumber(0, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MAX_VALUE, btsm1, btsm2, btsm3);
- setAndCheckSerialNumber(Integer.MIN_VALUE, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MAX_VALUE / 2, btsm1, btsm2, btsm3);
- setAndCheckSerialNumber(Integer.MIN_VALUE / 2, btsm1, btsm2, btsm3);
setAndCheckSerialNumber(Integer.MAX_VALUE / 3, btsm1, btsm2, btsm3);
- setAndCheckSerialNumber(Integer.MIN_VALUE / 3, btsm1, btsm2, btsm3);
+ setAndCheckSerialNumber(Integer.MAX_VALUE / 171717,
+ btsm1, btsm2, btsm3);
}
private void setAndCheckSerialNumber(int serialNumber, BlockTokenSecretManager... btsms) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org