You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cl...@apache.org on 2019/10/10 20:31:30 UTC
[hadoop] branch branch-2 updated: HDFS-14509. DN throws
InvalidToken due to inequality of password when upgrade NN 2.x to 3.x.
Contributed by Yuxuan Wang and Konstantin Shvachko.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 6566402 HDFS-14509. DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x. Contributed by Yuxuan Wang and Konstantin Shvachko.
6566402 is described below
commit 6566402a1b22d2fa8311db7c7583f7200a3de88d
Author: Chen Liang <cl...@apache.org>
AuthorDate: Thu Oct 10 13:29:30 2019 -0700
HDFS-14509. DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x. Contributed by Yuxuan Wang and Konstantin Shvachko.
---
.../security/token/block/BlockTokenIdentifier.java | 13 ++++++
.../hdfs/security/token/block/TestBlockToken.java | 50 ++++++++++++++++++++++
2 files changed, 63 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
index 87c831a..4d2037b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.hdfs.security.token.block;
import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.EOFException;
import java.io.IOException;
import java.util.EnumSet;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -116,6 +118,7 @@ public class BlockTokenIdentifier extends TokenIdentifier {
}
public void setHandshakeMsg(byte[] bytes) {
+ cache = null; // invalidate the cache
handshakeMsg = bytes;
}
@@ -159,6 +162,16 @@ public class BlockTokenIdentifier extends TokenIdentifier {
@Override
public void readFields(DataInput in) throws IOException {
this.cache = null;
+ if (in instanceof DataInputStream) {
+ final DataInputStream dis = (DataInputStream) in;
+ // this.cache should be assigned the raw bytes from the input data for
+ // upgrading compatibility. If we won't mutate fields and call getBytes()
+ // for something (e.g retrieve password), we should return the raw bytes
+ // instead of serializing the instance self fields to bytes, because we
+ // may lose newly added fields which we can't recognize.
+ this.cache = IOUtils.readFullyToByteArray(dis);
+ dis.reset();
+ }
expiryDate = WritableUtils.readVLong(in);
keyId = WritableUtils.readVInt(in);
userId = WritableUtils.readString(in);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
index 7d0c90f..07aaf09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs.security.token.block;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -30,6 +31,7 @@ import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
+import java.io.DataOutput;
import java.net.InetSocketAddress;
import java.util.EnumSet;
import java.util.Set;
@@ -76,6 +78,7 @@ import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -435,4 +438,51 @@ public class TestBlockToken {
}
}
}
+
+ @Test
+ public void testRetrievePasswordWithUnknownFields() throws IOException {
+ BlockTokenIdentifier id = new BlockTokenIdentifier();
+ BlockTokenIdentifier spyId = Mockito.spy(id);
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ DataOutput out = (DataOutput) invocation.getArguments()[0];
+ invocation.callRealMethod();
+ // write something at the end that BlockTokenIdentifier#readFields()
+ // will ignore, but which is still a part of the password
+ out.write(7);
+ return null;
+ }
+ }).when(spyId).write((DataOutput) Mockito.any());
+
+ BlockTokenSecretManager sm =
+ new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime,
+ 0, 1, "fake-pool", null, false);
+ // master create password
+ byte[] password = sm.createPassword(spyId);
+
+ BlockTokenIdentifier slaveId = new BlockTokenIdentifier();
+ slaveId.readFields(
+ new DataInputStream(new ByteArrayInputStream(spyId.getBytes())));
+
+ // slave retrieve password
+ assertArrayEquals(password, sm.retrievePassword(slaveId));
+ }
+
+ @Test
+ public void testRetrievePasswordWithRecognizableFieldsOnly()
+ throws IOException {
+ BlockTokenSecretManager sm =
+ new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime,
+ 0, 1, "fake-pool", null, false);
+ // master create password
+ BlockTokenIdentifier masterId = new BlockTokenIdentifier();
+ byte[] password = sm.createPassword(masterId);
+ // set cache to null, so that master getBytes() were only recognizable bytes
+ masterId.setExpiryDate(masterId.getExpiryDate());
+ BlockTokenIdentifier slaveId = new BlockTokenIdentifier();
+ slaveId.readFields(
+ new DataInputStream(new ByteArrayInputStream(masterId.getBytes())));
+ assertArrayEquals(password, sm.retrievePassword(slaveId));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org