You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/10/12 05:46:17 UTC
[01/50] [abbrv] hadoop git commit: HDDS-577. Support S3 buckets as
first class objects in Ozone Manager - 2. Contributed by Bharat Viswanadham.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 dc76e0f4d -> bba8aa3d9
HDDS-577. Support S3 buckets as first class objects in Ozone Manager - 2.
Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b7ba48c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b7ba48c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b7ba48c
Branch: refs/heads/HDFS-12943
Commit: 5b7ba48cedb0d70ca154771fec48e5c4129cf29a
Parents: bf04f19
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 13:37:42 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 13:37:42 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/ozone/client/ObjectStore.java | 48 +++++
.../ozone/client/protocol/ClientProtocol.java | 35 ++++
.../hadoop/ozone/client/rest/RestClient.java | 25 +++
.../hadoop/ozone/client/rpc/RpcClient.java | 32 ++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 25 +++
...neManagerProtocolClientSideTranslatorPB.java | 51 +++++
.../src/main/proto/OzoneManagerProtocol.proto | 32 ++++
.../ozone/client/rpc/TestOzoneRpcClient.java | 34 ++++
.../apache/hadoop/ozone/om/OzoneManager.java | 23 +++
.../hadoop/ozone/om/S3BucketManagerImpl.java | 11 ++
...neManagerProtocolServerSideTranslatorPB.java | 187 +++++++++++--------
11 files changed, 429 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 17d1938..4196556 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -83,6 +83,54 @@ public class ObjectStore {
}
/**
+ * Creates an S3 bucket inside Ozone manager and creates the mapping needed
+ * to access via both S3 and Ozone.
+ * @param userName - S3 user name.
+ * @param s3BucketName - S3 bucket Name.
+ * @throws IOException - On failure, throws an exception like Bucket exists.
+ */
+ public void createS3Bucket(String userName, String s3BucketName) throws
+ IOException {
+ proxy.createS3Bucket(userName, s3BucketName);
+ }
+
+ /**
+ * Returns the Ozone Namespace for the S3Bucket. It will return the
+ * OzoneVolume/OzoneBucketName.
+ * @param s3BucketName - S3 Bucket Name.
+ * @return String - The Ozone canonical name for this s3 bucket. This
+ * string is useful for mounting an OzoneFS.
+ * @throws IOException - Error is throw if the s3bucket does not exist.
+ */
+ public String getOzoneBucketMapping(String s3BucketName) throws IOException {
+ return proxy.getOzoneBucketMapping(s3BucketName);
+ }
+
+ /**
+ * Returns the corresponding Ozone volume given an S3 Bucket.
+ * @param s3BucketName - S3Bucket Name.
+ * @return String - Ozone Volume name.
+ * @throws IOException - Throws if the s3Bucket does not exist.
+ */
+ public String getOzoneVolumeName(String s3BucketName) throws IOException {
+ String mapping = getOzoneBucketMapping(s3BucketName);
+ return mapping.split("/")[0];
+
+ }
+
+ /**
+ * Returns the corresponding Ozone bucket name for the given S3 bucket.
+ * @param s3BucketName - S3Bucket Name.
+ * @return String - Ozone bucket Name.
+ * @throws IOException - Throws if the s3bucket does not exist.
+ */
+ public String getOzoneBucketName(String s3BucketName) throws IOException {
+ String mapping = getOzoneBucketMapping(s3BucketName);
+ return mapping.split("/")[1];
+ }
+
+
+ /**
* Returns the volume information.
* @param volumeName Name of the volume.
* @return OzoneVolume
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 008b69d..b750a5a 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -322,6 +322,41 @@ public interface ClientProtocol {
throws IOException;
/**
+ * Creates an S3 bucket inside Ozone manager and creates the mapping needed
+ * to access via both S3 and Ozone.
+ * @param userName - S3 user name.
+ * @param s3BucketName - S3 bucket Name.
+ * @throws IOException - On failure, throws an exception like Bucket exists.
+ */
+ void createS3Bucket(String userName, String s3BucketName) throws IOException;
+
+ /**
+ * Returns the Ozone Namespace for the S3Bucket. It will return the
+ * OzoneVolume/OzoneBucketName.
+ * @param s3BucketName - S3 Bucket Name.
+ * @return String - The Ozone canonical name for this s3 bucket. This
+ * string is useful for mounting an OzoneFS.
+ * @throws IOException - Error is throw if the s3bucket does not exist.
+ */
+ String getOzoneBucketMapping(String s3BucketName) throws IOException;
+
+ /**
+ * Returns the corresponding Ozone volume given an S3 Bucket.
+ * @param s3BucketName - S3Bucket Name.
+ * @return String - Ozone Volume name.
+ * @throws IOException - Throws if the s3Bucket does not exist.
+ */
+ String getOzoneVolumeName(String s3BucketName) throws IOException;
+
+ /**
+ * Returns the corresponding Ozone bucket name for the given S3 bucket.
+ * @param s3BucketName - S3Bucket Name.
+ * @return String - Ozone bucket Name.
+ * @throws IOException - Throws if the s3bucket does not exist.
+ */
+ String getOzoneBucketName(String s3BucketName) throws IOException;
+
+ /**
* Close and release the resources.
*/
void close() throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index fdd049a..8e75903 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -820,6 +820,31 @@ public class RestClient implements ClientProtocol {
}
}
+ @Override
+ public void createS3Bucket(String userName, String s3BucketName)
+ throws IOException {
+ throw new UnsupportedOperationException("Ozone REST protocol does not " +
+ "support this operation.");
+ }
+
+ @Override
+ public String getOzoneBucketMapping(String s3BucketName) throws IOException {
+ throw new UnsupportedOperationException("Ozone REST protocol does not " +
+ "support this operation.");
+ }
+
+ @Override
+ public String getOzoneVolumeName(String s3BucketName) throws IOException {
+ throw new UnsupportedOperationException("Ozone REST protocol does not " +
+ "support this operation.");
+ }
+
+ @Override
+ public String getOzoneBucketName(String s3BucketName) throws IOException {
+ throw new UnsupportedOperationException("Ozone REST protocol does not " +
+ "support this operation.");
+ }
+
/**
* Adds Ozone headers to http request.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 330eba8..11b5474 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdds.scm.protocolPB
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolPB;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -568,6 +569,37 @@ public class RpcClient implements ClientProtocol {
}
@Override
+ public void createS3Bucket(String userName, String s3BucketName)
+ throws IOException {
+ Preconditions.checkArgument(Strings.isNotBlank(userName), "user name " +
+ "cannot be null or empty.");
+
+ Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " +
+ "name cannot be null or empty.");
+ ozoneManagerClient.createS3Bucket(userName, s3BucketName);
+ }
+
+ @Override
+ public String getOzoneBucketMapping(String s3BucketName) throws IOException {
+ Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " +
+ "name cannot be null or empty.");
+ return ozoneManagerClient.getOzoneBucketMapping(s3BucketName);
+ }
+
+ @Override
+ public String getOzoneVolumeName(String s3BucketName) throws IOException {
+ String mapping = getOzoneBucketMapping(s3BucketName);
+ return mapping.split("/")[0];
+
+ }
+
+ @Override
+ public String getOzoneBucketName(String s3BucketName) throws IOException {
+ String mapping = getOzoneBucketMapping(s3BucketName);
+ return mapping.split("/")[1];
+ }
+
+ @Override
public void close() throws IOException {
IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
IOUtils.cleanupWithLogger(LOG, ozoneManagerClient);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index edb260a..d31f70b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -250,4 +250,29 @@ public interface OzoneManagerProtocol {
* @throws IOException
*/
List<ServiceInfo> getServiceList() throws IOException;
+
+ /*
+ * S3 Specific functionality that is supported by Ozone Manager.
+ */
+
+ /**
+ * Creates an S3 bucket inside Ozone manager and creates the mapping needed
+ * to access via both S3 and Ozone.
+ * @param userName - S3 user name.
+ * @param s3BucketName - S3 bucket Name.
+ * @throws IOException - On failure, throws an exception like Bucket exists.
+ */
+ void createS3Bucket(String userName, String s3BucketName) throws IOException;
+
+ /**
+ * Returns the Ozone Namespace for the S3Bucket. It will return the
+ * OzoneVolume/OzoneBucketName.
+ * @param s3BucketName - S3 Bucket Name.
+ * @return String - The Ozone canonical name for this s3 bucket. This
+ * string is useful for mounting an OzoneFS.
+ * @throws IOException - Error is throw if the s3bucket does not exist.
+ */
+ String getOzoneBucketMapping(String s3BucketName) throws IOException;
+
}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index c0829fa..dfe3c74 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -114,6 +114,17 @@ import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.ServiceListResponse;
+import org.apache.hadoop.ozone.protocol.proto
+ .OzoneManagerProtocolProtos.S3BucketRequest;
+import org.apache.hadoop.ozone.protocol.proto
+ .OzoneManagerProtocolProtos.S3BucketResponse;
+import org.apache.hadoop.ozone.protocol.proto
+ .OzoneManagerProtocolProtos.S3BucketInfoRequest;
+import org.apache.hadoop.ozone.protocol.proto
+ .OzoneManagerProtocolProtos.S3BucketInfoResponse;
+
+
+
import java.io.Closeable;
import java.io.IOException;
@@ -763,6 +774,46 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
}
}
+ @Override
+ public void createS3Bucket(String userName, String s3BucketName)
+ throws IOException {
+ S3BucketRequest request = S3BucketRequest.newBuilder()
+ .setUserName(userName)
+ .setS3Bucketname(s3BucketName)
+ .build();
+ final S3BucketResponse resp;
+ try {
+ resp = rpcProxy.createS3Bucket(NULL_RPC_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+
+ if(resp.getStatus() != Status.OK) {
+ throw new IOException("Creating S3 bucket failed, error: "
+ + resp.getStatus());
+ }
+
+ }
+
+ @Override
+ public String getOzoneBucketMapping(String s3BucketName)
+ throws IOException {
+ S3BucketInfoRequest request = S3BucketInfoRequest.newBuilder()
+ .setS3BucketName(s3BucketName)
+ .build();
+ final S3BucketInfoResponse resp;
+ try {
+ resp = rpcProxy.getS3Bucketinfo(NULL_RPC_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ if(resp.getStatus() != Status.OK) {
+ throw new IOException("GetOzoneBucketMapping failed, error:" + resp
+ .getStatus());
+ }
+ return resp.getOzoneMapping();
+ }
+
/**
* Return the proxy object underlying this protocol translator.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 823f36b..eef19a5 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -59,6 +59,8 @@ enum Status {
METADATA_ERROR = 19;
OM_NOT_INITIALIZED = 20;
SCM_VERSION_MISMATCH_ERROR = 21;
+ S3_BUCKET_NOT_FOUND = 22;
+ S3_BUCKET_ALREADY_EXISTS = 23;
}
@@ -362,6 +364,24 @@ message ServiceInfo {
repeated ServicePort servicePorts = 3;
}
+message S3BucketRequest {
+ required string userName = 1;
+ required string s3bucketname = 2;
+}
+
+message S3BucketResponse {
+ required Status status = 1;
+}
+
+message S3BucketInfoRequest {
+ required string s3bucketName = 1;
+}
+message S3BucketInfoResponse {
+ required Status status = 1;
+ optional string ozoneMapping = 2;
+}
+
+
/**
The OM service that takes care of Ozone namespace.
*/
@@ -479,4 +499,16 @@ service OzoneManagerService {
*/
rpc getServiceList(ServiceListRequest)
returns(ServiceListResponse);
+
+ /**
+ Creates an S3 bucket and creates an ozone mapping for that bucket.
+ */
+ rpc createS3Bucket(S3BucketRequest)
+ returns(S3BucketResponse);
+
+ /**
+ Gets the Ozone Mapping information for the S3Bucket.
+ */
+ rpc getS3Bucketinfo(S3BucketInfoRequest)
+ returns(S3BucketInfoResponse);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index 019cc52..cbc29f5 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -201,6 +201,40 @@ public class TestOzoneRpcClient {
}
@Test
+ public void testCreateS3Bucket()
+ throws IOException, OzoneException {
+ long currentTime = Time.now();
+ String userName = "ozone";
+ String bucketName = UUID.randomUUID().toString();
+ store.createS3Bucket(userName, bucketName);
+ String volumeName = store.getOzoneVolumeName(bucketName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ Assert.assertEquals(bucketName, bucket.getName());
+ Assert.assertTrue(bucket.getCreationTime() >= currentTime);
+ Assert.assertTrue(volume.getCreationTime() >= currentTime);
+ }
+
+ @Test
+ public void testCreateS3BucketMapping()
+ throws IOException, OzoneException {
+ long currentTime = Time.now();
+ String userName = "ozone";
+ String bucketName = UUID.randomUUID().toString();
+ store.createS3Bucket(userName, bucketName);
+ String volumeName = store.getOzoneVolumeName(bucketName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ Assert.assertEquals(bucketName, bucket.getName());
+
+ String mapping = store.getOzoneBucketMapping(bucketName);
+ Assert.assertEquals("s3"+userName+"/"+bucketName, mapping);
+ Assert.assertEquals(bucketName, store.getOzoneBucketName(bucketName));
+ Assert.assertEquals("s3"+userName, store.getOzoneVolumeName(bucketName));
+
+ }
+
+ @Test
public void testCreateBucketWithVersioning()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 7d07fcb..dbcd401 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -122,6 +122,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private final ScmBlockLocationProtocol scmBlockClient;
private final StorageContainerLocationProtocol scmContainerClient;
private ObjectName omInfoBeanName;
+ private final S3BucketManager s3BucketManager;
private OzoneManager(OzoneConfiguration conf) throws IOException {
Preconditions.checkNotNull(conf);
@@ -159,6 +160,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metadataManager = new OmMetadataManagerImpl(configuration);
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
bucketManager = new BucketManagerImpl(metadataManager);
+ s3BucketManager = new S3BucketManagerImpl(configuration, metadataManager,
+ volumeManager, bucketManager);
metrics = OMMetrics.create();
keyManager =
new KeyManagerImpl(scmBlockClient, metadataManager, configuration,
@@ -1128,6 +1131,26 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return services;
}
+ @Override
+ /**
+ * {@inheritDoc}
+ */
+ public void createS3Bucket(String userName, String s3BucketName)
+ throws IOException {
+ s3BucketManager.createS3Bucket(userName, s3BucketName);
+ }
+
+ @Override
+ /**
+ * {@inheritDoc}
+ */
+ public String getOzoneBucketMapping(String s3BucketName)
+ throws IOException {
+ return s3BucketManager.getOzoneBucketMapping(s3BucketName);
+ }
+
+
+
/**
* Startup options.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
index 907f054..89c529a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
@@ -74,6 +74,10 @@ public class S3BucketManagerImpl implements S3BucketManager {
Preconditions.checkArgument(Strings.isNotBlank(userName), "User name " +
"cannot be null or empty.");
+ Preconditions.checkArgument(bucketName.length() >=3 &&
+ bucketName.length() < 64, "Length of the S3 Bucket is not correct.");
+
+
// TODO: Decide if we want to enforce S3 Bucket Creation Rules in this
// code path?
// https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
@@ -105,6 +109,7 @@ public class S3BucketManagerImpl implements S3BucketManager {
bucketName.getBytes(StandardCharsets.UTF_8));
if (bucket != null) {
+ LOG.debug("Bucket already exists. {}", bucketName);
throw new OMException(
"Unable to create S3 bucket. " + bucketName + " already exists.",
OMException.ResultCodes.S3_BUCKET_ALREADY_EXISTS);
@@ -165,6 +170,12 @@ public class S3BucketManagerImpl implements S3BucketManager {
@Override
public String getOzoneBucketMapping(String s3BucketName) throws IOException {
+ Preconditions.checkArgument(
+ Strings.isNotBlank(s3BucketName),
+ "Bucket name cannot be null or empty.");
+ Preconditions.checkArgument(s3BucketName.length() >=3 &&
+ s3BucketName.length() < 64,
+ "Length of the S3 Bucket is not correct.");
omMetadataManager.getLock().acquireS3Lock(s3BucketName);
try {
byte[] mapping =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7ba48c/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index 06d782b..72ad9cb 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.protocolPB;
import com.google.common.collect.Lists;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -29,80 +31,86 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.AllocateBlockRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.AllocateBlockResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CommitKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CommitKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CreateBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CreateBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.InfoBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.InfoBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.SetBucketPropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.SetBucketPropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.DeleteBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.DeleteBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CreateVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CreateVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.LocateKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.LocateKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.RenameKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.RenameKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.KeyArgs;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.SetVolumePropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.SetVolumePropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.CheckVolumeAccessResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.InfoVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.InfoVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.DeleteVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.DeleteVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ListVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ListVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ListBucketsRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ListBucketsResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ListKeysRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ListKeysResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.Status;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ServiceListRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ServiceListResponse;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .AllocateBlockRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .AllocateBlockResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CheckVolumeAccessRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CheckVolumeAccessResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CommitKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CommitKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateBucketRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateBucketResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateVolumeRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .CreateVolumeResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .DeleteBucketRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .DeleteBucketResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .DeleteVolumeRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .DeleteVolumeResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .InfoBucketRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .InfoBucketResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .InfoVolumeRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .InfoVolumeResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .KeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ListBucketsRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ListBucketsResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ListKeysRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ListKeysResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ListVolumeRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ListVolumeResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .LocateKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .LocateKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .RenameKeyResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3BucketInfoRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3BucketInfoResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3BucketRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3BucketResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ServiceListRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .ServiceListResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .SetBucketPropertyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .SetBucketPropertyResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .SetVolumePropertyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .SetVolumePropertyResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -170,6 +178,10 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
return Status.OM_NOT_INITIALIZED;
case SCM_VERSION_MISMATCH_ERROR:
return Status.SCM_VERSION_MISMATCH_ERROR;
+ case S3_BUCKET_ALREADY_EXISTS:
+ return Status.S3_BUCKET_ALREADY_EXISTS;
+ case S3_BUCKET_NOT_FOUND:
+ return Status.S3_BUCKET_NOT_FOUND;
default:
return Status.INTERNAL_ERROR;
}
@@ -570,4 +582,31 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
}
return resp.build();
}
+
+ @Override
+ public S3BucketResponse createS3Bucket(RpcController controller,
+ S3BucketRequest request) throws ServiceException {
+ S3BucketResponse.Builder resp = S3BucketResponse.newBuilder();
+ try {
+ impl.createS3Bucket(request.getUserName(), request.getS3Bucketname());
+ resp.setStatus(Status.OK);
+ } catch (IOException e) {
+ resp.setStatus(exceptionToResponseStatus(e));
+ }
+ return resp.build();
+ }
+
+ @Override
+ public S3BucketInfoResponse getS3Bucketinfo(RpcController controller,
+ S3BucketInfoRequest request) throws ServiceException {
+ S3BucketInfoResponse.Builder resp = S3BucketInfoResponse.newBuilder();
+ try {
+ resp.setOzoneMapping(
+ impl.getOzoneBucketMapping(request.getS3BucketName()));
+ resp.setStatus(Status.OK);
+ } catch (IOException e) {
+ resp.setStatus(exceptionToResponseStatus(e));
+ }
+ return resp.build();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-8710. Service AM should set a
finite limit on NM container max retries. Contributed by Suma Shivaprasad
Posted by sh...@apache.org.
YARN-8710. Service AM should set a finite limit on NM container max retries. Contributed by Suma Shivaprasad
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aeeb0389
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aeeb0389
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aeeb0389
Branch: refs/heads/HDFS-12943
Commit: aeeb0389a58bf6bb4857bc0f246a63a4bd018d51
Parents: e787d65
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Oct 11 08:48:18 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Thu Oct 11 08:59:17 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/service/conf/YarnServiceConf.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aeeb0389/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 13ed1aa..b9568f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -34,12 +34,13 @@ public class YarnServiceConf {
// Retry settings for container failures
public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
- public static final int DEFAULT_CONTAINER_RETRY_MAX = -1;
+ public static final int DEFAULT_CONTAINER_RETRY_MAX = 10;
public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval-ms";
public static final int DEFAULT_CONTAINER_RETRY_INTERVAL = 30000;
public static final String CONTAINER_FAILURES_VALIDITY_INTERVAL =
"yarn.service.container-failure.validity-interval-ms";
- public static final long DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL = -1;
+ public static final long DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL =
+ 600000;
public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
public static final int DEFAULT_AM_RESTART_MAX = 20;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDDS-608. Access denied exception
for user hdfs while making getContainerWithPipeline call. Contributed by
Mukul Kumar Singh.
Posted by sh...@apache.org.
HDDS-608. Access denied exception for user hdfs while making getContainerWithPipeline call. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90552b1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90552b1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90552b1e
Branch: refs/heads/HDFS-12943
Commit: 90552b1ea8380c1155f7000bd9e4c23e709e6970
Parents: 5ce70e1
Author: Nanda kumar <na...@apache.org>
Authored: Wed Oct 10 20:50:49 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Oct 10 20:50:49 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90552b1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 4eedf1f..3523499 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -187,7 +187,7 @@ public class SCMClientProtocolServer implements
}
}
String remoteUser = getRpcRemoteUsername();
- getScm().checkAdminAccess(remoteUser);
+ getScm().checkAdminAccess(null);
return scm.getContainerManager()
.getContainerWithPipeline(containerID);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDDS-583. SCM returns zero as the
return code,
even when invalid options are passed. Contributed by Namit Maheshwari.
Posted by sh...@apache.org.
HDDS-583. SCM returns zero as the return code, even when invalid options are passed. Contributed by Namit Maheshwari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a397393
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a397393
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a397393
Branch: refs/heads/HDFS-12943
Commit: 6a39739316795a4828833e99d78aadc684270f98
Parents: 5b7ba48
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Tue Oct 9 15:21:02 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Tue Oct 9 15:21:19 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/server/StorageContainerManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a397393/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index efd5fc5..ce2725f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -513,7 +513,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private static StartupOption parseArguments(String[] args) {
int argsLen = (args == null) ? 0 : args.length;
- StartupOption startOpt = StartupOption.HELP;
+ StartupOption startOpt = null;
if (argsLen == 0) {
startOpt = StartupOption.REGULAR;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: MAPREDUCE-7149. Javadocs for
FileInputFormat and OutputFormat to mention DT collection. Contributed by
Steve Loughran.
Posted by sh...@apache.org.
MAPREDUCE-7149. Javadocs for FileInputFormat and OutputFormat to mention DT collection.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/735a36af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/735a36af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/735a36af
Branch: refs/heads/HDFS-12943
Commit: 735a36afd528862d7690bee2d71b2f60dd282d21
Parents: 96d28b4
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 20:01:42 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 20:01:42 2018 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapred/FileInputFormat.java | 8 ++++++--
.../main/java/org/apache/hadoop/mapred/OutputFormat.java | 3 +++
.../java/org/apache/hadoop/mapreduce/OutputFormat.java | 3 +++
.../hadoop/mapreduce/lib/input/FileInputFormat.java | 10 +++++++---
4 files changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index fe43991..afdc0ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -198,11 +198,15 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
}
}
- /** List input directories.
+ /**
+ * List input directories.
* Subclasses may override to, e.g., select only files matching a regular
* expression.
*
- * @param job the job to list input paths for
+ * If security is enabled, this method collects
+ * delegation tokens from the input paths and adds them to the job's
+ * credentials.
+ * @param job the job to list input paths for and attach tokens to.
* @return array of FileStatus objects
* @throws IOException if zero items.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
index 6ea6a71..adfda0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
@@ -71,6 +71,9 @@ public interface OutputFormat<K, V> {
* throwing an exception when it already exists, so that output is not
* overwritten.</p>
*
+ * Implementations which write to filesystems which support delegation
+ * tokens usually collect the tokens for the destination path(s)
+ * and attach them to the job configuration.
* @param ignored
* @param job job configuration.
* @throws IOException when output should not be attempted
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
index 6f9e525..901d5ee 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
@@ -66,6 +66,9 @@ public abstract class OutputFormat<K, V> {
* throwing an exception when it already exists, so that output is not
* overwritten.</p>
*
+ * Implementations which write to filesystems which support delegation
+ * tokens usually collect the tokens for the destination path(s)
+ * and attach them to the job context's JobConf.
* @param context information about the job
* @throws IOException when output should not be attempted
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index e2d8e6f..e2658ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -233,11 +233,15 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
(PathFilter) ReflectionUtils.newInstance(filterClass, conf) : null;
}
- /** List input directories.
+ /**
+ * List input directories.
* Subclasses may override to, e.g., select only files matching a regular
* expression.
- *
- * @param job the job to list input paths for
+ *
+ * If security is enabled, this method collects
+ * delegation tokens from the input paths and adds them to the job's
+ * credentials.
+ * @param job the job to list input paths for and attach tokens to.
* @return array of FileStatus objects
* @throws IOException if zero items.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDDS-550. Serialize
ApplyTransaction calls per Container in ContainerStateMachine. Contributed by
Shashikant Banerjee.
Posted by sh...@apache.org.
HDDS-550. Serialize ApplyTransaction calls per Container in ContainerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bc6d048
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bc6d048
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bc6d048
Branch: refs/heads/HDFS-12943
Commit: 0bc6d0484a1c3652234b76e4f99e2ef99173796d
Parents: 8d19818
Author: Jitendra Pandey <ji...@apache.org>
Authored: Thu Oct 11 16:59:59 2018 -0700
Committer: Jitendra Pandey <ji...@apache.org>
Committed: Thu Oct 11 17:12:47 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 6 +-
.../hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hdds/scm/storage/ChunkInputStream.java | 2 +-
.../hdds/scm/storage/ChunkOutputStream.java | 3 +-
hadoop-hdds/common/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-
.../scm/storage/ContainerProtocolCalls.java | 2 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 6 +-
.../org/apache/hadoop/utils/RocksDBStore.java | 3 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 3 +-
.../main/java/org/apache/ratis/RatisHelper.java | 14 +-
.../main/proto/DatanodeContainerProtocol.proto | 1 -
.../common/src/main/resources/ozone-default.xml | 8 +
.../transport/server/GrpcXceiverService.java | 2 +-
.../transport/server/XceiverServerGrpc.java | 8 +-
.../server/ratis/ContainerStateMachine.java | 440 +++++--------------
.../server/ratis/XceiverServerRatis.java | 15 +-
.../container/keyvalue/helpers/ChunkUtils.java | 2 +-
.../keyvalue/helpers/SmallFileUtils.java | 2 +-
.../background/BlockDeletingService.java | 2 +-
.../replication/GrpcReplicationClient.java | 6 +-
.../replication/GrpcReplicationService.java | 4 +-
.../common/impl/TestHddsDispatcher.java | 2 +-
.../ozone/client/io/ChunkGroupOutputStream.java | 5 +
.../rpc/TestCloseContainerHandlingByClient.java | 1 -
.../ozone/container/ContainerTestHelper.java | 3 +-
.../common/impl/TestCloseContainerHandler.java | 2 +-
.../container/server/TestContainerServer.java | 1 -
.../server/TestContainerStateMachine.java | 201 ---------
.../genesis/BenchMarkDatanodeDispatcher.java | 2 +-
hadoop-project/pom.xml | 2 +-
31 files changed, 192 insertions(+), 570 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index d353e7a..2f11872 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -31,9 +31,9 @@ import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.util.Time;
-import org.apache.ratis.shaded.io.grpc.ManagedChannel;
-import org.apache.ratis.shaded.io.grpc.netty.NettyChannelBuilder;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
+import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 0d301d9..4efe7ba 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.ratis.retry.RetryPolicy;
-import org.apache.ratis.shaded.com.google.protobuf
+import org.apache.ratis.thirdparty.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -39,7 +39,7 @@ import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.util.CheckedBiConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
index a483197..21b8974 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdds.scm.storage;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
index cc1ea8d..4547163 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
@@ -18,8 +18,9 @@
package org.apache.hadoop.hdds.scm.storage;
+
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index eea2264..bf2a6b9 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -172,10 +172,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<phase>generate-sources</phase>
<configuration>
<tasks>
- <replace token="com.google.protobuf" value="org.apache.ratis.shaded.com.google.protobuf"
+ <replace token="com.google.protobuf" value="org.apache.ratis.thirdparty.com.google.protobuf"
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
</replace>
- <replace token="io.grpc" value="org.apache.ratis.shaded.io.grpc"
+ <replace token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc"
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
</replace>
</tasks>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 63f5916..71d6e07 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.ratis.shaded.proto.RaftProtos.ReplicationLevel;
+import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
import org.apache.ratis.util.TimeDuration;
import java.util.concurrent.TimeUnit;
@@ -62,6 +62,10 @@ public final class ScmConfigKeys {
= "dfs.container.ratis.replication.level";
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT = ReplicationLevel.MAJORITY;
+ public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
+ = "dfs.container.ratis.num.container.op.threads";
+ public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
+ = 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
index 1df50b1..278b129 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdds.scm.storage;
import org.apache.hadoop.hdds.scm.container.common.helpers
.BlockNotCommittedException;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index e8aa22c..d6a6bf7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.ratis.shaded.proto.RaftProtos.ReplicationLevel;
+import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
import org.apache.ratis.util.TimeDuration;
/**
@@ -220,6 +220,10 @@ public final class OzoneConfigKeys {
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT;
+ public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
+ = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY;
+ public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
+ = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
index 379d9e9..377153a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
@@ -22,7 +22,8 @@ import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.ratis.thirdparty.com.google.common.annotations.
+ VisibleForTesting;
import org.rocksdb.DbPath;
import org.rocksdb.Options;
import org.rocksdb.RocksDB;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
index 5078b3e..cdee10b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
@@ -23,7 +23,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.utils.RocksDBStoreMBean;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.ratis.thirdparty.com.google.common.annotations.
+ VisibleForTesting;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 04bfeb2..2dbe2e6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -32,8 +32,8 @@ import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.retry.RetryPolicies;
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.proto.RaftProtos;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.util.Preconditions;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
@@ -103,7 +103,7 @@ public interface RatisHelper {
RaftGroupId DUMMY_GROUP_ID =
RaftGroupId.valueOf(ByteString.copyFromUtf8("AOzoneRatisGroup"));
- RaftGroup EMPTY_GROUP = new RaftGroup(DUMMY_GROUP_ID,
+ RaftGroup EMPTY_GROUP = RaftGroup.valueOf(DUMMY_GROUP_ID,
Collections.emptyList());
static RaftGroup emptyRaftGroup() {
@@ -112,7 +112,7 @@ public interface RatisHelper {
static RaftGroup newRaftGroup(Collection<RaftPeer> peers) {
return peers.isEmpty()? emptyRaftGroup()
- : new RaftGroup(DUMMY_GROUP_ID, peers);
+ : RaftGroup.valueOf(DUMMY_GROUP_ID, peers);
}
static RaftGroup newRaftGroup(RaftGroupId groupId,
@@ -120,12 +120,12 @@ public interface RatisHelper {
final List<RaftPeer> newPeers = peers.stream()
.map(RatisHelper::toRaftPeer)
.collect(Collectors.toList());
- return peers.isEmpty() ? new RaftGroup(groupId, Collections.emptyList())
- : new RaftGroup(groupId, newPeers);
+ return peers.isEmpty() ? RaftGroup.valueOf(groupId, Collections.emptyList())
+ : RaftGroup.valueOf(groupId, newPeers);
}
static RaftGroup newRaftGroup(Pipeline pipeline) {
- return new RaftGroup(pipeline.getId().getRaftGroupID(),
+ return RaftGroup.valueOf(pipeline.getId().getRaftGroupID(),
toRaftPeers(pipeline));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 4567750..662df8f 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -197,7 +197,6 @@ message ContainerCommandRequestProto {
optional PutSmallFileRequestProto putSmallFile = 19;
optional GetSmallFileRequestProto getSmallFile = 20;
-
optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 21;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index ef54ed2..0fde6bb 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -136,6 +136,14 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.num.container.op.executors</name>
+ <value>10</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>Number of executors that will be used by Ratis to execute
+ container ops.(10 by default).
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.segment.size</name>
<value>1073741824</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
index db4a86a..5fc3661 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto
.XceiverClientProtocolServiceGrpc;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index c51da98..8ebfe49 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -33,10 +33,10 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.ratis.shaded.io.grpc.BindableService;
-import org.apache.ratis.shaded.io.grpc.Server;
-import org.apache.ratis.shaded.io.grpc.ServerBuilder;
-import org.apache.ratis.shaded.io.grpc.netty.NettyServerBuilder;
+import org.apache.ratis.thirdparty.io.grpc.BindableService;
+import org.apache.ratis.thirdparty.io.grpc.Server;
+import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
+import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index f07c95f..7b7be91 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -26,7 +25,7 @@ import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.shaded.com.google.protobuf
+import org.apache.ratis.thirdparty.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Stage;
@@ -44,10 +43,10 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.protocol.Message;
import org.apache.ratis.protocol.RaftClientRequest;
import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.proto.RaftProtos.RoleInfoProto;
-import org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto;
-import org.apache.ratis.shaded.proto.RaftProtos.SMLogEntryProto;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
+import org.apache.ratis.proto.RaftProtos.LogEntryProto;
+import org.apache.ratis.proto.RaftProtos.SMLogEntryProto;
import org.apache.ratis.statemachine.StateMachineStorage;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.statemachine.impl.BaseStateMachine;
@@ -57,12 +56,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
/** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
@@ -98,44 +97,43 @@ import java.util.stream.Collectors;
*
* 2) Write chunk commit operation is executed after write chunk state machine
* operation. This will ensure that commit operation is sync'd with the state
- * machine operation.
- *
- * Synchronization between {@link #writeStateMachineData} and
- * {@link #applyTransaction} need to be enforced in the StateMachine
- * implementation. For example, synchronization between writeChunk and
+ * machine operation.For example, synchronization between writeChunk and
* createContainer in {@link ContainerStateMachine}.
- *
- * PutBlock is synchronized with WriteChunk operations, PutBlock for a block is
- * executed only after all the WriteChunk preceding the PutBlock have finished.
- *
- * CloseContainer is synchronized with WriteChunk and PutBlock operations,
- * CloseContainer for a container is processed after all the preceding write
- * operations for the container have finished.
- * */
+ **/
+
public class ContainerStateMachine extends BaseStateMachine {
- static final Logger LOG = LoggerFactory.getLogger(
- ContainerStateMachine.class);
- private final SimpleStateMachineStorage storage
- = new SimpleStateMachineStorage();
+ static final Logger LOG =
+ LoggerFactory.getLogger(ContainerStateMachine.class);
+ private final SimpleStateMachineStorage storage =
+ new SimpleStateMachineStorage();
private final ContainerDispatcher dispatcher;
private ThreadPoolExecutor chunkExecutor;
private final XceiverServerRatis ratisServer;
private final ConcurrentHashMap<Long, CompletableFuture<Message>>
writeChunkFutureMap;
- private final ConcurrentHashMap<Long, StateMachineHelper> stateMachineMap;
+ private final ConcurrentHashMap<Long, CompletableFuture<Message>>
+ createContainerFutureMap;
+ private ExecutorService[] executors;
+ private final int numExecutors;
/**
* CSM metrics.
*/
private final CSMMetrics metrics;
public ContainerStateMachine(ContainerDispatcher dispatcher,
- ThreadPoolExecutor chunkExecutor, XceiverServerRatis ratisServer) {
+ ThreadPoolExecutor chunkExecutor, XceiverServerRatis ratisServer,
+ int numOfExecutors) {
this.dispatcher = dispatcher;
this.chunkExecutor = chunkExecutor;
this.ratisServer = ratisServer;
this.writeChunkFutureMap = new ConcurrentHashMap<>();
- this.stateMachineMap = new ConcurrentHashMap<>();
metrics = CSMMetrics.create();
+ this.createContainerFutureMap = new ConcurrentHashMap<>();
+ this.numExecutors = numOfExecutors;
+ executors = new ExecutorService[numExecutors];
+ for (int i = 0; i < numExecutors; i++) {
+ executors[i] = Executors.newSingleThreadExecutor();
+ }
}
@Override
@@ -229,6 +227,41 @@ public class ContainerStateMachine extends BaseStateMachine {
return dispatchCommand(requestProto)::toByteString;
}
+ private ExecutorService getCommandExecutor(
+ ContainerCommandRequestProto requestProto) {
+ int executorId = (int)(requestProto.getContainerID() % numExecutors);
+ return executors[executorId];
+ }
+
+ private CompletableFuture<Message> handleWriteChunk(
+ ContainerCommandRequestProto requestProto, long entryIndex) {
+ final WriteChunkRequestProto write = requestProto.getWriteChunk();
+ long containerID = write.getBlockID().getContainerID();
+ CompletableFuture<Message> future =
+ createContainerFutureMap.get(containerID);
+ CompletableFuture<Message> writeChunkFuture;
+ if (future != null) {
+ writeChunkFuture = future.thenApplyAsync(
+ v -> runCommand(requestProto), chunkExecutor);
+ } else {
+ writeChunkFuture = CompletableFuture.supplyAsync(
+ () -> runCommand(requestProto), chunkExecutor);
+ }
+ writeChunkFutureMap.put(entryIndex, writeChunkFuture);
+ // Remove the future once it finishes execution from the
+ // writeChunkFutureMap.
+ writeChunkFuture.thenApply(r -> writeChunkFutureMap.remove(entryIndex));
+ return writeChunkFuture;
+ }
+
+ private CompletableFuture<Message> handleCreateContainer(
+ ContainerCommandRequestProto requestProto) {
+ long containerID = requestProto.getContainerID();
+ createContainerFutureMap.
+ computeIfAbsent(containerID, k -> new CompletableFuture<>());
+ return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
+ }
+
/*
* writeStateMachineData calls are not synchronized with each other
* and also with applyTransaction.
@@ -240,17 +273,15 @@ public class ContainerStateMachine extends BaseStateMachine {
final ContainerCommandRequestProto requestProto =
getRequestProto(entry.getSmLogEntry().getStateMachineData());
Type cmdType = requestProto.getCmdType();
- long containerId = requestProto.getContainerID();
- stateMachineMap
- .computeIfAbsent(containerId, k -> new StateMachineHelper());
- CompletableFuture<Message> stateMachineFuture =
- stateMachineMap.get(containerId)
- .handleStateMachineData(requestProto, entry.getIndex());
- if (stateMachineFuture == null) {
- throw new IllegalStateException(
- "Cmd Type:" + cmdType + " should not have state machine data");
+ switch (cmdType) {
+ case CreateContainer:
+ return handleCreateContainer(requestProto);
+ case WriteChunk:
+ return handleWriteChunk(requestProto, entry.getIndex());
+ default:
+ throw new IllegalStateException("Cmd Type:" + cmdType
+ + " should not have state machine data");
}
- return stateMachineFuture;
} catch (IOException e) {
metrics.incNumWriteStateMachineFails();
return completeExceptionally(e);
@@ -270,14 +301,14 @@ public class ContainerStateMachine extends BaseStateMachine {
}
}
- private LogEntryProto readStateMachineData(LogEntryProto entry,
+ private ByteString readStateMachineData(LogEntryProto entry,
ContainerCommandRequestProto requestProto) {
WriteChunkRequestProto writeChunkRequestProto =
requestProto.getWriteChunk();
// Assert that store log entry is for COMMIT_DATA, the WRITE_DATA is
// written through writeStateMachineData.
- Preconditions.checkArgument(writeChunkRequestProto.getStage()
- == Stage.COMMIT_DATA);
+ Preconditions
+ .checkArgument(writeChunkRequestProto.getStage() == Stage.COMMIT_DATA);
// prepare the chunk to be read
ReadChunkRequestProto.Builder readChunkRequestProto =
@@ -286,8 +317,7 @@ public class ContainerStateMachine extends BaseStateMachine {
.setChunkData(writeChunkRequestProto.getChunkData());
ContainerCommandRequestProto dataContainerCommandProto =
ContainerCommandRequestProto.newBuilder(requestProto)
- .setCmdType(Type.ReadChunk)
- .setReadChunk(readChunkRequestProto)
+ .setCmdType(Type.ReadChunk).setReadChunk(readChunkRequestProto)
.build();
// read the chunk
@@ -302,25 +332,13 @@ public class ContainerStateMachine extends BaseStateMachine {
final WriteChunkRequestProto.Builder dataWriteChunkProto =
WriteChunkRequestProto.newBuilder(writeChunkRequestProto)
// adding the state machine data
- .setData(responseProto.getData())
- .setStage(Stage.WRITE_DATA);
+ .setData(responseProto.getData()).setStage(Stage.WRITE_DATA);
ContainerCommandRequestProto.Builder newStateMachineProto =
ContainerCommandRequestProto.newBuilder(requestProto)
.setWriteChunk(dataWriteChunkProto);
- return recreateLogEntryProto(entry,
- newStateMachineProto.build().toByteString());
- }
-
- private LogEntryProto recreateLogEntryProto(LogEntryProto entry,
- ByteString stateMachineData) {
- // recreate the log entry
- final SMLogEntryProto log =
- SMLogEntryProto.newBuilder(entry.getSmLogEntry())
- .setStateMachineData(stateMachineData)
- .build();
- return LogEntryProto.newBuilder(entry).setSmLogEntry(log).build();
+ return newStateMachineProto.build().toByteString();
}
/**
@@ -347,11 +365,11 @@ public class ContainerStateMachine extends BaseStateMachine {
* evicted.
*/
@Override
- public CompletableFuture<LogEntryProto> readStateMachineData(
+ public CompletableFuture<ByteString> readStateMachineData(
LogEntryProto entry) {
SMLogEntryProto smLogEntryProto = entry.getSmLogEntry();
if (!smLogEntryProto.getStateMachineData().isEmpty()) {
- return CompletableFuture.completedFuture(entry);
+ return CompletableFuture.completedFuture(ByteString.EMPTY);
}
try {
@@ -365,9 +383,7 @@ public class ContainerStateMachine extends BaseStateMachine {
readStateMachineData(entry, requestProto),
chunkExecutor);
} else if (requestProto.getCmdType() == Type.CreateContainer) {
- LogEntryProto log =
- recreateLogEntryProto(entry, requestProto.toByteString());
- return CompletableFuture.completedFuture(log);
+ return CompletableFuture.completedFuture(requestProto.toByteString());
} else {
throw new IllegalStateException("Cmd type:" + requestProto.getCmdType()
+ " cannot have state machine data");
@@ -387,13 +403,44 @@ public class ContainerStateMachine extends BaseStateMachine {
metrics.incNumApplyTransactionsOps();
ContainerCommandRequestProto requestProto =
getRequestProto(trx.getSMLogEntry().getData());
- Preconditions.checkState(!HddsUtils.isReadOnly(requestProto));
- stateMachineMap.computeIfAbsent(requestProto.getContainerID(),
- k -> new StateMachineHelper());
- long index =
- trx.getLogEntry() == null ? -1 : trx.getLogEntry().getIndex();
- return stateMachineMap.get(requestProto.getContainerID())
- .executeContainerCommand(requestProto, index);
+ Type cmdType = requestProto.getCmdType();
+ CompletableFuture<Message> future;
+ if (cmdType == Type.PutBlock) {
+ BlockData blockData;
+ ContainerProtos.BlockData blockDataProto =
+ requestProto.getPutBlock().getBlockData();
+
+ // set the blockCommitSequenceId
+ try {
+ blockData = BlockData.getFromProtoBuf(blockDataProto);
+ } catch (IOException ioe) {
+ LOG.error("unable to retrieve blockData info for Block {}",
+ blockDataProto.getBlockID());
+ return completeExceptionally(ioe);
+ }
+ blockData.setBlockCommitSequenceId(trx.getLogEntry().getIndex());
+ final ContainerProtos.PutBlockRequestProto putBlockRequestProto =
+ ContainerProtos.PutBlockRequestProto
+ .newBuilder(requestProto.getPutBlock())
+ .setBlockData(blockData.getProtoBufMessage()).build();
+ ContainerCommandRequestProto containerCommandRequestProto =
+ ContainerCommandRequestProto.newBuilder(requestProto)
+ .setPutBlock(putBlockRequestProto).build();
+ future = CompletableFuture
+ .supplyAsync(() -> runCommand(containerCommandRequestProto),
+ getCommandExecutor(requestProto));
+ } else {
+ future = CompletableFuture.supplyAsync(() -> runCommand(requestProto),
+ getCommandExecutor(requestProto));
+ }
+ // Mark the createContainerFuture complete so that writeStateMachineData
+ // for WriteChunk gets unblocked
+ if (cmdType == Type.CreateContainer) {
+ long containerID = requestProto.getContainerID();
+ future.thenApply(
+ r -> createContainerFutureMap.remove(containerID).complete(null));
+ }
+ return future;
} catch (IOException e) {
metrics.incNumApplyTransactionsFails();
return completeExceptionally(e);
@@ -419,259 +466,8 @@ public class ContainerStateMachine extends BaseStateMachine {
@Override
public void close() throws IOException {
- }
-
- /**
- * Class to manage the future tasks for writeChunks.
- */
- static class CommitChunkFutureMap {
- private final ConcurrentHashMap<Long, CompletableFuture<Message>>
- block2ChunkMap = new ConcurrentHashMap<>();
-
- synchronized int removeAndGetSize(long index) {
- block2ChunkMap.remove(index);
- return block2ChunkMap.size();
+ for (int i = 0; i < numExecutors; i++){
+ executors[i].shutdown();
}
-
- synchronized CompletableFuture<Message> add(long index,
- CompletableFuture<Message> future) {
- return block2ChunkMap.put(index, future);
- }
-
- synchronized List<CompletableFuture<Message>> getAll() {
- return new ArrayList<>(block2ChunkMap.values());
- }
- }
-
- /**
- * This class maintains maps and provide utilities to enforce synchronization
- * among createContainer, writeChunk, putBlock and closeContainer.
- */
- private class StateMachineHelper {
-
- private CompletableFuture<Message> createContainerFuture;
-
- // Map for maintaining all writeChunk futures mapped to blockId
- private final ConcurrentHashMap<Long, CommitChunkFutureMap>
- block2ChunkMap;
-
- // Map for putBlock futures
- private final ConcurrentHashMap<Long, CompletableFuture<Message>>
- blockCommitMap;
-
- StateMachineHelper() {
- createContainerFuture = null;
- block2ChunkMap = new ConcurrentHashMap<>();
- blockCommitMap = new ConcurrentHashMap<>();
- }
-
- // The following section handles writeStateMachineData transactions
- // on a container
-
- // enqueue the create container future during writeStateMachineData
- // so that the write stateMachine data phase of writeChunk wait on
- // create container to finish.
- private CompletableFuture<Message> handleCreateContainer() {
- createContainerFuture = new CompletableFuture<>();
- return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
- }
-
- // This synchronizes on create container to finish
- private CompletableFuture<Message> handleWriteChunk(
- ContainerCommandRequestProto requestProto, long entryIndex) {
- CompletableFuture<Message> containerOpFuture;
-
- if (createContainerFuture != null) {
- containerOpFuture = createContainerFuture
- .thenApplyAsync(v -> runCommand(requestProto), chunkExecutor);
- } else {
- containerOpFuture = CompletableFuture
- .supplyAsync(() -> runCommand(requestProto), chunkExecutor);
- }
- writeChunkFutureMap.put(entryIndex, containerOpFuture);
- return containerOpFuture;
- }
-
- CompletableFuture<Message> handleStateMachineData(
- final ContainerCommandRequestProto requestProto, long index) {
- Type cmdType = requestProto.getCmdType();
- if (cmdType == Type.CreateContainer) {
- return handleCreateContainer();
- } else if (cmdType == Type.WriteChunk) {
- return handleWriteChunk(requestProto, index);
- } else {
- return null;
- }
- }
-
- // The following section handles applyTransaction transactions
- // on a container
-
- private CompletableFuture<Message> handlePutBlock(
- ContainerCommandRequestProto requestProto, long index) {
- List<CompletableFuture<Message>> futureList = new ArrayList<>();
- BlockData blockData = null;
- ContainerProtos.BlockData blockDataProto =
- requestProto.getPutBlock().getBlockData();
-
- // set the blockCommitSequenceId
- try {
- blockData = BlockData.getFromProtoBuf(blockDataProto);
- } catch (IOException ioe) {
- LOG.error("unable to retrieve blockData info for Block {}",
- blockDataProto.getBlockID());
- return completeExceptionally(ioe);
- }
- blockData.setBlockCommitSequenceId(index);
- final ContainerProtos.PutBlockRequestProto putBlockRequestProto =
- ContainerProtos.PutBlockRequestProto
- .newBuilder(requestProto.getPutBlock())
- .setBlockData(blockData.getProtoBufMessage()).build();
- ContainerCommandRequestProto containerCommandRequestProto =
- ContainerCommandRequestProto.newBuilder(requestProto)
- .setPutBlock(putBlockRequestProto).build();
- long localId = blockDataProto.getBlockID().getLocalID();
- // Need not wait for create container future here as it has already
- // finished.
- if (block2ChunkMap.get(localId) != null) {
- futureList.addAll(block2ChunkMap.get(localId).getAll());
- }
- CompletableFuture<Message> effectiveFuture =
- runCommandAfterFutures(futureList, containerCommandRequestProto);
-
- CompletableFuture<Message> putBlockFuture =
- effectiveFuture.thenApply(message -> {
- blockCommitMap.remove(localId);
- return message;
- });
- blockCommitMap.put(localId, putBlockFuture);
- return putBlockFuture;
- }
-
- // Close Container should be executed only if all pending WriteType
- // container cmds get executed. Transactions which can return a future
- // are WriteChunk and PutBlock.
- private CompletableFuture<Message> handleCloseContainer(
- ContainerCommandRequestProto requestProto) {
- List<CompletableFuture<Message>> futureList = new ArrayList<>();
-
- // No need to wait for create container future here as it should have
- // already finished.
- block2ChunkMap.values().forEach(b -> futureList.addAll(b.getAll()));
- futureList.addAll(blockCommitMap.values());
-
- // There are pending write Chunk/PutBlock type requests
- // Queue this closeContainer request behind all these requests
- CompletableFuture<Message> closeContainerFuture =
- runCommandAfterFutures(futureList, requestProto);
-
- return closeContainerFuture.thenApply(message -> {
- stateMachineMap.remove(requestProto.getContainerID());
- return message;
- });
- }
-
- private CompletableFuture<Message> handleChunkCommit(
- ContainerCommandRequestProto requestProto, long index) {
- WriteChunkRequestProto write = requestProto.getWriteChunk();
- // the data field has already been removed in start Transaction
- Preconditions.checkArgument(!write.hasData());
- CompletableFuture<Message> stateMachineFuture =
- writeChunkFutureMap.remove(index);
- CompletableFuture<Message> commitChunkFuture = stateMachineFuture
- .thenComposeAsync(v -> CompletableFuture
- .completedFuture(runCommand(requestProto)));
-
- long localId = requestProto.getWriteChunk().getBlockID().getLocalID();
- // Put the applyTransaction Future again to the Map.
- // closeContainer should synchronize with this.
- block2ChunkMap
- .computeIfAbsent(localId, id -> new CommitChunkFutureMap())
- .add(index, commitChunkFuture);
- return commitChunkFuture.thenApply(message -> {
- block2ChunkMap.computeIfPresent(localId, (containerId, chunks)
- -> chunks.removeAndGetSize(index) == 0? null: chunks);
- return message;
- });
- }
-
- private CompletableFuture<Message> runCommandAfterFutures(
- List<CompletableFuture<Message>> futureList,
- ContainerCommandRequestProto requestProto) {
- CompletableFuture<Message> effectiveFuture;
- if (futureList.isEmpty()) {
- effectiveFuture = CompletableFuture
- .supplyAsync(() -> runCommand(requestProto));
-
- } else {
- CompletableFuture<Void> allFuture = CompletableFuture.allOf(
- futureList.toArray(new CompletableFuture[futureList.size()]));
- effectiveFuture = allFuture
- .thenApplyAsync(v -> runCommand(requestProto));
- }
- return effectiveFuture;
- }
-
- CompletableFuture<Message> handleCreateContainer(
- ContainerCommandRequestProto requestProto) {
- CompletableFuture<Message> future =
- CompletableFuture.completedFuture(runCommand(requestProto));
- future.thenAccept(m -> {
- createContainerFuture.complete(m);
- createContainerFuture = null;
- });
- return future;
- }
-
- CompletableFuture<Message> handleOtherCommands(
- ContainerCommandRequestProto requestProto) {
- return CompletableFuture.completedFuture(runCommand(requestProto));
- }
-
- CompletableFuture<Message> executeContainerCommand(
- ContainerCommandRequestProto requestProto, long index) {
- Type cmdType = requestProto.getCmdType();
- switch (cmdType) {
- case WriteChunk:
- return handleChunkCommit(requestProto, index);
- case CloseContainer:
- return handleCloseContainer(requestProto);
- case PutBlock:
- return handlePutBlock(requestProto, index);
- case CreateContainer:
- return handleCreateContainer(requestProto);
- default:
- return handleOtherCommands(requestProto);
- }
- }
- }
-
- @VisibleForTesting
- public ConcurrentHashMap<Long, StateMachineHelper> getStateMachineMap() {
- return stateMachineMap;
- }
-
- @VisibleForTesting
- public CompletableFuture<Message> getCreateContainerFuture(long containerId) {
- StateMachineHelper helper = stateMachineMap.get(containerId);
- return helper == null ? null : helper.createContainerFuture;
- }
-
- @VisibleForTesting
- public List<CompletableFuture<Message>> getCommitChunkFutureMap(
- long containerId) {
- StateMachineHelper helper = stateMachineMap.get(containerId);
- if (helper != null) {
- List<CompletableFuture<Message>> futureList = new ArrayList<>();
- stateMachineMap.get(containerId).block2ChunkMap.values()
- .forEach(b -> futureList.addAll(b.getAll()));
- return futureList;
- }
- return null;
- }
-
- @VisibleForTesting
- public Collection<CompletableFuture<Message>> getWriteChunkFutureMap() {
- return writeChunkFutureMap.values();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index c2ef504..9094217 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -56,9 +56,9 @@ import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.shaded.proto.RaftProtos;
-import org.apache.ratis.shaded.proto.RaftProtos.RoleInfoProto;
-import org.apache.ratis.shaded.proto.RaftProtos.ReplicationLevel;
+import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
+import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
@@ -97,6 +97,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
private final StateContext context;
private final ReplicationLevel replicationLevel;
private long nodeFailureTimeoutMs;
+ private ContainerStateMachine stateMachine;
private XceiverServerRatis(DatanodeDetails dd, int port,
ContainerDispatcher dispatcher, Configuration conf, StateContext context)
@@ -112,12 +113,15 @@ public final class XceiverServerRatis implements XceiverServerSpi {
100, TimeUnit.SECONDS,
new ArrayBlockingQueue<>(1024),
new ThreadPoolExecutor.CallerRunsPolicy());
+ final int numContainerOpExecutors = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT);
this.context = context;
this.replicationLevel =
conf.getEnum(OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT);
- ContainerStateMachine stateMachine =
- new ContainerStateMachine(dispatcher, chunkExecutor, this);
+ stateMachine = new ContainerStateMachine(dispatcher, chunkExecutor, this,
+ numContainerOpExecutors);
this.server = RaftServer.newBuilder()
.setServerId(RatisHelper.toRaftPeerId(dd))
.setProperties(serverProperties)
@@ -292,6 +296,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
public void stop() {
try {
chunkExecutor.shutdown();
+ stateMachine.close();
server.close();
} catch (IOException e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 62e328e..8bdae0f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
index 3495363..dee0c11 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
@@ -21,7 +21,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index d96fbfa..61a303f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoo
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.ratis.shaded.com.google.protobuf
+import org.apache.ratis.thirdparty.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 3aafb0c..c8a40b2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
-import org.apache.ratis.shaded.io.grpc.ManagedChannel;
-import org.apache.ratis.shaded.io.grpc.netty.NettyChannelBuilder;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
+import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
index d8f696f..30a251d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index fc622b2..fc84ae7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 6580c2c9..de666ce 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -712,6 +712,11 @@ public class ChunkGroupOutputStream extends OutputStream {
if (this.outputStream instanceof ChunkOutputStream) {
ChunkOutputStream out = (ChunkOutputStream) this.outputStream;
return out.getBlockCommitSequenceId();
+ } else if (outputStream == null) {
+ // For a pre allocated block for which no write has been initiated,
+ // the OutputStream will be null here.
+ // In such cases, the default blockCommitSequenceId will be 0
+ return 0;
}
throw new IOException("Invalid Output Stream for Key: " + key);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index f606263..da8d334 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -341,7 +341,6 @@ public class TestCloseContainerHandlingByClient {
Assert.assertTrue(key.getOutputStream() instanceof ChunkGroupOutputStream);
// With the initial size provided, it should have pre allocated 4 blocks
Assert.assertEquals(2, groupOutputStream.getStreamEntries().size());
- Assert.assertEquals(2, groupOutputStream.getLocationInfoList().size());
String dataString = fixedLengthString(keyString, (1 * blockSize));
byte[] data = dataString.getBytes();
key.write(data);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index f278479..324187c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -439,6 +439,7 @@ public final class ContainerTestHelper {
List<ContainerProtos.ChunkInfo> newList = new LinkedList<>();
newList.add(writeRequest.getChunkData());
blockData.setChunks(newList);
+ blockData.setBlockCommitSequenceId(0);
putRequest.setBlockData(blockData.getProtoBufMessage());
ContainerCommandRequestProto.Builder request =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
index 78bf008..92bad27 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index de55d9e..e6ebbf1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
-import org.apache.ratis.shaded.io.netty.channel.embedded.EmbeddedChannel;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
deleted file mode 100644
index c875a7e..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.server;
-
-
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerType;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.container.ContainerTestHelper;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis
- .ContainerStateMachine;
-import org.apache.ratis.RatisHelper;
-import org.apache.ratis.protocol.ClientId;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientRequest;
-import org.apache.ratis.shaded.proto.RaftProtos;
-import org.apache.ratis.statemachine.TransactionContext;
-import org.apache.ratis.util.ProtoUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Random;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class tests ContainerStateMachine.
- */
-public class TestContainerStateMachine {
-
- private static final AtomicLong CALL_ID_COUNTER = new AtomicLong();
-
- private static long nextCallId() {
- return CALL_ID_COUNTER.getAndIncrement() & Long.MAX_VALUE;
- }
-
- private ThreadPoolExecutor executor =
- new ThreadPoolExecutor(4, 4, 100, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(1024),
- new ThreadPoolExecutor.CallerRunsPolicy());
- private ContainerStateMachine stateMachine =
- new ContainerStateMachine(new TestContainerDispatcher(), executor, null);
-
-
- @Test
- public void testCloseContainerSynchronization() throws Exception {
- Pipeline pipeline = ContainerTestHelper.createPipeline(3);
- long containerId = new Random().nextLong();
-
- //create container request
- RaftClientRequest createContainer = getRaftClientRequest(
- ContainerTestHelper.getCreateContainerRequest(containerId, pipeline));
-
- ContainerCommandRequestProto writeChunkProto = ContainerTestHelper
- .getWriteChunkRequest(pipeline, new BlockID(containerId, nextCallId()),
- 1024);
-
- RaftClientRequest writeChunkRequest = getRaftClientRequest(writeChunkProto);
-
- // add putKey request
- ContainerCommandRequestProto putKeyProto = ContainerTestHelper
- .getPutBlockRequest(pipeline, writeChunkProto.getWriteChunk());
- RaftClientRequest putKeyRequest = getRaftClientRequest(putKeyProto);
-
- TransactionContext createContainerCtxt =
- startAndWriteStateMachineData(createContainer);
- // Start and Write into the StateMachine
- TransactionContext writeChunkcontext =
- startAndWriteStateMachineData(writeChunkRequest);
-
- TransactionContext putKeyContext =
- stateMachine.startTransaction(putKeyRequest);
- Assert.assertEquals(1, stateMachine.getStateMachineMap().size());
- Assert.assertNotNull(stateMachine.getCreateContainerFuture(containerId));
- Assert.assertEquals(1,
- stateMachine.getWriteChunkFutureMap().size());
- Assert.assertTrue(
- stateMachine.getCommitChunkFutureMap(containerId).isEmpty());
-
- //Add a closeContainerRequest
- RaftClientRequest closeRequest = getRaftClientRequest(
- ContainerTestHelper.getCloseContainer(pipeline, containerId));
-
- TransactionContext closeCtx = stateMachine.startTransaction(closeRequest);
-
- // Now apply all the transaction for the CreateContainer Command.
- // This will unblock writeChunks as well
-
- stateMachine.applyTransaction(createContainerCtxt);
- stateMachine.applyTransaction(writeChunkcontext);
- CompletableFuture<Message> putKeyFuture =
- stateMachine.applyTransaction(putKeyContext);
- waitForTransactionCompletion(putKeyFuture);
- // Make sure the putKey transaction complete
- Assert.assertTrue(putKeyFuture.isDone());
-
- // Execute the closeContainer. This should ensure all prior Write Type
- // container requests finish execution
-
- CompletableFuture<Message> closeFuture =
- stateMachine.applyTransaction(closeCtx);
- waitForTransactionCompletion(closeFuture);
- // Make sure the closeContainer transaction complete
- Assert.assertTrue(closeFuture.isDone());
- Assert.assertNull(stateMachine.getCreateContainerFuture(containerId));
- Assert.assertNull(stateMachine.getCommitChunkFutureMap(containerId));
-
- }
-
- private RaftClientRequest getRaftClientRequest(
- ContainerCommandRequestProto req) throws IOException {
- ClientId clientId = ClientId.randomId();
- return new RaftClientRequest(clientId,
- RatisHelper.toRaftPeerId(ContainerTestHelper.createDatanodeDetails()),
- RatisHelper.emptyRaftGroup().getGroupId(), nextCallId(), 0,
- Message.valueOf(req.toByteString()), RaftClientRequest
- .writeRequestType(RaftProtos.ReplicationLevel.MAJORITY));
- }
-
- private void waitForTransactionCompletion(
- CompletableFuture<Message> future) throws Exception {
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- executorService
- .invokeAll(Collections.singleton(future::get), 10,
- TimeUnit.SECONDS); // Timeout of 10 minutes.
- executorService.shutdown();
- }
-
- private TransactionContext startAndWriteStateMachineData(
- RaftClientRequest request) throws IOException {
- TransactionContext ctx = stateMachine.startTransaction(request);
- RaftProtos.LogEntryProto e = ProtoUtils
- .toLogEntryProto(ctx.getSMLogEntry(), request.getSeqNum(),
- request.getCallId(), ClientId.randomId(), request.getCallId());
- ctx.setLogEntry(e);
- stateMachine.writeStateMachineData(e);
- return ctx;
- }
-
- // ContainerDispatcher for test only purpose.
- private static class TestContainerDispatcher implements ContainerDispatcher {
- /**
- * Dispatches commands to container layer.
- *
- * @param msg - Command Request
- * @return Command Response
- */
- @Override
- public ContainerCommandResponseProto dispatch(
- ContainerCommandRequestProto msg) {
- return ContainerTestHelper.getCreateContainerResponse(msg);
- }
-
- @Override
- public void init() {
- }
-
- @Override
- public void shutdown() {
- }
-
- @Override
- public void setScmId(String scmId) {
- }
-
- @Override
- public Handler getHandler(ContainerType containerType) {
- return null;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 8811d91..362e53f 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine.DatanodeStates;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 5f88371..7472a54 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -103,7 +103,7 @@
<ldap-api.version>1.0.0-M33</ldap-api.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-eca3531-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-9b84d79-SNAPSHOT</ratis.version>
<jcache.version>1.0-alpha-1</jcache.version>
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDDS-609. On restart,
SCM does not exit chill mode as it expects DNs to report containers
in ALLOCATED state. Contributed by Hanisha Koneru.
Posted by sh...@apache.org.
HDDS-609. On restart, SCM does not exit chill mode as it expects DNs to report containers in ALLOCATED state. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74db993a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74db993a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74db993a
Branch: refs/heads/HDFS-12943
Commit: 74db993a61e201d2e8d33bb42f787e0a71388d0d
Parents: c011810
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Thu Oct 11 21:03:01 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Thu Oct 11 21:03:01 2018 -0700
----------------------------------------------------------------------
.../hdds/scm/server/SCMChillModeManager.java | 12 +++--
.../scm/server/TestSCMChillModeManager.java | 53 +++++++++++++++++++-
2 files changed, 61 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74db993a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
index c11a60f..44744f0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -162,7 +163,7 @@ public class SCMChillModeManager implements
// Required cutoff % for containers with at least 1 reported replica.
private double chillModeCutoff;
- // Containers read from scm db.
+ // Containers read from scm db (excluding containers in ALLOCATED state).
private Map<Long, ContainerInfo> containerMap;
private double maxContainer;
@@ -174,11 +175,16 @@ public class SCMChillModeManager implements
containerMap = new ConcurrentHashMap<>();
if(containers != null) {
containers.forEach(c -> {
- if (c != null) {
+ // Containers in ALLOCATED state should not be included while
+ // calculating the total number of containers here. They are not
+ // reported by DNs and hence should not affect the chill mode exit
+ // rule.
+ if (c != null && c.getState() != null &&
+ !c.getState().equals(HddsProtos.LifeCycleState.ALLOCATED)) {
containerMap.put(c.getContainerID(), c);
}
});
- maxContainer = containers.size();
+ maxContainer = containerMap.size();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74db993a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
index 53d76e6..41708a4 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -70,6 +71,11 @@ public class TestSCMChillModeManager {
private void testChillMode(int numContainers) throws Exception {
containers = new ArrayList<>();
containers.addAll(HddsTestUtils.getContainerInfo(numContainers));
+ // Assign open state to containers to be included in the chill mode
+ // container list
+ for (ContainerInfo container : containers) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
scmChillModeManager = new SCMChillModeManager(config, containers, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
@@ -85,6 +91,11 @@ public class TestSCMChillModeManager {
public void testChillModeExitRule() throws Exception {
containers = new ArrayList<>();
containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ // Assign open state to containers to be included in the chill mode
+ // container list
+ for (ContainerInfo container : containers) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
scmChillModeManager = new SCMChillModeManager(config, containers, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
@@ -119,6 +130,46 @@ public class TestSCMChillModeManager {
testChillModeDataNodes(5);
}
+ /**
+ * Check that containers in Allocated state are not considered while
+ * computing percentage of containers with at least 1 reported replica in
+ * chill mode exit rule.
+ */
+ @Test
+ public void testContainerChillModeRule() throws Exception {
+ containers = new ArrayList<>();
+ // Add 100 containers to the list of containers in SCM
+ containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ // Assign OPEN state to first 25 containers and ALLLOCATED state to rest
+ // of the containers
+ for (ContainerInfo container : containers.subList(0, 25)) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
+ for (ContainerInfo container : containers.subList(25, 100)) {
+ container.setState(HddsProtos.LifeCycleState.ALLOCATED);
+ }
+
+ scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // When 10 OPEN containers are reported by DNs, the computed container
+ // threshold should be 10/25 as there are only 25 open containers.
+ // Containers in ALLOCATED state should not contribute towards list of
+ // containers while calculating container threshold in SCMChillNodeManager
+ testContainerThreshold(containers.subList(0, 10), 0.4);
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // When remaining 15 OPEN containers are reported by DNs, the container
+ // threshold should be (10+15)/25.
+ testContainerThreshold(containers.subList(10, 25), 1.0);
+
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 100, 1000 * 5);
+ }
+
private void testChillModeDataNodes(int numOfDns) throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(config);
conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
@@ -146,7 +197,7 @@ public class TestSCMChillModeManager {
queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
HddsTestUtils.createNodeRegistrationContainerReport(containers));
GenericTestUtils.waitFor(() -> {
- return scmChillModeManager.getInChillMode();
+ return !scmChillModeManager.getInChillMode();
}, 10, 1000 * 10);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDDS-568. ozone sh volume info,
update,
delete operations fail when volume name is not prefixed by /. Contributed by
Dinesh Chitlangia.
Posted by sh...@apache.org.
HDDS-568. ozone sh volume info, update, delete operations fail when volume name is not prefixed by /.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4de2dc26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4de2dc26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4de2dc26
Branch: refs/heads/HDFS-12943
Commit: 4de2dc2699fc371b2de83ba55ecbcecef1f0423b
Parents: 605622c
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:16:52 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 17:32:04 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/ozShell/TestOzoneShell.java | 48 ++++++++++++++++++++
.../hadoop/ozone/web/ozShell/Handler.java | 28 ++++++++++++
.../web/ozShell/volume/DeleteVolumeHandler.java | 12 +----
.../web/ozShell/volume/InfoVolumeHandler.java | 22 +--------
.../web/ozShell/volume/UpdateVolumeHandler.java | 12 +----
5 files changed, 79 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index 6e73b8c..d5f2554 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -283,6 +283,33 @@ public class TestOzoneShell {
GenericTestUtils.assertExceptionContains(
"Info Volume failed, error:VOLUME_NOT_FOUND", e);
}
+
+
+ volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ volumeArgs = VolumeArgs.newBuilder()
+ .setOwner("bilbo")
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
+ volume = client.getVolumeDetails(volumeName);
+ assertNotNull(volume);
+
+ //volumeName prefixed with /
+ String volumeNameWithSlashPrefix = "/" + volumeName;
+ args = new String[] {"volume", "delete",
+ url + "/" + volumeNameWithSlashPrefix};
+ execute(shell, args);
+ output = out.toString();
+ assertTrue(output.contains("Volume " + volumeName + " is deleted"));
+
+ // verify if volume has been deleted
+ try {
+ client.getVolumeDetails(volumeName);
+ fail("Get volume call should have thrown.");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Info Volume failed, error:VOLUME_NOT_FOUND", e);
+ }
}
@Test
@@ -295,6 +322,7 @@ public class TestOzoneShell {
.build();
client.createVolume(volumeName, volumeArgs);
+ //volumeName supplied as-is
String[] args = new String[] {"volume", "info", url + "/" + volumeName};
execute(shell, args);
@@ -303,6 +331,17 @@ public class TestOzoneShell {
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
+ //volumeName prefixed with /
+ String volumeNameWithSlashPrefix = "/" + volumeName;
+ args = new String[] {"volume", "info",
+ url + "/" + volumeNameWithSlashPrefix};
+ execute(shell, args);
+
+ output = out.toString();
+ assertTrue(output.contains(volumeName));
+ assertTrue(output.contains("createdOn")
+ && output.contains(OzoneConsts.OZONE_TIME_ZONE));
+
// test infoVolume with invalid volume name
args = new String[] {"volume", "info",
url + "/" + volumeName + "/invalid-name"};
@@ -365,6 +404,15 @@ public class TestOzoneShell {
vol = client.getVolumeDetails(volumeName);
assertEquals(newUser, vol.getOwner());
+ //volume with / prefix
+ String volumeWithPrefix = "/" + volumeName;
+ String newUser2 = "new-user2";
+ args = new String[] {"volume", "update", url + "/" + volumeWithPrefix,
+ "--user", newUser2};
+ execute(shell, args);
+ vol = client.getVolumeDetails(volumeName);
+ assertEquals(newUser2, vol.getOwner());
+
// test error conditions
args = new String[] {"volume", "update", url + "/invalid-volume",
"--user", newUser};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
index a9550c2..c8549c9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.ozone.web.ozShell;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
@@ -153,6 +155,32 @@ public abstract class Handler implements Callable<Void> {
}
}
+ /**
+ *
+ * @param uri
+ * @return volumeName
+ * @throws Exception
+ * @throws OzoneClientException when uri is null or invalid volume name
+ */
+ protected String parseVolumeName(String uri) throws Exception{
+ URI ozoneURI = verifyURI(uri);
+ Path path = Paths.get(ozoneURI.getPath());
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 1) {
+ String errorMessage;
+ if (pathNameCount < 1) {
+ errorMessage = "Volume name is required to perform volume " +
+ "operations like info, update, create and delete. ";
+ } else {
+ errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
+ "volume name";
+ }
+ throw new OzoneClientException(errorMessage);
+ }
+
+ return ozoneURI.getPath().replaceAll("^/+", "");
+ }
+
public boolean isVerbose() {
return parent.isVerbose();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
index d1e96fc..d757e20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
-import java.net.URI;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
@@ -43,14 +40,7 @@ public class DeleteVolumeHandler extends Handler {
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to delete a volume");
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
if (isVerbose()) {
System.out.printf("Volume name : %s%n", volumeName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
index 16c0342..48ed9f6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
@@ -18,11 +18,6 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
-import java.net.URI;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.web.ozShell.Handler;
@@ -48,22 +43,7 @@ public class InfoVolumeHandler extends Handler{
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- Path path = Paths.get(ozoneURI.getPath());
- int pathNameCount = path.getNameCount();
- if (pathNameCount != 1) {
- String errorMessage;
- if (pathNameCount < 1) {
- errorMessage = "Volume name is required to get info of a volume";
- } else {
- errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
- "volume name";
- }
- throw new OzoneClientException(errorMessage);
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
index 0336fc2..46803d8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
@@ -25,13 +25,10 @@ import picocli.CommandLine.Parameters;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import java.net.URI;
-
/**
* Executes update volume calls.
*/
@@ -57,14 +54,7 @@ public class UpdateVolumeHandler extends Handler {
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to update a volume");
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
if (quota != null && !quota.isEmpty()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HADOOP-15837. DynamoDB table
Update can fail S3A FS init. Contributed by Steve Loughran.
Posted by sh...@apache.org.
HADOOP-15837. DynamoDB table Update can fail S3A FS init.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee816f1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee816f1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee816f1f
Branch: refs/heads/HDFS-12943
Commit: ee816f1fd78b029b9efa567e8b1b62391563de14
Parents: 8a37983
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 14:57:38 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 14:57:38 2018 +0100
----------------------------------------------------------------------
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 89 +++++++++++++++---
.../s3a/s3guard/TestDynamoDBMiscOperations.java | 94 ++++++++++++++++++++
2 files changed, 169 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee816f1f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index f34afb5..6f892e5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -42,6 +42,7 @@ import java.util.stream.Collectors;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkBaseException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome;
@@ -66,6 +67,7 @@ import com.amazonaws.services.dynamodbv2.model.TableDescription;
import com.amazonaws.services.dynamodbv2.model.Tag;
import com.amazonaws.services.dynamodbv2.model.TagResourceRequest;
import com.amazonaws.services.dynamodbv2.model.WriteRequest;
+import com.amazonaws.waiters.WaiterTimedOutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
@@ -78,6 +80,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AWSClientIOException;
import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
import org.apache.hadoop.fs.s3a.AWSServiceThrottledException;
import org.apache.hadoop.fs.s3a.Constants;
@@ -1129,6 +1132,9 @@ public class DynamoDBMetadataStore implements MetadataStore {
* overall, this method is synchronous, and the table is guaranteed to exist
* after this method returns successfully.
*
+ * The wait for a table becoming active is Retry+Translated; it can fail
+ * while a table is not yet ready.
+ *
* @throws IOException if table does not exist and auto-creation is disabled;
* or table is being deleted, or any other I/O exception occurred.
*/
@@ -1144,7 +1150,6 @@ public class DynamoDBMetadataStore implements MetadataStore {
final String status = description.getTableStatus();
switch (status) {
case "CREATING":
- case "UPDATING":
LOG.debug("Table {} in region {} is being created/updated. This may"
+ " indicate that the table is being operated by another "
+ "concurrent thread or process. Waiting for active...",
@@ -1155,6 +1160,10 @@ public class DynamoDBMetadataStore implements MetadataStore {
throw new FileNotFoundException("DynamoDB table "
+ "'" + tableName + "' is being "
+ "deleted in region " + region);
+ case "UPDATING":
+ // table being updated; it can still be used.
+ LOG.debug("Table is being updated.");
+ break;
case "ACTIVE":
break;
default:
@@ -1285,24 +1294,34 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @throws InterruptedIOException if the wait was interrupted
* @throws IllegalArgumentException if an exception was raised in the waiter
*/
- @Retries.OnceRaw
- private void waitForTableActive(Table t) throws InterruptedIOException {
- try {
- t.waitForActive();
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting for table {} in region {} active",
- tableName, region, e);
- Thread.currentThread().interrupt();
- throw (InterruptedIOException)
- new InterruptedIOException("DynamoDB table '"
- + tableName + "' is not active yet in region " + region)
- .initCause(e);
- }
+ @Retries.RetryTranslated
+ private void waitForTableActive(Table t) throws IOException {
+ invoker.retry("Waiting for active state of table " + tableName,
+ null,
+ true,
+ () -> {
+ try {
+ t.waitForActive();
+ } catch (IllegalArgumentException ex) {
+ throw translateTableWaitFailure(tableName, ex);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for table {} in region {}"
+ + " active",
+ tableName, region, e);
+ Thread.currentThread().interrupt();
+ throw (InterruptedIOException)
+ new InterruptedIOException("DynamoDB table '"
+ + tableName + "' is not active yet in region " + region)
+ .initCause(e);
+ }
+ });
}
/**
* Create a table, wait for it to become active, then add the version
* marker.
+ * Creating an setting up the table isn't wrapped by any retry operations;
+ * the wait for a table to become available is RetryTranslated.
* @param capacity capacity to provision
* @throws IOException on any failure.
* @throws InterruptedIOException if the wait was interrupted
@@ -1604,4 +1623,46 @@ public class DynamoDBMetadataStore implements MetadataStore {
public Invoker getInvoker() {
return invoker;
}
+
+ /**
+ * Take an {@code IllegalArgumentException} raised by a DDB operation
+ * and if it contains an inner SDK exception, unwrap it.
+ * @param ex exception.
+ * @return the inner AWS exception or null.
+ */
+ public static SdkBaseException extractInnerException(
+ IllegalArgumentException ex) {
+ if (ex.getCause() instanceof SdkBaseException) {
+ return (SdkBaseException) ex.getCause();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Handle a table wait failure by extracting any inner cause and
+ * converting it, or, if unconvertable by wrapping
+ * the IllegalArgumentException in an IOE.
+ *
+ * @param name name of the table
+ * @param e exception
+ * @return an IOE to raise.
+ */
+ @VisibleForTesting
+ static IOException translateTableWaitFailure(
+ final String name, IllegalArgumentException e) {
+ final SdkBaseException ex = extractInnerException(e);
+ if (ex != null) {
+ if (ex instanceof WaiterTimedOutException) {
+ // a timeout waiting for state change: extract the
+ // message from the outer exception, but translate
+ // the inner one for the throttle policy.
+ return new AWSClientIOException(e.getMessage(), ex);
+ } else {
+ return translateException(e.getMessage(), name, ex);
+ }
+ } else {
+ return new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee816f1f/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
new file mode 100644
index 0000000..03be39f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
+import com.amazonaws.waiters.WaiterTimedOutException;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.s3a.AWSClientIOException;
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.translateTableWaitFailure;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Unit test suite for misc dynamoDB metastore operations.
+ */
+public class TestDynamoDBMiscOperations extends HadoopTestBase {
+
+ private static final String TIMEOUT_ERROR_MESSAGE
+ = "Table table-name did not transition into ACTIVE state.";
+
+ @Test
+ public void testUnwrapTableWaitTimeout() throws Throwable {
+ final Exception waiterTimedOut =
+ new WaiterTimedOutException("waiter timed out");
+ final AWSClientIOException ex = intercept(AWSClientIOException.class,
+ TIMEOUT_ERROR_MESSAGE,
+ () -> {
+ throw translateTableWaitFailure("example",
+ new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE,
+ waiterTimedOut));
+ });
+ assertEquals(waiterTimedOut, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateIllegalArgumentException() throws Throwable {
+ final IllegalArgumentException e =
+ new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE);
+ final IOException ex = intercept(IOException.class,
+ TIMEOUT_ERROR_MESSAGE,
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(e, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateWrappedDDBException() throws Throwable {
+ final Exception inner = new ResourceNotFoundException("ddb");
+ final IllegalArgumentException e =
+ new IllegalArgumentException("outer", inner);
+ final FileNotFoundException ex = intercept(FileNotFoundException.class,
+ "outer",
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(inner, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateWrappedOtherException() throws Throwable {
+ final Exception inner = new NullPointerException("npe");
+ final IllegalArgumentException e =
+ new IllegalArgumentException("outer", inner);
+ final IOException ex = intercept(IOException.class,
+ "outer",
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(e, ex.getCause());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDDS-604. Correct Ozone
getOzoneConf description. Contributed by Dinesh Chitlangia.
Posted by sh...@apache.org.
HDDS-604. Correct Ozone getOzoneConf description.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/794c0451
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/794c0451
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/794c0451
Branch: refs/heads/HDFS-12943
Commit: 794c0451cffbe147234a2417943709c121d06620
Parents: 4de2dc2
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:52:06 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 17:52:06 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/common/src/main/bin/ozone | 4 ++--
hadoop-ozone/common/src/main/bin/start-ozone.sh | 10 +++++-----
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 10 +++++-----
hadoop-ozone/docs/content/CommandShell.md | 2 +-
4 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 4b50771..2ba9ea7 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -39,7 +39,7 @@ function hadoop_usage
hadoop_add_subcommand "fs" client "run a filesystem command on Ozone file system. Equivalent to 'hadoop fs'"
hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
- hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration"
+ hadoop_add_subcommand "getconf" client "get ozone config values from configuration"
hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
hadoop_add_subcommand "om" daemon "Ozone Manager"
@@ -94,7 +94,7 @@ function ozonecmd_case
HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
- getozoneconf)
+ getconf)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf;
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index cfb54e0..4c022fb 100755
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -67,8 +67,8 @@ fi
#Add other possible options
nameStartOpt="$nameStartOpt $*"
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
echo "Ozone is not supported in a security enabled cluster."
@@ -77,7 +77,7 @@ fi
#---------------------------------------------------------
# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ "${OZONE_ENABLED}" != "true" ]]; then
echo "Operation is not supported because ozone is not enabled."
exit -1
@@ -96,7 +96,7 @@ hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone ozonemanager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
echo "Starting Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
@@ -113,7 +113,7 @@ HADOOP_JUMBO_RETCOUNTER=$?
#---------------------------------------------------------
# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
echo "Starting storage container manager nodes [${SCM_NODES}]"
hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index 97e1df4..fe902eb 100755
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -47,8 +47,8 @@ else
exit 1
fi
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
echo "Ozone is not supported in a security enabled cluster."
@@ -57,7 +57,7 @@ fi
#---------------------------------------------------------
# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ "${OZONE_ENABLED}" != "true" ]]; then
echo "Operation is not supported because ozone is not enabled."
exit -1
@@ -76,7 +76,7 @@ hadoop_uservar_su ozone datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone Manager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
echo "Stopping Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
@@ -91,7 +91,7 @@ hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
echo "Stopping storage container manager nodes [${SCM_NODES}]"
hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/docs/content/CommandShell.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/CommandShell.md b/hadoop-ozone/docs/content/CommandShell.md
index 74072a5..fdd6c01 100644
--- a/hadoop-ozone/docs/content/CommandShell.md
+++ b/hadoop-ozone/docs/content/CommandShell.md
@@ -36,7 +36,7 @@ The commands supported by ozone are:
* **envvars** - Display computed Hadoop environment variables.
* **freon** - Runs the ozone load generator.
* **genesis** - Developer Only, Ozone micro-benchmark application.
- * **getozoneconf** - Reads ozone config values from configuration.
+ * **getconf** - Reads ozone config values from configuration.
* **jmxget** - Get JMX exported values from NameNode or DataNode.
* **om** - Ozone Manager, via daemon command can be started or stopped.
* **sh** - Primary command line interface for ozone.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDDS-443. Create reusable
ProgressBar utility for freon tests. Contributed by Zsolt Horvath.
Posted by sh...@apache.org.
HDDS-443. Create reusable ProgressBar utility for freon tests.
Contributed by Zsolt Horvath.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f068296f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f068296f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f068296f
Branch: refs/heads/HDFS-12943
Commit: f068296f8a88fc2a4c7b1680bc190c5fa7fc2469
Parents: 6fa3feb
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 18:18:19 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 18:19:00 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/tools/pom.xml | 6 +
.../apache/hadoop/ozone/freon/ProgressBar.java | 210 +++++++++++++++++++
.../hadoop/ozone/freon/TestProgressBar.java | 112 ++++++++++
3 files changed, 328 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f068296f/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index ac819b5..2d273d1 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -77,6 +77,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.15.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f068296f/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
new file mode 100644
index 0000000..a8d7e73
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+/**
+ * Run an arbitrary code and print progress on the provided stream. The
+ * progressbar stops when: - the provided currentvalue is less the the maxvalue
+ * - exception thrown
+ */
+public class ProgressBar {
+
+ private static final long REFRESH_INTERVAL = 1000L;
+
+ private PrintStream stream;
+ private AtomicLong currentValue;
+ private long maxValue;
+ private Thread progressBar;
+ private volatile boolean exception = false;
+ private long startTime;
+
+ /**
+ * @param stream Used to display the progress
+ * @param maxValue Maximum value of the progress
+ */
+ ProgressBar(PrintStream stream, long maxValue) {
+ this.stream = stream;
+ this.maxValue = maxValue;
+ this.currentValue = new AtomicLong(0);
+ this.progressBar = new Thread(new ProgressBarThread());
+ }
+
+ /**
+ * Start a task with a progessbar without any in/out parameters Runnable used
+ * just a task wrapper.
+ *
+ * @param task Runnable
+ */
+ public void start(Runnable task) {
+
+ startTime = System.nanoTime();
+
+ try {
+
+ progressBar.start();
+ task.run();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * Start a task with only out parameters.
+ *
+ * @param task Supplier that represents the task
+ * @param <T> Generic return type
+ * @return Whatever the supllier produces
+ */
+ public <T> T start(Supplier<T> task) {
+
+ startTime = System.nanoTime();
+ T result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.get();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return result;
+ }
+ }
+
+ /**
+ * Start a task with in/out parameters.
+ *
+ * @param input Input of the function
+ * @param task A Function that does the task
+ * @param <T> type of the input
+ * @param <R> return type
+ * @return Whatever the Function returns
+ */
+ public <T, R> R start(T input, Function<T, R> task) {
+
+ startTime = System.nanoTime();
+ R result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.apply(input);
+
+ } catch (Exception e) {
+ exception = true;
+ throw e;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return result;
+ }
+ }
+
+ /**
+ * Increment the progress with one step.
+ */
+ public void incrementProgress() {
+ currentValue.incrementAndGet();
+ }
+
+ private class ProgressBarThread implements Runnable {
+
+ @Override
+ public void run() {
+ try {
+
+ stream.println();
+ long value;
+
+ while ((value = currentValue.get()) < maxValue) {
+ print(value);
+
+ if (exception) {
+ break;
+ }
+ Thread.sleep(REFRESH_INTERVAL);
+ }
+
+ if (exception) {
+ stream.println();
+ stream.println("Incomplete termination, " + "check log for " +
+ "exception.");
+ } else {
+ print(maxValue);
+ }
+ stream.println();
+ } catch (InterruptedException e) {
+ stream.println(e);
+ }
+ }
+
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('â–ˆ');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
+ }
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f068296f/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
new file mode 100644
index 0000000..cea7be8
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.PrintStream;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+import static org.mockito.Mockito.*;
+
+@RunWith(MockitoJUnitRunner.class)
+/**
+ * Tests for the Progressbar class for Freon.
+ */
+public class TestProgressBar {
+
+ private PrintStream stream;
+
+ @Before
+ public void setupMock() {
+ stream = mock(PrintStream.class);
+ }
+
+ @Test
+ public void testWithRunnable() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Runnable task = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+ };
+
+ progressbar.start(task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithSupplier() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Supplier<Long> tasks = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return 1L; //return the result of the dummy task
+ };
+
+ progressbar.start(tasks);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithFunction() {
+
+ int maxValue = 10;
+ Long result;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Function<Long, String> task = (Long l) -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return "dummy result"; //return the result of the dummy task
+ };
+
+ progressbar.start(1L, task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-3879 [Storage implementation]
Create HDFS backing storage implementation for ATS reads. Contributed by
Abhishek Modi.
Posted by sh...@apache.org.
YARN-3879 [Storage implementation] Create HDFS backing storage implementation for ATS reads. Contributed by Abhishek Modi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bca928d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bca928d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bca928d3
Branch: refs/heads/HDFS-12943
Commit: bca928d3c7b88f39e9bc1784889596f0b00964d4
Parents: 74db993
Author: Vrushali C <vr...@apache.org>
Authored: Thu Oct 11 21:13:52 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Thu Oct 11 21:14:06 2018 -0700
----------------------------------------------------------------------
.../storage/FileSystemTimelineReaderImpl.java | 87 +++++++++++---------
1 file changed, 50 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bca928d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index adb8821..ef08a9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -41,6 +40,11 @@ import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@@ -68,7 +72,9 @@ public class FileSystemTimelineReaderImpl extends AbstractService
private static final Logger LOG =
LoggerFactory.getLogger(FileSystemTimelineReaderImpl.class);
- private String rootPath;
+ private FileSystem fs;
+ private Path rootPath;
+ private Path entitiesPath;
private static final String ENTITIES_DIR = "entities";
/** Default extension for output files. */
@@ -94,7 +100,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
@VisibleForTesting
String getRootPath() {
- return rootPath;
+ return rootPath.toString();
}
private static ObjectMapper mapper;
@@ -162,12 +168,12 @@ public class FileSystemTimelineReaderImpl extends AbstractService
if (clusterId == null || appId == null) {
throw new IOException("Unable to get flow info");
}
- String appFlowMappingFile = rootPath + File.separator + ENTITIES_DIR +
- File.separator + clusterId + File.separator + APP_FLOW_MAPPING_FILE;
+ Path clusterIdPath = new Path(entitiesPath, clusterId);
+ Path appFlowMappingFilePath = new Path(clusterIdPath,
+ APP_FLOW_MAPPING_FILE);
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(
- new FileInputStream(
- appFlowMappingFile), Charset.forName("UTF-8")));
+ fs.open(appFlowMappingFilePath), Charset.forName("UTF-8")));
CSVParser parser = new CSVParser(reader, csvFormat)) {
for (CSVRecord record : parser.getRecords()) {
if (record.size() < 4) {
@@ -266,7 +272,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
return entity;
}
- private Set<TimelineEntity> getEntities(File dir, String entityType,
+ private Set<TimelineEntity> getEntities(Path dir, String entityType,
TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
throws IOException {
// First sort the selected entities based on created/start time.
@@ -280,15 +286,18 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
);
if (dir != null) {
- File[] files = dir.listFiles();
- if (files != null) {
- for (File entityFile : files) {
+ RemoteIterator<LocatedFileStatus> fileStatuses = fs.listFiles(dir,
+ false);
+ if (fileStatuses != null) {
+ while (fileStatuses.hasNext()) {
+ LocatedFileStatus locatedFileStatus = fileStatuses.next();
+ Path entityFile = locatedFileStatus.getPath();
if (!entityFile.getName()
.contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
continue;
}
try (BufferedReader reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(entityFile),
+ new InputStreamReader(fs.open(entityFile),
Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
if (!entity.getType().equals(entityType)) {
@@ -366,25 +375,30 @@ public class FileSystemTimelineReaderImpl extends AbstractService
@Override
public void serviceInit(Configuration conf) throws Exception {
- rootPath = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ String outputRoot = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
conf.get("hadoop.tmp.dir") + File.separator + STORAGE_DIR_ROOT);
+ rootPath = new Path(outputRoot);
+ entitiesPath = new Path(rootPath, ENTITIES_DIR);
+ fs = rootPath.getFileSystem(conf);
super.serviceInit(conf);
}
@Override
public TimelineEntity getEntity(TimelineReaderContext context,
TimelineDataToRetrieve dataToRetrieve) throws IOException {
- String flowRunPath = getFlowRunPath(context.getUserId(),
+ String flowRunPathStr = getFlowRunPath(context.getUserId(),
context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId());
- File dir = new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + File.separator + flowRunPath + File.separator +
- context.getAppId() + File.separator + context.getEntityType());
- File entityFile = new File(
- dir, context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
+ Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+ Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+ Path appIdPath = new Path(flowRunPath, context.getAppId());
+ Path entityTypePath = new Path(appIdPath, context.getEntityType());
+ Path entityFilePath = new Path(entityTypePath,
+ context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
+
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(
- new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
+ fs.open(entityFilePath), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
return createEntityToBeReturned(
entity, dataToRetrieve.getFieldsToRetrieve());
@@ -399,32 +413,31 @@ public class FileSystemTimelineReaderImpl extends AbstractService
public Set<TimelineEntity> getEntities(TimelineReaderContext context,
TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
throws IOException {
- String flowRunPath = getFlowRunPath(context.getUserId(),
+ String flowRunPathStr = getFlowRunPath(context.getUserId(),
context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId());
- File dir =
- new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + File.separator + flowRunPath +
- File.separator + context.getAppId() + File.separator +
- context.getEntityType());
- return getEntities(dir, context.getEntityType(), filters, dataToRetrieve);
+ Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+ Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+ Path appIdPath = new Path(flowRunPath, context.getAppId());
+ Path entityTypePath = new Path(appIdPath, context.getEntityType());
+
+ return getEntities(entityTypePath, context.getEntityType(), filters,
+ dataToRetrieve);
}
@Override public Set<String> getEntityTypes(TimelineReaderContext context)
throws IOException {
Set<String> result = new TreeSet<>();
- String flowRunPath = getFlowRunPath(context.getUserId(),
+ String flowRunPathStr = getFlowRunPath(context.getUserId(),
context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId());
- File dir = new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + File.separator + flowRunPath
- + File.separator + context.getAppId());
- File[] fileList = dir.listFiles();
- if (fileList != null) {
- for (File f : fileList) {
- if (f.isDirectory()) {
- result.add(f.getName());
- }
+ Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+ Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+ Path appIdPath = new Path(flowRunPath, context.getAppId());
+ FileStatus[] fileStatuses = fs.listStatus(appIdPath);
+ for (FileStatus fileStatus : fileStatuses) {
+ if (fileStatus.isDirectory()) {
+ result.add(fileStatus.getPath().getName());
}
}
return result;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDFS-13968. BlockReceiver
Array-Based Queue. Contributed by BELUGA BEHR.
Posted by sh...@apache.org.
HDFS-13968. BlockReceiver Array-Based Queue. Contributed by BELUGA BEHR.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3532aa38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3532aa38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3532aa38
Branch: refs/heads/HDFS-12943
Commit: 3532aa38867dc928b415c6c5eb47cca4fac349fe
Parents: eb34b5f
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Oct 11 13:49:39 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Oct 11 13:49:39 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockReceiver.java | 43 +++++++++-----------
1 file changed, 19 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3532aa38/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index afb9e2a..cb1f73d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -28,8 +28,9 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
import java.util.Arrays;
-import java.util.LinkedList;
+import java.util.Queue;
import java.util.zip.Checksum;
import org.apache.commons.logging.Log;
@@ -1208,9 +1209,9 @@ class BlockReceiver implements Closeable {
* Processes responses from downstream datanodes in the pipeline
* and sends back replies to the originator.
*/
- class PacketResponder implements Runnable, Closeable {
+ class PacketResponder implements Runnable, Closeable {
/** queue for packets waiting for ack - synchronization using monitor lock */
- private final LinkedList<Packet> ackQueue = new LinkedList<Packet>();
+ private final Queue<Packet> ackQueue = new ArrayDeque<>();
/** the thread that spawns this responder */
private final Thread receiverThread = Thread.currentThread();
/** is this responder running? - synchronization using monitor lock */
@@ -1264,12 +1265,10 @@ class BlockReceiver implements Closeable {
final long offsetInBlock, final Status ackStatus) {
final Packet p = new Packet(seqno, lastPacketInBlock, offsetInBlock,
System.nanoTime(), ackStatus);
- if(LOG.isDebugEnabled()) {
- LOG.debug(myString + ": enqueue " + p);
- }
- synchronized(ackQueue) {
+ LOG.debug("{}: enqueue {}", this, p);
+ synchronized (ackQueue) {
if (running) {
- ackQueue.addLast(p);
+ ackQueue.add(p);
ackQueue.notifyAll();
}
}
@@ -1321,15 +1320,13 @@ class BlockReceiver implements Closeable {
/** Wait for a packet with given {@code seqno} to be enqueued to ackQueue */
Packet waitForAckHead(long seqno) throws InterruptedException {
- synchronized(ackQueue) {
- while (isRunning() && ackQueue.size() == 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(myString + ": seqno=" + seqno +
- " waiting for local datanode to finish write.");
- }
+ synchronized (ackQueue) {
+ while (isRunning() && ackQueue.isEmpty()) {
+ LOG.debug("{}: seqno={} waiting for local datanode to finish write.",
+ myString, seqno);
ackQueue.wait();
}
- return isRunning() ? ackQueue.getFirst() : null;
+ return isRunning() ? ackQueue.element() : null;
}
}
@@ -1338,8 +1335,8 @@ class BlockReceiver implements Closeable {
*/
@Override
public void close() {
- synchronized(ackQueue) {
- while (isRunning() && ackQueue.size() != 0) {
+ synchronized (ackQueue) {
+ while (isRunning() && !ackQueue.isEmpty()) {
try {
ackQueue.wait();
} catch (InterruptedException e) {
@@ -1347,14 +1344,12 @@ class BlockReceiver implements Closeable {
Thread.currentThread().interrupt();
}
}
- if(LOG.isDebugEnabled()) {
- LOG.debug(myString + ": closing");
- }
+ LOG.debug("{}: closing", this);
running = false;
ackQueue.notifyAll();
}
- synchronized(this) {
+ synchronized (this) {
running = false;
notifyAll();
}
@@ -1657,12 +1652,12 @@ class BlockReceiver implements Closeable {
/**
* Remove a packet from the head of the ack queue
- *
+ *
* This should be called only when the ack queue is not empty
*/
private void removeAckHead() {
- synchronized(ackQueue) {
- ackQueue.removeFirst();
+ synchronized (ackQueue) {
+ ackQueue.remove();
ackQueue.notifyAll();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDDS-634. OzoneFS: support basic
user group and permission for file/dir. Contributed by Xiaoyu Yao.
Posted by sh...@apache.org.
HDDS-634. OzoneFS: support basic user group and permission for file/dir.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/604c27c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/604c27c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/604c27c3
Branch: refs/heads/HDFS-12943
Commit: 604c27c33a3bd30644debaf192cec3cf80dfa36d
Parents: 5055247
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Oct 11 17:19:38 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Oct 11 17:25:25 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 16 ++++++++++++++--
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 17 +++++++++++++++--
2 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/604c27c3/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 8fbd688..0b721fb 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -448,6 +448,14 @@ public class OzoneFileSystem extends FileSystem {
}
/**
+ * Get the username of the FS.
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
* Check whether the path is valid and then create directories.
* Directory is represented using a key with no value.
* All the non-existent parent directories are also created.
@@ -528,11 +536,15 @@ public class OzoneFileSystem extends FileSystem {
throw new FileNotFoundException(f + ": No such file or directory!");
} else if (isDirectory(meta)) {
return new FileStatus(0, true, 1, 0,
- meta.getModificationTime(), qualifiedPath);
+ meta.getModificationTime(), 0,
+ FsPermission.getDirDefault(), getUsername(), getUsername(),
+ qualifiedPath);
} else {
//TODO: Fetch replication count from ratis config
return new FileStatus(meta.getDataSize(), false, 1,
- getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
+ getDefaultBlockSize(f), meta.getModificationTime(), 0,
+ FsPermission.getFileDefault(), getUsername(), getUsername(),
+ qualifiedPath);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/604c27c3/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index a225702..7cf6e3d 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -23,6 +23,7 @@ import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.Before;
@@ -167,6 +168,10 @@ public class TestOzoneFileInterfaces {
assertTrue("Modification time has not been recorded: " + status,
status.getModificationTime() > currentTime);
+ assertEquals(false, status.isDirectory());
+ assertEquals(FsPermission.getFileDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
try (FSDataInputStream inputStream = fs.open(path)) {
byte[] buffer = new byte[stringLen];
inputStream.readFully(0, buffer);
@@ -175,6 +180,12 @@ public class TestOzoneFileInterfaces {
}
}
+ private void verifyOwnerGroup(FileStatus fileStatus) {
+ String owner = getCurrentUser();
+ assertEquals(owner, fileStatus.getOwner());
+ assertEquals(owner, fileStatus.getGroup());
+ }
+
@Test
public void testDirectory() throws IOException {
@@ -186,6 +197,10 @@ public class TestOzoneFileInterfaces {
FileStatus status = fs.getFileStatus(path);
assertTrue("The created path is not directory.", status.isDirectory());
+ assertEquals(true, status.isDirectory());
+ assertEquals(FsPermission.getDirDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
assertEquals(0, status.getLen());
FileStatus[] statusList = fs.listStatus(createPath("/"));
@@ -195,8 +210,6 @@ public class TestOzoneFileInterfaces {
FileStatus statusRoot = fs.getFileStatus(createPath("/"));
assertTrue("Root dir (/) is not a directory.", status.isDirectory());
assertEquals(0, status.getLen());
-
-
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDFS-13956. iNotify should include
information to identify a file as either replicated or erasure coded.
Contributed by Hrishikesh Gadre.
Posted by sh...@apache.org.
HDFS-13956. iNotify should include information to identify a file as either replicated or erasure coded. Contributed by Hrishikesh Gadre.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf3d591f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf3d591f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf3d591f
Branch: refs/heads/HDFS-12943
Commit: bf3d591f0cb0fedeab5d89cc8d2270d3b9a70313
Parents: 3ead525
Author: Hrishikesh Gadre <hg...@apache.org>
Authored: Wed Oct 10 10:23:07 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Oct 10 10:23:07 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/inotify/Event.java | 14 +++
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 60 ++++++-----
.../src/main/proto/inotify.proto | 1 +
.../namenode/InotifyFSEditLogOpTranslator.java | 3 +
.../hdfs/TestDFSInotifyEventInputStream.java | 100 +++++++++++++++++++
5 files changed, 152 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index 78a85b0..2ae8f0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import java.util.List;
+import java.util.Optional;
/**
* Events sent by the inotify system. Note that no events are necessarily sent
@@ -112,6 +113,7 @@ public abstract class Event {
private String symlinkTarget;
private boolean overwrite;
private long defaultBlockSize;
+ private Optional<Boolean> erasureCoded;
public static class Builder {
private INodeType iNodeType;
@@ -124,6 +126,7 @@ public abstract class Event {
private String symlinkTarget;
private boolean overwrite;
private long defaultBlockSize = 0;
+ private Optional<Boolean> erasureCoded = Optional.empty();
public Builder iNodeType(INodeType type) {
this.iNodeType = type;
@@ -175,6 +178,11 @@ public abstract class Event {
return this;
}
+ public Builder erasureCoded(boolean ecCoded) {
+ this.erasureCoded = Optional.of(ecCoded);
+ return this;
+ }
+
public CreateEvent build() {
return new CreateEvent(this);
}
@@ -192,6 +200,7 @@ public abstract class Event {
this.symlinkTarget = b.symlinkTarget;
this.overwrite = b.overwrite;
this.defaultBlockSize = b.defaultBlockSize;
+ this.erasureCoded = b.erasureCoded;
}
public INodeType getiNodeType() {
@@ -243,6 +252,10 @@ public abstract class Event {
return defaultBlockSize;
}
+ public Optional<Boolean> isErasureCoded() {
+ return erasureCoded;
+ }
+
@Override
@InterfaceStability.Unstable
public String toString() {
@@ -261,6 +274,7 @@ public abstract class Event {
content.append("overwrite=").append(overwrite)
.append(", defaultBlockSize=").append(defaultBlockSize)
+ .append(", erasureCoded=").append(erasureCoded)
.append("]");
return content.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 4a5a493..4b966bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -908,18 +908,22 @@ public class PBHelperClient {
case EVENT_CREATE:
InotifyProtos.CreateEventProto create =
InotifyProtos.CreateEventProto.parseFrom(p.getContents());
- events.add(new Event.CreateEvent.Builder()
- .iNodeType(createTypeConvert(create.getType()))
- .path(create.getPath())
- .ctime(create.getCtime())
- .ownerName(create.getOwnerName())
- .groupName(create.getGroupName())
- .perms(convert(create.getPerms()))
- .replication(create.getReplication())
- .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
- create.getSymlinkTarget())
- .defaultBlockSize(create.getDefaultBlockSize())
- .overwrite(create.getOverwrite()).build());
+ Event.CreateEvent.Builder builder = new Event.CreateEvent.Builder()
+ .iNodeType(createTypeConvert(create.getType()))
+ .path(create.getPath())
+ .ctime(create.getCtime())
+ .ownerName(create.getOwnerName())
+ .groupName(create.getGroupName())
+ .perms(convert(create.getPerms()))
+ .replication(create.getReplication())
+ .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
+ create.getSymlinkTarget())
+ .defaultBlockSize(create.getDefaultBlockSize())
+ .overwrite(create.getOverwrite());
+ if (create.hasErasureCoded()) {
+ builder.erasureCoded(create.getErasureCoded());
+ }
+ events.add(builder.build());
break;
case EVENT_METADATA:
InotifyProtos.MetadataUpdateEventProto meta =
@@ -2909,22 +2913,26 @@ public class PBHelperClient {
break;
case CREATE:
Event.CreateEvent ce2 = (Event.CreateEvent) e;
+ InotifyProtos.CreateEventProto.Builder pB =
+ (InotifyProtos.CreateEventProto.newBuilder());
+ pB.setType(createTypeConvert(ce2.getiNodeType()))
+ .setPath(ce2.getPath())
+ .setCtime(ce2.getCtime())
+ .setOwnerName(ce2.getOwnerName())
+ .setGroupName(ce2.getGroupName())
+ .setPerms(convert(ce2.getPerms()))
+ .setReplication(ce2.getReplication())
+ .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
+ "" : ce2.getSymlinkTarget())
+ .setDefaultBlockSize(ce2.getDefaultBlockSize())
+ .setOverwrite(ce2.getOverwrite());
+ if (ce2.isErasureCoded().isPresent()) {
+ pB.setErasureCoded(ce2.isErasureCoded().get());
+ }
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_CREATE)
- .setContents(
- InotifyProtos.CreateEventProto.newBuilder()
- .setType(createTypeConvert(ce2.getiNodeType()))
- .setPath(ce2.getPath())
- .setCtime(ce2.getCtime())
- .setOwnerName(ce2.getOwnerName())
- .setGroupName(ce2.getGroupName())
- .setPerms(convert(ce2.getPerms()))
- .setReplication(ce2.getReplication())
- .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
- "" : ce2.getSymlinkTarget())
- .setDefaultBlockSize(ce2.getDefaultBlockSize())
- .setOverwrite(ce2.getOverwrite()).build().toByteString()
- ).build());
+ .setContents(pB.build().toByteString())
+ .build());
break;
case METADATA:
Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
index 5339902..f193408 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
@@ -80,6 +80,7 @@ message CreateEventProto {
optional string symlinkTarget = 8;
optional bool overwrite = 9;
optional int64 defaultBlockSize = 10 [default=0];
+ optional bool erasureCoded = 11;
}
message CloseEventProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
index 0918107..8a54c8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.inotify.Event;
import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import java.util.List;
@@ -54,6 +55,8 @@ public class InotifyFSEditLogOpTranslator {
.perms(addOp.permissions.getPermission())
.overwrite(addOp.overwrite)
.defaultBlockSize(addOp.blockSize)
+ .erasureCoded(addOp.erasureCodingPolicyId
+ != ErasureCodeConstants.REPLICATION_POLICY_ID)
.iNodeType(Event.CreateEvent.INodeType.FILE).build() });
} else { // append
return new EventBatch(op.txid,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index b0b85e7..05d3c63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -140,6 +141,7 @@ public class TestDFSInotifyEventInputStream {
client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent
//TruncateOp -> TruncateEvent
client.truncate("/truncate_file", BLOCK_SIZE);
+ client.create("/file_ec_test1", false);
EventBatch batch = null;
// RenameOp
@@ -180,6 +182,8 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(ce.getSymlinkTarget() == null);
Assert.assertTrue(ce.getOverwrite());
Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize());
+ Assert.assertTrue(ce.isErasureCoded().isPresent());
+ Assert.assertFalse(ce.isErasureCoded().get());
LOG.info(ce.toString());
Assert.assertTrue(ce.toString().startsWith("CreateEvent [INodeType="));
@@ -395,6 +399,25 @@ public class TestDFSInotifyEventInputStream {
LOG.info(et.toString());
Assert.assertTrue(et.toString().startsWith("TruncateEvent [path="));
+ // CreateEvent without overwrite
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType()
+ == Event.EventType.CREATE);
+ ce = (Event.CreateEvent) batch.getEvents()[0];
+ Assert.assertTrue(ce.getiNodeType() == Event.CreateEvent.INodeType.FILE);
+ Assert.assertTrue(ce.getPath().equals("/file_ec_test1"));
+ Assert.assertTrue(ce.getCtime() > 0);
+ Assert.assertTrue(ce.getReplication() > 0);
+ Assert.assertTrue(ce.getSymlinkTarget() == null);
+ Assert.assertFalse(ce.getOverwrite());
+ Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize());
+ Assert.assertTrue(ce.isErasureCoded().isPresent());
+ Assert.assertFalse(ce.isErasureCoded().get());
+ LOG.info(ce.toString());
+ Assert.assertTrue(ce.toString().startsWith("CreateEvent [INodeType="));
+
// Returns null when there are no further events
Assert.assertTrue(eis.poll() == null);
@@ -411,6 +434,83 @@ public class TestDFSInotifyEventInputStream {
}
@Test(timeout = 120000)
+ public void testErasureCodedFiles() throws Exception {
+ ErasureCodingPolicy ecPolicy = StripedFileTestUtil.getDefaultECPolicy();
+ final int dataUnits = ecPolicy.getNumDataUnits();
+ final int parityUnits = ecPolicy.getNumParityUnits();
+
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, ecPolicy.getCellSize());
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ // so that we can get an atime change
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 1);
+
+ MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
+ builder.getDfsBuilder().numDataNodes(dataUnits + parityUnits);
+ MiniQJMHACluster cluster = builder.build();
+
+ try {
+ cluster.getDfsCluster().waitActive();
+ cluster.getDfsCluster().transitionToActive(0);
+ DFSClient client = new DFSClient(cluster.getDfsCluster().getNameNode(0)
+ .getNameNodeAddress(), conf);
+ DistributedFileSystem fs =
+ (DistributedFileSystem)cluster.getDfsCluster().getFileSystem(0);
+
+ Path ecDir = new Path("/ecdir");
+ fs.mkdirs(ecDir);
+ fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
+
+ DFSInotifyEventInputStream eis = client.getInotifyEventStream();
+
+ int sz = ecPolicy.getNumDataUnits() * ecPolicy.getCellSize();
+ byte[] contents = new byte[sz];
+ DFSTestUtil.writeFile(fs, new Path("/ecdir/file_ec_test2"), contents);
+
+ EventBatch batch = null;
+
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ long txid = batch.getTxid();
+ long eventsBehind = eis.getTxidsBehindEstimate();
+ Assert.assertTrue(batch.getEvents()[0].getEventType()
+ == Event.EventType.CREATE);
+ Event.CreateEvent ce = (Event.CreateEvent) batch.getEvents()[0];
+ Assert.assertTrue(ce.getiNodeType() == Event.CreateEvent.INodeType.FILE);
+ Assert.assertTrue(ce.getPath().equals("/ecdir/file_ec_test2"));
+ Assert.assertTrue(ce.getCtime() > 0);
+ Assert.assertEquals(1, ce.getReplication());
+ Assert.assertTrue(ce.getSymlinkTarget() == null);
+ Assert.assertTrue(ce.getOverwrite());
+ Assert.assertEquals(ecPolicy.getCellSize(), ce.getDefaultBlockSize());
+ Assert.assertTrue(ce.isErasureCoded().isPresent());
+ Assert.assertTrue(ce.isErasureCoded().get());
+ LOG.info(ce.toString());
+ Assert.assertTrue(ce.toString().startsWith("CreateEvent [INodeType="));
+
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType()
+ == Event.EventType.CLOSE);
+ Assert.assertTrue(((Event.CloseEvent) batch.getEvents()[0]).getPath()
+ .equals("/ecdir/file_ec_test2"));
+
+ // Returns null when there are no further events
+ Assert.assertTrue(eis.poll() == null);
+
+ // make sure the estimate hasn't changed since the above assertion
+ // tells us that we are fully caught up to the current namesystem state
+ // and we should not have been behind at all when eventsBehind was set
+ // either, since there were few enough events that they should have all
+ // been read to the client during the first poll() call
+ Assert.assertTrue(eis.getTxidsBehindEstimate() == eventsBehind);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 120000)
public void testNNFailover() throws IOException, URISyntaxException,
MissingEventsException {
Configuration conf = new HdfsConfiguration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HADOOP-15717. TGT renewal thread
does not log IOException (snemeth via rkanter)
Posted by sh...@apache.org.
HADOOP-15717. TGT renewal thread does not log IOException (snemeth via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d91d47bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d91d47bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d91d47bc
Branch: refs/heads/HDFS-12943
Commit: d91d47bc739f23ca22a7e44fc83d449db57ab130
Parents: 64f2b32
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Oct 11 15:34:41 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Oct 11 15:35:44 2018 -0700
----------------------------------------------------------------------
.../hadoop/security/UserGroupInformation.java | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d91d47bc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index db88106..915d6df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -920,8 +920,8 @@ public class UserGroupInformation {
final long now = Time.now();
if (tgt.isDestroyed()) {
- LOG.error("TGT is destroyed. Aborting renew thread for {}.",
- getUserName());
+ LOG.error(String.format("TGT is destroyed. " +
+ "Aborting renew thread for %s.", getUserName()), ie);
return;
}
@@ -933,16 +933,18 @@ public class UserGroupInformation {
try {
tgtEndTime = tgt.getEndTime().getTime();
} catch (NullPointerException npe) {
- LOG.error("NPE thrown while getting KerberosTicket endTime. "
- + "Aborting renew thread for {}.", getUserName());
+ LOG.error(String.format("NPE thrown while getting " +
+ "KerberosTicket endTime. Aborting renew thread for %s.",
+ getUserName()), ie);
return;
}
- LOG.warn("Exception encountered while running the renewal "
- + "command for {}. (TGT end time:{}, renewalFailures: {},"
- + "renewalFailuresTotal: {})", getUserName(), tgtEndTime,
+ LOG.warn(String.format("Exception encountered while running the " +
+ "renewal command for %s. " +
+ "(TGT end time:%d, renewalFailures: %d, " +
+ "renewalFailuresTotal: %d)", getUserName(), tgtEndTime,
metrics.renewalFailures.value(),
- metrics.renewalFailuresTotal.value(), ie);
+ metrics.renewalFailuresTotal.value()), ie);
if (rp == null) {
// Use a dummy maxRetries to create the policy. The policy will
// only be used to get next retry time with exponential back-off.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-625. putKey hangs for a long
time after completion, sometimes forever. Contributed by Arpit Agarwal
Posted by sh...@apache.org.
HDDS-625. putKey hangs for a long time after completion, sometimes forever. Contributed by Arpit Agarwal
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb18cc5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb18cc5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb18cc5e
Branch: refs/heads/HDFS-12943
Commit: fb18cc5eaa637772d5c8479cbae1bd19aa00e7fb
Parents: 604c27c
Author: Tsz Wo Nicholas Sze <sz...@apache.org>
Authored: Fri Oct 12 10:37:37 2018 +0800
Committer: Tsz Wo Nicholas Sze <sz...@apache.org>
Committed: Fri Oct 12 10:37:37 2018 +0800
----------------------------------------------------------------------
.../common/transport/server/ratis/XceiverServerRatis.java | 4 +++-
hadoop-project/pom.xml | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb18cc5e/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 9094217..0cd407a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -70,6 +70,7 @@ import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketAddress;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
@@ -240,7 +241,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
// Set the ratis storage directory
String storageDir = HddsServerUtil.getOzoneDatanodeRatisDirectory(conf);
- RaftServerConfigKeys.setStorageDir(properties, new File(storageDir));
+ RaftServerConfigKeys.setStorageDirs(properties,
+ Collections.singletonList(new File(storageDir)));
// For grpc set the maximum message size
GrpcConfigKeys.setMessageSizeMax(properties,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb18cc5e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7472a54..b850c7c 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -103,7 +103,7 @@
<ldap-api.version>1.0.0-M33</ldap-api.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-9b84d79-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-9b2d7b6-SNAPSHOT</ratis.version>
<jcache.version>1.0-alpha-1</jcache.version>
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-7644. NM gets backed up
deleting docker containers. Contributed by Chandni Singh
Posted by sh...@apache.org.
YARN-7644. NM gets backed up deleting docker containers. Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ce70e12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ce70e12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ce70e12
Branch: refs/heads/HDFS-12943
Commit: 5ce70e1211e624d58e8bb1181aec00729ebdc085
Parents: cd28051
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Oct 10 09:52:19 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Oct 10 09:52:19 2018 -0500
----------------------------------------------------------------------
.../launcher/ContainerCleanup.java | 229 +++++++++++++++++++
.../launcher/ContainerLaunch.java | 226 +++++-------------
.../launcher/ContainersLauncher.java | 14 +-
.../launcher/TestContainerCleanup.java | 108 +++++++++
.../launcher/TestContainersLauncher.java | 12 +-
5 files changed, 401 insertions(+), 188 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
new file mode 100644
index 0000000..963d28b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.EXIT_CODE_FILE_SUFFIX;
+
+/**
+ * Cleanup the container.
+ * Cancels the launch if launch has not started yet or signals
+ * the executor to not execute the process if not already done so.
+ * Also, sends a SIGTERM followed by a SIGKILL to the process if
+ * the process id is available.
+ */
+public class ContainerCleanup implements Runnable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerCleanup.class);
+
+ private final Context context;
+ private final Configuration conf;
+ private final Dispatcher dispatcher;
+ private final ContainerExecutor exec;
+ private final Container container;
+ private final ContainerLaunch launch;
+ private final long sleepDelayBeforeSigKill;
+
+
+ public ContainerCleanup(Context context, Configuration configuration,
+ Dispatcher dispatcher, ContainerExecutor exec,
+ Container container,
+ ContainerLaunch containerLaunch) {
+
+ this.context = Preconditions.checkNotNull(context, "context");
+ this.conf = Preconditions.checkNotNull(configuration, "config");
+ this.dispatcher = Preconditions.checkNotNull(dispatcher, "dispatcher");
+ this.exec = Preconditions.checkNotNull(exec, "exec");
+ this.container = Preconditions.checkNotNull(container, "container");
+ this.launch = Preconditions.checkNotNull(containerLaunch, "launch");
+ this.sleepDelayBeforeSigKill = conf.getLong(
+ YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
+ YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
+ }
+
+ @Override
+ public void run() {
+ ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
+ LOG.info("Cleaning up container " + containerIdStr);
+
+ try {
+ context.getNMStateStore().storeContainerKilled(containerId);
+ } catch (IOException e) {
+ LOG.error("Unable to mark container " + containerId
+ + " killed in store", e);
+ }
+
+ // launch flag will be set to true if process already launched
+ boolean alreadyLaunched = !launch.markLaunched();
+ if (!alreadyLaunched) {
+ LOG.info("Container " + containerIdStr + " not launched."
+ + " No cleanup needed to be done");
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Marking container " + containerIdStr + " as inactive");
+ }
+ // this should ensure that if the container process has not launched
+ // by this time, it will never be launched
+ exec.deactivateContainer(containerId);
+ Path pidFilePath = launch.getPidFilePath();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Getting pid for container {} to kill"
+ + " from pid file {}", containerIdStr, pidFilePath != null ?
+ pidFilePath : "null");
+ }
+
+ // however the container process may have already started
+ try {
+
+ // get process id from pid file if available
+ // else if shell is still active, get it from the shell
+ String processId = launch.getContainerPid();
+
+ // kill process
+ String user = container.getUser();
+ if (processId != null) {
+ signalProcess(processId, user, containerIdStr);
+ } else {
+ // Normally this means that the process was notified about
+ // deactivateContainer above and did not start.
+ // Since we already set the state to RUNNING or REINITIALIZING
+ // we have to send a killed event to continue.
+ if (!launch.isLaunchCompleted()) {
+ LOG.warn("Container clean up before pid file created "
+ + containerIdStr);
+ dispatcher.getEventHandler().handle(
+ new ContainerExitEvent(container.getContainerId(),
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ Shell.WINDOWS ?
+ ContainerExecutor.ExitCode.FORCE_KILLED.getExitCode() :
+ ContainerExecutor.ExitCode.TERMINATED.getExitCode(),
+ "Container terminated before pid file created."));
+ // There is a possibility that the launch grabbed the file name before
+ // the deactivateContainer above but it was slow enough to avoid
+ // getContainerPid.
+ // Increasing YarnConfiguration.NM_PROCESS_KILL_WAIT_MS
+ // reduces the likelihood of this race condition and process leak.
+ }
+ // The Docker container may not have fully started, reap the container.
+ if (DockerLinuxContainerRuntime.isDockerContainerRequested(conf,
+ container.getLaunchContext().getEnvironment())) {
+ reapDockerContainerNoPid(user);
+ }
+ }
+ } catch (Exception e) {
+ String message =
+ "Exception when trying to cleanup container " + containerIdStr
+ + ": " + StringUtils.stringifyException(e);
+ LOG.warn(message);
+ dispatcher.getEventHandler().handle(
+ new ContainerDiagnosticsUpdateEvent(containerId, message));
+ } finally {
+ // cleanup pid file if present
+ if (pidFilePath != null) {
+ try {
+ FileContext lfs = FileContext.getLocalFSFileContext();
+ lfs.delete(pidFilePath, false);
+ lfs.delete(pidFilePath.suffix(EXIT_CODE_FILE_SUFFIX), false);
+ } catch (IOException ioe) {
+ LOG.warn("{} exception trying to delete pid file {}. Ignoring.",
+ containerId, pidFilePath, ioe);
+ }
+ }
+ }
+
+ try {
+ // Reap the container
+ launch.reapContainer();
+ } catch (IOException ioe) {
+ LOG.warn("{} exception trying to reap container. Ignoring.", containerId,
+ ioe);
+ }
+ }
+
+ private void signalProcess(String processId, String user,
+ String containerIdStr) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending signal to pid " + processId + " as user " + user
+ + " for container " + containerIdStr);
+ }
+ final ContainerExecutor.Signal signal =
+ sleepDelayBeforeSigKill > 0 ? ContainerExecutor.Signal.TERM :
+ ContainerExecutor.Signal.KILL;
+
+ boolean result = sendSignal(user, processId, signal);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sent signal " + signal + " to pid " + processId + " as user "
+ + user + " for container " + containerIdStr + ", result="
+ + (result ? "success" : "failed"));
+ }
+ if (sleepDelayBeforeSigKill > 0) {
+ new ContainerExecutor.DelayedProcessKiller(container, user, processId,
+ sleepDelayBeforeSigKill, ContainerExecutor.Signal.KILL, exec).start();
+ }
+ }
+
+ private boolean sendSignal(String user, String processId,
+ ContainerExecutor.Signal signal)
+ throws IOException {
+ return exec.signalContainer(
+ new ContainerSignalContext.Builder().setContainer(container)
+ .setUser(user).setPid(processId).setSignal(signal).build());
+ }
+
+ private void reapDockerContainerNoPid(String user) throws IOException {
+ String containerIdStr =
+ container.getContainerTokenIdentifier().getContainerID().toString();
+ LOG.info("Unable to obtain pid, but docker container request detected. "
+ + "Attempting to reap container " + containerIdStr);
+ boolean result = exec.reapContainer(
+ new ContainerReapContext.Builder()
+ .setContainer(container)
+ .setUser(container.getUser())
+ .build());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sent signal to docker container " + containerIdStr
+ + " as user " + user + ", result=" + (result ? "success" : "failed"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 3fa73ec..6776836 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -85,7 +84,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
@@ -115,7 +113,7 @@ public class ContainerLaunch implements Callable<Integer> {
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
private static final String PID_FILE_NAME_FMT = "%s.pid";
- private static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";
+ static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";
protected final Dispatcher dispatcher;
protected final ContainerExecutor exec;
@@ -131,7 +129,6 @@ public class ContainerLaunch implements Callable<Integer> {
protected AtomicBoolean completed = new AtomicBoolean(false);
private volatile boolean killedBeforeStart = false;
- private long sleepDelayBeforeSigKill = 250;
private long maxKillWaitTime = 2000;
protected Path pidFilePath = null;
@@ -152,9 +149,6 @@ public class ContainerLaunch implements Callable<Integer> {
this.dispatcher = dispatcher;
this.dirsHandler = dirsHandler;
this.containerManager = containerManager;
- this.sleepDelayBeforeSigKill =
- conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
- YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
this.maxKillWaitTime =
conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
@@ -515,6 +509,25 @@ public class ContainerLaunch implements Callable<Integer> {
return launchPrep;
}
+ void reapContainer() throws IOException {
+ containerExecLock.lock();
+ try {
+ // Reap the container
+ boolean result = exec.reapContainer(
+ new ContainerReapContext.Builder()
+ .setContainer(container)
+ .setUser(container.getUser())
+ .build());
+ if (!result) {
+ throw new IOException("Reap container failed for container " +
+ container.getContainerId());
+ }
+ cleanupContainerFiles(getContainerWorkDir());
+ } finally {
+ containerExecLock.unlock();
+ }
+ }
+
protected int prepareForLaunch(ContainerStartContext ctx) throws IOException {
ContainerId containerId = container.getContainerId();
if (container.isMarkedForKilling()) {
@@ -721,121 +734,6 @@ public class ContainerLaunch implements Callable<Integer> {
return getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ String.format(ContainerLaunch.PID_FILE_NAME_FMT, containerIdStr);
}
-
- /**
- * Cleanup the container.
- * Cancels the launch if launch has not started yet or signals
- * the executor to not execute the process if not already done so.
- * Also, sends a SIGTERM followed by a SIGKILL to the process if
- * the process id is available.
- * @throws IOException
- */
- public void cleanupContainer() throws IOException {
- ContainerId containerId = container.getContainerId();
- String containerIdStr = containerId.toString();
- LOG.info("Cleaning up container " + containerIdStr);
-
- try {
- context.getNMStateStore().storeContainerKilled(containerId);
- } catch (IOException e) {
- LOG.error("Unable to mark container " + containerId
- + " killed in store", e);
- }
-
- // launch flag will be set to true if process already launched
- boolean alreadyLaunched =
- !containerAlreadyLaunched.compareAndSet(false, true);
- if (!alreadyLaunched) {
- LOG.info("Container " + containerIdStr + " not launched."
- + " No cleanup needed to be done");
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Marking container " + containerIdStr + " as inactive");
- }
- // this should ensure that if the container process has not launched
- // by this time, it will never be launched
- exec.deactivateContainer(containerId);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Getting pid for container " + containerIdStr + " to kill"
- + " from pid file "
- + (pidFilePath != null ? pidFilePath.toString() : "null"));
- }
-
- // however the container process may have already started
- try {
-
- // get process id from pid file if available
- // else if shell is still active, get it from the shell
- String processId = null;
- if (pidFilePath != null) {
- processId = getContainerPid(pidFilePath);
- }
-
- // kill process
- String user = container.getUser();
- if (processId != null) {
- signalProcess(processId, user, containerIdStr);
- } else {
- // Normally this means that the process was notified about
- // deactivateContainer above and did not start.
- // Since we already set the state to RUNNING or REINITIALIZING
- // we have to send a killed event to continue.
- if (!completed.get()) {
- LOG.warn("Container clean up before pid file created "
- + containerIdStr);
- dispatcher.getEventHandler().handle(
- new ContainerExitEvent(container.getContainerId(),
- ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
- Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
- ExitCode.TERMINATED.getExitCode(),
- "Container terminated before pid file created."));
- // There is a possibility that the launch grabbed the file name before
- // the deactivateContainer above but it was slow enough to avoid
- // getContainerPid.
- // Increasing YarnConfiguration.NM_PROCESS_KILL_WAIT_MS
- // reduces the likelihood of this race condition and process leak.
- }
- // The Docker container may not have fully started, reap the container.
- if (DockerLinuxContainerRuntime.isDockerContainerRequested(
- conf,
- container.getLaunchContext().getEnvironment())) {
- reapDockerContainerNoPid(user);
- }
- }
- } catch (Exception e) {
- String message =
- "Exception when trying to cleanup container " + containerIdStr
- + ": " + StringUtils.stringifyException(e);
- LOG.warn(message);
- dispatcher.getEventHandler().handle(
- new ContainerDiagnosticsUpdateEvent(containerId, message));
- } finally {
- // cleanup pid file if present
- if (pidFilePath != null) {
- FileContext lfs = FileContext.getLocalFSFileContext();
- lfs.delete(pidFilePath, false);
- lfs.delete(pidFilePath.suffix(EXIT_CODE_FILE_SUFFIX), false);
- }
- }
- containerExecLock.lock();
- try {
- // Reap the container
- boolean result = exec.reapContainer(
- new ContainerReapContext.Builder()
- .setContainer(container)
- .setUser(container.getUser())
- .build());
- if (!result) {
- throw new IOException("Reap container failed for container "
- + containerIdStr);
- }
- cleanupContainerFiles(getContainerWorkDir());
- } finally {
- containerExecLock.unlock();
- }
- }
/**
* Send a signal to the container.
@@ -874,11 +772,7 @@ public class ContainerLaunch implements Callable<Integer> {
try {
// get process id from pid file if available
// else if shell is still active, get it from the shell
- String processId = null;
- if (pidFilePath != null) {
- processId = getContainerPid(pidFilePath);
- }
-
+ String processId = getContainerPid();
if (processId != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Sending signal to pid " + processId
@@ -912,50 +806,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
}
- private boolean sendSignal(String user, String processId, Signal signal)
- throws IOException {
- return exec.signalContainer(
- new ContainerSignalContext.Builder().setContainer(container)
- .setUser(user).setPid(processId).setSignal(signal).build());
- }
-
- private void signalProcess(String processId, String user,
- String containerIdStr) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sending signal to pid " + processId + " as user " + user
- + " for container " + containerIdStr);
- }
- final Signal signal =
- sleepDelayBeforeSigKill > 0 ? Signal.TERM : Signal.KILL;
-
- boolean result = sendSignal(user, processId, signal);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sent signal " + signal + " to pid " + processId + " as user "
- + user + " for container " + containerIdStr + ", result="
- + (result ? "success" : "failed"));
- }
- if (sleepDelayBeforeSigKill > 0) {
- new DelayedProcessKiller(container, user, processId,
- sleepDelayBeforeSigKill, Signal.KILL, exec).start();
- }
- }
-
- private void reapDockerContainerNoPid(String user) throws IOException {
- String containerIdStr =
- container.getContainerTokenIdentifier().getContainerID().toString();
- LOG.info("Unable to obtain pid, but docker container request detected. "
- + "Attempting to reap container " + containerIdStr);
- boolean result = exec.reapContainer(
- new ContainerReapContext.Builder()
- .setContainer(container)
- .setUser(container.getUser())
- .build());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sent signal to docker container " + containerIdStr
- + " as user " + user + ", result=" + (result ? "success" : "failed"));
- }
- }
-
@VisibleForTesting
public static Signal translateCommandToSignal(
SignalContainerCommand command) {
@@ -1076,14 +926,16 @@ public class ContainerLaunch implements Callable<Integer> {
/**
* Loop through for a time-bounded interval waiting to
* read the process id from a file generated by a running process.
- * @param pidFilePath File from which to read the process id
- * @return Process ID
+ * @return Process ID; null when pidFilePath is null
* @throws Exception
*/
- private String getContainerPid(Path pidFilePath) throws Exception {
+ String getContainerPid() throws Exception {
+ if (pidFilePath == null) {
+ return null;
+ }
String containerIdStr =
container.getContainerId().toString();
- String processId = null;
+ String processId;
if (LOG.isDebugEnabled()) {
LOG.debug("Accessing pid for container " + containerIdStr
+ " from pid file " + pidFilePath);
@@ -1889,4 +1741,28 @@ public class ContainerLaunch implements Callable<Integer> {
LOG.warn("Failed to delete " + path, e);
}
}
+
+ /**
+ * Returns the PID File Path.
+ */
+ Path getPidFilePath() {
+ return pidFilePath;
+ }
+
+ /**
+ * Marks the container to be launched only if it was not launched.
+ *
+ * @return true if successful; false otherwise.
+ */
+ boolean markLaunched() {
+ return containerAlreadyLaunched.compareAndSet(false, true);
+ }
+
+ /**
+ * Returns if the launch is completed or not.
+ */
+ boolean isLaunchCompleted() {
+ return completed.get();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 7870f86..fdfe5b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.slf4j.Logger;
@@ -154,8 +153,8 @@ public class ContainersLauncher extends AbstractService
break;
case CLEANUP_CONTAINER:
case CLEANUP_CONTAINER_FOR_REINIT:
- ContainerLaunch launcher = running.remove(containerId);
- if (launcher == null) {
+ ContainerLaunch existingLaunch = running.remove(containerId);
+ if (existingLaunch == null) {
// Container not launched.
// triggering KILLING to CONTAINER_CLEANEDUP_AFTER_KILL transition.
dispatcher.getEventHandler().handle(
@@ -169,12 +168,9 @@ public class ContainersLauncher extends AbstractService
// Cleanup a container whether it is running/killed/completed, so that
// no sub-processes are alive.
- try {
- launcher.cleanupContainer();
- } catch (IOException e) {
- LOG.warn("Got exception while cleaning container " + containerId
- + ". Ignoring.");
- }
+ ContainerCleanup cleanup = new ContainerCleanup(context, getConfig(),
+ dispatcher, exec, event.getContainer(), existingLaunch);
+ containerLauncher.submit(cleanup);
break;
case SIGNAL_CONTAINER:
SignalContainersLauncherEvent signalEvent =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java
new file mode 100644
index 0000000..6c99379
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.InlineDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link ContainerCleanup}.
+ */
+public class TestContainerCleanup {
+
+ private YarnConfiguration conf;
+ private ContainerId containerId;
+ private ContainerExecutor executor;
+ private ContainerLaunch launch;
+ private ContainerCleanup cleanup;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new YarnConfiguration();
+ conf.setLong(NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, 60000);
+ Context context = mock(Context.class);
+ NMStateStoreService storeService = mock(NMStateStoreService.class);
+ when(context.getNMStateStore()).thenReturn(storeService);
+
+ Dispatcher dispatcher = new InlineDispatcher();
+ executor = mock(ContainerExecutor.class);
+ when(executor.signalContainer(Mockito.any(
+ ContainerSignalContext.class))).thenReturn(true);
+
+ ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
+ 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ containerId = ContainerId.newContainerId(attemptId, 1);
+ Container container = mock(Container.class);
+
+ when(container.getContainerId()).thenReturn(containerId);
+
+ launch = mock(ContainerLaunch.class);
+ launch.containerAlreadyLaunched = new AtomicBoolean(false);
+
+ launch.pidFilePath = new Path("target/" + containerId.toString() + ".pid");
+ when(launch.getContainerPid()).thenReturn(containerId.toString());
+
+ cleanup = new ContainerCleanup(context, conf, dispatcher, executor,
+ container, launch);
+ }
+
+ @Test
+ public void testNoCleanupWhenContainerNotLaunched() throws IOException {
+ cleanup.run();
+ verify(launch, Mockito.times(0)).signalContainer(
+ Mockito.any(SignalContainerCommand.class));
+ }
+
+ @Test
+ public void testCleanup() throws Exception {
+ launch.containerAlreadyLaunched.set(true);
+ cleanup.run();
+ ArgumentCaptor<ContainerSignalContext> captor =
+ ArgumentCaptor.forClass(ContainerSignalContext.class);
+
+ verify(executor, Mockito.times(1)).signalContainer(captor.capture());
+ Assert.assertEquals("signal", ContainerExecutor.Signal.TERM,
+ captor.getValue().getSignal());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
index 0234ac2..f2fafd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
@@ -192,12 +192,14 @@ public class TestContainersLauncher {
.synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
dummyMap.put(containerId, containerLaunch);
Whitebox.setInternalState(spy, "running", dummyMap);
+
when(event.getType())
.thenReturn(ContainersLauncherEventType.CLEANUP_CONTAINER);
- doNothing().when(containerLaunch).cleanupContainer();
+ assertEquals(1, dummyMap.size());
spy.handle(event);
assertEquals(0, dummyMap.size());
- Mockito.verify(containerLaunch, Mockito.times(1)).cleanupContainer();
+ Mockito.verify(containerLauncher, Mockito.times(1))
+ .submit(Mockito.any(ContainerCleanup.class));
}
@Test
@@ -207,12 +209,14 @@ public class TestContainersLauncher {
.synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
dummyMap.put(containerId, containerLaunch);
Whitebox.setInternalState(spy, "running", dummyMap);
+
when(event.getType())
.thenReturn(ContainersLauncherEventType.CLEANUP_CONTAINER_FOR_REINIT);
- doNothing().when(containerLaunch).cleanupContainer();
+ assertEquals(1, dummyMap.size());
spy.handle(event);
assertEquals(0, dummyMap.size());
- Mockito.verify(containerLaunch, Mockito.times(1)).cleanupContainer();
+ Mockito.verify(containerLauncher, Mockito.times(1))
+ .submit(Mockito.any(ContainerCleanup.class));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-12459. Fix revert: Add new op
GETFILEBLOCKLOCATIONS to WebHDFS REST API. Contributed by Weiwei Yang.
Posted by sh...@apache.org.
HDFS-12459. Fix revert: Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API. Contributed by Weiwei Yang.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ead525c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ead525c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ead525c
Branch: refs/heads/HDFS-12943
Commit: 3ead525c71cba068e7abf1c76ad629bfeec10852
Parents: cdc4350
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Oct 10 10:07:22 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Oct 10 10:11:10 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/web/resources/GetOpParam.java | 12 +-
.../web/resources/NamenodeWebHdfsMethods.java | 14 ++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 32 ++++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 173 +++++++++++++++++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 148 ++++++++++++++++
5 files changed, 378 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index 85bb11d..3f4f146 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -33,8 +33,18 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
GETHOMEDIRECTORY(false, HttpURLConnection.HTTP_OK),
GETDELEGATIONTOKEN(false, HttpURLConnection.HTTP_OK, true),
- /** GET_BLOCK_LOCATIONS is a private unstable op. */
+ /**
+ * GET_BLOCK_LOCATIONS is a private/stable API op. It returns a
+ * {@link org.apache.hadoop.hdfs.protocol.LocatedBlocks}
+ * json object.
+ */
GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK),
+ /**
+ * GETFILEBLOCKLOCATIONS is the public op that complies with
+ * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations}
+ * interface.
+ */
+ GETFILEBLOCKLOCATIONS(false, HttpURLConnection.HTTP_OK),
GETACLSTATUS(false, HttpURLConnection.HTTP_OK),
GETXATTRS(false, HttpURLConnection.HTTP_OK),
GETTRASHROOT(false, HttpURLConnection.HTTP_OK),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 619d439..d73fd45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -59,6 +59,7 @@ import javax.ws.rs.core.Response.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
@@ -72,6 +73,7 @@ import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -1075,6 +1077,18 @@ public class NamenodeWebHdfsMethods {
.build();
}
}
+ case GETFILEBLOCKLOCATIONS:
+ {
+ final long offsetValue = offset.getValue();
+ final Long lengthValue = length.getValue();
+ LocatedBlocks locatedBlocks = getRpcClientProtocol()
+ .getBlockLocations(fullpath, offsetValue, lengthValue != null ?
+ lengthValue : Long.MAX_VALUE);
+ BlockLocation[] locations =
+ DFSUtilClient.locatedBlocks2Locations(locatedBlocks);
+ final String js = JsonUtil.toJsonString(locations);
+ return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+ }
case GET_BLOCK_LOCATIONS:
{
final long offsetValue = offset.getValue();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 5c810bf..f966640 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -572,4 +572,36 @@ public class JsonUtil {
m.put("dirStatus", toJsonMap(snapshottableDirectoryStatus.getDirStatus()));
return m;
}
+
+ private static Map<String, Object> toJsonMap(
+ final BlockLocation blockLocation) throws IOException {
+ if (blockLocation == null) {
+ return null;
+ }
+
+ final Map<String, Object> m = new HashMap<>();
+ m.put("length", blockLocation.getLength());
+ m.put("offset", blockLocation.getOffset());
+ m.put("corrupt", blockLocation.isCorrupt());
+ m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes()));
+ m.put("cachedHosts", blockLocation.getCachedHosts());
+ m.put("hosts", blockLocation.getHosts());
+ m.put("names", blockLocation.getNames());
+ m.put("topologyPaths", blockLocation.getTopologyPaths());
+ return m;
+ }
+
+ public static String toJsonString(BlockLocation[] locations)
+ throws IOException {
+ if (locations == null) {
+ return null;
+ }
+ final Map<String, Object> m = new HashMap<>();
+ Object[] blockLocations = new Object[locations.length];
+ for(int i=0; i<locations.length; i++) {
+ blockLocations[i] = toJsonMap(locations[i]);
+ }
+ m.put(BlockLocation.class.getSimpleName(), blockLocations);
+ return toJsonString("BlockLocations", m);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index eb11815..e29b932 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -52,6 +52,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`GETSTORAGEPOLICY`](#Get_Storage_Policy) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStoragePolicy)
* [`GETSNAPSHOTDIFF`](#Get_Snapshot_Diff)
* [`GETSNAPSHOTTABLEDIRECTORYLIST`](#Get_Snapshottable_Directory_List)
+ * [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations)
* HTTP PUT
* [`CREATE`](#Create_and_Write_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create)
* [`MKDIRS`](#Make_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs)
@@ -1087,6 +1088,49 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).unsetStor
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStoragePolicy
+### Get File Block Locations
+
+* Submit a HTTP GET request.
+
+ curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILEBLOCKLOCATIONS
+
+ The client receives a response with a [`BlockLocations` JSON Object](#Block_Locations_JSON_Schema):
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {
+ "BlockLocations" :
+ {
+ "BlockLocation":
+ [
+ {
+ "cachedHosts" : [],
+ "corrupt" : false,
+ "hosts" : ["host"],
+ "length" : 134217728, // length of this block
+ "names" : ["host:ip"],
+ "offset" : 0, // offset of the block in the file
+ "storageTypes" : ["DISK"], // enum {RAM_DISK, SSD, DISK, ARCHIVE}
+ "topologyPaths" : ["/default-rack/hostname:ip"]
+ }, {
+ "cachedHosts" : [],
+ "corrupt" : false,
+ "hosts" : ["host"],
+ "length" : 62599364,
+ "names" : ["host:ip"],
+ "offset" : 134217728,
+ "storageTypes" : ["DISK"],
+ "topologyPaths" : ["/default-rack/hostname:ip"]
+ },
+ ...
+ ]
+ }
+ }
+
+See also: [`offset`](#Offset), [`length`](#Length), [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations
+
Extended Attributes(XAttrs) Operations
--------------------------------------
@@ -2227,6 +2271,135 @@ var snapshottableDirectoryStatus =
}
```
+### BlockLocations JSON Schema
+
+A `BlockLocations` JSON object represents an array of `BlockLocation` JSON objects.
+
+```json
+{
+ "name" : "BlockLocations",
+ "properties":
+ {
+ "BlockLocations":
+ {
+ "type" : "object",
+ "properties":
+ {
+ "BlockLocation":
+ {
+ "description": "An array of BlockLocation",
+ "type" : "array",
+ "items" : blockLocationProperties //See BlockLocation Properties
+ }
+ }
+ }
+ }
+}
+```
+
+See also [`BlockLocation` Properties](#BlockLocation_Properties), [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations), [BlockLocation](../../api/org/apache/hadoop/fs/BlockLocation.html)
+
+### BlockLocation JSON Schema
+
+```json
+{
+ "name" : "BlockLocation",
+ "properties":
+ {
+ "BlockLocation": blockLocationProperties //See BlockLocation Properties
+ }
+}
+```
+
+See also [`BlockLocation` Properties](#BlockLocation_Properties), [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations), [BlockLocation](../../api/org/apache/hadoop/fs/BlockLocation.html)
+
+#### BlockLocation Properties
+
+JavaScript syntax is used to define `blockLocationProperties` so that it can be referred in both `BlockLocation` and `BlockLocations` JSON schemas.
+
+```javascript
+var blockLocationProperties =
+{
+ "type" : "object",
+ "properties":
+ {
+ "cachedHosts":
+ {
+ "description": "Datanode hostnames with a cached replica",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "A datanode hostname",
+ "type" : "string"
+ }
+ },
+ "corrupt":
+ {
+ "description": "True if the block is corrupted",
+ "type" : "boolean",
+ "required" : "true"
+ },
+ "hosts":
+ {
+ "description": "Datanode hostnames store the block",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "A datanode hostname",
+ "type" : "string"
+ }
+ },
+ "length":
+ {
+ "description": "Length of the block",
+ "type" : "integer",
+ "required" : "true"
+ },
+ "names":
+ {
+ "description": "Datanode IP:xferPort for accessing the block",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "DatanodeIP:xferPort",
+ "type" : "string"
+ }
+ },
+ "offset":
+ {
+ "description": "Offset of the block in the file",
+ "type" : "integer",
+ "required" : "true"
+ },
+ "storageTypes":
+ {
+ "description": "Storage type of each replica",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "Storage type",
+ "enum" : ["RAM_DISK", "SSD", "DISK", "ARCHIVE"]
+ }
+ },
+ "topologyPaths":
+ {
+ "description": "Datanode addresses in network topology",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "/rack/host:ip",
+ "type" : "string"
+ }
+ }
+ }
+};
+```
+
HTTP Query Parameter Dictionary
-------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index d7ed46b..71948d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -47,6 +47,7 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
+import java.util.Map;
import java.util.Random;
import com.google.common.collect.ImmutableList;
@@ -118,6 +119,9 @@ import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.MapType;
+
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.doReturn;
@@ -1136,6 +1140,150 @@ public class TestWebHDFS {
}
}
+ @Test
+ public void testWebHdfsGetBlockLocations() throws Exception{
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ final int offset = 42;
+ final int length = 512;
+ final Path path = new Path("/foo");
+ byte[] contents = new byte[1024];
+ RANDOM.nextBytes(contents);
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsConstants.WEBHDFS_SCHEME);
+ try (OutputStream os = fs.create(path)) {
+ os.write(contents);
+ }
+ BlockLocation[] locations = fs.getFileBlockLocations(path, offset,
+ length);
+
+ // Query webhdfs REST API to get block locations
+ InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
+
+ // Case 1
+ // URL without length or offset parameters
+ URL url1 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS");
+
+ String response1 = getResponse(url1, "GET");
+ // Parse BlockLocation array from json output using object mapper
+ BlockLocation[] locationArray1 = toBlockLocationArray(response1);
+
+ // Verify the result from rest call is same as file system api
+ verifyEquals(locations, locationArray1);
+
+ // Case 2
+ // URL contains length and offset parameters
+ URL url2 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&length=" + length + "&offset=" + offset);
+
+ String response2 = getResponse(url2, "GET");
+ BlockLocation[] locationArray2 = toBlockLocationArray(response2);
+
+ verifyEquals(locations, locationArray2);
+
+ // Case 3
+ // URL contains length parameter but without offset parameters
+ URL url3 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&length=" + length);
+
+ String response3 = getResponse(url3, "GET");
+ BlockLocation[] locationArray3 = toBlockLocationArray(response3);
+
+ verifyEquals(locations, locationArray3);
+
+ // Case 4
+ // URL contains offset parameter but without length parameter
+ URL url4 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&offset=" + offset);
+
+ String response4 = getResponse(url4, "GET");
+ BlockLocation[] locationArray4 = toBlockLocationArray(response4);
+
+ verifyEquals(locations, locationArray4);
+
+ // Case 5
+ // URL specifies offset exceeds the file length
+ URL url5 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&offset=1200");
+
+ String response5 = getResponse(url5, "GET");
+ BlockLocation[] locationArray5 = toBlockLocationArray(response5);
+
+ // Expected an empty array of BlockLocation
+ verifyEquals(new BlockLocation[] {}, locationArray5);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private BlockLocation[] toBlockLocationArray(String json)
+ throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ MapType subType = mapper.getTypeFactory().constructMapType(
+ Map.class,
+ String.class,
+ BlockLocation[].class);
+ MapType rootType = mapper.getTypeFactory().constructMapType(
+ Map.class,
+ mapper.constructType(String.class),
+ mapper.constructType(subType));
+
+ Map<String, Map<String, BlockLocation[]>> jsonMap = mapper
+ .readValue(json, rootType);
+ Map<String, BlockLocation[]> locationMap = jsonMap
+ .get("BlockLocations");
+ BlockLocation[] locationArray = locationMap.get(
+ BlockLocation.class.getSimpleName());
+ return locationArray;
+ }
+
+ private void verifyEquals(BlockLocation[] locations1,
+ BlockLocation[] locations2) throws IOException {
+ for(int i=0; i<locations1.length; i++) {
+ BlockLocation location1 = locations1[i];
+ BlockLocation location2 = locations2[i];
+ Assert.assertEquals(location1.getLength(),
+ location2.getLength());
+ Assert.assertEquals(location1.getOffset(),
+ location2.getOffset());
+ Assert.assertArrayEquals(location1.getCachedHosts(),
+ location2.getCachedHosts());
+ Assert.assertArrayEquals(location1.getHosts(),
+ location2.getHosts());
+ Assert.assertArrayEquals(location1.getNames(),
+ location2.getNames());
+ Assert.assertArrayEquals(location1.getTopologyPaths(),
+ location2.getTopologyPaths());
+ Assert.assertArrayEquals(location1.getStorageTypes(),
+ location2.getStorageTypes());
+ }
+ }
+
+ private static String getResponse(URL url, String httpRequestType)
+ throws IOException {
+ HttpURLConnection conn = null;
+ try {
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod(httpRequestType);
+ conn.setInstanceFollowRedirects(false);
+ return IOUtils.toString(conn.getInputStream(),
+ StandardCharsets.UTF_8);
+ } finally {
+ if(conn != null) {
+ conn.disconnect();
+ }
+ }
+ }
+
private WebHdfsFileSystem createWebHDFSAsTestUser(final Configuration conf,
final URI uri, final String userName) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-8666. [UI2] Remove
application tab from YARN Queue Page. Contributed by Yesha Vora.
Posted by sh...@apache.org.
YARN-8666. [UI2] Remove application tab from YARN Queue Page. Contributed by Yesha Vora.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3da741f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3da741f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3da741f
Branch: refs/heads/HDFS-12943
Commit: d3da741fa27c14f3d1391a76d6ade5d1a8365aa5
Parents: 7b57f2f
Author: Sunil G <su...@apache.org>
Authored: Thu Oct 11 16:05:53 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Oct 11 16:05:53 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/templates/yarn-queue.hbs | 11 -----------
1 file changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3da741f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
index 5cced1c..756ef69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
@@ -60,17 +60,6 @@
</div>
</div>
</div>
- <div class="panel-heading">
- <div class="clearfix">
- <ul class="nav nav-pills">
- <ul class="nav nav-pills collapse in">
- {{#link-to 'yarn-queue.apps' tagName="li" class=(if (eq target.currentPath 'yarn-queue.apps') "active")}}
- {{#link-to 'yarn-queue.apps' appId (query-params service=model.serviceName)}}Applications{{/link-to}}
- {{/link-to}}
- </ul>
- </ul>
- </div>
- </div>
<div class="panel-body yarn-app-body">
{{outlet}}
</div>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDDS-619. hdds.db.profile should
not be tagged as a required setting & should default to DISK. Contributed by
Dinesh Chitlangia.
Posted by sh...@apache.org.
HDDS-619. hdds.db.profile should not be tagged as a required setting & should default to DISK. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bd000c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bd000c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bd000c8
Branch: refs/heads/HDFS-12943
Commit: 2bd000c85195416b9bcd06a3abb5100aeeda9727
Parents: 045069e
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Oct 10 17:37:20 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Oct 10 17:37:20 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd000c8/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 13b3bb7..210b075 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -72,7 +72,7 @@ public final class HddsConfigKeys {
// DB Profiles used by ROCKDB instances.
public static final String HDDS_DB_PROFILE = "hdds.db.profile";
- public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.SSD;
+ public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.DISK;
// Once a container usage crosses this threshold, it is eligible for
// closing.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd000c8/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aee471e..ef54ed2 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1184,8 +1184,8 @@
<property>
<name>hdds.db.profile</name>
- <value>SSD</value>
- <tag>OZONE, OM, PERFORMANCE, REQUIRED</tag>
+ <value>DISK</value>
+ <tag>OZONE, OM, PERFORMANCE</tag>
<description>This property allows user to pick a configuration
that tunes the RocksDB settings for the hardware it is running
on. Right now, we have SSD and DISK as profile options.</description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-15839. Review + update
cloud store sensitive keys in hadoop.security.sensitive-config-keys.
Contributed by Steve Loughran.
Posted by sh...@apache.org.
HADOOP-15839. Review + update cloud store sensitive keys in hadoop.security.sensitive-config-keys.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdc43507
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdc43507
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdc43507
Branch: refs/heads/HDFS-12943
Commit: cdc4350718055189fef8c70e31314607001d4009
Parents: 10185d9
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 10 17:00:37 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 10 17:00:37 2018 +0100
----------------------------------------------------------------------
.../hadoop-common/src/main/resources/core-default.xml | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdc43507/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 3ea2797..32dd622 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -599,9 +599,17 @@
secret$
password$
ssl.keystore.pass$
- fs.s3.*[Ss]ecret.?[Kk]ey
+ fs.s3a.server-side-encryption.key
fs.s3a.*.server-side-encryption.key
+ fs.s3a.secret.key
+ fs.s3a.*.secret.key
+ fs.s3a.session.key
+ fs.s3a.*.session.key
+ fs.s3a.session.token
+ fs.s3a.*.session.token
fs.azure.account.key.*
+ fs.azure.oauth2.*
+ fs.adl.oauth2.*
credential$
oauth.*secret
oauth.*password
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDDS-517. Implement HeadObject
REST endpoint. Contributed by LiXin Ge.
Posted by sh...@apache.org.
HDDS-517. Implement HeadObject REST endpoint. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd280514
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd280514
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd280514
Branch: refs/heads/HDFS-12943
Commit: cd280514b84e8a2915f274883a07f152b772ebad
Parents: b39b802
Author: Márton Elek <el...@apache.org>
Authored: Wed Oct 10 15:23:00 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Wed Oct 10 15:28:41 2018 +0200
----------------------------------------------------------------------
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +-
.../hadoop/ozone/s3/object/HeadObject.java | 38 +++++---
.../hadoop/ozone/client/OzoneBucketStub.java | 6 +-
.../hadoop/ozone/s3/object/TestHeadObject.java | 95 ++++++++++++++++++++
4 files changed, 131 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
index 3fee7be..99ae7a0 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@ -42,6 +42,8 @@ public final class S3ErrorTable {
"BucketNotEmpty", "The bucket you tried to delete is not empty.",
HTTP_CONFLICT);
+ public static final OS3Exception NO_SUCH_OBJECT = new OS3Exception(
+ "NoSuchObject", "The specified object does not exist", HTTP_NOT_FOUND);
/**
* Create a new instance of Error.
@@ -60,7 +62,8 @@ public final class S3ErrorTable {
* Resources, which can be defined in OS3Exception.
*/
public enum Resource {
- BUCKET("Bucket");
+ BUCKET("Bucket"),
+ OBJECT("Object");
private final String resource;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
index e284100..8bbdf76 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
@@ -18,41 +18,57 @@
package org.apache.hadoop.ozone.s3.object;
import javax.ws.rs.HEAD;
-import javax.ws.rs.HeaderParam;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.IOException;
-import java.io.InputStream;
-import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.s3.EndpointBase;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+
+import org.apache.http.HttpStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Get object info rest endpoint.
*/
@Path("/{volume}/{bucket}/{path:.+}")
public class HeadObject extends EndpointBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HeadObject.class);
@HEAD
@Produces(MediaType.APPLICATION_XML)
public Response head(
@PathParam("volume") String volumeName,
@PathParam("bucket") String bucketName,
- @PathParam("path") String keyPath,
- @HeaderParam("Content-Length") long length,
- InputStream body) throws IOException {
+ @PathParam("path") String keyPath) throws Exception {
+ OzoneKeyDetails key;
- OzoneBucket bucket = getBucket(volumeName, bucketName);
- OzoneKeyDetails key = bucket.getKey(keyPath);
+ try {
+ key = getVolume(volumeName).getBucket(bucketName).getKey(keyPath);
+ // TODO: return the specified range bytes of this object.
+ } catch (IOException ex) {
+ LOG.error("Exception occurred in HeadObject", ex);
+ if (ex.getMessage().contains("KEY_NOT_FOUND")) {
+ OS3Exception os3Exception = S3ErrorTable.newError(S3ErrorTable
+ .NO_SUCH_OBJECT, S3ErrorTable.Resource.OBJECT);
+ throw os3Exception;
+ } else {
+ throw ex;
+ }
+ }
- return Response.
- ok()
+ return Response.ok().status(HttpStatus.SC_OK)
+ .header("Last-Modified", key.getModificationTime())
+ .header("ETag", "" + key.getModificationTime())
.header("Content-Length", key.getDataSize())
+ .header("Content-Type", "binary/octet-stream")
.build();
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 18dff71..ff0f706 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -107,7 +107,11 @@ public class OzoneBucketStub extends OzoneBucket {
@Override
public OzoneKeyDetails getKey(String key) throws IOException {
- return keyDetails.get(key);
+ if (keyDetails.containsKey(key)) {
+ return keyDetails.get(key);
+ } else {
+ throw new IOException("Lookup key failed, error:KEY_NOT_FOUND");
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
new file mode 100644
index 0000000..5b65dac
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.hadoop.ozone.s3.object;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.Response;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import java.io.IOException;
+
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+
+/**
+ * Test head object.
+ */
+public class TestHeadObject {
+ private String volName = "vol1";
+ private String bucketName = "b1";
+ private OzoneClientStub clientStub;
+ private ObjectStore objectStoreStub;
+ private HeadObject headObject;
+ private OzoneBucket bucket;
+
+ @Before
+ public void setup() throws IOException {
+ //Create client stub and object store stub.
+ clientStub = new OzoneClientStub();
+ objectStoreStub = clientStub.getObjectStore();
+
+ // Create volume and bucket
+ objectStoreStub.createVolume(volName);
+ OzoneVolume volumeStub = objectStoreStub.getVolume(volName);
+ volumeStub.createBucket(bucketName);
+ bucket = objectStoreStub.getVolume(volName).getBucket(bucketName);
+
+ // Create HeadBucket and setClient to OzoneClientStub
+ headObject = new HeadObject();
+ headObject.setClient(clientStub);
+ }
+
+ @Test
+ public void testHeadObject() throws Exception {
+ //GIVEN
+ String value = RandomStringUtils.randomAlphanumeric(32);
+ OzoneOutputStream out = bucket.createKey("key1",
+ value.getBytes().length, ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE);
+ out.write(value.getBytes());
+ out.close();
+
+ //WHEN
+ Response response = headObject.head(volName, bucketName, "key1");
+
+ //THEN
+ Assert.assertEquals(200, response.getStatus());
+ Assert.assertEquals(value.getBytes().length,
+ Long.parseLong(response.getHeaderString("Content-Length")));
+ }
+
+ @Test
+ public void testHeadFailByBadName() throws Exception {
+ //Head an object that doesn't exist.
+ try {
+ headObject.head(volName, bucketName, "badKeyName");
+ } catch (OS3Exception ex) {
+ Assert.assertTrue(ex.getCode().contains("NoSuchObject"));
+ Assert.assertTrue(ex.getErrorMessage().contains("object does not exist"));
+ Assert.assertEquals(HTTP_NOT_FOUND, ex.getHttpCode());
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDFS-13951. HDFS
DelegationTokenFetcher can't print non-HDFS tokens in a tokenfile.
Contributed by Steve Loughran.
Posted by sh...@apache.org.
HDFS-13951. HDFS DelegationTokenFetcher can't print non-HDFS tokens in a tokenfile.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2addebb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2addebb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2addebb9
Branch: refs/heads/HDFS-12943
Commit: 2addebb94f592e46b261e2edd9e95a82e83bd761
Parents: 735a36a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 20:03:56 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 20:03:56 2018 +0100
----------------------------------------------------------------------
.../hdfs/tools/DelegationTokenFetcher.java | 58 +++++++++++++++-----
1 file changed, 43 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2addebb9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 7d1e88f..0dc4bcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.tools;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
@@ -36,12 +34,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.GenericOptionsParser;
@@ -64,10 +62,29 @@ public class DelegationTokenFetcher {
private static final String RENEWER = "renewer";
private static final String VERBOSE = "verbose";
/**
- * Command-line interface
+ * Command-line interface.
+ * @param args argument list.
+ * @throws Exception on a failure.
+ * @throws org.apache.hadoop.util.ExitUtil.ExitException if the command
+ * failed and exiting was disabled.
*/
public static void main(final String[] args) throws Exception {
- final Configuration conf = new HdfsConfiguration();
+ main(new HdfsConfiguration(), args);
+ }
+
+ /**
+ * Command line interface with a specific configuration.
+ * Errors in this operation will call {@link ExitUtil#terminate(int)} to
+ * exit the process.
+ * @param conf configuration to create filesystems with.
+ * @param args argument list.
+ * @throws Exception on a failure.
+ * @throws org.apache.hadoop.util.ExitUtil.ExitException if the command
+ * failed and exiting was disabled.
+ */
+ @VisibleForTesting
+ public static void main(Configuration conf, final String[] args)
+ throws Exception {
Options fetcherOptions = new Options();
fetcherOptions
.addOption(WEBSERVICE, true, "HTTP url to reach the NameNode at")
@@ -96,17 +113,19 @@ public class DelegationTokenFetcher {
// check option validity
if (help) {
printUsage(System.out);
- System.exit(0);
+ return;
}
int commandCount = (cancel ? 1 : 0) + (renew ? 1 : 0) + (print ? 1 : 0);
if (commandCount > 1) {
System.err.println("ERROR: Only specify cancel, renew or print.");
printUsage(System.err);
+ return;
}
if (remaining.length != 1 || remaining[0].charAt(0) == '-') {
System.err.println("ERROR: Must specify exactly one token file");
printUsage(System.err);
+ return;
}
// default to using the local file system
FileSystem local = FileSystem.getLocal(conf);
@@ -195,19 +214,22 @@ public class DelegationTokenFetcher {
}
@VisibleForTesting
- static String printTokensToString(
+ public static String printTokensToString(
final Configuration conf,
final Path tokenFile,
final boolean verbose) throws IOException {
StringBuilder sbld = new StringBuilder();
final String nl = System.getProperty("line.separator");
- DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0,
- 0, null).createIdentifier();
for (Token<?> token : readTokens(tokenFile, conf)) {
- DataInputStream in = new DataInputStream(new ByteArrayInputStream(token
- .getIdentifier()));
- id.readFields(in);
- String idStr = (verbose? id.toString() : id.toStringStable());
+ TokenIdentifier tokenId = token.decodeIdentifier();
+
+ String idStr;
+ if (tokenId instanceof DelegationTokenIdentifier) {
+ DelegationTokenIdentifier id = (DelegationTokenIdentifier) tokenId;
+ idStr = (verbose? id.toString() : id.toStringStable());
+ } else {
+ idStr = tokenId.toString();
+ }
sbld
.append("Token (").append(idStr)
.append(") for ").append(token.getService()).append(nl);
@@ -222,13 +244,19 @@ public class DelegationTokenFetcher {
System.out.print(printTokensToString(conf, tokenFile, verbose));
}
+ /**
+ * Print usage to the error stream, then
+ * call {@link ExitUtil#terminate(int)} with status code 1.
+ * This will exit or raise an exception if that's been disabled.
+ * @param err stream for the messages.
+ */
private static void printUsage(PrintStream err) {
err.println("fetchdt retrieves delegation tokens from the NameNode");
err.println();
err.println("fetchdt <opts> <token file>");
err.println("Options:");
- err.println(" --webservice <url> Url to contact NN on (starts with " +
- "http:// or https://)");
+ err.println(" --webservice <url> URL to contact NN on (starts with " +
+ "http:// or https://), or other filesystem URL");
err.println(" --renewer <name> Name of the delegation token renewer");
err.println(" --cancel Cancel the delegation token");
err.println(" --renew Renew the delegation token. " +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HADOOP-15832. Upgrade BouncyCastle
to 1.60. Contributed by Robert Kanter.
Posted by sh...@apache.org.
HADOOP-15832. Upgrade BouncyCastle to 1.60. Contributed by Robert Kanter.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fa3feb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fa3feb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fa3feb5
Branch: refs/heads/HDFS-12943
Commit: 6fa3feb577d05d73a2eb1bc8e39800326f678c31
Parents: 794c045
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 10 10:16:57 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 10 10:16:57 2018 +0900
----------------------------------------------------------------------
.../hadoop-client-check-invariants/pom.xml | 2 ++
.../hadoop-client-check-test-invariants/pom.xml | 2 ++
.../hadoop-client-minicluster/pom.xml | 2 ++
.../hadoop-client-runtime/pom.xml | 2 ++
hadoop-common-project/hadoop-common/pom.xml | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 2 +-
hadoop-hdds/server-scm/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 +-
.../hadoop-mapreduce-client-app/pom.xml | 20 ++++++++++++++++++++
.../hadoop-mapreduce-client-jobclient/pom.xml | 7 ++++++-
hadoop-ozone/ozone-manager/pom.xml | 2 +-
hadoop-project/pom.xml | 12 +++++++++---
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../pom.xml | 2 +-
.../hadoop-yarn-server-tests/pom.xml | 2 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 8 ++++++++
18 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
index 4319a8b..f062986 100644
--- a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
@@ -90,6 +90,8 @@
<exclude>log4j:log4j</exclude>
<!-- Leave javax annotations we need exposed -->
<exclude>com.google.code.findbugs:jsr305</exclude>
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
index 70a987f..c02ce56 100644
--- a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
@@ -98,6 +98,8 @@
<exclude> org.hamcrest:hamcrest-core</exclude>
<!-- Leave javax annotations we need exposed -->
<exclude>com.google.code.findbugs:jsr305</exclude>
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index dac5e69..70af1ed 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -671,6 +671,8 @@
<exclude>javax.annotation:javax.annotation-api</exclude>
<exclude>org.eclipse.jetty:jetty-jndi</exclude>
<!-- We need a filter that matches just those things that are included in the above artiacts -->
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</artifactSet>
<filters>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml
index 8afc89e..1e98a61 100644
--- a/hadoop-client-modules/hadoop-client-runtime/pom.xml
+++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml
@@ -162,6 +162,8 @@
<exclude>org.eclipse.jetty:jetty-servlet</exclude>
<exclude>org.eclipse.jetty:jetty-security</exclude>
<exclude>org.ow2.asm:*</exclude>
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</artifactSet>
<filters>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index a9e0fb1..3036b64 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -303,7 +303,7 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 9b1936f..3a6ed54 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -171,7 +171,7 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdds/server-scm/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
index 92563e7..5fcd2bb 100644
--- a/hadoop-hdds/server-scm/pom.xml
+++ b/hadoop-hdds/server-scm/pom.xml
@@ -93,7 +93,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index c0078da..c3d112e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -204,7 +204,7 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 8d796ab..d99e495 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -165,7 +165,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index a0ea04f..f97ca17 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -190,7 +190,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 0a354c8..fab6466 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -46,6 +46,16 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -88,6 +98,16 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index b961cc7..b956fd4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -108,7 +108,12 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-ozone/ozone-manager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
index e2062ef..76e3afb 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -42,7 +42,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0f54a55..5f88371 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -96,6 +96,8 @@
<guice.version>4.0</guice.version>
<joda-time.version>2.9.9</joda-time.version>
+ <bouncycastle.version>1.60</bouncycastle.version>
+
<!-- Required for testing LDAP integration -->
<apacheds.version>2.0.0-M21</apacheds.version>
<ldap-api.version>1.0.0-M33</ldap-api.version>
@@ -1424,10 +1426,14 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
- <version>1.46</version>
- <scope>test</scope>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <version>${bouncycastle.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <version>${bouncycastle.version}</version>
+ </dependency>
<dependency>
<groupId>joda-time</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index de0d78b..133003a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -139,7 +139,7 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 5ea8c9e..bfac673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -177,7 +177,7 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 8c559b9..a05cdc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -127,7 +127,7 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index aa9f753..d08c25f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -115,6 +115,14 @@
<artifactId>jersey-test-framework-grizzly2</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ </dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDFS-13878. HttpFS: Implement
GETSNAPSHOTTABLEDIRECTORYLIST. Contributed by Siyao Meng.
Posted by sh...@apache.org.
HDFS-13878. HttpFS: Implement GETSNAPSHOTTABLEDIRECTORYLIST. Contributed by Siyao Meng.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6dcfef79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6dcfef79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6dcfef79
Branch: refs/heads/HDFS-12943
Commit: 6dcfef79afe97eda6d09fe2567bb4d4074223141
Parents: 3532aa3
Author: Siyao Meng <sm...@cloudera.com>
Authored: Thu Oct 11 15:01:50 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Oct 11 15:01:50 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/http/client/HttpFSFileSystem.java | 16 +++++-
.../hadoop/fs/http/server/FSOperations.java | 35 +++++++++++++
.../http/server/HttpFSParametersProvider.java | 1 +
.../hadoop/fs/http/server/HttpFSServer.java | 8 +++
.../fs/http/client/BaseTestHttpFSWith.java | 54 +++++++++++++++++++-
.../hadoop/fs/http/server/TestHttpFSServer.java | 53 +++++++++++++++++++
6 files changed, 165 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 3904a87..d80c276 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.web.JsonUtilClient;
import org.apache.hadoop.lib.wsrs.EnumSetParam;
import org.apache.hadoop.security.UserGroupInformation;
@@ -234,7 +235,8 @@ public class HttpFSFileSystem extends FileSystem
SETSTORAGEPOLICY(HTTP_PUT), UNSETSTORAGEPOLICY(HTTP_POST),
ALLOWSNAPSHOT(HTTP_PUT), DISALLOWSNAPSHOT(HTTP_PUT),
CREATESNAPSHOT(HTTP_PUT), DELETESNAPSHOT(HTTP_DELETE),
- RENAMESNAPSHOT(HTTP_PUT), GETSNAPSHOTDIFF(HTTP_GET);
+ RENAMESNAPSHOT(HTTP_PUT), GETSNAPSHOTDIFF(HTTP_GET),
+ GETSNAPSHOTTABLEDIRECTORYLIST(HTTP_GET);
private String httpMethod;
@@ -1482,4 +1484,16 @@ public class HttpFSFileSystem extends FileSystem
return JsonUtilClient.toSnapshotDiffReport(json);
}
+ public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList()
+ throws IOException {
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(OP_PARAM, Operation.GETSNAPSHOTTABLEDIRECTORYLIST.toString());
+ HttpURLConnection conn = getConnection(
+ Operation.GETSNAPSHOTTABLEDIRECTORYLIST.getMethod(),
+ params, new Path(getUri().toString(), "/"), true);
+ HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+ return JsonUtilClient.toSnapshottableDirectoryList(json);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index ed7628f..7989895 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.lib.service.FileSystemAccess;
@@ -1701,4 +1702,38 @@ public class FSOperations {
}
}
}
+
+ /**
+ * Executor that performs a getSnapshottableDirListing operation.
+ */
+ @InterfaceAudience.Private
+ public static class FSGetSnapshottableDirListing implements
+ FileSystemAccess.FileSystemExecutor<String> {
+
+ /**
+ * Creates a getSnapshottableDirListing executor.
+ */
+ public FSGetSnapshottableDirListing() {
+ }
+
+ /**
+ * Executes the filesystem operation.
+ * @param fs filesystem instance to use.
+ * @return A JSON string of all snapshottable directories.
+ * @throws IOException thrown if an IO error occurred.
+ */
+ @Override
+ public String execute(FileSystem fs) throws IOException {
+ SnapshottableDirectoryStatus[] sds = null;
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem dfs = (DistributedFileSystem) fs;
+ sds = dfs.getSnapshottableDirListing();
+ } else {
+ throw new UnsupportedOperationException("getSnapshottableDirListing is "
+ + "not supported for HttpFs on " + fs.getClass()
+ + ". Please check your fs.defaultFS configuration");
+ }
+ return JsonUtil.toJsonString(sds);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
index 5301527..754ae2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
@@ -113,6 +113,7 @@ public class HttpFSParametersProvider extends ParametersProvider {
PARAMS_DEF.put(Operation.GETSNAPSHOTDIFF,
new Class[] {OldSnapshotNameParam.class,
SnapshotNameParam.class});
+ PARAMS_DEF.put(Operation.GETSNAPSHOTTABLEDIRECTORYLIST, new Class[] {});
}
public HttpFSParametersProvider() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index f510061..2895210 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -379,6 +379,14 @@ public class HttpFSServer {
response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
break;
}
+ case GETSNAPSHOTTABLEDIRECTORYLIST: {
+ FSOperations.FSGetSnapshottableDirListing command =
+ new FSOperations.FSGetSnapshottableDirListing();
+ String js = fsExecute(user, command);
+ AUDIT_LOG.info("[{}]", "/");
+ response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+ break;
+ }
default: {
throw new IOException(
MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index cd9d3b9..a23ca7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -41,6 +41,8 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -1071,7 +1073,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING,
CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT,
ALLOW_SNAPSHOT, DISALLOW_SNAPSHOT, DISALLOW_SNAPSHOT_EXCEPTION,
- FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF
+ FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST
}
private void operation(Operation op) throws Exception {
@@ -1185,6 +1187,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
testGetSnapshotDiff();
testGetSnapshotDiffIllegalParam();
break;
+ case GET_SNAPSHOTTABLE_DIRECTORY_LIST:
+ testGetSnapshottableDirListing();
+ break;
}
}
@@ -1529,4 +1534,51 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
fs.delete(path, true);
}
}
+
+ private void verifyGetSnapshottableDirListing(
+ FileSystem fs, DistributedFileSystem dfs) throws Exception {
+ // Get snapshottable directory list
+ SnapshottableDirectoryStatus[] sds = null;
+ if (fs instanceof HttpFSFileSystem) {
+ HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
+ sds = httpFS.getSnapshottableDirectoryList();
+ } else if (fs instanceof WebHdfsFileSystem) {
+ WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
+ sds = webHdfsFileSystem.getSnapshottableDirectoryList();
+ } else {
+ Assert.fail(fs.getClass().getSimpleName() +
+ " doesn't support getSnapshottableDirListing");
+ }
+ // Verify result with DFS
+ SnapshottableDirectoryStatus[] dfssds = dfs.getSnapshottableDirListing();
+ Assert.assertEquals(JsonUtil.toJsonString(sds),
+ JsonUtil.toJsonString(dfssds));
+ }
+
+ private void testGetSnapshottableDirListing() throws Exception {
+ if (!this.isLocalFS()) {
+ FileSystem fs = this.getHttpFSFileSystem();
+ // Create directories with snapshot allowed
+ Path path1 = new Path("/tmp/tmp-snap-dirlist-test-1");
+ DistributedFileSystem dfs = (DistributedFileSystem)
+ FileSystem.get(path1.toUri(), this.getProxiedFSConf());
+ // Verify response when there is no snapshottable directory
+ verifyGetSnapshottableDirListing(fs, dfs);
+ createSnapshotTestsPreconditions(path1);
+ Assert.assertTrue(fs.getFileStatus(path1).isSnapshotEnabled());
+ // Verify response when there is one snapshottable directory
+ verifyGetSnapshottableDirListing(fs, dfs);
+ Path path2 = new Path("/tmp/tmp-snap-dirlist-test-2");
+ createSnapshotTestsPreconditions(path2);
+ Assert.assertTrue(fs.getFileStatus(path2).isSnapshotEnabled());
+ // Verify response when there are two snapshottable directories
+ verifyGetSnapshottableDirListing(fs, dfs);
+
+ // Clean up and verify
+ fs.delete(path2, true);
+ verifyGetSnapshottableDirListing(fs, dfs);
+ fs.delete(path1, true);
+ verifyGetSnapshottableDirListing(fs, dfs);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index f024c95..8f7662f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.http.server;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
@@ -1400,4 +1401,56 @@ public class TestHttpFSServer extends HFSTestCase {
// Clean up
dfs.delete(path, true);
}
+
+ private void verifyGetSnapshottableDirectoryList(DistributedFileSystem dfs)
+ throws Exception {
+ // Send a request
+ HttpURLConnection conn = sendRequestToHttpFSServer("/",
+ "GETSNAPSHOTTABLEDIRECTORYLIST", "");
+ // Should return HTTP_OK
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
+ // Verify the response
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(conn.getInputStream()));
+ // The response should be a one-line JSON string.
+ String dirLst = reader.readLine();
+ // Verify the content of diff with DFS API.
+ SnapshottableDirectoryStatus[] dfsDirLst = dfs.getSnapshottableDirListing();
+ Assert.assertEquals(dirLst, JsonUtil.toJsonString(dfsDirLst));
+ }
+
+ @Test
+ @TestDir
+ @TestJetty
+ @TestHdfs
+ public void testGetSnapshottableDirectoryList() throws Exception {
+ createHttpFSServer(false, false);
+ // Create test directories
+ String pathStr1 = "/tmp/tmp-snap-dirlist-test-1";
+ createDirWithHttp(pathStr1, "700", null);
+ Path path1 = new Path(pathStr1);
+ String pathStr2 = "/tmp/tmp-snap-dirlist-test-2";
+ createDirWithHttp(pathStr2, "700", null);
+ Path path2 = new Path(pathStr2);
+ DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+ path1.toUri(), TestHdfsHelper.getHdfsConf());
+ // Verify response when there is no snapshottable directory
+ verifyGetSnapshottableDirectoryList(dfs);
+ // Enable snapshot for path1
+ dfs.allowSnapshot(path1);
+ Assert.assertTrue(dfs.getFileStatus(path1).isSnapshotEnabled());
+ // Verify response when there is one snapshottable directory
+ verifyGetSnapshottableDirectoryList(dfs);
+ // Enable snapshot for path2
+ dfs.allowSnapshot(path2);
+ Assert.assertTrue(dfs.getFileStatus(path2).isSnapshotEnabled());
+ // Verify response when there are two snapshottable directories
+ verifyGetSnapshottableDirectoryList(dfs);
+
+ // Clean up and verify
+ dfs.delete(path2, true);
+ verifyGetSnapshottableDirectoryList(dfs);
+ dfs.delete(path1, true);
+ verifyGetSnapshottableDirectoryList(dfs);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-478. Log files related to
each daemon doesn't have proper startup and shutdown logs. Contributed by
Dinesh Chitlangia.
Posted by sh...@apache.org.
HDDS-478. Log files related to each daemon doesn't have proper startup and shutdown logs.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1fe657a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1fe657a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1fe657a
Branch: refs/heads/HDFS-12943
Commit: c1fe657a106aaae3bdf81fa4add70962aaee165b
Parents: 6a39739
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 16:44:32 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 16:44:32 2018 -0700
----------------------------------------------------------------------
.../common/src/main/conf/om-audit-log4j2.properties | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1fe657a/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties b/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
index 7d097a0..7be51ac 100644
--- a/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
+++ b/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
@@ -52,11 +52,15 @@ filter.write.onMismatch=NEUTRAL
# TRACE (least specific, a lot of data)
# ALL (least specific, all data)
-appenders=console, rolling
-appender.console.type=Console
-appender.console.name=STDOUT
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+# Uncomment following section to enable logging to console appender also
+#appenders=console, rolling
+#appender.console.type=Console
+#appender.console.name=STDOUT
+#appender.console.layout.type=PatternLayout
+#appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+
+# Comment this line when using both console and rolling appenders
+appenders=rolling
#Rolling File Appender with size & time thresholds.
#Rolling is triggered when either threshold is breached.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-8834 Provide Java client for
fetching Yarn specific entities from TimelineReader. Contributed by Abhishek
Modi
Posted by sh...@apache.org.
YARN-8834 Provide Java client for fetching Yarn specific entities from TimelineReader. Contributed by Abhishek Modi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3edfddc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3edfddc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3edfddc
Branch: refs/heads/HDFS-12943
Commit: a3edfddcf7822ea13bdf4858672eb82cea5e0b5f
Parents: fb18cc5
Author: Vrushali C <vr...@apache.org>
Authored: Thu Oct 11 20:21:00 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Thu Oct 11 20:21:00 2018 -0700
----------------------------------------------------------------------
.../yarn/client/api/TimelineReaderClient.java | 120 ++++++++++
.../api/impl/TimelineReaderClientImpl.java | 239 +++++++++++++++++++
.../api/impl/TestTimelineReaderClientImpl.java | 157 ++++++++++++
3 files changed, 516 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3edfddc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
new file mode 100644
index 0000000..f73c2d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A client library that can be used to get Timeline Entities associated with
+ * application, application attempt or containers. This client library needs to
+ * be used along with time line v.2 server version.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class TimelineReaderClient extends CompositeService {
+
+ /**
+ * Create a new instance of Timeline Reader Client.
+ */
+ @InterfaceAudience.Public
+ public static TimelineReaderClient createTimelineReaderClient() {
+ return new TimelineReaderClientImpl();
+ }
+
+ @InterfaceAudience.Private
+ public TimelineReaderClient(String name) {
+ super(name);
+ }
+
+ /**
+ * Gets application entity.
+ * @param appId application id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @return entity of the application
+ * @throws IOException
+ */
+ public abstract TimelineEntity getApplicationEntity(
+ ApplicationId appId, String fields, Map<String, String> filters)
+ throws IOException;
+
+ /**
+ * Gets application attempt entity.
+ * @param appAttemptId application attempt id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @return entity associated with application attempt
+ * @throws IOException
+ */
+ public abstract TimelineEntity getApplicationAttemptEntity(
+ ApplicationAttemptId appAttemptId, String fields,
+ Map<String, String> filters) throws IOException;
+
+ /**
+ * Gets application attempt entities.
+ * @param appId application id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @param limit Number of entities to return.
+ * @param fromId Retrieve next set of generic ids from given fromId
+ * @return list of application attempt entities
+ * @throws IOException
+ */
+ public abstract List<TimelineEntity> getApplicationAttemptEntities(
+ ApplicationId appId, String fields, Map<String, String> filters,
+ long limit, String fromId) throws IOException;
+
+ /**
+ * Gets Timeline entity for the container.
+ * @param containerId container id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @return timeline entity for container
+ * @throws IOException
+ */
+ public abstract TimelineEntity getContainerEntity(
+ ContainerId containerId, String fields, Map<String, String> filters)
+ throws IOException;
+
+ /**
+ * Gets container entities for an application.
+ * @param appId application id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @param limit Number of entities to return.
+ * @param fromId Retrieve next set of generic ids from given fromId
+ * @return list of entities
+ * @throws IOException
+ */
+ public abstract List<TimelineEntity> getContainerEntities(
+ ApplicationId appId, String fields,
+ Map<String, String> filters,
+ long limit, String fromId) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3edfddc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
new file mode 100644
index 0000000..b138a54
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.api.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.TimelineReaderClient;
+import com.sun.jersey.api.client.ClientResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_APPLICATION_ATTEMPT;
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_CONTAINER;
+import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
+
+/**
+ * Implementation of TimelineReaderClient interface.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TimelineReaderClientImpl extends TimelineReaderClient {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineReaderClientImpl.class);
+
+ private static final String RESOURCE_URI_STR_V2 = "/ws/v2/timeline/";
+
+ private TimelineConnector connector;
+ private URI baseUri;
+ private String clusterId;
+
+ public TimelineReaderClientImpl() {
+ super(TimelineReaderClientImpl.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ throw new IOException("Timeline V2 client is not properly configured. "
+ + "Either timeline service is not enabled or version is not set to"
+ + " 2");
+ }
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation realUgi = ugi.getRealUser();
+ String doAsUser;
+ UserGroupInformation authUgi;
+ if (realUgi != null) {
+ authUgi = realUgi;
+ doAsUser = ugi.getShortUserName();
+ } else {
+ authUgi = ugi;
+ doAsUser = null;
+ }
+ DelegationTokenAuthenticatedURL.Token token =
+ new DelegationTokenAuthenticatedURL.Token();
+ connector = new TimelineConnector(false, authUgi, doAsUser, token);
+ addIfService(connector);
+ String timelineReaderWebAppAddress =
+ WebAppUtils.getTimelineReaderWebAppURLWithoutScheme(conf);
+ baseUri = TimelineConnector.constructResURI(
+ conf, timelineReaderWebAppAddress, RESOURCE_URI_STR_V2);
+ clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public TimelineEntity getApplicationEntity(ApplicationId appId, String fields,
+ Map<String, String> filters)
+ throws IOException {
+ String path = PATH_JOINER.join("clusters", clusterId, "apps", appId);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity entity = response.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ @Override
+ public TimelineEntity getApplicationAttemptEntity(
+ ApplicationAttemptId appAttemptId,
+ String fields, Map<String, String> filters) throws IOException {
+ ApplicationId appId = appAttemptId.getApplicationId();
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_APPLICATION_ATTEMPT, appAttemptId);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity entity = response.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ @Override
+ public List<TimelineEntity> getApplicationAttemptEntities(
+ ApplicationId appId, String fields, Map<String, String> filters,
+ long limit, String fromId) throws IOException {
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_APPLICATION_ATTEMPT);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ if (limit > 0) {
+ params.add("limit", Long.toString(limit));
+ }
+ if (fromId != null && !fromId.isEmpty()) {
+ params.add("fromid", fromId);
+ }
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity[] entities = response.getEntity(TimelineEntity[].class);
+ return Arrays.asList(entities);
+ }
+
+ @Override
+ public TimelineEntity getContainerEntity(ContainerId containerId,
+ String fields, Map<String, String> filters) throws IOException {
+ ApplicationId appId = containerId.getApplicationAttemptId().
+ getApplicationId();
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_CONTAINER, containerId);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity entity = response.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ @Override
+ public List<TimelineEntity> getContainerEntities(
+ ApplicationId appId, String fields,
+ Map<String, String> filters,
+ long limit, String fromId) throws IOException {
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_CONTAINER);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ if (limit > 0) {
+ params.add("limit", Long.toString(limit));
+ }
+ if (fromId != null && !fromId.isEmpty()) {
+ params.add("fromid", fromId);
+ }
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity[] entity = response.getEntity(TimelineEntity[].class);
+ return Arrays.asList(entity);
+ }
+
+ private void mergeFilters(MultivaluedMap<String, String> defaults,
+ Map<String, String> filters) {
+ if (filters != null && !filters.isEmpty()) {
+ for (Map.Entry<String, String> entry : filters.entrySet()) {
+ if (!defaults.containsKey(entry.getKey())) {
+ defaults.add(entry.getKey(), filters.get(entry.getValue()));
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ protected ClientResponse doGetUri(URI base, String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ ClientResponse resp = connector.getClient().resource(base).path(path)
+ .queryParams(params).accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ if (resp == null ||
+ resp.getStatusInfo().getStatusCode() != ClientResponse.Status.OK
+ .getStatusCode()) {
+ String msg =
+ "Response from the timeline reader server is " +
+ ((resp == null) ? "null" : "not successful," +
+ " HTTP error code: " + resp.getStatus() +
+ ", Server response:\n" + resp.getEntity(String.class));
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ return resp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3edfddc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
new file mode 100644
index 0000000..f668472
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_APPLICATION_ATTEMPT;
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_CONTAINER;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.TimelineReaderClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.MultivaluedMap;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test class for Timeline Reader Client.
+ */
+public class TestTimelineReaderClientImpl {
+
+ private TimelineReaderClient client;
+
+ @Before
+ public void setup() {
+ client = new MockTimelineReaderClient();
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ client.init(conf);
+ client.start();
+ }
+
+ @Test
+ public void testGetApplication() throws Exception {
+ ApplicationId applicationId =
+ ApplicationId.fromString("application_1234_0001");
+ TimelineEntity entity = client.getApplicationEntity(applicationId,
+ null, null);
+ Assert.assertEquals("mockApp1", entity.getId());
+ }
+
+ @Test
+ public void getApplicationAttemptEntity() throws Exception {
+ ApplicationAttemptId attemptId =
+ ApplicationAttemptId.fromString("appattempt_1234_0001_000001");
+ TimelineEntity entity = client.getApplicationAttemptEntity(attemptId,
+ null, null);
+ Assert.assertEquals("mockAppAttempt1", entity.getId());
+ }
+
+ @Test
+ public void getApplicationAttemptEntities() throws Exception {
+ ApplicationId applicationId =
+ ApplicationId.fromString("application_1234_0001");
+ List<TimelineEntity> entities =
+ client.getApplicationAttemptEntities(applicationId, null,
+ null, 0, null);
+ Assert.assertEquals(2, entities.size());
+ Assert.assertEquals("mockAppAttempt2", entities.get(1).getId());
+ }
+
+ @Test
+ public void testGetContainer() throws Exception {
+ ContainerId containerId =
+ ContainerId.fromString("container_1234_0001_01_000001");
+ TimelineEntity entity = client.getContainerEntity(containerId,
+ null, null);
+ Assert.assertEquals("mockContainer1", entity.getId());
+ }
+
+ @Test
+ public void testGetContainers() throws Exception {
+ ApplicationId appId =
+ ApplicationId.fromString("application_1234_0001");
+ List<TimelineEntity> entities = client.getContainerEntities(appId,
+ null, null, 0, null);
+ Assert.assertEquals(2, entities.size());
+ Assert.assertEquals("mockContainer2", entities.get(1).getId());
+ }
+
+ @After
+ public void tearDown() {
+ if (client != null) {
+ client.stop();
+ }
+ }
+
+ private static TimelineEntity createTimelineEntity(String id) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ return entity;
+ }
+
+ private static TimelineEntity[] createTimelineEntities(String... ids) {
+ List<TimelineEntity> entities = new ArrayList<>();
+ for (String id : ids) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ entities.add(entity);
+ }
+ return entities.toArray(new TimelineEntity[entities.size()]);
+ }
+
+ private class MockTimelineReaderClient extends TimelineReaderClientImpl {
+ @Override
+ protected ClientResponse doGetUri(URI base, String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ ClientResponse mockClientResponse = mock(ClientResponse.class);
+ if (path.contains(YARN_CONTAINER.toString())) {
+ when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
+ createTimelineEntity("mockContainer1"));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockContainer1", "mockContainer2"));
+ } else if (path.contains(YARN_APPLICATION_ATTEMPT.toString())) {
+ when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
+ createTimelineEntity("mockAppAttempt1"));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockAppAttempt1", "mockAppAttempt2"));
+ } else {
+ when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
+ createTimelineEntity("mockApp1"));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockApp1", "mockApp2"));
+ }
+ return mockClientResponse;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HADOOP-15785. [JDK10] Javadoc
build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
index a16f439..a67cafe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
@@ -42,7 +42,7 @@ public final class ScopedAclEntries {
* list is already sorted such that all access entries precede all default
* entries.
*
- * @param aclEntries List<AclEntry> to separate
+ * @param aclEntries List<AclEntry> to separate
*/
public ScopedAclEntries(List<AclEntry> aclEntries) {
int pivot = calculatePivotOnDefaultEntries(aclEntries);
@@ -59,8 +59,8 @@ public final class ScopedAclEntries {
/**
* Returns access entries.
*
- * @return List<AclEntry> containing just access entries, or an empty list if
- * there are no access entries
+ * @return List<AclEntry> containing just access entries, or an empty
+ * list if there are no access entries
*/
public List<AclEntry> getAccessEntries() {
return accessEntries;
@@ -69,8 +69,8 @@ public final class ScopedAclEntries {
/**
* Returns default entries.
*
- * @return List<AclEntry> containing just default entries, or an empty list if
- * there are no default entries
+ * @return List<AclEntry> containing just default entries, or an empty
+ * list if there are no default entries
*/
public List<AclEntry> getDefaultEntries() {
return defaultEntries;
@@ -78,8 +78,8 @@ public final class ScopedAclEntries {
/**
* Returns the pivot point in the list between the access entries and the
- * default entries. This is the index of the first element in the list that is
- * a default entry.
+ * default entries. This is the index of the first element in the list that
+ * is a default entry.
*
* @param aclBuilder ArrayList<AclEntry> containing entries to build
* @return int pivot point, or -1 if list contains no default entries
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index a4746cf..3eef278 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -148,16 +148,16 @@ abstract public class Command extends Configured {
* expand arguments, and then process each argument.
* <pre>
* run
- * |-> {@link #processOptions(LinkedList)}
- * \-> {@link #processRawArguments(LinkedList)}
- * |-> {@link #expandArguments(LinkedList)}
- * | \-> {@link #expandArgument(String)}*
- * \-> {@link #processArguments(LinkedList)}
- * |-> {@link #processArgument(PathData)}*
- * | |-> {@link #processPathArgument(PathData)}
- * | \-> {@link #processPaths(PathData, PathData...)}
- * | \-> {@link #processPath(PathData)}*
- * \-> {@link #processNonexistentPath(PathData)}
+ * |{@literal ->} {@link #processOptions(LinkedList)}
+ * \{@literal ->} {@link #processRawArguments(LinkedList)}
+ * |{@literal ->} {@link #expandArguments(LinkedList)}
+ * | \{@literal ->} {@link #expandArgument(String)}*
+ * \{@literal ->} {@link #processArguments(LinkedList)}
+ * |{@literal ->} {@link #processArgument(PathData)}*
+ * | |{@literal ->} {@link #processPathArgument(PathData)}
+ * | \{@literal ->} {@link #processPaths(PathData, PathData...)}
+ * | \{@literal ->} {@link #processPath(PathData)}*
+ * \{@literal ->} {@link #processNonexistentPath(PathData)}
* </pre>
* Most commands will chose to implement just
* {@link #processOptions(LinkedList)} and {@link #processPath(PathData)}
@@ -292,8 +292,8 @@ abstract public class Command extends Configured {
/**
* This is the last chance to modify an argument before going into the
* (possibly) recursive {@link #processPaths(PathData, PathData...)}
- * -> {@link #processPath(PathData)} loop. Ex. ls and du use this to
- * expand out directories.
+ * {@literal ->} {@link #processPath(PathData)} loop. Ex. ls and du use
+ * this to expand out directories.
* @param item a {@link PathData} representing a path which exists
* @throws IOException if anything goes wrong...
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
index bf30b22..4dd20d1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
@@ -162,7 +162,7 @@ public class CommandFormat {
/** Returns all the options that are set
*
- * @return Set<String> of the enabled options
+ * @return Set{@literal <}String{@literal >} of the enabled options
*/
public Set<String> getOpts() {
Set<String> optSet = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 463b14c..8366eac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -75,16 +75,16 @@ import org.apache.hadoop.util.Time;
* one or more individual file systems (a localFs or Hdfs, S3fs, etc).
* For example one could have a mount table that provides links such as
* <ul>
- * <li> /user -> hdfs://nnContainingUserDir/user
- * <li> /project/foo -> hdfs://nnProject1/projects/foo
- * <li> /project/bar -> hdfs://nnProject2/projects/bar
- * <li> /tmp -> hdfs://nnTmp/privateTmpForUserXXX
+ * <li> /user {@literal ->} hdfs://nnContainingUserDir/user
+ * <li> /project/foo {@literal ->} hdfs://nnProject1/projects/foo
+ * <li> /project/bar {@literal ->} hdfs://nnProject2/projects/bar
+ * <li> /tmp {@literal ->} hdfs://nnTmp/privateTmpForUserXXX
* </ul>
*
* ViewFs is specified with the following URI: <b>viewfs:///</b>
* <p>
* To use viewfs one would typically set the default file system in the
- * config (i.e. fs.defaultFS < = viewfs:///) along with the
+ * config (i.e. fs.defaultFS {@literal <} = viewfs:///) along with the
* mount table config variables as described below.
*
* <p>
@@ -132,7 +132,7 @@ import org.apache.hadoop.util.Time;
* (because they do not fit on one) then one could specify a mount
* entry such as following merges two dirs:
* <ul>
- * <li> /user -> hdfs://nnUser1/user,hdfs://nnUser2/user
+ * <li> /user {@literal ->} hdfs://nnUser1/user,hdfs://nnUser2/user
* </ul>
* Such a mergeLink can be specified with the following config var where ","
* is used as the separator for each of links to be merged:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index d099ca7..12de2ef 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -54,10 +54,10 @@ import org.slf4j.LoggerFactory;
* Apache Zookeeper. Using Zookeeper as a coordination service, leader election
* can be performed by atomically creating an ephemeral lock file (znode) on
* Zookeeper. The service instance that successfully creates the znode becomes
- * active and the rest become standbys. <br/>
+ * active and the rest become standbys. <br>
* This election mechanism is only efficient for small number of election
* candidates (order of 10's) because contention on single znode by a large
- * number of candidates can result in Zookeeper overload. <br/>
+ * number of candidates can result in Zookeeper overload. <br>
* The elector does not guarantee fencing (protection of shared resources) among
* service instances. After it has notified an instance about becoming a leader,
* then that instance must ensure that it meets the service consistency
@@ -70,10 +70,10 @@ import org.slf4j.LoggerFactory;
public class ActiveStandbyElector implements StatCallback, StringCallback {
/**
- * Callback interface to interact with the ActiveStandbyElector object. <br/>
+ * Callback interface to interact with the ActiveStandbyElector object. <br>
* The application will be notified with a callback only on state changes
* (i.e. there will never be successive calls to becomeActive without an
- * intermediate call to enterNeutralMode). <br/>
+ * intermediate call to enterNeutralMode). <br>
* The callbacks will be running on Zookeeper client library threads. The
* application should return from these callbacks quickly so as not to impede
* Zookeeper client library performance and notifications. The app will
@@ -105,7 +105,7 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
* interface. The service may choose to ignore this or stop doing state
* changing operations. Upon reconnection, the elector verifies the leader
* status and calls back on the becomeActive and becomeStandby app
- * interfaces. <br/>
+ * interfaces. <br>
* Zookeeper disconnects can happen due to network issues or loss of
* Zookeeper quorum. Thus enterNeutralMode can be used to guard against
* split-brain issues. In such situations it might be prudent to call
@@ -178,12 +178,12 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
private ZooKeeper monitorLockNodeClient;
/**
- * Create a new ActiveStandbyElector object <br/>
+ * Create a new ActiveStandbyElector object <br>
* The elector is created by providing to it the Zookeeper configuration, the
* parent znode under which to create the znode and a reference to the
- * callback interface. <br/>
+ * callback interface. <br>
* The parent znode name must be the same for all service instances and
- * different across services. <br/>
+ * different across services. <br>
* After the leader has been lost, a new leader will be elected after the
* session timeout expires. Hence, the app must set this parameter based on
* its needs for failure response time. The session timeout must be greater
@@ -217,12 +217,12 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
}
/**
- * Create a new ActiveStandbyElector object <br/>
+ * Create a new ActiveStandbyElector object <br>
* The elector is created by providing to it the Zookeeper configuration, the
* parent znode under which to create the znode and a reference to the
- * callback interface. <br/>
+ * callback interface. <br>
* The parent znode name must be the same for all service instances and
- * different across services. <br/>
+ * different across services. <br>
* After the leader has been lost, a new leader will be elected after the
* session timeout expires. Hence, the app must set this parameter based on
* its needs for failure response time. The session timeout must be greater
@@ -278,9 +278,9 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
/**
* To participate in election, the app will call joinElection. The result will
* be notified by a callback on either the becomeActive or becomeStandby app
- * interfaces. <br/>
+ * interfaces. <br>
* After this the elector will automatically monitor the leader status and
- * perform re-election if necessary<br/>
+ * perform re-election if necessary<br>
* The app could potentially start off in standby mode and ignore the
* becomeStandby call.
*
@@ -397,11 +397,11 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
/**
* Any service instance can drop out of the election by calling quitElection.
- * <br/>
+ * <br>
* This will lose any leader status, if held, and stop monitoring of the lock
- * node. <br/>
+ * node. <br>
* If the instance wants to participate in election again, then it needs to
- * call joinElection(). <br/>
+ * call joinElection(). <br>
* This allows service instances to take themselves out of rotation for known
* impending unavailable states (e.g. long GC pause or software upgrade).
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index e955979..2107bf1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -372,7 +372,7 @@ public abstract class HAAdmin extends Configured implements Tool {
/**
* Return the serviceId as is, we are assuming it was
- * given as a service address of form <host:ipcport>.
+ * given as a service address of form {@literal <}host:ipcport{@literal >}.
*/
protected String getServiceAddr(String serviceId) {
return serviceId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
index 2247a34..64e7315 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
@@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
* <code>com.example.foo.MyMethod</code>
* The class provided must implement the {@link FenceMethod} interface.
* The fencing methods that ship with Hadoop may also be referred to
- * by shortened names:<p>
+ * by shortened names:<br>
* <ul>
* <li><code>shell(/path/to/some/script.sh args...)</code></li>
* <li><code>sshfence(...)</code> (see {@link SshFenceByTcpPort})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
index 9ae113b..e0c2f4d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
@@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory;
* with ssh.
* <p>
* In order to achieve passwordless SSH, the operator must also configure
- * <code>dfs.ha.fencing.ssh.private-key-files<code> to point to an
+ * <code>dfs.ha.fencing.ssh.private-key-files</code> to point to an
* SSH key that has passphrase-less access to the given username and host.
*/
public class SshFenceByTcpPort extends Configured
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index d2ba469..a9c2319 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -107,9 +107,9 @@ import org.slf4j.LoggerFactory;
/**
* Create a Jetty embedded server to answer http requests. The primary goal is
* to serve up status information for the server. There are three contexts:
- * "/logs/" -> points to the log directory "/static/" -> points to common static
- * files (src/webapps/static) "/" -> the jsp server code from
- * (src/webapps/<name>)
+ * "/logs/" {@literal ->} points to the log directory "/static/" {@literal ->}
+ * points to common static files (src/webapps/static) "/" {@literal ->} the
+ * jsp server code from (src/webapps/{@literal <}name{@literal >})
*
* This class is a fork of the old HttpServer. HttpServer exists for
* compatibility reasons. See HBASE-10336 for more details.
@@ -1364,10 +1364,10 @@ public final class HttpServer2 implements FilterContainer {
/**
* Checks the user has privileges to access to instrumentation servlets.
- * <p/>
+ * <p>
* If <code>hadoop.security.instrumentation.requires.admin</code> is set to FALSE
* (default value) it always returns TRUE.
- * <p/>
+ * <p>
* If <code>hadoop.security.instrumentation.requires.admin</code> is set to TRUE
* it will check that if the current user is in the admin ACLS. If the user is
* in the admin ACLs it returns TRUE, otherwise it returns FALSE.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
index dc430cc..be86159 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
@@ -83,7 +83,7 @@ public class EnumSetWritable<E extends Enum<E>> extends AbstractCollection<E>
/**
* reset the EnumSetWritable with specified
- * <tt>value</value> and <tt>elementType</tt>. If the <tt>value</tt> argument
+ * <tt>value</tt> and <tt>elementType</tt>. If the <tt>value</tt> argument
* is null or its size is zero, the <tt>elementType</tt> argument must not be
* null. If the argument <tt>value</tt>'s size is bigger than zero, the
* argument <tt>elementType</tt> is not be used.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
index 3708a3b..5bbfba3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
@@ -370,7 +370,7 @@ public class IOUtils {
}
/**
- * Return the complete list of files in a directory as strings.<p/>
+ * Return the complete list of files in a directory as strings.<p>
*
* This is better than File#listDir because it does not ignore IOExceptions.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
index 7cd7f98..804d365 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
@@ -80,7 +80,7 @@ public class ReadaheadPool {
* @param readaheadLength the configured length to read ahead
* @param maxOffsetToRead the maximum offset that will be readahead
* (useful if, for example, only some segment of the file is
- * requested by the user). Pass {@link Long.MAX_VALUE} to allow
+ * requested by the user). Pass {@link Long#MAX_VALUE} to allow
* readahead to the end of the file.
* @param lastReadahead the result returned by the previous invocation
* of this function on this file descriptor, or null if this is
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
index 252ee4c..9d3c3c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
@@ -37,7 +37,7 @@ import com.google.common.annotations.VisibleForTesting;
/**
* This class provides secure APIs for opening and creating files on the local
* disk. The main issue this class tries to handle is that of symlink traversal.
- * <br/>
+ * <br>
* An example of such an attack is:
* <ol>
* <li> Malicious user removes his task's syslog file, and puts a link to the
@@ -50,7 +50,7 @@ import com.google.common.annotations.VisibleForTesting;
* </ol>
* A similar attack is possible involving task log truncation, but in that case
* due to an insecure write to a file.
- * <br/>
+ * <br>
*/
public class SecureIOUtils {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index f42848b..9afa621 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -79,7 +79,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* values.
* </li>
* <li>
- * <code>BlockCompressWriter</code> : Block-compressed files, both keys &
+ * <code>BlockCompressWriter</code> : Block-compressed files, both keys &
* values are collected in 'blocks'
* separately and compressed. The size of
* the 'block' is configurable.
@@ -94,13 +94,13 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* <p>The {@link SequenceFile.Reader} acts as the bridge and can read any of the
* above <code>SequenceFile</code> formats.</p>
*
- * <h4 id="Formats">SequenceFile Formats</h4>
+ * <h3 id="Formats">SequenceFile Formats</h3>
*
* <p>Essentially there are 3 different formats for <code>SequenceFile</code>s
* depending on the <code>CompressionType</code> specified. All of them share a
* <a href="#Header">common header</a> described below.
*
- * <h5 id="Header">SequenceFile Header</h5>
+ * <h4 id="Header">SequenceFile Header</h4>
* <ul>
* <li>
* version - 3 bytes of magic header <b>SEQ</b>, followed by 1 byte of actual
@@ -133,7 +133,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </li>
* </ul>
*
- * <h5 id="#UncompressedFormat">Uncompressed SequenceFile Format</h5>
+ * <h5>Uncompressed SequenceFile Format</h5>
* <ul>
* <li>
* <a href="#Header">Header</a>
@@ -152,7 +152,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </li>
* </ul>
*
- * <h5 id="#RecordCompressedFormat">Record-Compressed SequenceFile Format</h5>
+ * <h5>Record-Compressed SequenceFile Format</h5>
* <ul>
* <li>
* <a href="#Header">Header</a>
@@ -171,7 +171,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </li>
* </ul>
*
- * <h5 id="#BlockCompressedFormat">Block-Compressed SequenceFile Format</h5>
+ * <h5>Block-Compressed SequenceFile Format</h5>
* <ul>
* <li>
* <a href="#Header">Header</a>
@@ -1935,8 +1935,8 @@ public class SequenceFile {
* @param fs The file system used to open the file.
* @param file The file being read.
* @param bufferSize The buffer size used to read the file.
- * @param length The length being read if it is >= 0. Otherwise,
- * the length is not available.
+ * @param length The length being read if it is {@literal >=} 0.
+ * Otherwise, the length is not available.
* @return The opened stream.
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
index f0fe6fb..b94de6c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* and returns the instance.</p>
*
* <p>Example:</p>
- * <p><blockquote><pre>
+ * <blockquote><pre>
* public class MyWritable implements Writable {
* // Some data
* private int counter;
@@ -62,7 +62,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* return w;
* }
* }
- * </pre></blockquote></p>
+ * </pre></blockquote>
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
index b030481..c1208aa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
@@ -36,8 +36,9 @@ import org.apache.hadoop.classification.InterfaceStability;
* satisfy this property.</p>
*
* <p>Example:</p>
- * <p><blockquote><pre>
- * public class MyWritableComparable implements WritableComparable<MyWritableComparable> {
+ * <blockquote><pre>
+ * public class MyWritableComparable implements
+ * WritableComparable{@literal <MyWritableComparable>} {
* // Some data
* private int counter;
* private long timestamp;
@@ -66,7 +67,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* return result
* }
* }
- * </pre></blockquote></p>
+ * </pre></blockquote>
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
index e58e0e1..2062fb6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
@@ -236,7 +236,8 @@ public final class WritableUtils {
/**
* Serializes an integer to a binary stream with zero-compressed encoding.
- * For -112 <= i <= 127, only one byte is used with the actual value.
+ * For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ * actual value.
* For other values of i, the first byte value indicates whether the
* integer is positive or negative, and the number of bytes that follow.
* If the first byte value v is between -113 and -116, the following integer
@@ -255,7 +256,8 @@ public final class WritableUtils {
/**
* Serializes a long to a binary stream with zero-compressed encoding.
- * For -112 <= i <= 127, only one byte is used with the actual value.
+ * For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ * actual value.
* For other values of i, the first byte value indicates whether the
* long is positive or negative, and the number of bytes that follow.
* If the first byte value v is between -113 and -120, the following long
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
index 3701f20..e248120 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
@@ -227,9 +227,9 @@ public class CompressionCodecFactory {
/**
* Find the relevant compression codec for the codec's canonical class name
* or by codec alias.
- * <p/>
+ * <p>
* Codec aliases are case insensitive.
- * <p/>
+ * <p>
* The code alias is the short class name (without the package name).
* If the short class name ends with 'Codec', then there are two aliases for
* the codec, the complete short class name and the short class name without
@@ -255,9 +255,9 @@ public class CompressionCodecFactory {
/**
* Find the relevant compression codec for the codec's canonical class name
* or by codec alias and returns its implemetation class.
- * <p/>
+ * <p>
* Codec aliases are case insensitive.
- * <p/>
+ * <p>
* The code alias is the short class name (without the package name).
* If the short class name ends with 'Codec', then there are two aliases for
* the codec, the complete short class name and the short class name without
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
index 61462c0..45b5e9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
@@ -61,9 +61,9 @@ public class Lz4Codec implements Configurable, CompressionCodec {
}
/**
- * Are the native lz4 libraries loaded & initialized?
+ * Are the native lz4 libraries loaded & initialized?
*
- * @return true if loaded & initialized, otherwise false
+ * @return true if loaded & initialized, otherwise false
*/
public static boolean isNativeCodeLoaded() {
return NativeCodeLoader.isNativeCodeLoaded();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
index 20a4cd6..cd0c788 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
@@ -57,7 +57,7 @@ public class SnappyCodec implements Configurable, CompressionCodec, DirectDecomp
}
/**
- * Are the native snappy libraries loaded & initialized?
+ * Are the native snappy libraries loaded & initialized?
*/
public static void checkNativeCodeLoaded() {
if (!NativeCodeLoader.buildSupportsSnappy()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
index d4a9787..5713c56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
@@ -247,7 +247,7 @@ public class Bzip2Compressor implements Compressor {
}
/**
- * Returns the total number of uncompressed bytes input so far.</p>
+ * Returns the total number of uncompressed bytes input so far.
*
* @return the total (non-negative) number of uncompressed bytes input so far
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
index 96693ad..72ba976 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
@@ -183,7 +183,7 @@ public class Bzip2Decompressor implements Decompressor {
}
/**
- * Returns the total number of compressed bytes input so far.</p>
+ * Returns the total number of compressed bytes input so far.
*
* @return the total (non-negative) number of compressed bytes input so far
*/
@@ -195,7 +195,7 @@ public class Bzip2Decompressor implements Decompressor {
/**
* Returns the number of bytes remaining in the input buffers; normally
* called when finished() is true to determine amount of post-gzip-stream
- * data.</p>
+ * data.
*
* @return the total (non-negative) number of unprocessed bytes in input
*/
@@ -206,7 +206,7 @@ public class Bzip2Decompressor implements Decompressor {
}
/**
- * Resets everything including the input buffers (user and direct).</p>
+ * Resets everything including the input buffers (user and direct).
*/
@Override
public synchronized void reset() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
index d24b4bf..3af5309 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
@@ -37,11 +37,11 @@ public class Bzip2Factory {
private static boolean nativeBzip2Loaded;
/**
- * Check if native-bzip2 code is loaded & initialized correctly and
+ * Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
- * @return <code>true</code> if native-bzip2 is loaded & initialized
+ * @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static synchronized boolean isNativeBzip2Loaded(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
index bb02cf2..8426d25 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
@@ -200,20 +200,18 @@ public class CBZip2InputStream extends InputStream implements BZip2Constants {
}
/**
- * This method tries to find the marker (passed to it as the first parameter)
- * in the stream. It can find bit patterns of length <= 63 bits. Specifically
- * this method is used in CBZip2InputStream to find the end of block (EOB)
- * delimiter in the stream, starting from the current position of the stream.
- * If marker is found, the stream position will be at the byte containing
- * the starting bit of the marker.
- *
- * @param marker The bit pattern to be found in the stream
- * @param markerBitLength No of bits in the marker
- * @return true if the marker was found otherwise false
- *
- * @throws IOException
- * @throws IllegalArgumentException if marketBitLength is greater than 63
- */
+ * This method tries to find the marker (passed to it as the first parameter)
+ * in the stream. It can find bit patterns of length <= 63 bits.
+ * Specifically this method is used in CBZip2InputStream to find the end of
+ * block (EOB) delimiter in the stream, starting from the current position
+ * of the stream. If marker is found, the stream position will be at the
+ * byte containing the starting bit of the marker.
+ * @param marker The bit pattern to be found in the stream
+ * @param markerBitLength No of bits in the marker
+ * @return true if the marker was found otherwise false
+ * @throws IOException
+ * @throws IllegalArgumentException if marketBitLength is greater than 63
+ */
public boolean skipToNextMarker(long marker, int markerBitLength)
throws IOException, IllegalArgumentException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
index ca4e5cd..850fec7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
@@ -64,7 +64,8 @@ import org.apache.hadoop.io.IOUtils;
* </pre>
*
* <table width="100%" border="1">
- * <colgroup> <col width="33%" /> <col width="33%" /> <col width="33%" />
+ * <caption></caption>
+ * <colgroup> <col width="33%" > <col width="33%" > <col width="33%" >
* </colgroup>
* <tr>
* <th colspan="3">Memory usage by blocksize</th>
@@ -614,9 +615,9 @@ public class CBZip2OutputStream extends OutputStream implements BZip2Constants {
* @throws IOException
* if an I/O error occurs in the specified stream.
* @throws IllegalArgumentException
- * if <code>(blockSize < 1) || (blockSize > 9)</code>.
+ * if {@code (blockSize < 1) || (blockSize > 9)}
* @throws NullPointerException
- * if <code>out == null</code>.
+ * if {@code out == null}.
*
* @see #MIN_BLOCKSIZE
* @see #MAX_BLOCKSIZE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
index b4c6659..896d35e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
@@ -404,7 +404,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
/**
* Returns the total number of compressed bytes input so far, including
- * gzip header/trailer bytes.</p>
+ * gzip header/trailer bytes.
*
* @return the total (non-negative) number of compressed bytes read so far
*/
@@ -420,7 +420,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
* non-zero value unless called after {@link #setInput(byte[] b, int off,
* int len)} and before {@link #decompress(byte[] b, int off, int len)}.
* (That is, after {@link #decompress(byte[] b, int off, int len)} it
- * always returns zero, except in finished state with concatenated data.)</p>
+ * always returns zero, except in finished state with concatenated data.)
*
* @return the total (non-negative) number of unprocessed bytes in input
*/
@@ -441,7 +441,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
/**
* Returns true if the end of the gzip substream (single "member") has been
- * reached.</p>
+ * reached.
*/
@Override
public synchronized boolean finished() {
@@ -450,7 +450,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
/**
* Resets everything, including the input buffer, regardless of whether the
- * current gzip substream is finished.</p>
+ * current gzip substream is finished.
*/
@Override
public synchronized void reset() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
index 438c8be..da8a90b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
@@ -435,7 +435,7 @@ public class ZlibCompressor implements Compressor {
}
/**
- * Returns the total number of uncompressed bytes input so far.</p>
+ * Returns the total number of uncompressed bytes input so far.
*
* @return the total (non-negative) number of uncompressed bytes input so far
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
index dd550b9..f642d77 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
@@ -243,7 +243,7 @@ public class ZlibDecompressor implements Decompressor {
}
/**
- * Returns the total number of compressed bytes input so far.</p>
+ * Returns the total number of compressed bytes input so far.
*
* @return the total (non-negative) number of compressed bytes input so far
*/
@@ -255,7 +255,7 @@ public class ZlibDecompressor implements Decompressor {
/**
* Returns the number of bytes remaining in the input buffers; normally
* called when finished() is true to determine amount of post-gzip-stream
- * data.</p>
+ * data.
*
* @return the total (non-negative) number of unprocessed bytes in input
*/
@@ -266,7 +266,7 @@ public class ZlibDecompressor implements Decompressor {
}
/**
- * Resets everything including the input buffers (user and direct).</p>
+ * Resets everything including the input buffers (user and direct).
*/
@Override
public void reset() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
index 93b3b6d..07afbab 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
@@ -73,11 +73,11 @@ public class ZlibFactory {
ZlibFactory.nativeZlibLoaded = isLoaded;
}
/**
- * Check if native-zlib code is loaded & initialized correctly and
+ * Check if native-zlib code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
- * @return <code>true</code> if native-zlib is loaded & initialized
+ * @return <code>true</code> if native-zlib is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeZlibLoaded(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 8ec0e72..5ba6e9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -36,7 +36,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
/**
- * A codec & coder utility to help create coders conveniently.
+ * A codec & coder utility to help create coders conveniently.
*
* {@link CodecUtil} includes erasure coder configurations key and default
* values such as coder class name and erasure codec option values included
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
index fdb47be..f80fcec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
@@ -518,7 +518,7 @@ public class GaloisField {
/**
* Perform Gaussian elimination on the given matrix. This matrix has to be a
- * fat matrix (number of rows > number of columns).
+ * fat matrix (number of rows > number of columns).
*/
public void gaussianElimination(int[][] matrix) {
assert(matrix != null && matrix.length > 0 && matrix[0].length > 0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
index c63baa5..09cd282 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
@@ -1308,11 +1308,11 @@ public class TFile {
* @param reader
* The TFile reader object.
* @param beginKey
- * Begin key of the scan. If null, scan from the first <K,V>
- * entry of the TFile.
+ * Begin key of the scan. If null, scan from the first
+ * <K, V> entry of the TFile.
* @param endKey
- * End key of the scan. If null, scan up to the last <K, V> entry
- * of the TFile.
+ * End key of the scan. If null, scan up to the last <K, V>
+ * entry of the TFile.
* @throws IOException
*/
protected Scanner(Reader reader, RawComparable beginKey,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
index 8cb6e0d..17a27f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
@@ -62,27 +62,33 @@ public final class Utils {
* <li>if n in [-32, 127): encode in one byte with the actual value.
* Otherwise,
* <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52;
- * byte[1]=n&0xff. Otherwise,
+ * byte[1]=n&0xff. Otherwise,
* <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 -
- * 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
+ * 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
* <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112;
- * byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise:
+ * byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff;
+ * byte[3]=n&0xff.
+ * Otherwise:
* <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] =
- * (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
+ * (n>>24)&0xff; byte[2]=(n>>16)&0xff;
+ * byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
* <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] =
- * (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff;
- * byte[4]=(n>>8)&0xff; byte[5]=n&0xff
+ * (n>>32)&0xff; byte[2]=(n>>24)&0xff;
+ * byte[3]=(n>>16)&0xff; byte[4]=(n>>8)&0xff;
+ * byte[5]=n&0xff
* <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] =
- * (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff;
- * byte[4]=(n>>16)&0xff; byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
+ * (n>>40)&0xff; byte[2]=(n>>32)&0xff;
+ * byte[3]=(n>>24)&0xff; byte[4]=(n>>16)&0xff;
+ * byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
* <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] =
- * (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff;
- * byte[4]=(n>>24)&0xff; byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff;
- * byte[7]=n&0xff;
+ * (n>>48)&0xff; byte[2] = (n>>40)&0xff;
+ * byte[3]=(n>>32)&0xff; byte[4]=(n>>24)&0xff; byte[5]=
+ * (n>>16)&0xff; byte[6]=(n>>8)&0xff; byte[7]=n&0xff;
* <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] =
- * (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff;
- * byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff;
- * byte[7]=(n>>8)&0xff; byte[8]=n&0xff;
+ * (n>>54)&0xff; byte[2] = (n>>48)&0xff;
+ * byte[3] = (n>>40)&0xff; byte[4]=(n>>32)&0xff;
+ * byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; byte[7]=
+ * (n>>8)&0xff; byte[8]=n&0xff;
* </ul>
*
* @param out
@@ -181,15 +187,15 @@ public final class Utils {
* Decoding the variable-length integer. Suppose the value of the first byte
* is FB, and the following bytes are NB[*].
* <ul>
- * <li>if (FB >= -32), return (long)FB;
- * <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
- * <li>if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 +
- * NB[1]&0xff;
- * <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 +
- * (NB[1]&0xff)<<8 + NB[2]&0xff;
+ * <li>if (FB >= -32), return (long)FB;
+ * <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
+ * <li>if (FB in [-104, -73]), return (FB+88)<<16 +
+ * (NB[0]&0xff)<<8 + NB[1]&0xff;
+ * <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)
+ * <<16 + (NB[1]&0xff)<<8 + NB[2]&0xff;
* <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed
* big-endian integer.
- *
+ * </ul>
* @param in
* input stream
* @return the decoded long integer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
index 9875bcd..7fcd5fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
@@ -89,12 +89,12 @@ public class RetryProxy {
*
* @param iface the interface that the retry will implement
* @param proxyProvider provides implementation instances whose methods should be retried
- * @param methodNameToPolicyMapa map of method names to retry policies
+ * @param methodNameToPolicyMap map of method names to retry policies
* @return the retry proxy
*/
public static <T> Object create(Class<T> iface,
FailoverProxyProvider<T> proxyProvider,
- Map<String,RetryPolicy> methodNameToPolicyMap,
+ Map<String, RetryPolicy> methodNameToPolicyMap,
RetryPolicy defaultPolicy) {
return Proxy.newProxyInstance(
proxyProvider.getInterface().getClassLoader(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
index 3b727d9..3c8dfcc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* <p>
- * Provides a facility for deserializing objects of type <T> from an
+ * Provides a facility for deserializing objects of type {@literal <T>} from an
* {@link InputStream}.
* </p>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
index 63d3738..5ada541 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* <p>
- * Provides a facility for serializing objects of type <T> to an
+ * Provides a facility for serializing objects of type <T> to an
* {@link OutputStream}.
* </p>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index b156d1f..0be5939 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -131,7 +131,7 @@ public final class CallerContext {
/**
* The thread local current caller context.
- * <p/>
+ * <p>
* Internal class for defered singleton idiom.
* https://en.wikipedia.org/wiki/Initialization_on_demand_holder_idiom
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 07a2f13..49f1e49 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -1357,7 +1357,7 @@ public class Client implements AutoCloseable {
* @param remoteId - the target rpc server
* @param fallbackToSimpleAuth - set to true or false during this method to
* indicate if a secure client falls back to simple auth
- * @returns the rpc response
+ * @return the rpc response
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
*/
@@ -1392,7 +1392,7 @@ public class Client implements AutoCloseable {
* @param serviceClass - service class for RPC
* @param fallbackToSimpleAuth - set to true or false during this method to
* indicate if a secure client falls back to simple auth
- * @returns the rpc response
+ * @return the rpc response
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
*/
@@ -1461,7 +1461,7 @@ public class Client implements AutoCloseable {
/**
* Check if RPC is in asynchronous mode or not.
*
- * @returns true, if RPC is in asynchronous mode, otherwise false for
+ * @return true, if RPC is in asynchronous mode, otherwise false for
* synchronous mode.
*/
@Unstable
@@ -1575,7 +1575,8 @@ public class Client implements AutoCloseable {
/**
* This class holds the address and the user ticket. The client connections
- * to servers are uniquely identified by <remoteAddress, protocol, ticket>
+ * to servers are uniquely identified by {@literal <}remoteAddress, protocol,
+ * ticket{@literal >}
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
index d0fb8fd..00d9a79 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
@@ -37,7 +37,7 @@ public class ClientCache {
new HashMap<SocketFactory, Client>();
/**
- * Construct & cache an IPC client with the user-provided SocketFactory
+ * Construct & cache an IPC client with the user-provided SocketFactory
* if no cached client exists.
*
* @param conf Configuration
@@ -66,7 +66,7 @@ public class ClientCache {
}
/**
- * Construct & cache an IPC client with the default SocketFactory
+ * Construct & cache an IPC client with the default SocketFactory
* and default valueClass if no cached client exists.
*
* @param conf Configuration
@@ -77,7 +77,7 @@ public class ClientCache {
}
/**
- * Construct & cache an IPC client with the user-provided SocketFactory
+ * Construct & cache an IPC client with the user-provided SocketFactory
* if no cached client exists. Default response type is ObjectWritable.
*
* @param conf Configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index 8bb0ce4..512b0b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -76,7 +76,8 @@ public class DecayRpcScheduler implements RpcScheduler,
/**
* Decay factor controls how much each count is suppressed by on each sweep.
- * Valid numbers are > 0 and < 1. Decay factor works in tandem with period
+ * Valid numbers are > 0 and < 1. Decay factor works in tandem with
+ * period
* to control how long the scheduler remembers an identity.
*/
public static final String IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
index 3fe9eb7..3622d2c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
@@ -28,7 +28,6 @@ public interface RefreshHandler {
* Implement this method to accept refresh requests from the administrator.
* @param identifier is the identifier you registered earlier
* @param args contains a list of string args from the administrator
- * @throws Exception as a shorthand for a RefreshResponse(-1, message)
* @return a RefreshResponse
*/
RefreshResponse handleRefresh(String identifier, String[] args);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
index 620e100..36e280f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
@@ -102,7 +102,7 @@ public class RemoteException extends IOException {
* a <code>String</code> as a parameter.
* Otherwise it returns this.
*
- * @return <code>Throwable
+ * @return <code>Throwable</code>
*/
public IOException unwrapRemoteException() {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 76d9c40..d162d2f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -2747,7 +2747,7 @@ public abstract class Server {
/**
* Constructs a server listening on the named port and address. Parameters passed must
- * be of the named class. The <code>handlerCount</handlerCount> determines
+ * be of the named class. The <code>handlerCount</code> determines
* the number of handler threads that will be used to process calls.
* If queueSizePerHandler or numReaders are not -1 they will be used instead of parameters
* from configuration. Otherwise the configuration will be picked up.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
index 093d0af..c404ebe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
@@ -70,7 +70,7 @@ import java.util.Set;
* <p>
* The optional <code>get</code> parameter is used to query an specific
* attribute of a JMX bean. The format of the URL is
- * <code>http://.../jmx?get=MXBeanName::AttributeName<code>
+ * <code>http://.../jmx?get=MXBeanName::AttributeName</code>
* <p>
* For example
* <code>
@@ -85,7 +85,7 @@ import java.util.Set;
* <p>
* The return format is JSON and in the form
* <p>
- * <code><pre>
+ * <pre><code>
* {
* "beans" : [
* {
@@ -94,7 +94,7 @@ import java.util.Set;
* }
* ]
* }
- * </pre></code>
+ * </code></pre>
* <p>
* The servlet attempts to convert the the JMXBeans into JSON. Each
* bean's attributes will be converted to a JSON object member.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
index aa4e61c..41bee04 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
@@ -62,10 +62,10 @@ import org.apache.hadoop.util.Timer;
* still maintaining overall information about how many large requests were
* received.
*
- * <p/>This class can also be used to coordinate multiple logging points; see
+ * <p>This class can also be used to coordinate multiple logging points; see
* {@link #record(String, long, double...)} for more details.
*
- * <p/>This class is not thread-safe.
+ * <p>This class is not thread-safe.
*/
public class LogThrottlingHelper {
@@ -175,7 +175,7 @@ public class LogThrottlingHelper {
* about the values specified since the last time the caller was expected to
* write to its log.
*
- * <p/>Specifying multiple values will maintain separate summary statistics
+ * <p>Specifying multiple values will maintain separate summary statistics
* about each value. For example:
* <pre>{@code
* helper.record(1, 0);
@@ -230,7 +230,7 @@ public class LogThrottlingHelper {
* iteration as "pre", yet each one is able to maintain its own summary
* information.
*
- * <p/>Other behavior is the same as {@link #record(double...)}.
+ * <p>Other behavior is the same as {@link #record(double...)}.
*
* @param recorderName The name of the recorder. This is used to check if the
* current recorder is the primary. Other names are
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
index 22c288a..6803d11 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
@@ -251,7 +251,7 @@ public class MutableRollingAverages extends MutableMetric implements Closeable {
}
/**
- * Retrieve a map of metric name -> (aggregate).
+ * Retrieve a map of metric name {@literal ->} (aggregate).
* Filter out entries that don't have at least minSamples.
*
* @return a map of peer DataNode Id to the average latency to that
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
index ff7cd25..8fd3b33 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
@@ -87,6 +87,7 @@
<h2><a name="gettingstarted">Getting started</a></h2>
<h3>Implementing metrics sources</h3>
<table width="99%" border="1" cellspacing="0" cellpadding="4">
+ <caption></caption>
<tbody>
<tr>
<th>Using annotations</th><th>Using MetricsSource interface</th>
@@ -289,6 +290,7 @@
backend that can handle multiple contexts (file, gangalia etc.):
</p>
<table width="99%" border="1" cellspacing="0" cellpadding="4">
+ <caption></caption>
<tbody>
<tr>
<th width="40%">Before</th><th>After</th>
@@ -310,6 +312,7 @@
using the context option in the sink options like the following:
</p>
<table width="99%" border="1" cellspacing="0" cellpadding="4">
+ <caption></caption>
<tbody>
<tr>
<th width="40%">Before</th><th>After</th>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
index 92ac952..1d330c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
@@ -111,7 +111,7 @@ import org.apache.hadoop.security.UserGroupInformation;
* <i>unknown</i>.</p>
*
* <p>Instead of appending to an existing file, by default the sink
- * will create a new file with a suffix of ".<n>&quet;, where
+ * will create a new file with a suffix of ".<n>", where
* <i>n</i> is the next lowest integer that isn't already used in a file name,
* similar to the Hadoop daemon logs. NOTE: the file with the <b>highest</b>
* sequence number is the <b>newest</b> file, unlike the Hadoop daemon logs.</p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
index b2be0a2..c1dbf7e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
@@ -47,10 +47,10 @@ import org.slf4j.LoggerFactory;
* a daemon that is running on the localhost and will add the
* hostname to the metric (such as the
* <a href="https://collectd.org/">CollectD</a> StatsD plugin).
- * <br/>
+ * <br>
* To configure this plugin, you will need to add the following
* entries to your hadoop-metrics2.properties file:
- * <br/>
+ * <br>
* <pre>
* *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
* [prefix].sink.statsd.server.host=
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
index 916367f..1b50498 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
@@ -59,8 +59,9 @@ public final class MBeans {
/**
* Register the MBean using our standard MBeanName format
- * "hadoop:service=<serviceName>,name=<nameName>"
- * Where the <serviceName> and <nameName> are the supplied parameters.
+ * "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ * Where the {@literal <serviceName> and <nameName>} are the supplied
+ * parameters.
*
* @param serviceName
* @param nameName
@@ -75,8 +76,9 @@ public final class MBeans {
/**
* Register the MBean using our standard MBeanName format
- * "hadoop:service=<serviceName>,name=<nameName>"
- * Where the <serviceName> and <nameName> are the supplied parameters.
+ * "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ * Where the {@literal <serviceName> and <nameName>} are the supplied
+ * parameters.
*
* @param serviceName
* @param nameName
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
index b2d803c..97723c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
@@ -29,11 +29,11 @@ import java.util.Map;
import java.util.Set;
/**
- * This is a base class for DNS to Switch mappings. <p/> It is not mandatory to
+ * This is a base class for DNS to Switch mappings. <p> It is not mandatory to
* derive {@link DNSToSwitchMapping} implementations from it, but it is strongly
* recommended, as it makes it easy for the Hadoop developers to add new methods
* to this base class that are automatically picked up by all implementations.
- * <p/>
+ * <p>
*
* This class does not extend the <code>Configured</code>
* base class, and should not be changed to do so, as it causes problems
@@ -81,7 +81,7 @@ public abstract class AbstractDNSToSwitchMapping
* multi-rack. Subclasses may override this with methods that are more aware
* of their topologies.
*
- * <p/>
+ * <p>
*
* This method is used when parts of Hadoop need know whether to apply
* single rack vs multi-rack policies, such as during block placement.
@@ -140,7 +140,7 @@ public abstract class AbstractDNSToSwitchMapping
/**
* Query for a {@link DNSToSwitchMapping} instance being on a single
* switch.
- * <p/>
+ * <p>
* This predicate simply assumes that all mappings not derived from
* this class are multi-switch.
* @param mapping the mapping to query
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
index 2fb4d3e..061971c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
@@ -141,7 +141,7 @@ public class DNS {
}
/**
- * Like {@link DNS#getIPs(String, boolean), but returns all
+ * Like {@link DNS#getIPs(String, boolean)}, but returns all
* IPs associated with the given interface and its subinterfaces.
*/
public static String[] getIPs(String strInterface)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
index 7b1b332..1e6f5f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
@@ -40,7 +40,7 @@ public interface DNSToSwitchMapping {
* Note the hostname/ip-address is not part of the returned path.
* The network topology of the cluster would determine the number of
* components in the network path.
- * <p/>
+ * <p>
*
* If a name cannot be resolved to a rack, the implementation
* should return {@link NetworkTopology#DEFAULT_RACK}. This
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index 0f9cfc3..ceb8ec2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -146,8 +146,8 @@ public class NetUtils {
/**
* Util method to build socket addr from either:
- * <host>:<port>
- * <fs>://<host>:<port>/<path>
+ * {@literal <host>:<port>}
+ * {@literal <fs>://<host>:<port>/<path>}
*/
public static InetSocketAddress createSocketAddr(String target) {
return createSocketAddr(target, -1);
@@ -155,9 +155,9 @@ public class NetUtils {
/**
* Util method to build socket addr from either:
- * <host>
- * <host>:<port>
- * <fs>://<host>:<port>/<path>
+ * {@literal <host>}
+ * {@literal <host>:<port>}
+ * {@literal <fs>://<host>:<port>/<path>}
*/
public static InetSocketAddress createSocketAddr(String target,
int defaultPort) {
@@ -938,7 +938,7 @@ public class NetUtils {
* Return a free port number. There is no guarantee it will remain free, so
* it should be used immediately.
*
- * @returns A free port for binding a local socket
+ * @return A free port for binding a local socket
*/
public static int getFreeSocketPort() {
int port = 0;
@@ -959,7 +959,7 @@ public class NetUtils {
*
* @param localAddr
* @param bindWildCardAddress
- * @returns InetAddress
+ * @return InetAddress
*/
public static InetAddress bindToLocalAddress(InetAddress localAddr, boolean
bindWildCardAddress) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 1f077a7..4d4f2bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -114,7 +114,7 @@ public class NetworkTopology {
}
/** Add a leaf node
- * Update node counter & rack counter if necessary
+ * Update node counter & rack counter if necessary
* @param node node to be added; can be null
* @exception IllegalArgumentException if add a node to a leave
or node to be added is not a leaf
@@ -858,12 +858,12 @@ public class NetworkTopology {
/**
* Sort nodes array by network distance to <i>reader</i>.
- * <p/>
+ * <p>
* In a three-level topology, a node can be either local, on the same rack,
* or on a different rack from the reader. Sorting the nodes based on network
* distance from the reader reduces network traffic and improves
* performance.
- * <p/>
+ * <p>
* As an additional twist, we also randomize the nodes at each network
* distance. This helps with load balancing when there is data skew.
*
@@ -881,11 +881,11 @@ public class NetworkTopology {
/**
* Sort nodes array by network distance to <i>reader</i>.
- * <p/> using network location. This is used when the reader
+ * <p> using network location. This is used when the reader
* is not a datanode. Sorting the nodes based on network distance
* from the reader reduces network traffic and improves
* performance.
- * <p/>
+ * <p>
*
* @param reader Node where data will be read
* @param nodes Available replicas with the requested data
@@ -902,7 +902,7 @@ public class NetworkTopology {
/**
* Sort nodes array by network distance to <i>reader</i>.
- * <p/>
+ * <p>
* As an additional twist, we also randomize the nodes at each network
* distance. This helps with load balancing when there is data skew.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HDDS-559. fs.default.name is
deprecated. Contributed by Dinesh Chitlangia.
Posted by sh...@apache.org.
HDDS-559. fs.default.name is deprecated.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a06bc30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a06bc30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a06bc30
Branch: refs/heads/HDFS-12943
Commit: 6a06bc309d72c766694eb6296d5f3fb5c3c597c5
Parents: c1fe657
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 16:57:39 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 16:57:39 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/OzoneFS.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a06bc30/hadoop-ozone/docs/content/OzoneFS.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md
index d0621be..6853992 100644
--- a/hadoop-ozone/docs/content/OzoneFS.md
+++ b/hadoop-ozone/docs/content/OzoneFS.md
@@ -46,7 +46,7 @@ Please add the following entry to the core-site.xml.
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
</property>
<property>
- <name>fs.default.name</name>
+ <name>fs.defaultFS</name>
<value>o3://localhost:9864/volume/bucket</value>
</property>
{{< /highlight >}}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-8861. executorLock is
misleading in ContainerLaunch. Contributed by Chandni Singh
Posted by sh...@apache.org.
YARN-8861. executorLock is misleading in ContainerLaunch. Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e787d65a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e787d65a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e787d65a
Branch: refs/heads/HDFS-12943
Commit: e787d65a08f5d5245d2313fc34f2dde518bfaa5b
Parents: ee816f1
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Oct 11 10:54:57 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Oct 11 10:54:57 2018 -0500
----------------------------------------------------------------------
.../containermanager/launcher/ContainerLaunch.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e787d65a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6776836..f198e83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -135,7 +135,7 @@ public class ContainerLaunch implements Callable<Integer> {
protected final LocalDirsHandlerService dirsHandler;
- private final Lock containerExecLock = new ReentrantLock();
+ private final Lock launchLock = new ReentrantLock();
public ContainerLaunch(Context context, Configuration configuration,
Dispatcher dispatcher, ContainerExecutor exec, Application app,
@@ -485,11 +485,11 @@ public class ContainerLaunch implements Callable<Integer> {
throws IOException, ConfigurationException {
int launchPrep = prepareForLaunch(ctx);
if (launchPrep == 0) {
- containerExecLock.lock();
+ launchLock.lock();
try {
return exec.launchContainer(ctx);
} finally {
- containerExecLock.unlock();
+ launchLock.unlock();
}
}
return launchPrep;
@@ -499,18 +499,18 @@ public class ContainerLaunch implements Callable<Integer> {
throws IOException, ConfigurationException {
int launchPrep = prepareForLaunch(ctx);
if (launchPrep == 0) {
- containerExecLock.lock();
+ launchLock.lock();
try {
return exec.relaunchContainer(ctx);
} finally {
- containerExecLock.unlock();
+ launchLock.unlock();
}
}
return launchPrep;
}
void reapContainer() throws IOException {
- containerExecLock.lock();
+ launchLock.lock();
try {
// Reap the container
boolean result = exec.reapContainer(
@@ -524,7 +524,7 @@ public class ContainerLaunch implements Callable<Integer> {
}
cleanupContainerFiles(getContainerWorkDir());
} finally {
- containerExecLock.unlock();
+ launchLock.unlock();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-8858. CapacityScheduler
should respect maximum node resource when per-queue maximum-allocation is
being used. Contributed by Wangda Tan.
Posted by sh...@apache.org.
YARN-8858. CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edce8664
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edce8664
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edce8664
Branch: refs/heads/HDFS-12943
Commit: edce866489d83744f3f47a3b884b0c6136885e4a
Parents: f068296
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Oct 10 09:32:17 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Oct 10 09:32:27 2018 +0800
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 11 +++++
.../scheduler/capacity/CapacityScheduler.java | 12 ++++-
.../capacity/TestContainerAllocation.java | 52 ++++++++++++++++++++
3 files changed, 74 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edce8664/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 8c7e447..0f72c76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -258,6 +259,16 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
}
+ @VisibleForTesting
+ public void setForceConfiguredMaxAllocation(boolean flag) {
+ writeLock.lock();
+ try {
+ forceConfiguredMaxAllocation = flag;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
private void updateMaxResources(SchedulerNode node, boolean add) {
Resource totalResource = node.getTotalResource();
ResourceInformation[] totalResources;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edce8664/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 75d6144..fddd361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2581,7 +2581,17 @@ public class CapacityScheduler extends
LOG.error("queue " + queueName + " is not an leaf queue");
return getMaximumResourceCapability();
}
- return ((LeafQueue)queue).getMaximumAllocation();
+
+ // queue.getMaxAllocation returns *configured* maximum allocation.
+ // getMaximumResourceCapability() returns maximum allocation considers
+ // per-node maximum resources. So return (component-wise) min of the two.
+
+ Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation();
+ Resource clusterMaxAllocationConsiderNodeMax =
+ getMaximumResourceCapability();
+
+ return Resources.componentwiseMin(queueMaxAllocation,
+ clusterMaxAllocationConsiderNodeMax);
}
private String handleMoveToPlanQueue(String targetQueueName) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edce8664/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index f1b4444..d5f6ab1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@@ -68,6 +69,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
public class TestContainerAllocation {
@@ -1130,4 +1132,54 @@ public class TestContainerAllocation {
rm1.close();
}
+
+ @Test(timeout = 60000)
+ public void testContainerRejectionWhenAskBeyondDynamicMax()
+ throws Exception {
+ CapacitySchedulerConfiguration newConf =
+ (CapacitySchedulerConfiguration) TestUtils
+ .getConfigurationWithMultipleQueues(conf);
+ newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+ newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+ + MAXIMUM_ALLOCATION_MB, "4096");
+
+ MockRM rm1 = new MockRM(newConf);
+ rm1.start();
+
+ // before any node registered or before registration timeout,
+ // submit an app beyond queue max leads to failure.
+ boolean submitFailed = false;
+ MockNM nm1 = rm1.registerNode("h1:1234", 2 * GB, 1);
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ try {
+ am1.allocate("*", 5 * GB, 1, null);
+ } catch (InvalidResourceRequestException e) {
+ submitFailed = true;
+ }
+ Assert.assertTrue(submitFailed);
+
+ // Ask 4GB succeeded.
+ am1.allocate("*", 4 * GB, 1, null);
+
+ // Add a new node, now the cluster maximum should be refreshed to 3GB.
+ CapacityScheduler cs = (CapacityScheduler)rm1.getResourceScheduler();
+ cs.getNodeTracker().setForceConfiguredMaxAllocation(false);
+ rm1.registerNode("h2:1234", 3 * GB, 1);
+
+ // Now ask 4 GB will fail
+ submitFailed = false;
+ try {
+ am1.allocate("*", 4 * GB, 1, null);
+ } catch (InvalidResourceRequestException e) {
+ submitFailed = true;
+ }
+ Assert.assertTrue(submitFailed);
+
+ // But ask 3 GB succeeded.
+ am1.allocate("*", 3 * GB, 1, null);
+
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-627. OzoneFS read from an MR
Job throws java.lang.ArrayIndexOutOfBoundsException. Contributed by Mukul
Kumar Singh.
Posted by sh...@apache.org.
HDDS-627. OzoneFS read from an MR Job throws java.lang.ArrayIndexOutOfBoundsException. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c05b2600
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c05b2600
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c05b2600
Branch: refs/heads/HDFS-12943
Commit: c05b260069511f2993a67683555ed2e1aea49a0a
Parents: 6dcfef7
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Oct 11 15:02:18 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Oct 11 15:08:22 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b2600/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
index 2b10578..2f17035 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
@@ -129,7 +129,8 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
totalReadLen += numBytesRead;
off += numBytesRead;
len -= numBytesRead;
- if (current.getRemaining() <= 0) {
+ if (current.getRemaining() <= 0 &&
+ ((currentStreamIndex + 1) < streamEntries.size())) {
currentStreamIndex += 1;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HADOOP-15676. Cleanup
TestSSLHttpServer. Contributed by Szilard Nemeth.
Posted by sh...@apache.org.
HADOOP-15676. Cleanup TestSSLHttpServer. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64f2b32d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64f2b32d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64f2b32d
Branch: refs/heads/HDFS-12943
Commit: 64f2b32d57f35864b5c47b7e80f02e9c939f592a
Parents: c05b260
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Oct 11 15:08:22 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Oct 11 15:12:36 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/http/TestSSLHttpServer.java | 204 ++++++++++---------
1 file changed, 103 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64f2b32d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
index 2166464..3f6ee7b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
@@ -24,8 +24,9 @@ import java.io.InputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
-import java.net.UnknownHostException;
+import java.security.GeneralSecurityException;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLHandshakeException;
@@ -34,6 +35,7 @@ import javax.net.ssl.SSLSocketFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.http.TestHttpServer.EchoServlet;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
@@ -60,23 +62,34 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
LoggerFactory.getLogger(TestSSLHttpServer.class);
private static final String HTTPS_CIPHER_SUITES_KEY = "https.cipherSuites";
private static final String JAVAX_NET_DEBUG_KEY = "javax.net.debug";
- private static Configuration conf;
+ private static final String SSL_SERVER_KEYSTORE_PROP_PREFIX = "ssl.server" +
+ ".keystore";
+ private static final String SSL_SERVER_TRUSTSTORE_PROP_PREFIX = "ssl.server" +
+ ".truststore";
+
+ private static final String SERVLET_NAME_LONGHEADER = "longheader";
+ private static final String SERVLET_PATH_LONGHEADER =
+ "/" + SERVLET_NAME_LONGHEADER;
+ private static final String SERVLET_NAME_ECHO = "echo";
+ private static final String SERVLET_PATH_ECHO = "/" + SERVLET_NAME_ECHO;
+
private static HttpServer2 server;
- private static String keystoresDir;
+ private static String keystoreDir;
private static String sslConfDir;
private static SSLFactory clientSslFactory;
private static String cipherSuitesPropertyValue;
private static String sslDebugPropertyValue;
- private static final String excludeCiphers = "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ private static final String EXCLUDED_CIPHERS =
+ "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, \n"
+ "SSL_RSA_WITH_DES_CBC_SHA,"
+ "SSL_DHE_RSA_WITH_DES_CBC_SHA, "
+ "SSL_RSA_EXPORT_WITH_RC4_40_MD5,\t \n"
+ "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ "SSL_RSA_WITH_RC4_128_MD5 \t";
- private static final String oneEnabledCiphers = excludeCiphers
+ private static final String ONE_ENABLED_CIPHERS = EXCLUDED_CIPHERS
+ ",TLS_RSA_WITH_AES_128_CBC_SHA";
- private static final String exclusiveEnabledCiphers
+ private static final String EXCLUSIVE_ENABLED_CIPHERS
= "\tTLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, \n"
+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_RSA_WITH_AES_128_CBC_SHA,"
@@ -90,49 +103,54 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
turnOnSSLDebugLogging();
storeHttpsCipherSuites();
- conf = new Configuration();
+ Configuration conf = new Configuration();
conf.setInt(HttpServer2.HTTP_MAX_THREADS_KEY, 10);
File base = new File(BASEDIR);
FileUtil.fullyDelete(base);
base.mkdirs();
- keystoresDir = new File(BASEDIR).getAbsolutePath();
+ keystoreDir = new File(BASEDIR).getAbsolutePath();
sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class);
- KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false, true,
- excludeCiphers);
+ KeyStoreTestUtil.setupSSLConfig(keystoreDir, sslConfDir, conf, false, true,
+ EXCLUDED_CIPHERS);
Configuration sslConf = KeyStoreTestUtil.getSslConfig();
clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, sslConf);
clientSslFactory.init();
- server = new HttpServer2.Builder()
- .setName("test")
- .addEndpoint(new URI("https://localhost"))
- .setConf(conf)
- .keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
- .keyStore(sslConf.get("ssl.server.keystore.location"),
- sslConf.get("ssl.server.keystore.password"),
- sslConf.get("ssl.server.keystore.type", "jks"))
- .trustStore(sslConf.get("ssl.server.truststore.location"),
- sslConf.get("ssl.server.truststore.password"),
- sslConf.get("ssl.server.truststore.type", "jks"))
- .excludeCiphers(
- sslConf.get("ssl.server.exclude.cipher.list")).build();
- server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
- server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
- server.start();
+ setupServer(conf, sslConf);
baseUrl = new URL("https://"
+ NetUtils.getHostPortString(server.getConnectorAddress(0)));
LOG.info("HTTP server started: " + baseUrl);
}
+ private static void setupServer(Configuration conf, Configuration sslConf)
+ throws IOException, URISyntaxException {
+ server = new HttpServer2.Builder().setName("test")
+ .addEndpoint(new URI("https://localhost")).setConf(conf)
+ .keyPassword(
+ sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".keypassword"))
+ .keyStore(sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".location"),
+ sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".password"),
+ sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".type", "jks"))
+ .trustStore(
+ sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".location"),
+ sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".password"),
+ sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".type", "jks"))
+ .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")).build();
+ server.addServlet(SERVLET_NAME_ECHO, SERVLET_PATH_ECHO, EchoServlet.class);
+ server.addServlet(SERVLET_NAME_LONGHEADER, SERVLET_PATH_LONGHEADER,
+ LongHeaderServlet.class);
+ server.start();
+ }
+
@AfterClass
public static void cleanup() throws Exception {
server.stop();
FileUtil.fullyDelete(new File(BASEDIR));
- KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
+ KeyStoreTestUtil.cleanupSSLConfig(keystoreDir, sslConfDir);
clientSslFactory.destroy();
restoreHttpsCipherSuites();
restoreSSLDebugLogging();
@@ -181,11 +199,27 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
}
}
+ private HttpsURLConnection getConnectionWithSSLSocketFactory(URL url,
+ String ciphers) throws IOException, GeneralSecurityException {
+ HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
+ SSLSocketFactory sslSocketFactory = clientSslFactory
+ .createSSLSocketFactory();
+ LOG.info("Creating " +
+ PreferredCipherSSLSocketFactory.class.getCanonicalName() +
+ " with ciphers: " + ciphers);
+ PreferredCipherSSLSocketFactory cipherSSLSocketFactory
+ = new PreferredCipherSSLSocketFactory(sslSocketFactory,
+ StringUtils.getTrimmedStrings(ciphers));
+ conn.setSSLSocketFactory(cipherSSLSocketFactory);
+ return conn;
+ }
+
@Test
public void testEcho() throws Exception {
- assertEquals("a:b\nc:d\n", readOut(new URL(baseUrl, "/echo?a=b&c=d")));
- assertEquals("a:b\nc<:d\ne:>\n", readOut(new URL(baseUrl,
- "/echo?a=b&c<=d&e=>")));
+ assertEquals("a:b\nc:d\n",
+ readFromURL(new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d")));
+ assertEquals("a:b\nc<:d\ne:>\n",
+ readFromURL(new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c<=d&e=>")));
}
/**
@@ -195,15 +229,20 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testLongHeader() throws Exception {
- URL url = new URL(baseUrl, "/longheader");
+ URL url = new URL(baseUrl, SERVLET_PATH_LONGHEADER);
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());
testLongHeader(conn);
}
- private static String readOut(URL url) throws Exception {
+ private static String readFromURL(URL url) throws Exception {
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());
+ return readFromConnection(conn);
+ }
+
+ private static String readFromConnection(HttpsURLConnection conn)
+ throws IOException {
InputStream in = conn.getInputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copyBytes(in, out, 1024);
@@ -218,21 +257,15 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testExcludedCiphers() throws Exception {
- URL url = new URL(baseUrl, "/echo?a=b&c=d");
- HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
- SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
- PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
- = new PrefferedCipherSSLSocketFactory(sslSocketF,
- StringUtils.getTrimmedStrings(excludeCiphers));
- conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
- assertFalse("excludedCipher list is empty", excludeCiphers.isEmpty());
+ URL url = new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d");
+ HttpsURLConnection conn = getConnectionWithSSLSocketFactory(url,
+ EXCLUDED_CIPHERS);
+ assertFalse("excludedCipher list is empty", EXCLUDED_CIPHERS.isEmpty());
try {
- InputStream in = conn.getInputStream();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(in, out, 1024);
+ readFromConnection(conn);
fail("No Ciphers in common, SSLHandshake must fail.");
} catch (SSLHandshakeException ex) {
- LOG.info("No Ciphers in common, expected succesful test result.", ex);
+ LOG.info("No Ciphers in common, expected successful test result.", ex);
}
}
@@ -242,25 +275,7 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testOneEnabledCiphers() throws Exception {
- URL url = new URL(baseUrl, "/echo?a=b&c=d");
- HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
- SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
- PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
- = new PrefferedCipherSSLSocketFactory(sslSocketF,
- StringUtils.getTrimmedStrings(oneEnabledCiphers));
- conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
- assertFalse("excludedCipher list is empty", oneEnabledCiphers.isEmpty());
- try {
- InputStream in = conn.getInputStream();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(in, out, 1024);
- assertEquals(out.toString(), "a:b\nc:d\n");
- LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
- + " expected successful test result.");
- } catch (SSLHandshakeException ex) {
- fail("Atleast one additional cipher available for successful handshake."
- + " Unexpected test failure: " + ex);
- }
+ testEnabledCiphers(ONE_ENABLED_CIPHERS);
}
/** Test verifies that mutually exclusive server's disabled cipher suites and
@@ -268,33 +283,25 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testExclusiveEnabledCiphers() throws Exception {
- URL url = new URL(baseUrl, "/echo?a=b&c=d");
- HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
- SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
- PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
- = new PrefferedCipherSSLSocketFactory(sslSocketF,
- StringUtils.getTrimmedStrings(exclusiveEnabledCiphers));
- conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
- assertFalse("excludedCipher list is empty",
- exclusiveEnabledCiphers.isEmpty());
- try {
- InputStream in = conn.getInputStream();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(in, out, 1024);
- assertEquals(out.toString(), "a:b\nc:d\n");
- LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
- + " expected successful test result.");
- } catch (SSLHandshakeException ex) {
- fail("Atleast one additional cipher available for successful handshake."
- + " Unexpected test failure: " + ex);
- }
+ testEnabledCiphers(EXCLUSIVE_ENABLED_CIPHERS);
}
- private class PrefferedCipherSSLSocketFactory extends SSLSocketFactory {
+ private void testEnabledCiphers(String ciphers) throws
+ IOException, GeneralSecurityException {
+ URL url = new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d");
+ HttpsURLConnection conn = getConnectionWithSSLSocketFactory(url, ciphers);
+ assertFalse("excludedCipher list is empty", ciphers.isEmpty());
+ String out = readFromConnection(conn);
+ assertEquals(out, "a:b\nc:d\n");
+ LOG.info("At least one additional enabled cipher than excluded ciphers,"
+ + " expected successful test result.");
+ }
+
+ private class PreferredCipherSSLSocketFactory extends SSLSocketFactory {
private final SSLSocketFactory delegateSocketFactory;
private final String[] enabledCipherSuites;
- public PrefferedCipherSSLSocketFactory(SSLSocketFactory sslSocketFactory,
+ PreferredCipherSSLSocketFactory(SSLSocketFactory sslSocketFactory,
String[] pEnabledCipherSuites) {
delegateSocketFactory = sslSocketFactory;
if (null != pEnabledCipherSuites && pEnabledCipherSuites.length > 0) {
@@ -319,31 +326,24 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
socket, string, i, bln);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@Override
- public Socket createSocket(String string, int i) throws IOException,
- UnknownHostException {
+ public Socket createSocket(String string, int i) throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
string, i);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@Override
public Socket createSocket(String string, int i, InetAddress ia, int i1)
- throws IOException, UnknownHostException {
+ throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
string, i, ia, i1);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@@ -351,9 +351,7 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
public Socket createSocket(InetAddress ia, int i) throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
i);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@@ -362,10 +360,14 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
i, ia1, i1);
+ setEnabledCipherSuites(sslSocket);
+ return sslSocket;
+ }
+
+ private void setEnabledCipherSuites(SSLSocket sslSocket) {
if (null != enabledCipherSuites) {
sslSocket.setEnabledCipherSuites(enabledCipherSuites);
}
- return sslSocket;
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HADOOP-15785. [JDK10] Javadoc
build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
index bec0fe1..487a457 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
@@ -168,7 +168,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
}
/** Add a leaf node
- * Update node counter & rack counter if necessary
+ * Update node counter & rack counter if necessary
* @param node node to be added; can be null
* @exception IllegalArgumentException if add a node to a leave
* or node to be added is not a leaf
@@ -272,7 +272,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
/**
* Sort nodes array by their distances to <i>reader</i>.
- * <p/>
+ * <p>
* This is the same as {@link NetworkTopology#sortByDistance(Node, Node[],
* int)} except with a four-level network topology which contains the
* additional network distance of a "node group" which is between local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
index 02b44a5..4db8155 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
@@ -33,13 +33,13 @@ import org.slf4j.LoggerFactory;
* This class implements the {@link DNSToSwitchMapping} interface using a
* script configured via the
* {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
- * <p/>
+ * <p>
* It contains a static class <code>RawScriptBasedMapping</code> that performs
* the work: reading the configuration parameters, executing any defined
* script, handling errors and such like. The outer
* class extends {@link CachedDNSToSwitchMapping} to cache the delegated
* queries.
- * <p/>
+ * <p>
* This DNS mapper's {@link #isSingleSwitch()} predicate returns
* true if and only if a script is defined.
*/
@@ -78,7 +78,7 @@ public class ScriptBasedMapping extends CachedDNSToSwitchMapping {
/**
* Create an instance with the default configuration.
- * </p>
+ * <p>
* Calling {@link #setConf(Configuration)} will trigger a
* re-evaluation of the configuration settings and so be used to
* set up the mapping script.
@@ -125,7 +125,7 @@ public class ScriptBasedMapping extends CachedDNSToSwitchMapping {
/**
* {@inheritDoc}
- * <p/>
+ * <p>
* This will get called in the superclass constructor, so a check is needed
* to ensure that the raw mapping is defined before trying to relaying a null
* configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
index 086650b..e05fae6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
@@ -32,10 +32,9 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
* the {@link DNSToSwitchMappingWithDependency} interface using
* a script configured via the
* {@link CommonConfigurationKeys#NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY} option.
- * <p/>
+ * <p>
* It contains a static class <code>RawScriptBasedMappingWithDependency</code>
* that performs the getDependency work.
- * <p/>
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -52,7 +51,7 @@ public class ScriptBasedMappingWithDependency extends ScriptBasedMapping
/**
* Create an instance with the default configuration.
- * </p>
+ * <p>
* Calling {@link #setConf(Configuration)} will trigger a
* re-evaluation of the configuration settings and so be used to
* set up the mapping script.
@@ -76,7 +75,7 @@ public class ScriptBasedMappingWithDependency extends ScriptBasedMapping
/**
* {@inheritDoc}
- * <p/>
+ * <p>
* This will get called in the superclass constructor, so a check is needed
* to ensure that the raw mapping is defined before trying to relaying a null
* configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
index ead1d7b2..93f4f56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
@@ -32,7 +32,6 @@ import java.nio.channels.WritableByteChannel;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.metrics2.lib.MutableRate;
/**
* This implements an output stream that can have a timeout while writing.
@@ -187,7 +186,7 @@ public class SocketOutputStream extends OutputStream
* @param count number of bytes to transfer.
* @param waitForWritableTime nanoseconds spent waiting for the socket
* to become writable
- * @param transferTime nanoseconds spent transferring data
+ * @param transferToTime nanoseconds spent transferring data
*
* @throws EOFException
* If end of input file is reached before requested number of
@@ -253,7 +252,8 @@ public class SocketOutputStream extends OutputStream
/**
* Call
- * {@link #transferToFully(FileChannel, long, int, MutableRate, MutableRate)}
+ * {@link #transferToFully(FileChannel, long, int, LongWritable, LongWritable)
+ * }
* with null <code>waitForWritableTime</code> and <code>transferToTime</code>
*/
public void transferToFully(FileChannel fileCh, long position, int count)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
index 7e6b3a8..b25d5d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
@@ -31,11 +31,11 @@ import java.util.Map;
/**
* Initializes hadoop-auth AuthenticationFilter which provides support for
* Kerberos HTTP SPNEGO authentication.
- * <p/>
+ * <p>
* It enables anonymous access, simple/speudo and Kerberos HTTP SPNEGO
* authentication for Hadoop JobTracker, NameNode, DataNodes and
* TaskTrackers.
- * <p/>
+ * <p>
* Refer to the <code>core-default.xml</code> file, after the comment
* 'HTTP Authentication' for details on the configuration options.
* All related configuration properties have 'hadoop.http.authentication.'
@@ -47,7 +47,7 @@ public class AuthenticationFilterInitializer extends FilterInitializer {
/**
* Initializes hadoop-auth AuthenticationFilter.
- * <p/>
+ * <p>
* Propagates to hadoop-auth AuthenticationFilter configuration all Hadoop
* configuration properties prefixed with "hadoop.http.authentication."
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
index 55b2786..d7f0e81 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
@@ -46,7 +46,7 @@ public class HadoopKerberosName extends KerberosName {
}
/**
* Set the static configuration to get the rules.
- * <p/>
+ * <p>
* IMPORTANT: This method does a NOP if the rules have been set already.
* If there is a need to reset the rules, the {@link KerberosName#setRules(String)}
* method should be invoked directly.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
index 4a1185e..86edab7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
@@ -25,8 +25,9 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/**
- * An interface for the implementation of <userId, userName> mapping
- * and <groupId, groupName> mapping
+ * An interface for the implementation of {@literal <}userId,
+ * userName{@literal >} mapping and {@literal <}groupId, groupName{@literal >}
+ * mapping.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index 11714b1..d236ab0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -343,13 +343,9 @@ public class SaslRpcClient {
}
/**
- * Do client side SASL authentication with server via the given InputStream
- * and OutputStream
- *
- * @param inS
- * InputStream to use
- * @param outS
- * OutputStream to use
+ * Do client side SASL authentication with server via the given IpcStreams.
+ *
+ * @param ipcStreams
* @return AuthMethod used to negotiate the connection
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 2313119..aa12b93 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -332,7 +332,8 @@ public final class SecurityUtil {
}
/**
- * Get the host name from the principal name of format <service>/host@realm.
+ * Get the host name from the principal name of format {@literal <}service
+ * {@literal >}/host@realm.
* @param principalName principal name of format as described above
* @return host name if the the string conforms to the above format, else null
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 6ce72edb..db88106 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -676,7 +676,7 @@ public class UserGroupInformation {
/**
* remove the login method that is followed by a space from the username
- * e.g. "jack (auth:SIMPLE)" -> "jack"
+ * e.g. "jack (auth:SIMPLE)" {@literal ->} "jack"
*
* @param userName
* @return userName without login method
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
index 52f39ef..5028482 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
@@ -33,9 +33,9 @@ import java.net.URI;
/**
* CredentialProvider based on Java's KeyStore file format. The file may be
* stored in any Hadoop FileSystem using the following name mangling:
- * jceks://hdfs@nn1.example.com/my/creds.jceks ->
- * hdfs://nn1.example.com/my/creds.jceks jceks://file/home/larry/creds.jceks ->
- * file:///home/larry/creds.jceks
+ * jceks://hdfs@nn1.example.com/my/creds.jceks {@literal ->}
+ * hdfs://nn1.example.com/my/creds.jceks jceks://file/home/larry/creds.jceks
+ * {@literal ->} file:///home/larry/creds.jceks
*/
@InterfaceAudience.Private
public class JavaKeyStoreProvider extends AbstractJavaKeyStoreProvider {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
index 9ea9a57..ce0eb7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
@@ -44,7 +44,8 @@ import java.util.EnumSet;
/**
* CredentialProvider based on Java's KeyStore file format. The file may be
* stored only on the local filesystem using the following name mangling:
- * localjceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
+ * localjceks://file/home/larry/creds.jceks {@literal ->}
+ * file:///home/larry/creds.jceks
*/
@InterfaceAudience.Private
public final class LocalJavaKeyStoreProvider extends
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
index 60d82cb..97a7f08 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
@@ -108,12 +108,12 @@ public class ProxyUsers {
* @param remoteAddress
* @param conf
* @throws AuthorizationException
- * @deprecated use {@link #authorize(UserGroupInformation, String) instead.
+ * @deprecated use {@link #authorize(UserGroupInformation, String)} instead.
*/
@Deprecated
public static void authorize(UserGroupInformation user,
String remoteAddress, Configuration conf) throws AuthorizationException {
- authorize(user,remoteAddress);
+ authorize(user, remoteAddress);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
index b0df8f0..e5bdab3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
@@ -38,7 +38,7 @@ import java.text.MessageFormat;
/**
* {@link KeyStoresFactory} implementation that reads the certificates from
* keystore files.
- * <p/>
+ * <p>
* if the trust certificates keystore file changes, the {@link TrustManager}
* is refreshed with the new trust certificate entries (using a
* {@link ReloadingX509TrustManager} trustmanager).
@@ -87,7 +87,7 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
/**
* Resolves a property name to its client/server version if applicable.
- * <p/>
+ * <p>
* NOTE: This method is public for testing purposes.
*
* @param mode client/server mode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index f05274a..10c1d7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -44,10 +44,10 @@ import java.util.List;
/**
* Factory that creates SSLEngine and SSLSocketFactory instances using
* Hadoop configuration information.
- * <p/>
+ * <p>
* This SSLFactory uses a {@link ReloadingX509TrustManager} instance,
* which reloads public keys if the truststore file changes.
- * <p/>
+ * <p>
* This factory is used to configure HTTPS in Hadoop HTTP based endpoints, both
* client and server.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
index 47546b3..f9236b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
@@ -58,15 +58,15 @@ import org.slf4j.LoggerFactory;
* or X509Certificate, or ultimately (they all end up calling this one),
* String. (It's easier to supply JUnit with Strings instead of mock
* SSLSession objects!)
- * </p><p>Our check() methods throw exceptions if the name is
+ * <p>Our check() methods throw exceptions if the name is
* invalid, whereas javax.net.ssl.HostnameVerifier just returns true/false.
- * <p/>
+ * <p>
* We provide the HostnameVerifier.DEFAULT, HostnameVerifier.STRICT, and
* HostnameVerifier.ALLOW_ALL implementations. We also provide the more
* specialized HostnameVerifier.DEFAULT_AND_LOCALHOST, as well as
* HostnameVerifier.STRICT_IE6. But feel free to define your own
* implementations!
- * <p/>
+ * <p>
* Inspired by Sebastian Hauer's original StrictSSLProtocolSocketFactory in the
* HttpClient "contrib" repository.
*/
@@ -109,10 +109,10 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
/**
* The DEFAULT HostnameVerifier works the same way as Curl and Firefox.
- * <p/>
+ * <p>
* The hostname must match either the first CN, or any of the subject-alts.
* A wildcard can occur in the CN, and in any of the subject-alts.
- * <p/>
+ * <p>
* The only difference between DEFAULT and STRICT is that a wildcard (such
* as "*.foo.com") with DEFAULT matches all subdomains, including
* "a.b.foo.com".
@@ -158,13 +158,13 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
* Java 1.4, Sun Java 5, Sun Java 6. It's also pretty close to IE6.
* This implementation appears to be compliant with RFC 2818 for dealing
* with wildcards.
- * <p/>
+ * <p>
* The hostname must match either the first CN, or any of the subject-alts.
* A wildcard can occur in the CN, and in any of the subject-alts. The
* one divergence from IE6 is how we only check the first CN. IE6 allows
* a match against any of the CNs present. We decided to follow in
* Sun Java 1.4's footsteps and only check the first CN.
- * <p/>
+ * <p>
* A wildcard such as "*.foo.com" matches only subdomains in the same
* level, for example "a.foo.com". It does not match deeper subdomains
* such as "a.b.foo.com".
@@ -229,7 +229,7 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
* This contains a list of 2nd-level domains that aren't allowed to
* have wildcards when combined with country-codes.
* For example: [*.co.uk].
- * <p/>
+ * <p>
* The [*.co.uk] problem is an interesting one. Should we just hope
* that CA's would never foolishly allow such a certificate to happen?
* Looks like we're the only implementation guarding against this.
@@ -564,11 +564,11 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
/**
* Extracts the array of SubjectAlt DNS names from an X509Certificate.
* Returns null if there aren't any.
- * <p/>
+ * <p>
* Note: Java doesn't appear able to extract international characters
* from the SubjectAlts. It can only extract international characters
* from the CN field.
- * <p/>
+ * <p>
* (Or maybe the version of OpenSSL I'm using to test isn't storing the
* international characters correctly in the SubjectAlts?).
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index 0b1fdf8..35589a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -44,19 +44,19 @@ import java.util.Map;
* The <code>DelegationTokenAuthenticatedURL</code> is a
* {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token
* functionality.
- * <p/>
+ * <p>
* The authentication mechanisms supported by default are Hadoop Simple
* authentication (also known as pseudo authentication) and Kerberos SPNEGO
* authentication.
- * <p/>
+ * <p>
* Additional authentication mechanisms can be supported via {@link
* DelegationTokenAuthenticator} implementations.
- * <p/>
+ * <p>
* The default {@link DelegationTokenAuthenticator} is the {@link
* KerberosDelegationTokenAuthenticator} class which supports
* automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via
* the {@link PseudoDelegationTokenAuthenticator} class.
- * <p/>
+ * <p>
* <code>AuthenticatedURL</code> instances are not thread-safe.
*/
@InterfaceAudience.Public
@@ -115,7 +115,7 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
* Returns the default {@link DelegationTokenAuthenticator} class to use when
* an {@link DelegationTokenAuthenticatedURL} instance is created without
* specifying one.
- * <p/>
+ * <p>
* The default class is {@link KerberosDelegationTokenAuthenticator}
*
* @return the delegation token authenticator class to use as default.
@@ -143,7 +143,7 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
/**
* Creates an <code>DelegationTokenAuthenticatedURL</code>.
- * <p/>
+ * <p>
* An instance of the default {@link DelegationTokenAuthenticator} will be
* used.
*/
@@ -191,7 +191,7 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
* Sets if delegation token should be transmitted in the URL query string.
* By default it is transmitted using the
* {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
- * <p/>
+ * <p>
* This method is provided to enable WebHDFS backwards compatibility.
*
* @param useQueryString <code>TRUE</code> if the token is transmitted in the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index f5e798e..5275526 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -60,7 +60,7 @@ import java.util.Properties;
/**
* The <code>DelegationTokenAuthenticationFilter</code> filter is a
* {@link AuthenticationFilter} with Hadoop Delegation Token support.
- * <p/>
+ * <p>
* By default it uses it own instance of the {@link
* AbstractDelegationTokenSecretManager}. For situations where an external
* <code>AbstractDelegationTokenSecretManager</code> is required (i.e. one that
@@ -86,7 +86,7 @@ public class DelegationTokenAuthenticationFilter
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
- * <p/>
+ * <p>
* This is useful for use cases where secrets must be shared across multiple
* services.
*/
@@ -148,7 +148,7 @@ public class DelegationTokenAuthenticationFilter
/**
* Returns the proxyuser configuration. All returned properties must start
* with <code>proxyuser.</code>'
- * <p/>
+ * <p>
* Subclasses may override this method if the proxyuser configuration is
* read from other place than the filter init parameters.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index 6ee59f1..284044f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -59,7 +59,7 @@ import com.google.common.annotations.VisibleForTesting;
/**
* An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism
* for HTTP and supports Delegation Token functionality.
- * <p/>
+ * <p>
* In addition to the wrapped {@link AuthenticationHandler} configuration
* properties, this handler supports the following properties prefixed
* with the type of the wrapped <code>AuthenticationHandler</code>:
@@ -135,7 +135,7 @@ public abstract class DelegationTokenAuthenticationHandler
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
- * <p/>
+ * <p>
* This is useful for use cases where secrets must be shared across multiple
* services.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
index fd19b67..e1445fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
@@ -127,7 +127,7 @@ public class DelegationTokenManager {
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
- * <p/>
+ * <p>
* This is useful for use cases where secrets must be shared across multiple
* services.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
index 395d2f2..28509e1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHa
/**
* An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism
* for HTTP and supports Delegation Token functionality.
- * <p/>
+ * <p>
* In addition to the {@link KerberosAuthenticationHandler} configuration
* properties, this handler supports:
* <ul>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
index 7e0e266..8b8a4bd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
* The <code>KerberosDelegationTokenAuthenticator</code> provides support for
* Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation
* Token operations.
- * <p/>
+ * <p>
* It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP
* endpoint does not trigger a SPNEGO authentication
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
index fc32a19..0661fb2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
import org.apache.hadoop.security.authentication.server.AuthenticationHandlerUtil;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.hadoop.security.authentication.server.CompositeAuthenticationHandler;
@@ -52,7 +53,7 @@ import com.google.common.base.Splitter;
* required to ensure that only schemes with strongest level of security should
* be used for delegation token management.
*
- * <p/>
+ * <p>
* In addition to the wrapped {@link AuthenticationHandler} configuration
* properties, this handler supports the following properties prefixed with the
* type of the wrapped <code>AuthenticationHandler</code>:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
index 6846fdb..9a4527a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHand
/**
* An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism
* for HTTP and supports Delegation Token functionality.
- * <p/>
+ * <p>
* In addition to the {@link KerberosAuthenticationHandler} configuration
* properties, this handler supports:
* <ul>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
index 8713aa4..3478f39 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
@@ -29,7 +29,7 @@ import java.io.IOException;
* Hadoop's pseudo authentication mechanism that accepts
* the user name specified as a query string parameter and support for Hadoop
* Delegation Token operations.
- * <p/>
+ * <p>
* This mimics the model of Hadoop Simple authentication trusting the
* {@link UserGroupInformation#getCurrentUser()} value.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
index d064ef9..726a83d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
@@ -42,9 +42,9 @@ public final class ServiceOperations {
/**
* Stop a service.
- * <p/>Do nothing if the service is null or not
+ * <p>Do nothing if the service is null or not
* in a state in which it can be/needs to be stopped.
- * <p/>
+ * <p>
* The service state is checked <i>before</i> the operation begins.
* This process is <i>not</i> thread safe.
* @param service a service or null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
index 6b0b4e8..da91a3d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
@@ -268,7 +268,7 @@ public class ServiceLauncher<S extends Service>
* <ol>
* <li>Parse the command line.</li>
* <li>Build the service configuration from it.</li>
- * <li>Start the service.</li>.
+ * <li>Start the service.</li>
* <li>If it is a {@link LaunchableService}: execute it</li>
* <li>Otherwise: wait for it to finish.</li>
* <li>Exit passing the status code to the {@link #exit(int, String)}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
index 8516357..f582fa2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
@@ -379,7 +379,7 @@
a new configuration is created:
<pre>
- public Configuration bindArgs(Configuration config, List<String> args)
+ public Configuration bindArgs(Configuration config, List<String> args)
throws Exception {
Configuration newConf = new YarnConfiguration(config);
return newConf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
index 6f94989..44c9466 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
@@ -34,7 +34,6 @@ public class ClassUtil {
*
* @param clazz the class to find.
* @return a jar file that contains the class, or null.
- * @throws IOException
*/
public static String findContainingJar(Class<?> clazz) {
ClassLoader loader = clazz.getClassLoader();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
index 1f34291..bcb17d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
@@ -64,7 +64,7 @@ import java.util.Stack;
* Unknown qualifiers are considered after known qualifiers, with lexical order (always case insensitive),
* </li>
* <li>a dash usually precedes a qualifier, and is always less important than something preceded with a dot.</li>
- * </ul></p>
+ * </ul><p>
*
* @see <a href="https://cwiki.apache.org/confluence/display/MAVENOLD/Versioning">"Versioning" on Maven Wiki</a>
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
index b7feb22..690d097 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
@@ -33,9 +33,9 @@ import java.security.CodeSource;
* This entry point exists for diagnosing classloader problems:
* is a class or resource present -and if so, where?
*
- * <p/>
+ * <p>
* Actions
- * <p/>
+ * <br>
* <ul>
* <li><pre>load</pre>: load a class but do not attempt to create it </li>
* <li><pre>create</pre>: load and create a class, print its string value</li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
index a8a513d..8ca7a90 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.util;import java.io.File;
+package org.apache.hadoop.util;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
@@ -54,9 +55,9 @@ import org.slf4j.LoggerFactory;
* line arguments, enabling applications to easily specify a namenode, a
* ResourceManager, additional configuration resources etc.
*
- * <h4 id="GenericOptions">Generic Options</h4>
+ * <h3 id="GenericOptions">Generic Options</h3>
*
- * <p>The supported generic options are:</p>
+ * <p>The supported generic options are:
* <p><blockquote><pre>
* -conf <configuration file> specify a configuration file
* -D <property=value> use value for given property
@@ -69,12 +70,12 @@ import org.slf4j.LoggerFactory;
* -archives <comma separated list of archives> specify comma
* separated archives to be unarchived on the compute machines.
- * </pre></blockquote></p>
+ * </pre></blockquote><p>
*
* <p>The general command line syntax is:</p>
- * <p><tt><pre>
+ * <p><pre><code>
* bin/hadoop command [genericOptions] [commandOptions]
- * </pre></tt></p>
+ * </code></pre><p>
*
* <p>Generic command line arguments <strong>might</strong> modify
* <code>Configuration </code> objects, given to constructors.</p>
@@ -104,7 +105,7 @@ import org.slf4j.LoggerFactory;
* $ bin/hadoop jar -libjars testlib.jar
* -archives test.tgz -files file.txt inputjar args
* job submission with libjars, files and archives
- * </pre></blockquote></p>
+ * </pre></blockquote><p>
*
* @see Tool
* @see ToolRunner
@@ -141,8 +142,8 @@ public class GenericOptionsParser {
}
/**
- * Create a <code>GenericOptionsParser<code> to parse only the generic Hadoop
- * arguments.
+ * Create a <code>GenericOptionsParser</code> to parse only the generic
+ * Hadoop arguments.
*
* The array of string arguments other than the generic arguments can be
* obtained by {@link #getRemainingArgs()}.
@@ -217,7 +218,7 @@ public class GenericOptionsParser {
/**
* Specify properties of each generic option.
- * <i>Important</i?: as {@link OptionBuilder} is not thread safe, subclasses
+ * <i>Important</i>: as {@link OptionBuilder} is not thread safe, subclasses
* must synchronize use on {@code OptionBuilder.class}
*/
@SuppressWarnings("static-access")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
index 50be1c3..366c8c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
@@ -34,10 +34,10 @@ import java.util.Map;
/**
* HTTP utility class to help propagate server side exception to the client
* over HTTP as a JSON payload.
- * <p/>
+ * <p>
* It creates HTTP Servlet and JAX-RPC error responses including details of the
* exception that allows a client to recreate the remote exception.
- * <p/>
+ * <p>
* It parses HTTP client connections and recreates the exception.
*/
@InterfaceAudience.Private
@@ -125,7 +125,7 @@ public class HttpExceptionUtils {
* expected HTTP status code. If the current status code is not the expected
* one it throws an exception with a detail message using Server side error
* messages if available.
- * <p/>
+ * <p>
* <b>NOTE:</b> this method will throw the deserialized exception even if not
* declared in the <code>throws</code> of the method signature.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
index cbc8560..7e09a61 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -185,7 +185,7 @@ public class JsonSerialization<T> {
* Save to a local file. Any existing file is overwritten unless
* the OS blocks that.
* @param file file
- * @param path path
+ * @param instance instance
* @throws IOException IO exception
*/
public void save(File file, T instance) throws
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
index d79aade..79de1ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
@@ -99,11 +99,11 @@ public class LightWeightCache<K, E extends K> extends LightWeightGSet<K, E> {
/**
* @param recommendedLength Recommended size of the internal array.
* @param sizeLimit the limit of the size of the cache.
- * The limit is disabled if it is <= 0.
- * @param creationExpirationPeriod the time period C > 0 in nanoseconds that
- * the creation of an entry is expired if it is added to the cache
- * longer than C.
- * @param accessExpirationPeriod the time period A >= 0 in nanoseconds that
+ * The limit is disabled if it is <= 0.
+ * @param creationExpirationPeriod the time period C > 0 in nanoseconds
+ * that the creation of an entry is expired if it is added to the
+ * cache longer than C.
+ * @param accessExpirationPeriod the time period A >= 0 in nanoseconds that
* the access of an entry is expired if it is not accessed
* longer than A.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
index a1cf709..e2cd304 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
@@ -62,7 +62,6 @@ public class LineReader implements Closeable {
* Create a line reader that reads from the given stream using the
* default buffer-size (64k).
* @param in The input stream
- * @throws IOException
*/
public LineReader(InputStream in) {
this(in, DEFAULT_BUFFER_SIZE);
@@ -73,7 +72,6 @@ public class LineReader implements Closeable {
* given buffer-size.
* @param in The input stream
* @param bufferSize Size of the read buffer
- * @throws IOException
*/
public LineReader(InputStream in, int bufferSize) {
this.in = in;
@@ -115,7 +113,6 @@ public class LineReader implements Closeable {
* @param in The input stream
* @param bufferSize Size of the read buffer
* @param recordDelimiterBytes The delimiter
- * @throws IOException
*/
public LineReader(InputStream in, int bufferSize,
byte[] recordDelimiterBytes) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
index b01330f..20931de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
@@ -187,7 +187,7 @@ public class MachineList {
}
/**
- * returns the contents of the MachineList as a Collection<String>
+ * returns the contents of the MachineList as a Collection<String>
* This can be used for testing
* @return contents of the MachineList
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
index 2ca8e55..556f4e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
@@ -46,7 +46,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOW
/**
* The <code>ShutdownHookManager</code> enables running shutdownHook
* in a deterministic order, higher priority first.
- * <p/>
+ * <p>
* The JVM runs ShutdownHooks in a non-deterministic order or in parallel.
* This class registers a single JVM shutdownHook and run all the
* shutdownHooks registered to it (to this class) in order based on their
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
index 5405d77..50a728e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
@@ -39,7 +39,6 @@ public class ShutdownThreadsHelper {
* @param thread {@link Thread to be shutdown}
* @return <tt>true</tt> if the thread is successfully interrupted,
* <tt>false</tt> otherwise
- * @throws InterruptedException
*/
public static boolean shutdownThread(Thread thread) {
return shutdownThread(thread, SHUTDOWN_WAIT_MS);
@@ -51,7 +50,6 @@ public class ShutdownThreadsHelper {
* interrupted
* @return <tt>true</tt> if the thread is successfully interrupted,
* <tt>false</tt> otherwise
- * @throws InterruptedException
*/
public static boolean shutdownThread(Thread thread,
long timeoutInMilliSeconds) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index f49698c..d19ced9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -428,7 +428,7 @@ public class StringUtils {
* Splits a comma separated value <code>String</code>, trimming leading and
* trailing whitespace on each value. Duplicate and empty values are removed.
*
- * @param str a comma separated <String> with values, may be null
+ * @param str a comma separated <code>String</code> with values, may be null
* @return a <code>Collection</code> of <code>String</code> values, empty
* Collection if null String input
*/
@@ -1006,8 +1006,8 @@ public class StringUtils {
* @param template String template to receive replacements
* @param pattern Pattern to match for identifying tokens, must use a capturing
* group
- * @param replacements Map<String, String> mapping tokens identified by the
- * capturing group to their replacement values
+ * @param replacements Map<String, String> mapping tokens identified by
+ * the capturing group to their replacement values
* @return String template with replacements
*/
public static String replaceTokens(String template, Pattern pattern,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
index b6e3d7d..a4fbce4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.conf.Configurable;
* System.exit(res);
* }
* }
- * </pre></blockquote></p>
+ * </pre></blockquote><p>
*
* @see GenericOptionsParser
* @see ToolRunner
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
index d23df79..fe64adb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
@@ -90,7 +90,7 @@ public class ZKUtil {
* <code>sasl:hdfs/host1@MY.DOMAIN:cdrwa,sasl:hdfs/host2@MY.DOMAIN:cdrwa</code>
*
* @return ACL list
- * @throws {@link BadAclFormatException} if an ACL is invalid
+ * @throws BadAclFormatException if an ACL is invalid
*/
public static List<ACL> parseACLs(String aclString) throws
BadAclFormatException {
@@ -128,7 +128,7 @@ public class ZKUtil {
*
* @param authString the comma-separated auth mechanisms
* @return a list of parsed authentications
- * @throws {@link BadAuthFormatException} if the auth format is invalid
+ * @throws BadAuthFormatException if the auth format is invalid
*/
public static List<ZKAuthInfo> parseAuth(String authString) throws
BadAuthFormatException{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
index be9a407..8a7ec69 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
@@ -64,8 +64,9 @@ import org.apache.hadoop.classification.InterfaceStability;
* process of a DBF is iterative. At the start, the DBF is a <code>1 * m</code>
* bit matrix, i.e., it is composed of a single standard Bloom filter.
* It assumes that <code>n<sub>r</sub></code> elements are recorded in the
- * initial bit vector, where <code>n<sub>r</sub> <= n</code> (<code>n</code> is
- * the cardinality of the set <code>A</code> to record in the filter).
+ * initial bit vector, where <code>n<sub>r</sub> {@literal <=} n</code>
+ * (<code>n</code> is the cardinality of the set <code>A</code> to record in
+ * the filter).
* <p>
* As the size of <code>A</code> grows during the execution of the application,
* several keys must be inserted in the DBF. When inserting a key into the DBF,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
index f124890..9304b48 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
@@ -37,7 +37,7 @@ public interface AsyncGet<R, E extends Throwable> {
*
* @param timeout The maximum time period to wait.
* When timeout == 0, it does not wait at all.
- * When timeout < 0, it waits indefinitely.
+ * When timeout < 0, it waits indefinitely.
* @param unit The unit of the timeout value
* @return the result, which is possibly null.
* @throws E an exception thrown by the underlying implementation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
index f3895d0..8c3b9da 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
@@ -69,11 +69,11 @@ public class JenkinsHash extends Hash {
* <p>The best hash table sizes are powers of 2. There is no need to do mod
* a prime (mod is sooo slow!). If you need less than 32 bits, use a bitmask.
* For example, if you need only 10 bits, do
- * <code>h = (h & hashmask(10));</code>
+ * <code>h = (h & hashmask(10));</code>
* In which case, the hash table should have hashsize(10) elements.
*
* <p>If you are hashing n strings byte[][] k, do it like this:
- * for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h);
+ * for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h);
*
* <p>By Bob Jenkins, 2006. bob_jenkins@burtleburtle.net. You may use this
* code any way you wish, private, educational, or commercial. It's free.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
index 8823f5c..a7a8fa5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +20,7 @@ package org.apache.hadoop.cli.util;
import org.apache.hadoop.conf.Configuration;
/**
- * This interface is to generalize types of test command for upstream projects
+ * This interface is to generalize types of test command for upstream projects.
*/
public interface CLICommand {
public CommandExecutor getExecutor(String tag, Configuration conf)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
index eb96a06..e22c200 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
index 44e0c30..8efe70c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,7 +18,7 @@
package org.apache.hadoop.cli.util;
/**
- * This interface is to provide command type for test commands enums
+ * This interface is to provide command type for test commands enums.
*/
public interface CLICommandTypes {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
index df8c54b..362bde9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
index 56dec3a..6b82077 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
index 493d860..eb51820 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
@@ -44,9 +44,9 @@ public class StaticMapping extends AbstractDNSToSwitchMapping {
/**
* Key to define the node mapping as a comma-delimited list of host=rack
* mappings, e.g. <code>host1=r1,host2=r1,host3=r2</code>.
- * <p/>
+ * <p>
* Value: {@value}
- * <p/>
+ * <p>
* <b>Important: </b>spaces not trimmed and are considered significant.
*/
public static final String KEY_HADOOP_CONFIGURED_NODE_MAPPING =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
index 2bc68ce..d4599b0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
@@ -35,7 +35,7 @@ import org.junit.Assert;
/**
* Span receiver that puts all spans into a single set.
* This is useful for testing.
- * <p/>
+ * <p>
* We're not using HTrace's POJOReceiver here so as that doesn't
* push all the metrics to a static place, and would make testing
* SpanReceiverHost harder.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HDFS-13523. Support observer nodes
in MiniDFSCluster. Contributed by Konstantin Shvachko.
Posted by sh...@apache.org.
HDFS-13523. Support observer nodes in MiniDFSCluster. Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bba8aa3d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bba8aa3d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bba8aa3d
Branch: refs/heads/HDFS-12943
Commit: bba8aa3d940a2234cd9b7edbb3c56c76d9afb05e
Parents: fc09352
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Oct 11 22:39:17 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Oct 11 22:39:17 2018 -0700
----------------------------------------------------------------------
.../hdfs/TestStateAlignmentContextWithHA.java | 61 +++++++++-----------
.../hdfs/server/namenode/ha/HATestUtil.java | 40 +++++++++----
.../ha/TestConsistentReadsObserver.java | 19 +-----
.../namenode/ha/TestMultiObserverNode.java | 16 +----
.../server/namenode/ha/TestObserverNode.java | 16 +----
5 files changed, 67 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bba8aa3d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
index a642872..3dbeea7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStateAlignmentContextWithHA.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.hdfs;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
import org.apache.hadoop.hdfs.server.namenode.ha.HAProxyFactory;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider;
@@ -66,6 +67,7 @@ public class TestStateAlignmentContextWithHA {
private static final Configuration CONF = new HdfsConfiguration();
private static final List<ClientGSIContext> AC_LIST = new ArrayList<>();
+ private static MiniQJMHACluster qjmhaCluster;
private static MiniDFSCluster cluster;
private static List<Worker> clients;
@@ -87,33 +89,26 @@ public class TestStateAlignmentContextWithHA {
@BeforeClass
public static void startUpCluster() throws IOException {
- // disable block scanner
- CONF.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
// Set short retry timeouts so this test runs faster
CONF.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
- CONF.setBoolean("fs.hdfs.impl.disable.cache", true);
+ CONF.setBoolean(String.format(
+ "fs.%s.impl.disable.cache", HdfsConstants.HDFS_URI_SCHEME), true);
+ CONF.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, NUMDATANODES);
- cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(NUMDATANODES)
- .nnTopology(MiniDFSNNTopology.simpleHATopology(3))
- .build();
- cluster.waitActive();
- cluster.transitionToActive(0);
- cluster.transitionToObserver(2);
-
- HATestUtil.setupHAConfiguration(
- cluster, CONF, 0, ORPPwithAlignmentContexts.class);
+ qjmhaCluster = HATestUtil.setUpObserverCluster(CONF, 1, NUMDATANODES, true);
+ cluster = qjmhaCluster.getDfsCluster();
}
@Before
public void before() throws IOException, URISyntaxException {
- dfs = (DistributedFileSystem) FileSystem.get(CONF);
+ dfs = HATestUtil.configureObserverReadFs(
+ cluster, CONF, ORPPwithAlignmentContexts.class, true);
}
@AfterClass
public static void shutDownCluster() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
- cluster = null;
+ if (qjmhaCluster != null) {
+ qjmhaCluster.shutdown();
}
}
@@ -144,9 +139,9 @@ public class TestStateAlignmentContextWithHA {
long postWriteState =
cluster.getNamesystem(active).getLastWrittenTransactionId();
// Write(s) should have increased state. Check for greater than.
- assertThat(clientState > preWriteState, is(true));
+ assertTrue(clientState > preWriteState);
// Client and server state should be equal.
- assertThat(clientState, is(postWriteState));
+ assertEquals(clientState, postWriteState);
}
/**
@@ -161,7 +156,7 @@ public class TestStateAlignmentContextWithHA {
DFSTestUtil.readFile(dfs, new Path("/testFile2"));
// Read should catch client up to last written state.
long clientState = getContext(0).getLastSeenStateId();
- assertThat(clientState, is(lastWrittenId));
+ assertEquals(clientState, lastWrittenId);
}
/**
@@ -173,12 +168,12 @@ public class TestStateAlignmentContextWithHA {
DFSTestUtil.writeFile(dfs, new Path("/testFile3"), "ezpz");
long lastWrittenId =
cluster.getNamesystem(active).getLastWrittenTransactionId();
- try (DistributedFileSystem clearDfs =
- (DistributedFileSystem) FileSystem.get(CONF)) {
+ try (DistributedFileSystem clearDfs = HATestUtil.configureObserverReadFs(
+ cluster, CONF, ORPPwithAlignmentContexts.class, true);) {
ClientGSIContext clientState = getContext(1);
- assertThat(clientState.getLastSeenStateId(), is(Long.MIN_VALUE));
+ assertEquals(clientState.getLastSeenStateId(), Long.MIN_VALUE);
DFSTestUtil.readFile(clearDfs, new Path("/testFile3"));
- assertThat(clientState.getLastSeenStateId(), is(lastWrittenId));
+ assertEquals(clientState.getLastSeenStateId(), lastWrittenId);
}
}
@@ -196,9 +191,9 @@ public class TestStateAlignmentContextWithHA {
long postWriteState =
cluster.getNamesystem(active).getLastWrittenTransactionId();
// Write(s) should have increased state. Check for greater than.
- assertThat(clientState > preWriteState, is(true));
+ assertTrue(clientState > preWriteState);
// Client and server state should be equal.
- assertThat(clientState, is(postWriteState));
+ assertEquals(clientState, postWriteState);
// Failover NameNode.
failOver();
@@ -210,9 +205,9 @@ public class TestStateAlignmentContextWithHA {
cluster.getNamesystem(active).getLastWrittenTransactionId();
// Write(s) should have increased state. Check for greater than.
- assertThat(clientStateFO > postWriteState, is(true));
+ assertTrue(clientStateFO > postWriteState);
// Client and server state should be equal.
- assertThat(clientStateFO, is(writeStateFO));
+ assertEquals(clientStateFO, writeStateFO);
}
@Test(timeout=300000)
@@ -230,8 +225,8 @@ public class TestStateAlignmentContextWithHA {
ExecutorService execService = Executors.newFixedThreadPool(2);
clients = new ArrayList<>(numClients);
for (int i = clientStartId; i <= numClients; i++) {
- DistributedFileSystem haClient =
- (DistributedFileSystem) FileSystem.get(CONF);
+ DistributedFileSystem haClient = HATestUtil.configureObserverReadFs(
+ cluster, CONF, ORPPwithAlignmentContexts.class, true);
clients.add(new Worker(haClient, numFiles, "/testFile3FO_", i));
}
@@ -248,7 +243,7 @@ public class TestStateAlignmentContextWithHA {
// Validation.
for (Future<STATE> future : futures) {
- assertThat(future.get(), is(STATE.SUCCESS));
+ assertEquals(future.get(), STATE.SUCCESS);
}
clients.clear();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bba8aa3d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
index f4a766d..d1095ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
@@ -17,7 +17,10 @@
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSUtil.createUri;
@@ -29,6 +32,7 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.base.Function;
@@ -164,17 +168,18 @@ public abstract class HATestUtil {
return (DistributedFileSystem)fs;
}
- public static DistributedFileSystem configureObserverReadFs(
+ public static <P extends ObserverReadProxyProvider<?>>
+ DistributedFileSystem configureObserverReadFs(
MiniDFSCluster cluster, Configuration conf,
- boolean isObserverReadEnabled)
+ Class<P> classFPP, boolean isObserverReadEnabled)
throws IOException, URISyntaxException {
conf = new Configuration(conf);
- setupHAConfiguration(cluster, conf, 0, ObserverReadProxyProvider.class);
+ setupHAConfiguration(cluster, conf, 0, classFPP);
DistributedFileSystem dfs = (DistributedFileSystem)
FileSystem.get(getLogicalUri(cluster), conf);
- ObserverReadProxyProvider<?> provider = (ObserverReadProxyProvider<?>)
- ((RetryInvocationHandler<?>) Proxy.getInvocationHandler(
- dfs.getClient().getNamenode())).getProxyProvider();
+ @SuppressWarnings("unchecked")
+ P provider = (P) ((RetryInvocationHandler<?>) Proxy.getInvocationHandler(
+ dfs.getClient().getNamenode())).getProxyProvider();
provider.setObserverReadEnabled(isObserverReadEnabled);
return dfs;
}
@@ -196,10 +201,25 @@ public abstract class HATestUtil {
}
public static MiniQJMHACluster setUpObserverCluster(
- Configuration conf, int numObservers) throws IOException {
- MiniQJMHACluster qjmhaCluster = new MiniQJMHACluster.Builder(conf)
- .setNumNameNodes(2 + numObservers)
- .build();
+ Configuration conf, int numObservers, int numDataNodes,
+ boolean fastTailing) throws IOException {
+ // disable block scanner
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
+
+ conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, fastTailing);
+ if(fastTailing) {
+ conf.setTimeDuration(
+ DFS_HA_TAILEDITS_PERIOD_KEY, 100, TimeUnit.MILLISECONDS);
+ } else {
+ // disable fast tailing so that coordination takes time.
+ conf.setTimeDuration(DFS_HA_LOGROLL_PERIOD_KEY, 300, TimeUnit.SECONDS);
+ conf.setTimeDuration(DFS_HA_TAILEDITS_PERIOD_KEY, 200, TimeUnit.SECONDS);
+ }
+
+ MiniQJMHACluster.Builder qjmBuilder = new MiniQJMHACluster.Builder(conf)
+ .setNumNameNodes(2 + numObservers);
+ qjmBuilder.getDfsBuilder().numDataNodes(numDataNodes);
+ MiniQJMHACluster qjmhaCluster = qjmBuilder.build();
MiniDFSCluster dfsCluster = qjmhaCluster.getDfsCluster();
dfsCluster.transitionToActive(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bba8aa3d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
index 26ad3a2..3048842 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
@@ -17,21 +17,16 @@
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -46,7 +41,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* Test consistency of reads while accessing an ObserverNode.
* The tests are based on traditional (non fast path) edits tailing.
@@ -65,19 +59,11 @@ public class TestConsistentReadsObserver {
@BeforeClass
public static void startUpCluster() throws Exception {
conf = new Configuration();
- // disable block scanner
- conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
// disable fast tailing here because this test's assertions are based on the
// timing of explicitly called rollEditLogAndTail. Although this means this
// test takes some time to run
// TODO: revisit if there is a better way.
- conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, false);
-
- // disable fast tailing so that coordination takes time.
- conf.setTimeDuration(DFS_HA_LOGROLL_PERIOD_KEY, 300, TimeUnit.SECONDS);
- conf.setTimeDuration(DFS_HA_TAILEDITS_PERIOD_KEY, 200, TimeUnit.SECONDS);
-
- qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 1);
+ qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 1, 0, false);
dfsCluster = qjmhaCluster.getDfsCluster();
}
@@ -177,6 +163,7 @@ public class TestConsistentReadsObserver {
}
private static void setObserverRead(boolean flag) throws Exception {
- dfs = HATestUtil.configureObserverReadFs(dfsCluster, conf, flag);
+ dfs = HATestUtil.configureObserverReadFs(
+ dfsCluster, conf, ObserverReadProxyProvider.class, flag);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bba8aa3d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestMultiObserverNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestMultiObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestMultiObserverNode.java
index ab1251e..4aa3133 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestMultiObserverNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestMultiObserverNode.java
@@ -17,17 +17,12 @@
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
@@ -50,15 +45,10 @@ public class TestMultiObserverNode {
@BeforeClass
public static void startUpCluster() throws Exception {
conf = new Configuration();
- // disable block scanner
- conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
- conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
- conf.setTimeDuration(
- DFS_HA_TAILEDITS_PERIOD_KEY, 100, TimeUnit.MILLISECONDS);
-
- qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 2);
+ qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 2, 0, true);
dfsCluster = qjmhaCluster.getDfsCluster();
- dfs = HATestUtil.configureObserverReadFs(dfsCluster, conf, true);
+ dfs = HATestUtil.configureObserverReadFs(
+ dfsCluster, conf, ObserverReadProxyProvider.class, true);
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bba8aa3d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
index 2c826e6..28fd330 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -32,14 +30,11 @@ import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -77,13 +72,7 @@ public class TestObserverNode {
@BeforeClass
public static void startUpCluster() throws Exception {
conf = new Configuration();
- // disable block scanner
- conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
- conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
- conf.setTimeDuration(
- DFS_HA_TAILEDITS_PERIOD_KEY, 100, TimeUnit.MILLISECONDS);
-
- qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 1);
+ qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 1, 0, true);
dfsCluster = qjmhaCluster.getDfsCluster();
}
@@ -302,6 +291,7 @@ public class TestObserverNode {
}
private static void setObserverRead(boolean flag) throws Exception {
- dfs = HATestUtil.configureObserverReadFs(dfsCluster, conf, flag);
+ dfs = HATestUtil.configureObserverReadFs(
+ dfsCluster, conf, ObserverReadProxyProvider.class, flag);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDFS-13882. Set a maximum delay
for retrying locateFollowingBlock. Contributed by Kitti Nanasi.
Posted by sh...@apache.org.
HDFS-13882. Set a maximum delay for retrying locateFollowingBlock. Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10185d9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10185d9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10185d9a
Branch: refs/heads/HDFS-12943
Commit: 10185d9a77ce07080588f3c77399a07cd7ccf427
Parents: 90552b1
Author: Kitti Nanasi <kn...@cloudera.com>
Authored: Wed Oct 10 08:54:50 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Oct 10 08:55:32 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 21 ++++++++-
.../hdfs/client/HdfsClientConfigKeys.java | 3 ++
.../hadoop/hdfs/client/impl/DfsClientConf.java | 8 ++++
.../src/main/resources/hdfs-default.xml | 13 +++++-
.../hadoop/hdfs/TestDFSClientRetries.java | 47 +++++++++++++++-----
.../hadoop/tools/TestHdfsConfigFields.java | 1 +
6 files changed, 80 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index e977054..b8aae97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -942,6 +942,7 @@ public class DFSOutputStream extends FSOutputSummer
long localstart = Time.monotonicNow();
final DfsClientConf conf = dfsClient.getConf();
long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs();
+ long maxSleepTime = conf.getBlockWriteLocateFollowingMaxDelayMs();
boolean fileComplete = false;
int retries = conf.getNumBlockWriteLocateFollowingRetry();
while (!fileComplete) {
@@ -965,7 +966,7 @@ public class DFSOutputStream extends FSOutputSummer
}
retries--;
Thread.sleep(sleeptime);
- sleeptime *= 2;
+ sleeptime = calculateDelayForNextRetry(sleeptime, maxSleepTime);
if (Time.monotonicNow() - localstart > 5000) {
DFSClient.LOG.info("Could not complete " + src + " retrying...");
}
@@ -1075,6 +1076,7 @@ public class DFSOutputStream extends FSOutputSummer
final DfsClientConf conf = dfsClient.getConf();
int retries = conf.getNumBlockWriteLocateFollowingRetry();
long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs();
+ long maxSleepTime = conf.getBlockWriteLocateFollowingMaxDelayMs();
long localstart = Time.monotonicNow();
while (true) {
try {
@@ -1106,7 +1108,7 @@ public class DFSOutputStream extends FSOutputSummer
LOG.warn("NotReplicatedYetException sleeping " + src
+ " retries left " + retries);
Thread.sleep(sleeptime);
- sleeptime *= 2;
+ sleeptime = calculateDelayForNextRetry(sleeptime, maxSleepTime);
} catch (InterruptedException ie) {
LOG.warn("Caught exception", ie);
}
@@ -1117,4 +1119,19 @@ public class DFSOutputStream extends FSOutputSummer
}
}
}
+
+ /**
+ * Calculates the delay for the next retry.
+ *
+ * The delay is increased exponentially until the maximum delay is reached.
+ *
+ * @param previousDelay delay for the previous retry
+ * @param maxDelay maximum delay
+ * @return the minimum of the double of the previous delay
+ * and the maximum delay
+ */
+ private static long calculateDelayForNextRetry(long previousDelay,
+ long maxDelay) {
+ return Math.min(previousDelay * 2, maxDelay);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index a812670..b1ce78d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -322,6 +322,9 @@ public interface HdfsClientConfigKeys {
String LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY =
PREFIX + "locateFollowingBlock.initial.delay.ms";
int LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_DEFAULT = 400;
+ String LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_KEY =
+ PREFIX + "locateFollowingBlock.max.delay.ms";
+ int LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_DEFAULT = 60000;
interface ReplaceDatanodeOnFailure {
String PREFIX = BlockWrite.PREFIX + "replace-datanode-on-failure.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index e63e3f5..4110f94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -122,6 +122,7 @@ public class DfsClientConf {
private final int numBlockWriteRetry;
private final int numBlockWriteLocateFollowingRetry;
private final int blockWriteLocateFollowingInitialDelayMs;
+ private final int blockWriteLocateFollowingMaxDelayMs;
private final long defaultBlockSize;
private final long prefetchSize;
private final short defaultReplication;
@@ -237,6 +238,9 @@ public class DfsClientConf {
blockWriteLocateFollowingInitialDelayMs = conf.getInt(
BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY,
BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_DEFAULT);
+ blockWriteLocateFollowingMaxDelayMs = conf.getInt(
+ BlockWrite.LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_KEY,
+ BlockWrite.LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_DEFAULT);
uMask = FsPermission.getUMask(conf);
connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
@@ -349,6 +353,10 @@ public class DfsClientConf {
return blockWriteLocateFollowingInitialDelayMs;
}
+ public int getBlockWriteLocateFollowingMaxDelayMs() {
+ return blockWriteLocateFollowingMaxDelayMs;
+ }
+
/**
* @return the hdfsTimeout
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 503ba8f..cb770f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3135,11 +3135,22 @@
<name>dfs.client.block.write.locateFollowingBlock.initial.delay.ms</name>
<value>400</value>
<description>The initial delay (unit is ms) for locateFollowingBlock,
- the delay time will increase exponentially(double) for each retry.
+ the delay time will increase exponentially(double) for each retry
+ until dfs.client.block.write.locateFollowingBlock.max.delay.ms is reached,
+ after that the delay for each retry will be
+ dfs.client.block.write.locateFollowingBlock.max.delay.ms.
</description>
</property>
<property>
+ <name>dfs.client.block.write.locateFollowingBlock.max.delay.ms</name>
+ <value>60000</value>
+ <description>
+ The maximum delay (unit is ms) before retrying locateFollowingBlock.
+ </description>
+</property>
+
+<property>
<name>dfs.ha.zkfc.nn.http.timeout.ms</name>
<value>20000</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 14d0ee0..49de91d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -1229,27 +1229,54 @@ public class TestDFSClientRetries {
}
}
+ /**
+ * Tests default configuration values and configuration setting
+ * of locate following block delays and number of retries.
+ *
+ * Configuration values tested:
+ * - dfs.client.block.write.locateFollowingBlock.initial.delay.ms
+ * - dfs.client.block.write.locateFollowingBlock.max.delay.ms
+ * - dfs.client.block.write.locateFollowingBlock.retries
+ */
@Test
- public void testDFSClientConfigurationLocateFollowingBlockInitialDelay()
+ public void testDFSClientConfigurationLocateFollowingBlock()
throws Exception {
- // test if HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY
- // is not configured, verify DFSClient uses the default value 400.
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ final int initialDelayTestValue = 1000;
+ final int maxDelayTestValue = 35000;
+ final int retryTestValue = 7;
+
+ final int defaultInitialDelay = 400;
+ final int defaultMaxDelay = 60000;
+ final int defultRetry = 5;
+
try {
cluster.waitActive();
NamenodeProtocols nn = cluster.getNameNodeRpc();
DFSClient client = new DFSClient(null, nn, conf, null);
- assertEquals(client.getConf().
- getBlockWriteLocateFollowingInitialDelayMs(), 400);
+ assertEquals(defaultInitialDelay, client.getConf().
+ getBlockWriteLocateFollowingInitialDelayMs());
+ assertEquals(defaultMaxDelay, client.getConf().
+ getBlockWriteLocateFollowingMaxDelayMs());
+ assertEquals(defultRetry, client.getConf().
+ getNumBlockWriteLocateFollowingRetry());
- // change HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY,
- // verify DFSClient uses the configured value 1000.
conf.setInt(
HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY,
- 1000);
+ initialDelayTestValue);
+ conf.setInt(
+ HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_KEY,
+ maxDelayTestValue);
+ conf.setInt(
+ HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_RETRIES_KEY,
+ retryTestValue);
client = new DFSClient(null, nn, conf, null);
- assertEquals(client.getConf().
- getBlockWriteLocateFollowingInitialDelayMs(), 1000);
+ assertEquals(initialDelayTestValue, client.getConf().
+ getBlockWriteLocateFollowingInitialDelayMs());
+ assertEquals(maxDelayTestValue, client.getConf().
+ getBlockWriteLocateFollowingMaxDelayMs());
+ assertEquals(retryTestValue, client.getConf().
+ getNumBlockWriteLocateFollowingRetry());
} finally {
cluster.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 5cae2fc..9fd505f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -43,6 +43,7 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
configurationClasses = new Class[] { HdfsClientConfigKeys.class,
HdfsClientConfigKeys.Failover.class,
HdfsClientConfigKeys.StripedRead.class, DFSConfigKeys.class,
+ HdfsClientConfigKeys.BlockWrite.class,
HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.class };
// Set error modes
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDDS-630. Rename KSM to OM in
Hdds.md. Contributed by Takanobu Asanuma.
Posted by sh...@apache.org.
HDDS-630. Rename KSM to OM in Hdds.md. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47eeb3eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47eeb3eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47eeb3eb
Branch: refs/heads/HDFS-12943
Commit: 47eeb3eb7bffa20f2c3526a5cbb85cae9af667bd
Parents: aeeb038
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Oct 11 09:20:11 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Oct 11 09:20:11 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/Hdds.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47eeb3eb/hadoop-ozone/docs/content/Hdds.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/Hdds.md b/hadoop-ozone/docs/content/Hdds.md
index 9978c26..35654cd 100644
--- a/hadoop-ozone/docs/content/Hdds.md
+++ b/hadoop-ozone/docs/content/Hdds.md
@@ -33,7 +33,7 @@ to maintain the replication levels needed by the cluster.
It is easier to look at a putKey operation to understand the role that SCM plays.
-To put a key, a client makes a call to KSM with the following arguments.
+To put a key, a client makes a call to OM with the following arguments.
-- putKey(keyName, data, pipeline type, replication count)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HADOOP-15708. Reading values from
Configuration before adding deprecations make it impossible to read value
with deprecated key (zsiegl via rkanter)
Posted by sh...@apache.org.
HADOOP-15708. Reading values from Configuration before adding deprecations make it impossible to read value with deprecated key (zsiegl via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f261c319
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f261c319
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f261c319
Branch: refs/heads/HDFS-12943
Commit: f261c319375c5a8c298338752ee77214c22f4e29
Parents: 2bd000c
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Oct 10 18:51:37 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Oct 10 18:51:37 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 26 +++++-
.../conf/TestConfigurationDeprecation.java | 88 ++++++++++++++------
2 files changed, 88 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f261c319/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index a78e311..c004cb5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -709,6 +709,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* deprecated key, the value of the deprecated key is set as the value for
* the provided property name.
*
+ * Also updates properties and overlays with deprecated keys, if the new
+ * key does not already exist.
+ *
* @param deprecations deprecation context
* @param name the property name
* @return the first property in the list of properties mapping
@@ -730,6 +733,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
// Override return value for deprecated keys
names = keyInfo.newKeys;
}
+
+ // Update properties with deprecated key if already loaded and new
+ // deprecation has been added
+ updatePropertiesWithDeprecatedKeys(deprecations, names);
+
// If there are no overlay values we can return early
Properties overlayProperties = getOverlay();
if (overlayProperties.isEmpty()) {
@@ -748,6 +756,19 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
return names;
}
+
+ private void updatePropertiesWithDeprecatedKeys(
+ DeprecationContext deprecations, String[] newNames) {
+ for (String newName : newNames) {
+ String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(newName);
+ if (deprecatedKey != null && !getProps().containsKey(newName)) {
+ String deprecatedValue = getProps().getProperty(deprecatedKey);
+ if (deprecatedValue != null) {
+ getProps().setProperty(newName, deprecatedValue);
+ }
+ }
+ }
+ }
private void handleDeprecation() {
LOG.debug("Handling deprecation for all properties in config...");
@@ -1187,7 +1208,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* the first key which replaces the deprecated key and is not null.
*
* Values are processed for <a href="#VariableExpansion">variable expansion</a>
- * before being returned.
+ * before being returned.
+ *
+ * As a side effect get loads the properties from the sources if called for
+ * the first time as a lazy init.
*
* @param name the property name, will be trimmed before get value.
* @return the value of the <code>name</code> or its replacing property,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f261c319/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
index 93a235a..4014b60 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
@@ -38,6 +38,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.junit.Assert;
import org.apache.hadoop.fs.Path;
@@ -52,9 +53,14 @@ import com.google.common.util.concurrent.Uninterruptibles;
public class TestConfigurationDeprecation {
private Configuration conf;
- final static String CONFIG = new File("./test-config-TestConfigurationDeprecation.xml").getAbsolutePath();
- final static String CONFIG2 = new File("./test-config2-TestConfigurationDeprecation.xml").getAbsolutePath();
- final static String CONFIG3 = new File("./test-config3-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG = new File("./test-config" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG2 = new File("./test-config2" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG3 = new File("./test-config3" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG4 = new File("./test-config4" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
BufferedWriter out;
static {
@@ -288,14 +294,14 @@ public class TestConfigurationDeprecation {
@Test
public void testIteratorWithDeprecatedKeys() {
Configuration conf = new Configuration();
- Configuration.addDeprecation("dK", new String[]{"nK"});
+ Configuration.addDeprecation("dK_iterator", new String[]{"nK_iterator"});
conf.set("k", "v");
- conf.set("dK", "V");
- assertEquals("V", conf.get("dK"));
- assertEquals("V", conf.get("nK"));
- conf.set("nK", "VV");
- assertEquals("VV", conf.get("dK"));
- assertEquals("VV", conf.get("nK"));
+ conf.set("dK_iterator", "V");
+ assertEquals("V", conf.get("dK_iterator"));
+ assertEquals("V", conf.get("nK_iterator"));
+ conf.set("nK_iterator", "VV");
+ assertEquals("VV", conf.get("dK_iterator"));
+ assertEquals("VV", conf.get("nK_iterator"));
boolean kFound = false;
boolean dKFound = false;
boolean nKFound = false;
@@ -304,11 +310,11 @@ public class TestConfigurationDeprecation {
assertEquals("v", entry.getValue());
kFound = true;
}
- if (entry.getKey().equals("dK")) {
+ if (entry.getKey().equals("dK_iterator")) {
assertEquals("VV", entry.getValue());
dKFound = true;
}
- if (entry.getKey().equals("nK")) {
+ if (entry.getKey().equals("nK_iterator")) {
assertEquals("VV", entry.getValue());
nKFound = true;
}
@@ -321,19 +327,19 @@ public class TestConfigurationDeprecation {
@Test
public void testUnsetWithDeprecatedKeys() {
Configuration conf = new Configuration();
- Configuration.addDeprecation("dK", new String[]{"nK"});
- conf.set("nK", "VV");
- assertEquals("VV", conf.get("dK"));
- assertEquals("VV", conf.get("nK"));
- conf.unset("dK");
- assertNull(conf.get("dK"));
- assertNull(conf.get("nK"));
- conf.set("nK", "VV");
- assertEquals("VV", conf.get("dK"));
- assertEquals("VV", conf.get("nK"));
- conf.unset("nK");
- assertNull(conf.get("dK"));
- assertNull(conf.get("nK"));
+ Configuration.addDeprecation("dK_unset", new String[]{"nK_unset"});
+ conf.set("nK_unset", "VV");
+ assertEquals("VV", conf.get("dK_unset"));
+ assertEquals("VV", conf.get("nK_unset"));
+ conf.unset("dK_unset");
+ assertNull(conf.get("dK_unset"));
+ assertNull(conf.get("nK_unset"));
+ conf.set("nK_unset", "VV");
+ assertEquals("VV", conf.get("dK_unset"));
+ assertEquals("VV", conf.get("nK_unset"));
+ conf.unset("nK_unset");
+ assertNull(conf.get("dK_unset"));
+ assertNull(conf.get("nK_unset"));
}
private static String getTestKeyName(int threadIndex, int testIndex) {
@@ -426,4 +432,36 @@ public class TestConfigurationDeprecation {
assertEquals(null, conf.get("Z"));
assertEquals(null, conf.get("X"));
}
+
+ @Test
+ public void testGetPropertyBeforeDeprecetionsAreSet() throws Exception {
+ // SETUP
+ final String oldZkAddressKey = "yarn.resourcemanager.zk-address";
+ final String newZkAddressKey = CommonConfigurationKeys.ZK_ADDRESS;
+ final String zkAddressValue = "dummyZkAddress";
+
+ try{
+ out = new BufferedWriter(new FileWriter(CONFIG4));
+ startConfig();
+ appendProperty(oldZkAddressKey, zkAddressValue);
+ endConfig();
+
+ Path fileResource = new Path(CONFIG4);
+ conf.addResource(fileResource);
+ } finally {
+ out.close();
+ }
+
+ // ACT
+ conf.get(oldZkAddressKey);
+ Configuration.addDeprecations(new Configuration.DeprecationDelta[] {
+ new Configuration.DeprecationDelta(oldZkAddressKey, newZkAddressKey)});
+
+ // ASSERT
+ assertEquals("Property should be accessible through deprecated key",
+ zkAddressValue, conf.get(oldZkAddressKey));
+ assertEquals("Property should be accessible through new key",
+ zkAddressValue, conf.get(newZkAddressKey));
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HDFS-12943
Posted by sh...@apache.org.
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc093528
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc093528
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc093528
Branch: refs/heads/HDFS-12943
Commit: fc093528247985ca3e539784c103a034cc9c684e
Parents: dc76e0f bca928d
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Oct 11 22:34:56 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Oct 11 22:34:56 2018 -0700
----------------------------------------------------------------------
.../hadoop-client-check-invariants/pom.xml | 2 +
.../hadoop-client-check-test-invariants/pom.xml | 2 +
.../hadoop-client-minicluster/pom.xml | 12 +
.../hadoop-client-runtime/pom.xml | 13 +
hadoop-common-project/hadoop-common/pom.xml | 7 +-
.../org/apache/hadoop/conf/Configurable.java | 10 +-
.../org/apache/hadoop/conf/Configuration.java | 71 +-
.../hadoop/conf/ConfigurationWithLogging.java | 14 +-
.../org/apache/hadoop/crypto/CryptoCodec.java | 7 +-
.../apache/hadoop/crypto/CryptoInputStream.java | 4 +-
.../hadoop/crypto/CryptoOutputStream.java | 4 +-
.../org/apache/hadoop/crypto/Decryptor.java | 14 +-
.../org/apache/hadoop/crypto/Encryptor.java | 14 +-
.../org/apache/hadoop/crypto/OpensslCipher.java | 18 +-
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 15 +-
.../apache/hadoop/crypto/key/KeyProvider.java | 6 +-
.../crypto/key/KeyProviderCryptoExtension.java | 9 +-
.../KeyProviderDelegationTokenExtension.java | 2 +-
.../crypto/key/kms/KMSClientProvider.java | 6 +-
.../hadoop/crypto/key/kms/ValueQueue.java | 2 +-
.../crypto/random/OpensslSecureRandom.java | 10 +-
.../apache/hadoop/fs/AbstractFileSystem.java | 45 +-
.../apache/hadoop/fs/BufferedFSInputStream.java | 4 +-
.../apache/hadoop/fs/ByteBufferReadable.java | 8 +-
.../apache/hadoop/fs/ChecksumFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/ChecksumFs.java | 2 +-
.../hadoop/fs/CommonConfigurationKeys.java | 2 +-
.../java/org/apache/hadoop/fs/CreateFlag.java | 2 +-
.../org/apache/hadoop/fs/FSDataInputStream.java | 1 +
.../org/apache/hadoop/fs/FSInputChecker.java | 2 +-
.../java/org/apache/hadoop/fs/FileContext.java | 51 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 11 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 4 +-
.../org/apache/hadoop/fs/HarFileSystem.java | 2 +-
.../hadoop/fs/HasEnhancedByteBufferAccess.java | 15 +-
.../org/apache/hadoop/fs/LocalDirAllocator.java | 3 +-
.../org/apache/hadoop/fs/LocalFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/fs/Options.java | 2 +-
.../java/org/apache/hadoop/fs/QuotaUsage.java | 8 +-
.../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 6 +-
.../org/apache/hadoop/fs/ftp/FtpConfigKeys.java | 1 +
.../apache/hadoop/fs/local/LocalConfigKeys.java | 1 +
.../apache/hadoop/fs/permission/AclStatus.java | 2 +-
.../apache/hadoop/fs/permission/AclUtil.java | 10 +-
.../hadoop/fs/permission/ScopedAclEntries.java | 14 +-
.../org/apache/hadoop/fs/shell/Command.java | 24 +-
.../apache/hadoop/fs/shell/CommandFormat.java | 2 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +-
.../apache/hadoop/ha/ActiveStandbyElector.java | 32 +-
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 2 +-
.../java/org/apache/hadoop/ha/NodeFencer.java | 2 +-
.../org/apache/hadoop/ha/SshFenceByTcpPort.java | 2 +-
.../org/apache/hadoop/http/HttpServer2.java | 10 +-
.../org/apache/hadoop/io/EnumSetWritable.java | 2 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 2 +-
.../org/apache/hadoop/io/ReadaheadPool.java | 2 +-
.../org/apache/hadoop/io/SecureIOUtils.java | 4 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 16 +-
.../java/org/apache/hadoop/io/Writable.java | 4 +-
.../apache/hadoop/io/WritableComparable.java | 7 +-
.../org/apache/hadoop/io/WritableUtils.java | 6 +-
.../io/compress/CompressionCodecFactory.java | 8 +-
.../org/apache/hadoop/io/compress/Lz4Codec.java | 4 +-
.../apache/hadoop/io/compress/SnappyCodec.java | 2 +-
.../io/compress/bzip2/Bzip2Compressor.java | 2 +-
.../io/compress/bzip2/Bzip2Decompressor.java | 6 +-
.../hadoop/io/compress/bzip2/Bzip2Factory.java | 4 +-
.../io/compress/bzip2/CBZip2InputStream.java | 26 +-
.../io/compress/bzip2/CBZip2OutputStream.java | 7 +-
.../compress/zlib/BuiltInGzipDecompressor.java | 8 +-
.../hadoop/io/compress/zlib/ZlibCompressor.java | 2 +-
.../io/compress/zlib/ZlibDecompressor.java | 6 +-
.../hadoop/io/compress/zlib/ZlibFactory.java | 4 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 2 +-
.../erasurecode/rawcoder/util/GaloisField.java | 2 +-
.../org/apache/hadoop/io/file/tfile/TFile.java | 8 +-
.../org/apache/hadoop/io/file/tfile/Utils.java | 48 +-
.../org/apache/hadoop/io/retry/RetryProxy.java | 4 +-
.../hadoop/io/serializer/Deserializer.java | 2 +-
.../apache/hadoop/io/serializer/Serializer.java | 2 +-
.../org/apache/hadoop/ipc/CallerContext.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 9 +-
.../java/org/apache/hadoop/ipc/ClientCache.java | 6 +-
.../apache/hadoop/ipc/DecayRpcScheduler.java | 3 +-
.../org/apache/hadoop/ipc/RefreshHandler.java | 1 -
.../org/apache/hadoop/ipc/RemoteException.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 2 +-
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +-
.../apache/hadoop/log/LogThrottlingHelper.java | 8 +-
.../metrics2/lib/MutableRollingAverages.java | 2 +-
.../apache/hadoop/metrics2/package-info.java | 3 +
.../metrics2/sink/RollingFileSystemSink.java | 2 +-
.../apache/hadoop/metrics2/sink/StatsDSink.java | 4 +-
.../org/apache/hadoop/metrics2/util/MBeans.java | 10 +-
.../hadoop/net/AbstractDNSToSwitchMapping.java | 8 +-
.../main/java/org/apache/hadoop/net/DNS.java | 2 +-
.../apache/hadoop/net/DNSToSwitchMapping.java | 2 +-
.../java/org/apache/hadoop/net/NetUtils.java | 14 +-
.../org/apache/hadoop/net/NetworkTopology.java | 12 +-
.../net/NetworkTopologyWithNodeGroup.java | 4 +-
.../apache/hadoop/net/ScriptBasedMapping.java | 8 +-
.../net/ScriptBasedMappingWithDependency.java | 7 +-
.../apache/hadoop/net/SocketOutputStream.java | 6 +-
.../AuthenticationFilterInitializer.java | 6 +-
.../hadoop/security/HadoopKerberosName.java | 2 +-
.../security/IdMappingServiceProvider.java | 5 +-
.../apache/hadoop/security/SaslRpcClient.java | 10 +-
.../apache/hadoop/security/SecurityUtil.java | 3 +-
.../hadoop/security/UserGroupInformation.java | 20 +-
.../security/alias/JavaKeyStoreProvider.java | 6 +-
.../alias/LocalJavaKeyStoreProvider.java | 3 +-
.../hadoop/security/authorize/ProxyUsers.java | 4 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 4 +-
.../apache/hadoop/security/ssl/SSLFactory.java | 4 +-
.../security/ssl/SSLHostnameVerifier.java | 20 +-
.../web/DelegationTokenAuthenticatedURL.java | 14 +-
.../DelegationTokenAuthenticationFilter.java | 6 +-
.../DelegationTokenAuthenticationHandler.java | 4 +-
.../delegation/web/DelegationTokenManager.java | 2 +-
...rosDelegationTokenAuthenticationHandler.java | 2 +-
.../KerberosDelegationTokenAuthenticator.java | 2 +-
...emeDelegationTokenAuthenticationHandler.java | 3 +-
...udoDelegationTokenAuthenticationHandler.java | 2 +-
.../web/PseudoDelegationTokenAuthenticator.java | 2 +-
.../hadoop/service/ServiceOperations.java | 4 +-
.../service/launcher/ServiceLauncher.java | 2 +-
.../hadoop/service/launcher/package-info.java | 2 +-
.../java/org/apache/hadoop/util/ClassUtil.java | 1 -
.../apache/hadoop/util/ComparableVersion.java | 2 +-
.../java/org/apache/hadoop/util/FindClass.java | 4 +-
.../hadoop/util/GenericOptionsParser.java | 21 +-
.../apache/hadoop/util/HttpExceptionUtils.java | 6 +-
.../apache/hadoop/util/JsonSerialization.java | 2 +-
.../apache/hadoop/util/LightWeightCache.java | 10 +-
.../java/org/apache/hadoop/util/LineReader.java | 3 -
.../org/apache/hadoop/util/MachineList.java | 2 +-
.../apache/hadoop/util/ShutdownHookManager.java | 2 +-
.../hadoop/util/ShutdownThreadsHelper.java | 2 -
.../org/apache/hadoop/util/StringUtils.java | 6 +-
.../main/java/org/apache/hadoop/util/Tool.java | 2 +-
.../java/org/apache/hadoop/util/ZKUtil.java | 4 +-
.../hadoop/util/bloom/DynamicBloomFilter.java | 5 +-
.../apache/hadoop/util/concurrent/AsyncGet.java | 2 +-
.../apache/hadoop/util/hash/JenkinsHash.java | 4 +-
.../src/main/resources/core-default.xml | 26 +-
.../org/apache/hadoop/cli/util/CLICommand.java | 6 +-
.../apache/hadoop/cli/util/CLICommandFS.java | 4 +-
.../apache/hadoop/cli/util/CLICommandTypes.java | 6 +-
.../conf/TestConfigurationDeprecation.java | 88 ++-
.../apache/hadoop/http/TestSSLHttpServer.java | 204 ++---
.../rawcoder/RawErasureCoderBenchmark.java | 4 +-
.../hadoop/io/retry/TestDefaultRetryPolicy.java | 4 +-
.../org/apache/hadoop/net/StaticMapping.java | 4 +-
.../apache/hadoop/tracing/SetSpanReceiver.java | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 5 +-
hadoop-hdds/client/pom.xml | 4 +-
.../hadoop/hdds/scm/XceiverClientGrpc.java | 6 +-
.../hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hdds/scm/storage/ChunkInputStream.java | 2 +-
.../hdds/scm/storage/ChunkOutputStream.java | 15 +-
hadoop-hdds/common/pom.xml | 10 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-
.../scm/storage/ContainerProtocolCalls.java | 27 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 8 +-
.../container/common/helpers/BlockData.java | 12 +
.../org/apache/hadoop/utils/RocksDBStore.java | 3 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 3 +-
.../main/java/org/apache/ratis/RatisHelper.java | 14 +-
.../main/proto/DatanodeContainerProtocol.proto | 3 +-
.../common/src/main/resources/ozone-default.xml | 35 +-
hadoop-hdds/container-service/pom.xml | 4 +-
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../transport/server/GrpcXceiverService.java | 2 +-
.../transport/server/XceiverServerGrpc.java | 8 +-
.../server/ratis/ContainerStateMachine.java | 423 +++--------
.../server/ratis/XceiverServerRatis.java | 19 +-
.../container/keyvalue/helpers/BlockUtils.java | 5 +-
.../container/keyvalue/helpers/ChunkUtils.java | 2 +-
.../keyvalue/helpers/SmallFileUtils.java | 2 +-
.../keyvalue/impl/BlockManagerImpl.java | 1 -
.../background/BlockDeletingService.java | 2 +-
.../replication/GrpcReplicationClient.java | 6 +-
.../replication/GrpcReplicationService.java | 4 +-
.../protocol/StorageContainerNodeProtocol.java | 8 +
.../common/impl/TestHddsDispatcher.java | 2 +-
hadoop-hdds/framework/pom.xml | 4 +-
hadoop-hdds/pom.xml | 2 +-
hadoop-hdds/server-scm/pom.xml | 6 +-
.../hadoop/hdds/scm/node/NodeManager.java | 7 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 26 +-
.../hdds/scm/server/SCMChillModeManager.java | 12 +-
.../scm/server/SCMClientProtocolServer.java | 2 +-
.../server/SCMDatanodeHeartbeatDispatcher.java | 107 ++-
.../scm/server/StorageContainerManager.java | 2 +-
.../hdds/scm/container/MockNodeManager.java | 11 +
.../hadoop/hdds/scm/node/TestNodeManager.java | 53 --
.../scm/server/TestSCMChillModeManager.java | 53 +-
.../TestSCMDatanodeHeartbeatDispatcher.java | 41 +-
.../testutils/ReplicationNodeManagerMock.java | 11 +
hadoop-hdds/tools/pom.xml | 4 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 8 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 21 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 22 +
.../org/apache/hadoop/hdfs/ReaderStrategy.java | 15 -
.../org/apache/hadoop/hdfs/StripeReader.java | 23 +-
.../hdfs/client/HdfsClientConfigKeys.java | 3 +
.../hadoop/hdfs/client/impl/DfsClientConf.java | 8 +
.../org/apache/hadoop/hdfs/inotify/Event.java | 14 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 60 +-
.../apache/hadoop/hdfs/util/IOUtilsClient.java | 10 +-
.../hadoop/hdfs/util/StripedBlockUtil.java | 65 +-
.../hadoop/hdfs/web/resources/GetOpParam.java | 12 +-
.../src/main/proto/inotify.proto | 1 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 16 +-
.../hadoop/fs/http/server/FSOperations.java | 35 +
.../http/server/HttpFSParametersProvider.java | 1 +
.../hadoop/fs/http/server/HttpFSServer.java | 8 +
.../fs/http/client/BaseTestHttpFSWith.java | 54 +-
.../hadoop/fs/http/server/TestHttpFSServer.java | 53 ++
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 2 +-
.../hdfs/server/federation/router/Quota.java | 54 +-
.../driver/impl/StateStoreFileSystemImpl.java | 16 +-
.../router/web/RouterWebHDFSContract.java | 1 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 +-
.../hdfs/server/datanode/BlockReceiver.java | 43 +-
.../erasurecode/ErasureCodingWorker.java | 3 +-
.../erasurecode/StripedBlockReader.java | 14 +-
.../datanode/erasurecode/StripedReader.java | 17 +-
.../erasurecode/StripedReconstructor.java | 3 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/ReplicaMap.java | 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../namenode/InotifyFSEditLogOpTranslator.java | 3 +
.../web/resources/NamenodeWebHdfsMethods.java | 14 +
.../hdfs/tools/DelegationTokenFetcher.java | 58 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 32 +
.../src/main/resources/hdfs-default.xml | 22 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 173 +++++
.../hadoop/hdfs/TestDFSClientRetries.java | 47 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 100 +++
.../TestDistributedFileSystemWithECFile.java | 44 ++
.../TestNameNodeMetadataConsistency.java | 2 +
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 148 ++++
.../hadoop/tools/TestHdfsConfigFields.java | 1 +
.../hadoop-mapreduce-client-app/pom.xml | 20 +
.../hadoop-mapreduce-client-common/pom.xml | 7 +
.../hadoop-mapreduce-client-core/pom.xml | 8 +
.../apache/hadoop/mapred/FileInputFormat.java | 8 +-
.../org/apache/hadoop/mapred/OutputFormat.java | 3 +
.../apache/hadoop/mapreduce/OutputFormat.java | 3 +
.../mapreduce/lib/input/FileInputFormat.java | 10 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 7 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 7 +
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 48 ++
.../ozone/client/io/ChunkGroupInputStream.java | 3 +-
.../ozone/client/io/ChunkGroupOutputStream.java | 21 +-
.../ozone/client/protocol/ClientProtocol.java | 35 +
.../hadoop/ozone/client/rest/RestClient.java | 25 +
.../hadoop/ozone/client/rpc/RpcClient.java | 32 +
hadoop-ozone/common/pom.xml | 4 +-
hadoop-ozone/common/src/main/bin/ozone | 4 +-
hadoop-ozone/common/src/main/bin/start-ozone.sh | 10 +-
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 10 +-
.../src/main/conf/om-audit-log4j2.properties | 14 +-
.../ozone/om/helpers/OmKeyLocationInfo.java | 17 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 25 +
...neManagerProtocolClientSideTranslatorPB.java | 51 ++
.../src/main/proto/OzoneManagerProtocol.proto | 33 +
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dist/pom.xml | 4 +-
hadoop-ozone/docs/content/CommandShell.md | 2 +-
hadoop-ozone/docs/content/Hdds.md | 2 +-
hadoop-ozone/docs/content/OzoneFS.md | 2 +-
hadoop-ozone/docs/pom.xml | 4 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../ozone/TestStorageContainerManager.java | 13 +
.../rpc/TestCloseContainerHandlingByClient.java | 1 -
.../ozone/client/rpc/TestOzoneRpcClient.java | 34 +
.../ozone/container/ContainerTestHelper.java | 3 +-
.../common/impl/TestCloseContainerHandler.java | 2 +-
.../container/server/TestContainerServer.java | 1 -
.../server/TestContainerStateMachine.java | 201 -----
.../hadoop/ozone/ozShell/TestOzoneShell.java | 48 ++
.../TestGetCommittedBlockLengthAndPutKey.java | 4 +-
hadoop-ozone/objectstore-service/pom.xml | 4 +-
hadoop-ozone/ozone-manager/pom.xml | 6 +-
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +
.../apache/hadoop/ozone/om/OzoneManager.java | 23 +
.../hadoop/ozone/om/S3BucketManagerImpl.java | 11 +
...neManagerProtocolServerSideTranslatorPB.java | 187 +++--
.../hadoop/ozone/web/ozShell/Handler.java | 28 +
.../web/ozShell/volume/DeleteVolumeHandler.java | 12 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 22 +-
.../web/ozShell/volume/UpdateVolumeHandler.java | 12 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 16 +-
.../fs/ozone/TestOzoneFileInterfaces.java | 17 +-
hadoop-ozone/pom.xml | 6 +-
hadoop-ozone/s3gateway/pom.xml | 4 +-
.../CommonHeadersContainerResponseFilter.java | 9 +
.../apache/hadoop/ozone/s3/EndpointBase.java | 23 +-
.../hadoop/ozone/s3/RequestIdentifier.java | 48 ++
.../hadoop/ozone/s3/bucket/DeleteBucket.java | 32 +-
.../hadoop/ozone/s3/bucket/HeadBucket.java | 21 +-
.../ozone/s3/exception/OS3ExceptionMapper.java | 9 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 15 +-
.../hadoop/ozone/s3/object/HeadObject.java | 38 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 6 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 23 +-
.../ozone/s3/bucket/TestDeleteBucket.java | 106 +++
.../hadoop/ozone/s3/bucket/TestHeadBucket.java | 9 +-
.../ozone/s3/exception/TestOS3Exception.java | 3 +-
.../hadoop/ozone/s3/object/TestHeadObject.java | 95 +++
hadoop-ozone/tools/pom.xml | 10 +-
.../apache/hadoop/ozone/freon/ProgressBar.java | 210 +++++
.../genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../hadoop/ozone/freon/TestProgressBar.java | 112 +++
hadoop-project/pom.xml | 34 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 91 ++-
.../s3a/s3guard/TestDynamoDBMiscOperations.java | 94 +++
.../azurebfs/AbstractAbfsIntegrationTest.java | 3 +-
.../ITestAzureBlobFileSystemBackCompat.java | 6 +-
.../ITestAzureBlobFileSystemFileStatus.java | 12 +-
.../ITestAzureBlobFileSystemFinalize.java | 8 +-
.../azurebfs/ITestAzureBlobFileSystemFlush.java | 5 -
.../ITestAzureBlobFileSystemInitAndCreate.java | 3 +-
.../ITestAzureBlobFileSystemRandomRead.java | 12 +-
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 14 +-
.../tools/rumen/Pre21JobHistoryConstants.java | 11 +-
.../yarn/service/conf/YarnServiceConf.java | 5 +-
.../runtimes/yarnservice/YarnServiceUtils.java | 17 +-
.../src/site/markdown/Index.md | 4 +
.../src/site/markdown/InstallationGuide.md | 760 +++++++++++++++++++
.../markdown/InstallationGuideChineseVersion.md | 757 ++++++++++++++++++
.../hadoop/yarn/client/cli/RMAdminCLI.java | 9 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 7 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../yarn/client/api/TimelineReaderClient.java | 120 +++
.../api/impl/TimelineReaderClientImpl.java | 239 ++++++
.../api/impl/TestTimelineReaderClientImpl.java | 157 ++++
.../registry/client/api/RegistryConstants.java | 11 -
.../pom.xml | 2 +-
.../webapp/AHSWebServices.java | 250 ++----
.../webapp/TestAHSWebServices.java | 7 +-
.../yarn/server/AMHeartbeatRequestHandler.java | 10 +-
.../hadoop/yarn/server/AMRMClientRelayer.java | 6 +-
.../server/uam/UnmanagedApplicationManager.java | 20 +-
.../yarn/server/webapp/LogWebService.java | 506 ++++++++++++
.../yarn/server/webapp/LogWebServiceUtils.java | 258 +++++++
.../server/webapp/YarnWebServiceParams.java | 1 +
.../yarn/server/MockResourceManagerFacade.java | 29 +-
.../uam/TestUnmanagedApplicationManager.java | 51 +-
.../yarn/server/webapp/TestLogWebService.java | 126 +++
.../hadoop-yarn-server-nodemanager/pom.xml | 4 +
.../server/nodemanager/ContainerExecutor.java | 12 +
.../nodemanager/DefaultContainerExecutor.java | 16 +
.../nodemanager/LinuxContainerExecutor.java | 33 +
.../amrmproxy/FederationInterceptor.java | 291 +++----
.../launcher/ContainerCleanup.java | 229 ++++++
.../launcher/ContainerLaunch.java | 236 ++----
.../launcher/ContainersLauncher.java | 14 +-
.../executor/ContainerExecContext.java | 85 +++
.../webapp/ContainerShellWebSocket.java | 106 +++
.../webapp/ContainerShellWebSocketServlet.java | 36 +
.../server/nodemanager/webapp/WebServer.java | 5 +
.../impl/container-executor.c | 164 +++-
.../impl/container-executor.h | 7 +-
.../main/native/container-executor/impl/util.h | 3 +-
.../container-executor/impl/utils/docker-util.c | 53 ++
.../container-executor/impl/utils/docker-util.h | 10 +
.../test/utils/test_docker_util.cc | 29 +
.../nodemanager/TestContainerExecutor.java | 15 +
.../nodemanager/TestLinuxContainerExecutor.java | 12 +
.../amrmproxy/TestFederationInterceptor.java | 16 +-
.../TestableFederationInterceptor.java | 26 +-
.../launcher/TestContainerCleanup.java | 108 +++
.../launcher/TestContainersLauncher.java | 12 +-
.../TestContainersMonitorResourceChange.java | 8 +
.../webapp/ContainerShellClientSocketTest.java | 80 ++
.../webapp/TestNMContainerWebSocket.java | 149 ++++
.../server/resourcemanager/ClientRMService.java | 47 +-
.../scheduler/ClusterNodeTracker.java | 11 +
.../scheduler/capacity/CapacityScheduler.java | 12 +-
.../capacity/TestContainerAllocation.java | 52 ++
.../webapp/TestRMWebServicesApps.java | 211 +++--
.../hadoop-yarn-server-tests/pom.xml | 2 +-
.../reader/TimelineReaderServer.java | 4 +-
.../reader/TimelineReaderWebServicesUtils.java | 21 +-
...elineReaderWhitelistAuthorizationFilter.java | 2 +-
.../storage/FileSystemTimelineReaderImpl.java | 87 ++-
.../hadoop-yarn-server-web-proxy/pom.xml | 8 +
.../markdown/registry/registry-configuration.md | 56 --
.../src/site/markdown/registry/yarn-registry.md | 15 -
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 8 +
.../main/webapp/app/models/cluster-metric.js | 6 +-
.../src/main/webapp/app/styles/app.scss | 8 +
.../src/main/webapp/app/styles/compose-box.scss | 6 +-
.../src/main/webapp/app/styles/layout.scss | 8 +
.../webapp/app/templates/yarn-app-attempt.hbs | 80 +-
.../webapp/app/templates/yarn-app/attempts.hbs | 2 +-
.../webapp/app/templates/yarn-app/configs.hbs | 93 ++-
.../main/webapp/app/templates/yarn-app/info.hbs | 2 +-
.../app/templates/yarn-component-instance.hbs | 22 +-
.../app/templates/yarn-component-instances.hbs | 33 +-
.../main/webapp/app/templates/yarn-queue.hbs | 11 -
.../webapp/app/templates/yarn-queue/apps.hbs | 5 +-
pom.xml | 2 +-
410 files changed, 8870 insertions(+), 2842 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc093528/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc093528/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --cc hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 99c1f3e,49f1e49..ea79887
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@@ -1421,8 -1392,7 +1421,8 @@@ public class Client implements AutoClos
* @param serviceClass - service class for RPC
* @param fallbackToSimpleAuth - set to true or false during this method to
* indicate if a secure client falls back to simple auth
+ * @param alignmentContext - state alignment context
- * @returns the rpc response
+ * @return the rpc response
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc093528/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc093528/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc093528/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc093528/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-5742 Serve aggregated logs of
historical apps from timeline service. Contributed by Rohith Sharma KS
Posted by sh...@apache.org.
YARN-5742 Serve aggregated logs of historical apps from timeline service. Contributed by Rohith Sharma KS
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d198180
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d198180
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d198180
Branch: refs/heads/HDFS-12943
Commit: 8d1981806feb8278966c02a9eff42d72541bb35e
Parents: d91d47b
Author: Vrushali C <vr...@apache.org>
Authored: Thu Oct 11 16:26:07 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Thu Oct 11 16:26:07 2018 -0700
----------------------------------------------------------------------
.../webapp/AHSWebServices.java | 250 ++-------
.../webapp/TestAHSWebServices.java | 7 +-
.../yarn/server/webapp/LogWebService.java | 506 +++++++++++++++++++
.../yarn/server/webapp/LogWebServiceUtils.java | 258 ++++++++++
.../server/webapp/YarnWebServiceParams.java | 1 +
.../yarn/server/webapp/TestLogWebService.java | 126 +++++
.../reader/TimelineReaderServer.java | 4 +-
7 files changed, 945 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
index 9aa71a7..d94605f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
@@ -18,13 +18,7 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
@@ -35,15 +29,16 @@ import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
-import javax.ws.rs.core.GenericEntity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.Response.ResponseBuilder;
import javax.ws.rs.core.Response.Status;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -54,10 +49,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
-import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
-import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
-import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
+import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils;
import org.apache.hadoop.yarn.server.webapp.WebServices;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
@@ -65,21 +58,14 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.UniformInterfaceException;
+import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -271,15 +257,18 @@ public class AHSWebServices extends WebServices {
appInfo = super.getApp(req, res, appId.toString());
} catch (Exception ex) {
// directly find logs from HDFS.
- return getContainerLogMeta(appId, null, null, containerIdStr, false);
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
}
// if the application finishes, directly find logs
// from HDFS.
- if (isFinishedState(appInfo.getAppState())) {
- return getContainerLogMeta(appId, null, null,
- containerIdStr, false);
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
}
- if (isRunningState(appInfo.getAppState())) {
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
String appOwner = appInfo.getUser();
String nodeHttpAddress = null;
if (nmId != null && !nmId.isEmpty()) {
@@ -301,8 +290,9 @@ public class AHSWebServices extends WebServices {
} catch (Exception ex) {
// return log meta for the aggregated logs if exists.
// It will also return empty log meta for the local logs.
- return getContainerLogMeta(appId, appOwner, null,
- containerIdStr, true);
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
}
nodeHttpAddress = containerInfo.getNodeHttpAddress();
// make sure nodeHttpAddress is not null and not empty. Otherwise,
@@ -314,12 +304,14 @@ public class AHSWebServices extends WebServices {
// It will also return empty log meta for the local logs.
// If this is the redirect request from NM, we should not
// re-direct the request back. Simply output the aggregated log meta.
- return getContainerLogMeta(appId, appOwner, null,
- containerIdStr, true);
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
}
}
String uri = "/" + containerId.toString() + "/logs";
- String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress),
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(conf, nodeHttpAddress),
NM_DOWNLOAD_URI_STR, uri);
String query = req.getQueryString();
if (query != null && !query.isEmpty()) {
@@ -397,11 +389,11 @@ public class AHSWebServices extends WebServices {
try {
containerId = ContainerId.fromString(containerIdStr);
} catch (IllegalArgumentException ex) {
- return createBadResponse(Status.NOT_FOUND,
+ return LogWebServiceUtils.createBadResponse(Status.NOT_FOUND,
"Invalid ContainerId: " + containerIdStr);
}
- final long length = parseLongParam(size);
+ final long length = LogWebServiceUtils.parseLongParam(size);
ApplicationId appId = containerId.getApplicationAttemptId()
.getApplicationId();
@@ -410,17 +402,19 @@ public class AHSWebServices extends WebServices {
appInfo = super.getApp(req, res, appId.toString());
} catch (Exception ex) {
// directly find logs from HDFS.
- return sendStreamOutputResponse(appId, null, null, containerIdStr,
- filename, format, length, false);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, null, null, containerIdStr,
+ filename, format, length, false);
}
String appOwner = appInfo.getUser();
- if (isFinishedState(appInfo.getAppState())) {
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
// directly find logs from HDFS.
- return sendStreamOutputResponse(appId, appOwner, null, containerIdStr,
- filename, format, length, false);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, false);
}
- if (isRunningState(appInfo.getAppState())) {
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
String nodeHttpAddress = null;
if (nmId != null && !nmId.isEmpty()) {
try {
@@ -440,8 +434,9 @@ public class AHSWebServices extends WebServices {
containerId.toString());
} catch (Exception ex) {
// output the aggregated logs
- return sendStreamOutputResponse(appId, appOwner, null,
- containerIdStr, filename, format, length, true);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
}
nodeHttpAddress = containerInfo.getNodeHttpAddress();
// make sure nodeHttpAddress is not null and not empty. Otherwise,
@@ -452,12 +447,14 @@ public class AHSWebServices extends WebServices {
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()
|| redirected_from_node) {
// output the aggregated logs
- return sendStreamOutputResponse(appId, appOwner, null,
- containerIdStr, filename, format, length, true);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
}
}
String uri = "/" + containerId.toString() + "/logs/" + filename;
- String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress),
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(conf, nodeHttpAddress),
NM_DOWNLOAD_URI_STR, uri);
String query = req.getQueryString();
if (query != null && !query.isEmpty()) {
@@ -468,170 +465,15 @@ public class AHSWebServices extends WebServices {
response.header("Location", resURI);
return response.build();
} else {
- return createBadResponse(Status.NOT_FOUND,
+ return LogWebServiceUtils.createBadResponse(Status.NOT_FOUND,
"The application is not at Running or Finished State.");
}
}
- private boolean isRunningState(YarnApplicationState appState) {
- return appState == YarnApplicationState.RUNNING;
- }
-
- private boolean isFinishedState(YarnApplicationState appState) {
- return appState == YarnApplicationState.FINISHED
- || appState == YarnApplicationState.FAILED
- || appState == YarnApplicationState.KILLED;
- }
-
- private Response createBadResponse(Status status, String errMessage) {
- Response response = Response.status(status)
- .entity(DOT_JOINER.join(status.toString(), errMessage)).build();
- return response;
- }
-
- private Response sendStreamOutputResponse(ApplicationId appId,
- String appOwner, String nodeId, String containerIdStr,
- String fileName, String format, long bytes,
- boolean printEmptyLocalContainerLog) {
- String contentType = WebAppUtils.getDefaultLogContentType();
- if (format != null && !format.isEmpty()) {
- contentType = WebAppUtils.getSupportedLogContentType(format);
- if (contentType == null) {
- String errorMessage = "The valid values for the parameter : format "
- + "are " + WebAppUtils.listSupportedLogContentType();
- return Response.status(Status.BAD_REQUEST).entity(errorMessage)
- .build();
- }
- }
- StreamingOutput stream = null;
- try {
- stream = getStreamingOutput(appId, appOwner, nodeId,
- containerIdStr, fileName, bytes, printEmptyLocalContainerLog);
- } catch (Exception ex) {
- return createBadResponse(Status.INTERNAL_SERVER_ERROR,
- ex.getMessage());
- }
- ResponseBuilder response = Response.ok(stream);
- response.header("Content-Type", contentType);
- // Sending the X-Content-Type-Options response header with the value
- // nosniff will prevent Internet Explorer from MIME-sniffing a response
- // away from the declared content-type.
- response.header("X-Content-Type-Options", "nosniff");
- return response.build();
- }
-
- private StreamingOutput getStreamingOutput(final ApplicationId appId,
- final String appOwner, final String nodeId, final String containerIdStr,
- final String logFile, final long bytes,
- final boolean printEmptyLocalContainerLog) throws IOException{
- StreamingOutput stream = new StreamingOutput() {
-
- @Override
- public void write(OutputStream os) throws IOException,
- WebApplicationException {
- ContainerLogsRequest request = new ContainerLogsRequest();
- request.setAppId(appId);
- request.setAppOwner(appOwner);
- request.setContainerId(containerIdStr);
- request.setBytes(bytes);
- request.setNodeId(nodeId);
- Set<String> logTypes = new HashSet<>();
- logTypes.add(logFile);
- request.setLogTypes(logTypes);
- boolean findLogs = factory.getFileControllerForRead(appId, appOwner)
- .readAggregatedLogs(request, os);
- if (!findLogs) {
- os.write(("Can not find logs for container:"
- + containerIdStr).getBytes(Charset.forName("UTF-8")));
- } else {
- if (printEmptyLocalContainerLog) {
- StringBuilder sb = new StringBuilder();
- sb.append(containerIdStr + "\n");
- sb.append("LogAggregationType: "
- + ContainerLogAggregationType.LOCAL + "\n");
- sb.append("LogContents:\n");
- sb.append(getNoRedirectWarning() + "\n");
- os.write(sb.toString().getBytes(Charset.forName("UTF-8")));
- }
- }
- }
- };
- return stream;
- }
-
- private long parseLongParam(String bytes) {
- if (bytes == null || bytes.isEmpty()) {
- return Long.MAX_VALUE;
- }
- return Long.parseLong(bytes);
- }
-
- private Response getContainerLogMeta(ApplicationId appId, String appOwner,
- final String nodeId, final String containerIdStr,
- boolean emptyLocalContainerLogMeta) {
- try {
- ContainerLogsRequest request = new ContainerLogsRequest();
- request.setAppId(appId);
- request.setAppOwner(appOwner);
- request.setContainerId(containerIdStr);
- request.setNodeId(nodeId);
- List<ContainerLogMeta> containerLogMeta = factory
- .getFileControllerForRead(appId, appOwner)
- .readAggregatedLogsMeta(request);
- if (containerLogMeta.isEmpty()) {
- throw new NotFoundException(
- "Can not get log meta for container: " + containerIdStr);
- }
- List<ContainerLogsInfo> containersLogsInfo = new ArrayList<>();
- for (ContainerLogMeta meta : containerLogMeta) {
- ContainerLogsInfo logInfo = new ContainerLogsInfo(meta,
- ContainerLogAggregationType.AGGREGATED);
- containersLogsInfo.add(logInfo);
- }
- if (emptyLocalContainerLogMeta) {
- ContainerLogMeta emptyMeta = new ContainerLogMeta(
- containerIdStr, "N/A");
- ContainerLogsInfo empty = new ContainerLogsInfo(emptyMeta,
- ContainerLogAggregationType.LOCAL);
- containersLogsInfo.add(empty);
- }
- GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<
- ContainerLogsInfo>>(containersLogsInfo){};
- ResponseBuilder response = Response.ok(meta);
- // Sending the X-Content-Type-Options response header with the value
- // nosniff will prevent Internet Explorer from MIME-sniffing a response
- // away from the declared content-type.
- response.header("X-Content-Type-Options", "nosniff");
- return response.build();
- } catch (Exception ex) {
- throw new WebApplicationException(ex);
- }
- }
-
- @Private
- @VisibleForTesting
- public static String getNoRedirectWarning() {
- return "We do not have NodeManager web address, so we can not "
- + "re-direct the request to related NodeManager "
- + "for local container logs.";
- }
-
- private String getAbsoluteNMWebAddress(String nmWebAddress) {
- if (nmWebAddress.contains(WebAppUtils.HTTP_PREFIX) ||
- nmWebAddress.contains(WebAppUtils.HTTPS_PREFIX)) {
- return nmWebAddress;
- }
- return WebAppUtils.getHttpSchemePrefix(conf) + nmWebAddress;
- }
-
- @VisibleForTesting
- @Private
+ @VisibleForTesting @InterfaceAudience.Private
public String getNMWebAddressFromRM(Configuration configuration,
- String nodeId) throws ClientHandlerException,
- UniformInterfaceException, JSONException {
- JSONObject nodeInfo = YarnWebServiceUtils.getNodeInfoFromRMWebService(
- configuration, nodeId).getJSONObject("node");
- return nodeInfo.has("nodeHTTPAddress") ?
- nodeInfo.getString("nodeHTTPAddress") : null;
+ String nodeId)
+ throws ClientHandlerException, UniformInterfaceException, JSONException {
+ return LogWebServiceUtils.getNMWebAddressFromRM(configuration, nodeId);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index a4f56ff..e72714b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
+import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
@@ -774,7 +775,8 @@ public class TestAHSWebServices extends JerseyTestBase {
// the warning message.
assertTrue(responseText.contains("LogAggregationType: "
+ ContainerLogAggregationType.LOCAL));
- assertTrue(responseText.contains(AHSWebServices.getNoRedirectWarning()));
+ assertTrue(
+ responseText.contains(LogWebServiceUtils.getNoRedirectWarning()));
// If we can not container information from ATS, and we specify the NM id,
// but we can not get nm web address, we would still try to
@@ -790,7 +792,8 @@ public class TestAHSWebServices extends JerseyTestBase {
assertTrue(responseText.contains(content));
assertTrue(responseText.contains("LogAggregationType: "
+ ContainerLogAggregationType.LOCAL));
- assertTrue(responseText.contains(AHSWebServices.getNoRedirectWarning()));
+ assertTrue(
+ responseText.contains(LogWebServiceUtils.getNoRedirectWarning()));
// If this is the redirect request, we would not re-direct the request
// back and get the aggregated logs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java
new file mode 100644
index 0000000..246ee9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java
@@ -0,0 +1,506 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.inject.Singleton;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * Support only ATSv2 client only.
+ */
+@Singleton @Path("/ws/v2/applicationlog") public class LogWebService {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LogWebService.class);
+ private static final String RESOURCE_URI_STR_V2 = "/ws/v2/timeline/";
+ private static final String NM_DOWNLOAD_URI_STR = "/ws/v1/node/containers";
+ private static final Joiner JOINER = Joiner.on("");
+ private static Configuration yarnConf = new YarnConfiguration();
+ private static LogAggregationFileControllerFactory factory;
+ private static String base;
+ private static String defaultClusterid;
+ private volatile Client webTimelineClient;
+
+ static {
+ init();
+ }
+
+ // initialize all the common resources - order is important
+ private static void init() {
+ factory = new LogAggregationFileControllerFactory(yarnConf);
+ base = JOINER.join(WebAppUtils.getHttpSchemePrefix(yarnConf),
+ WebAppUtils.getTimelineReaderWebAppURLWithoutScheme(yarnConf),
+ RESOURCE_URI_STR_V2);
+ defaultClusterid = yarnConf.get(YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
+ LOG.info("Initialized LogWeService with clusterid " + defaultClusterid
+ + " for URI: " + base);
+ }
+
+ private Client createTimelineWebClient() {
+ ClientConfig cfg = new DefaultClientConfig();
+ cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+ Client client = new Client(
+ new URLConnectionClientHandler(new HttpURLConnectionFactory() {
+ @Override public HttpURLConnection getHttpURLConnection(URL url)
+ throws IOException {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ HttpURLConnection conn = null;
+ try {
+ conn = new AuthenticatedURL().openConnection(url, token);
+ LOG.info("LogWeService:Connecetion created.");
+ } catch (AuthenticationException e) {
+ throw new IOException(e);
+ }
+ return conn;
+ }
+ }), cfg);
+
+ return client;
+ }
+
+ private void initForReadableEndpoints(HttpServletResponse response) {
+ // clear content type
+ response.setContentType(null);
+ }
+
+ /**
+ * Returns log file's name as well as current file size for a container.
+ *
+ * @param req HttpServletRequest
+ * @param res HttpServletResponse
+ * @param containerIdStr The container ID
+ * @param nmId The Node Manager NodeId
+ * @param redirectedFromNode Whether this is a redirected request from NM
+ * @return The log file's name and current file size
+ */
+ @GET @Path("/containers/{containerid}/logs")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response getContainerLogsInfo(@Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr,
+ @QueryParam(YarnWebServiceParams.NM_ID) String nmId,
+ @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE)
+ @DefaultValue("false") boolean redirectedFromNode,
+ @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) {
+ ContainerId containerId = null;
+ initForReadableEndpoints(res);
+ try {
+ containerId = ContainerId.fromString(containerIdStr);
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("invalid container id, " + containerIdStr);
+ }
+
+ ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+ AppInfo appInfo;
+ try {
+ appInfo = getApp(req, appId.toString(), clusterId);
+ } catch (Exception ex) {
+ // directly find logs from HDFS.
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
+ }
+ // if the application finishes, directly find logs
+ // from HDFS.
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
+ }
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
+ String appOwner = appInfo.getUser();
+ String nodeHttpAddress = null;
+ if (nmId != null && !nmId.isEmpty()) {
+ try {
+ nodeHttpAddress =
+ LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
+ } catch (Exception ex) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(ex.getMessage());
+ }
+ }
+ }
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ ContainerInfo containerInfo;
+ try {
+ containerInfo =
+ getContainer(req, appId.toString(), containerId.toString(),
+ clusterId);
+ } catch (Exception ex) {
+ // return log meta for the aggregated logs if exists.
+ // It will also return empty log meta for the local logs.
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
+ }
+ nodeHttpAddress = containerInfo.getNodeHttpAddress();
+ // make sure nodeHttpAddress is not null and not empty. Otherwise,
+ // we would only get log meta for aggregated logs instead of
+ // re-directing the request
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()
+ || redirectedFromNode) {
+ // return log meta for the aggregated logs if exists.
+ // It will also return empty log meta for the local logs.
+ // If this is the redirect request from NM, we should not
+ // re-direct the request back. Simply output the aggregated log meta.
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
+ }
+ }
+ String uri = "/" + containerId.toString() + "/logs";
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(yarnConf, nodeHttpAddress),
+ NM_DOWNLOAD_URI_STR, uri);
+ String query = req.getQueryString();
+ if (query != null && !query.isEmpty()) {
+ resURI += "?" + query;
+ }
+ Response.ResponseBuilder response =
+ Response.status(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ response.header("Location", resURI);
+ return response.build();
+ } else {
+ throw new NotFoundException(
+ "The application is not at Running or Finished State.");
+ }
+ }
+
+ protected ContainerInfo getContainer(HttpServletRequest req, String appId,
+ String containerId, String clusterId) {
+ UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req);
+ String cId = clusterId != null ? clusterId : defaultClusterid;
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", "INFO");
+ String path = JOINER.join("clusters/", cId, "/apps/", appId, "/entities/",
+ TimelineEntityType.YARN_CONTAINER.toString(), "/", containerId);
+ TimelineEntity conEntity = null;
+ try {
+ if (callerUGI == null) {
+ conEntity = getEntity(path, params);
+ } else {
+ setUserName(params, callerUGI.getShortUserName());
+ conEntity =
+ callerUGI.doAs(new PrivilegedExceptionAction<TimelineEntity>() {
+ @Override public TimelineEntity run() throws Exception {
+ return getEntity(path, params);
+ }
+ });
+ }
+ } catch (Exception e) {
+ LogWebServiceUtils.rewrapAndThrowException(e);
+ }
+ if (conEntity == null) {
+ return null;
+ }
+ String nodeHttpAddress = (String) conEntity.getInfo()
+ .get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO);
+
+ ContainerInfo info = new ContainerInfo(nodeHttpAddress);
+ return info;
+ }
+
+ protected AppInfo getApp(HttpServletRequest req, String appId,
+ String clusterId) {
+ UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req);
+
+ String cId = clusterId != null ? clusterId : defaultClusterid;
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", "INFO");
+ String path = JOINER.join("clusters/", cId, "/apps/", appId);
+ TimelineEntity appEntity = null;
+
+ try {
+ if (callerUGI == null) {
+ appEntity = getEntity(path, params);
+ } else {
+ setUserName(params, callerUGI.getShortUserName());
+ appEntity =
+ callerUGI.doAs(new PrivilegedExceptionAction<TimelineEntity>() {
+ @Override public TimelineEntity run() throws Exception {
+ return getEntity(path, params);
+ }
+ });
+ }
+ } catch (Exception e) {
+ LogWebServiceUtils.rewrapAndThrowException(e);
+ }
+
+ if (appEntity == null) {
+ return null;
+ }
+ String appOwner = (String) appEntity.getInfo()
+ .get(ApplicationMetricsConstants.USER_ENTITY_INFO);
+ String state = (String) appEntity.getInfo()
+ .get(ApplicationMetricsConstants.STATE_EVENT_INFO);
+ YarnApplicationState appState = YarnApplicationState.valueOf(state);
+ AppInfo info = new AppInfo(appState, appOwner);
+ return info;
+ }
+
+ /**
+ * Returns the contents of a container's log file in plain text.
+ *
+ * @param req HttpServletRequest
+ * @param res HttpServletResponse
+ * @param containerIdStr The container ID
+ * @param filename The name of the log file
+ * @param format The content type
+ * @param size the size of the log file
+ * @param nmId The Node Manager NodeId
+ * @param redirectedFromNode Whether this is the redirect request from NM
+ * @return The contents of the container's log file
+ */
+ @GET @Path("/containers/{containerid}/logs/{filename}")
+ @Produces({ MediaType.TEXT_PLAIN }) @InterfaceAudience.Public
+ @InterfaceStability.Unstable public Response getContainerLogFile(
+ @Context HttpServletRequest req, @Context HttpServletResponse res,
+ @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr,
+ @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size,
+ @QueryParam(YarnWebServiceParams.NM_ID) String nmId,
+ @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE)
+ boolean redirectedFromNode,
+ @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) {
+ return getLogs(req, res, containerIdStr, filename, format, size, nmId,
+ redirectedFromNode, clusterId);
+ }
+
+ //TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and
+ // container log webservice introduced in AHS to minimize
+ // the duplication.
+ @GET @Path("/containerlogs/{containerid}/{filename}")
+ @Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 })
+ @InterfaceAudience.Public @InterfaceStability.Unstable
+ public Response getLogs(@Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr,
+ @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size,
+ @QueryParam(YarnWebServiceParams.NM_ID) String nmId,
+ @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE)
+ @DefaultValue("false") boolean redirectedFromNode,
+ @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) {
+ initForReadableEndpoints(res);
+ ContainerId containerId;
+ try {
+ containerId = ContainerId.fromString(containerIdStr);
+ } catch (IllegalArgumentException ex) {
+ return LogWebServiceUtils.createBadResponse(Response.Status.NOT_FOUND,
+ "Invalid ContainerId: " + containerIdStr);
+ }
+
+ final long length = LogWebServiceUtils.parseLongParam(size);
+
+ ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+ AppInfo appInfo;
+ try {
+ appInfo = getApp(req, appId.toString(), clusterId);
+ } catch (Exception ex) {
+ // directly find logs from HDFS.
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, null, null, containerIdStr,
+ filename, format, length, false);
+ }
+ String appOwner = appInfo.getUser();
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
+ // directly find logs from HDFS.
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, false);
+ }
+
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
+ String nodeHttpAddress = null;
+ if (nmId != null && !nmId.isEmpty()) {
+ try {
+ nodeHttpAddress =
+ LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
+ } catch (Exception ex) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(ex.getMessage());
+ }
+ }
+ }
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ ContainerInfo containerInfo;
+ try {
+ containerInfo =
+ getContainer(req, appId.toString(), containerId.toString(),
+ clusterId);
+ } catch (Exception ex) {
+ // output the aggregated logs
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
+ }
+ nodeHttpAddress = containerInfo.getNodeHttpAddress();
+ // make sure nodeHttpAddress is not null and not empty. Otherwise,
+ // we would only get aggregated logs instead of re-directing the
+ // request.
+ // If this is the redirect request from NM, we should not re-direct the
+ // request back. Simply output the aggregated logs.
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()
+ || redirectedFromNode) {
+ // output the aggregated logs
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
+ }
+ }
+ String uri = "/" + containerId.toString() + "/logs/" + filename;
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(yarnConf, nodeHttpAddress),
+ NM_DOWNLOAD_URI_STR, uri);
+ String query = req.getQueryString();
+ if (query != null && !query.isEmpty()) {
+ resURI += "?" + query;
+ }
+ Response.ResponseBuilder response =
+ Response.status(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ response.header("Location", resURI);
+ return response.build();
+ } else {
+ return LogWebServiceUtils.createBadResponse(Response.Status.NOT_FOUND,
+ "The application is not at Running or Finished State.");
+ }
+ }
+
+ protected static class AppInfo {
+ private YarnApplicationState appState;
+ private String user;
+
+ AppInfo(YarnApplicationState appState, String user) {
+ this.appState = appState;
+ this.user = user;
+ }
+
+ public YarnApplicationState getAppState() {
+ return this.appState;
+ }
+
+ public String getUser() {
+ return this.user;
+ }
+ }
+
+ protected static class ContainerInfo {
+ private String nodeHttpAddress;
+
+ ContainerInfo(String nodeHttpAddress) {
+ this.nodeHttpAddress = nodeHttpAddress;
+ }
+
+ public String getNodeHttpAddress() {
+ return nodeHttpAddress;
+ }
+ }
+
+ @VisibleForTesting protected TimelineEntity getEntity(String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ ClientResponse resp =
+ getClient().resource(base).path(path).queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ if (resp == null
+ || resp.getStatusInfo().getStatusCode() != ClientResponse.Status.OK
+ .getStatusCode()) {
+ String msg =
+ "Response from the timeline reader server is " + ((resp == null) ?
+ "null" :
+ "not successful," + " HTTP error code: " + resp.getStatus()
+ + ", Server response:\n" + resp.getEntity(String.class));
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ private Client getClient() {
+ if (webTimelineClient == null) {
+ synchronized (LogWebService.class) {
+ if (webTimelineClient == null) {
+ webTimelineClient = createTimelineWebClient();
+ }
+ }
+ }
+ return webTimelineClient;
+ }
+
+ /**
+ * Set user.name in non-secure mode to delegate to next rest call.
+ */
+ private void setUserName(MultivaluedMap<String, String> params, String user) {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ params.add("user.name", user);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java
new file mode 100644
index 0000000..bc301bb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import com.google.common.base.Joiner;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
+import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
+import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.GenericEntity;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Log web service utils class.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class LogWebServiceUtils {
+
+ private LogWebServiceUtils() {
+ }
+
+ private static final Joiner DOT_JOINER = Joiner.on(". ");
+
+ public static Response getContainerLogMeta(
+ LogAggregationFileControllerFactory factory, ApplicationId appId,
+ String appOwner, final String nodeId, final String containerIdStr,
+ boolean emptyLocalContainerLogMeta) {
+ try {
+ ContainerLogsRequest request = new ContainerLogsRequest();
+ request.setAppId(appId);
+ request.setAppOwner(appOwner);
+ request.setContainerId(containerIdStr);
+ request.setNodeId(nodeId);
+ List<ContainerLogMeta> containerLogMeta =
+ factory.getFileControllerForRead(appId, appOwner)
+ .readAggregatedLogsMeta(request);
+ if (containerLogMeta.isEmpty()) {
+ throw new NotFoundException(
+ "Can not get log meta for container: " + containerIdStr);
+ }
+ List<ContainerLogsInfo> containersLogsInfo = new ArrayList<>();
+ for (ContainerLogMeta meta : containerLogMeta) {
+ ContainerLogsInfo logInfo =
+ new ContainerLogsInfo(meta, ContainerLogAggregationType.AGGREGATED);
+ containersLogsInfo.add(logInfo);
+ }
+ if (emptyLocalContainerLogMeta) {
+ ContainerLogMeta emptyMeta =
+ new ContainerLogMeta(containerIdStr, "N/A");
+ ContainerLogsInfo empty =
+ new ContainerLogsInfo(emptyMeta, ContainerLogAggregationType.LOCAL);
+ containersLogsInfo.add(empty);
+ }
+ GenericEntity<List<ContainerLogsInfo>> meta =
+ new GenericEntity<List<ContainerLogsInfo>>(containersLogsInfo) {
+ };
+ Response.ResponseBuilder response = Response.ok(meta);
+ // Sending the X-Content-Type-Options response header with the value
+ // nosniff will prevent Internet Explorer from MIME-sniffing a response
+ // away from the declared content-type.
+ response.header("X-Content-Type-Options", "nosniff");
+ return response.build();
+ } catch (Exception ex) {
+ throw new WebApplicationException(ex);
+ }
+ }
+
+ public static Response sendStreamOutputResponse(
+ LogAggregationFileControllerFactory factory, ApplicationId appId,
+ String appOwner, String nodeId, String containerIdStr, String fileName,
+ String format, long bytes, boolean printEmptyLocalContainerLog) {
+ String contentType = WebAppUtils.getDefaultLogContentType();
+ if (format != null && !format.isEmpty()) {
+ contentType = WebAppUtils.getSupportedLogContentType(format);
+ if (contentType == null) {
+ String errorMessage =
+ "The valid values for the parameter : format " + "are "
+ + WebAppUtils.listSupportedLogContentType();
+ return Response.status(Response.Status.BAD_REQUEST).entity(errorMessage)
+ .build();
+ }
+ }
+ StreamingOutput stream = null;
+ try {
+ stream =
+ getStreamingOutput(factory, appId, appOwner, nodeId, containerIdStr,
+ fileName, bytes, printEmptyLocalContainerLog);
+ } catch (Exception ex) {
+ return createBadResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ ex.getMessage());
+ }
+ Response.ResponseBuilder response = Response.ok(stream);
+ response.header("Content-Type", contentType);
+ // Sending the X-Content-Type-Options response header with the value
+ // nosniff will prevent Internet Explorer from MIME-sniffing a response
+ // away from the declared content-type.
+ response.header("X-Content-Type-Options", "nosniff");
+ return response.build();
+ }
+
+ private static StreamingOutput getStreamingOutput(
+ final LogAggregationFileControllerFactory factory,
+ final ApplicationId appId, final String appOwner, final String nodeId,
+ final String containerIdStr, final String logFile, final long bytes,
+ final boolean printEmptyLocalContainerLog) throws IOException {
+ StreamingOutput stream = new StreamingOutput() {
+
+ @Override public void write(OutputStream os)
+ throws IOException, WebApplicationException {
+ ContainerLogsRequest request = new ContainerLogsRequest();
+ request.setAppId(appId);
+ request.setAppOwner(appOwner);
+ request.setContainerId(containerIdStr);
+ request.setBytes(bytes);
+ request.setNodeId(nodeId);
+ Set<String> logTypes = new HashSet<>();
+ logTypes.add(logFile);
+ request.setLogTypes(logTypes);
+ boolean findLogs = factory.getFileControllerForRead(appId, appOwner)
+ .readAggregatedLogs(request, os);
+ if (!findLogs) {
+ os.write(("Can not find logs for container:" + containerIdStr)
+ .getBytes(Charset.forName("UTF-8")));
+ } else {
+ if (printEmptyLocalContainerLog) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(containerIdStr + "\n");
+ sb.append("LogAggregationType: " + ContainerLogAggregationType.LOCAL
+ + "\n");
+ sb.append("LogContents:\n");
+ sb.append(getNoRedirectWarning() + "\n");
+ os.write(sb.toString().getBytes(Charset.forName("UTF-8")));
+ }
+ }
+ }
+ };
+ return stream;
+ }
+
+ public static String getNoRedirectWarning() {
+ return "We do not have NodeManager web address, so we can not "
+ + "re-direct the request to related NodeManager "
+ + "for local container logs.";
+ }
+
+ public static void rewrapAndThrowException(Exception e) {
+ if (e instanceof UndeclaredThrowableException) {
+ rewrapAndThrowThrowable(e.getCause());
+ } else {
+ rewrapAndThrowThrowable(e);
+ }
+ }
+
+ public static void rewrapAndThrowThrowable(Throwable t) {
+ if (t instanceof AuthorizationException) {
+ throw new ForbiddenException(t);
+ } else {
+ throw new WebApplicationException(t);
+ }
+ }
+
+ public static long parseLongParam(String bytes) {
+ if (bytes == null || bytes.isEmpty()) {
+ return Long.MAX_VALUE;
+ }
+ return Long.parseLong(bytes);
+ }
+
+ public static Response createBadResponse(Response.Status status,
+ String errMessage) {
+ Response response = Response.status(status)
+ .entity(DOT_JOINER.join(status.toString(), errMessage)).build();
+ return response;
+ }
+
+ public static boolean isRunningState(YarnApplicationState appState) {
+ return appState == YarnApplicationState.RUNNING;
+ }
+
+ public static boolean isFinishedState(YarnApplicationState appState) {
+ return appState == YarnApplicationState.FINISHED
+ || appState == YarnApplicationState.FAILED
+ || appState == YarnApplicationState.KILLED;
+ }
+
+ protected static UserGroupInformation getUser(HttpServletRequest req) {
+ String remoteUser = req.getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return callerUGI;
+ }
+
+ public static String getNMWebAddressFromRM(Configuration yarnConf,
+ String nodeId)
+ throws ClientHandlerException, UniformInterfaceException, JSONException {
+ JSONObject nodeInfo =
+ YarnWebServiceUtils.getNodeInfoFromRMWebService(yarnConf, nodeId)
+ .getJSONObject("node");
+ return nodeInfo.has("nodeHTTPAddress") ?
+ nodeInfo.getString("nodeHTTPAddress") : null;
+ }
+
+ public static String getAbsoluteNMWebAddress(Configuration yarnConf,
+ String nmWebAddress) {
+ if (nmWebAddress.contains(WebAppUtils.HTTP_PREFIX) || nmWebAddress
+ .contains(WebAppUtils.HTTPS_PREFIX)) {
+ return nmWebAddress;
+ }
+ return WebAppUtils.getHttpSchemePrefix(yarnConf) + nmWebAddress;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
index 479cc75..5f96f23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
@@ -35,4 +35,5 @@ public interface YarnWebServiceParams {
String RESPONSE_CONTENT_SIZE = "size";
String NM_ID = "nm.id";
String REDIRECTED_FROM_NODE = "redirected_from_node";
+ String CLUSTER_ID = "clusterid";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java
new file mode 100644
index 0000000..9ceb629
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.MultivaluedMap;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * test class for log web service.
+ */
+public class TestLogWebService {
+
+ private HttpServletRequest request;
+ private LogWebServiceTest logWebService;
+ private static TimelineEntity entity;
+ private ApplicationId appId;
+ private ContainerId cId;
+ private String user = "user1";
+ private Map<String, TimelineEntity> entities;
+ private String nodeHttpAddress = "localhost:0";
+
+ @Before public void setup() throws Exception {
+ appId = ApplicationId.fromString("application_1518143905142_509690");
+ cId =
+ ContainerId.fromString("container_e138_1518143905142_509690_01_000001");
+ entities = new HashMap<>();
+ generateEntity();
+ request = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(request.getRemoteUser())
+ .thenReturn(System.getProperty("user.name"));
+ logWebService = new LogWebServiceTest();
+
+ }
+
+ @Test public void testGetApp() {
+
+ LogWebService.AppInfo app =
+ logWebService.getApp(request, appId.toString(), null);
+ Assert.assertEquals("RUNNING", app.getAppState().toString());
+ Assert.assertEquals(user, app.getUser());
+ }
+
+ @Test public void testGetContainer() {
+ LogWebService.ContainerInfo container = logWebService
+ .getContainer(request, appId.toString(), cId.toString(), null);
+ Assert.assertEquals(nodeHttpAddress, container.getNodeHttpAddress());
+ }
+
+ class LogWebServiceTest extends LogWebService {
+
+ @Override protected TimelineEntity getEntity(String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ if (path.endsWith(cId.toString())) {
+ return entities.get(cId.toString());
+ } else if (path.endsWith(appId.toString())) {
+ return entities.get(appId.toString());
+ } else {
+ throw new IOException();
+ }
+ }
+ }
+
+ private void generateEntity() {
+ createAppEntities();
+ createContainerEntities();
+ }
+
+ private void createContainerEntities() {
+ TimelineEntity timelineEntity =
+ generateEntity(TimelineEntityType.YARN_APPLICATION.toString(),
+ appId.toString());
+ timelineEntity.addInfo(ApplicationMetricsConstants.USER_ENTITY_INFO, user);
+ timelineEntity
+ .addInfo(ApplicationMetricsConstants.STATE_EVENT_INFO, "RUNNING");
+ entities.put(appId.toString(), timelineEntity);
+ }
+
+ private void createAppEntities() {
+ TimelineEntity timelineEntity =
+ generateEntity(TimelineEntityType.YARN_CONTAINER.toString(),
+ cId.toString());
+ timelineEntity
+ .addInfo(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO,
+ nodeHttpAddress);
+ entities.put(cId.toString(), timelineEntity);
+ }
+
+ private TimelineEntity generateEntity(String entityType,
+ String entityId) {
+ TimelineEntity timelineEntity = new TimelineEntity();
+ timelineEntity.setId(entityId);
+ timelineEntity.setType(entityType);
+ timelineEntity.setCreatedTime(System.currentTimeMillis());
+ return timelineEntity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index bd2c428..8f1e7d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineRea
import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderWhitelistAuthorizationFilterInitializer;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils;
+import org.apache.hadoop.yarn.server.webapp.LogWebService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -201,7 +202,8 @@ public class TimelineReaderServer extends CompositeService {
readerWebServer.addJerseyResourcePackage(
TimelineReaderWebServices.class.getPackage().getName() + ";"
+ GenericExceptionHandler.class.getPackage().getName() + ";"
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName()+ ";"
+ + LogWebService.class.getPackage().getName(),
"/*");
readerWebServer.setAttribute(TIMELINE_READER_MANAGER_ATTR,
timelineReaderManager);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-8777. Container Executor C
binary change to execute interactive docker command. Contributed by Eric Yang
Posted by sh...@apache.org.
YARN-8777. Container Executor C binary change to execute interactive docker command. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96d28b47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96d28b47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96d28b47
Branch: refs/heads/HDFS-12943
Commit: 96d28b4750f1088f2a10c83991b5a8149c97ef86
Parents: 47eeb3e
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Oct 11 09:25:21 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Thu Oct 11 09:25:21 2018 -0700
----------------------------------------------------------------------
.../impl/container-executor.c | 164 ++++++++++++++++++-
.../impl/container-executor.h | 7 +-
.../main/native/container-executor/impl/util.h | 3 +-
.../container-executor/impl/utils/docker-util.c | 53 ++++++
.../container-executor/impl/utils/docker-util.h | 10 ++
.../test/utils/test_docker_util.cc | 29 ++++
6 files changed, 263 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 7765308..1f7ae3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -47,6 +47,7 @@
#include <sys/wait.h>
#include <getopt.h>
#include <sys/param.h>
+#include <termios.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -1357,8 +1358,25 @@ char **construct_docker_command(const char *command_file) {
}
int run_docker(const char *command_file) {
+ struct configuration command_config = {0, NULL};
+
+ int ret = read_config(command_file, &command_config);
+ if (ret != 0) {
+ free_configuration(&command_config);
+ return INVALID_COMMAND_FILE;
+ }
+ char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (value != NULL && strcasecmp(value, "exec") == 0) {
+ free(value);
+ free_configuration(&command_config);
+ return run_docker_with_pty(command_file);
+ }
+ free_configuration(&command_config);
+ free(value);
+
char **args = construct_docker_command(command_file);
char* docker_binary = get_docker_binary(&CFG);
+
int exit_code = -1;
if (execvp(docker_binary, args) != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
@@ -1375,6 +1393,150 @@ int run_docker(const char *command_file) {
return exit_code;
}
+int run_docker_with_pty(const char *command_file) {
+ int exit_code = -1;
+ char **args = construct_docker_command(command_file);
+ char* docker_binary = get_docker_binary(&CFG);
+ int fdm, fds, rc;
+ char input[4000];
+
+ fdm = posix_openpt(O_RDWR);
+ if (fdm < 0) {
+ fprintf(stderr, "Error %d on posix_openpt()\n", errno);
+ return DOCKER_EXEC_FAILED;
+ }
+
+ rc = grantpt(fdm);
+ if (rc != 0) {
+ fprintf(stderr, "Error %d on grantpt()\n", errno);
+ return DOCKER_EXEC_FAILED;
+ }
+
+ rc = unlockpt(fdm);
+ if (rc != 0) {
+ fprintf(stderr, "Error %d on unlockpt()\n", errno);
+ return DOCKER_EXEC_FAILED;
+ }
+
+ // Open the slave PTY
+ fds = open(ptsname(fdm), O_RDWR);
+
+ // Creation of a child process
+ if (fork()) {
+ fd_set fd_in;
+ // Parent
+
+ // Close the slave side of the PTY
+ close(fds);
+ while (1) {
+ // Wait for data from standard input and master side of PTY
+ FD_ZERO(&fd_in);
+ FD_SET(0, &fd_in);
+ FD_SET(fdm, &fd_in);
+ rc = select(fdm + 1, &fd_in, NULL, NULL, NULL);
+ switch(rc) {
+ case -1 : fprintf(stderr, "Error %d on select()\n", errno);
+ exit(1);
+ default :
+ {
+ // If data on standard input
+ if (FD_ISSET(0, &fd_in)) {
+ rc = read(0, input, sizeof(input));
+ if (rc > 0) {
+ // Send data on the master side of PTY
+ ssize_t written = write(fdm, input, rc);
+ if (written == -1) {
+ fprintf(stderr, "Error %d writing to container.\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ } else {
+ if (rc < 0) {
+ fprintf(stderr, "Error %d on read standard input\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ }
+ }
+
+ // If data on master side of PTY
+ if (FD_ISSET(fdm, &fd_in)) {
+ rc = read(fdm, input, sizeof(input));
+ if (rc > 0) {
+ // Send data on standard output
+ ssize_t written = write(1, input, rc);
+ if (written == -1) {
+ fprintf(stderr, "Error %d writing to terminal.\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ } else {
+ if (rc < 0) {
+ fprintf(stderr, "Error %d on read master PTY\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ }
+ }
+ }
+ } // End switch
+ } // End while
+ } else {
+ struct termios slave_orig_term_settings; // Saved terminal settings
+ struct termios new_term_settings; // Current terminal settings
+
+ // Child
+
+ // Close the master side of the PTY
+ close(fdm);
+
+ // Save the default parameters of the slave side of the PTY
+ rc = tcgetattr(fds, &slave_orig_term_settings);
+
+ // Set raw mode on the slave side of the PTY
+ new_term_settings = slave_orig_term_settings;
+ cfmakeraw (&new_term_settings);
+ tcsetattr (fds, TCSANOW, &new_term_settings);
+
+ // The slave side of the PTY becomes the standard input and outputs of the child process
+ close(0); // Close standard input (current terminal)
+ close(1); // Close standard output (current terminal)
+ close(2); // Close standard error (current terminal)
+
+ if (dup(fds) == -1) {
+ // PTY becomes standard input (0)
+ exit(DOCKER_EXEC_FAILED);
+ }
+ if (dup(fds) == -1) {
+ // PTY becomes standard output (1)
+ exit(DOCKER_EXEC_FAILED);
+ }
+ if (dup(fds) == -1) {
+ // PTY becomes standard error (2)
+ exit(DOCKER_EXEC_FAILED);
+ }
+
+ // Now the original file descriptor is useless
+ close(fds);
+
+ // Make the current process a new session leader
+ setsid();
+
+ // As the child is a session leader, set the controlling terminal to be the slave side of the PTY
+ // (Mandatory for programs like the shell to make them manage correctly their outputs)
+ ioctl(0, TIOCSCTTY, 1);
+
+ if (execvp(docker_binary, args) != 0) {
+ fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
+ docker_binary, strerror(errno));
+ free(docker_binary);
+ free_values(args);
+ exit_code = DOCKER_EXEC_FAILED;
+ } else {
+ free_values(args);
+ exit_code = 0;
+ }
+ }
+
+ return exit_code;
+}
+
int exec_docker_command(char *docker_command, char **argv, int argc) {
int i;
char* docker_binary = get_docker_binary(&CFG);
@@ -2708,4 +2870,4 @@ int remove_docker_container(char**argv, int argc) {
exit_code = clean_docker_cgroups(yarn_hierarchy, container_id);
}
return exit_code;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index cd09e1e..ce05c57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -269,6 +269,11 @@ int is_docker_support_enabled();
int run_docker(const char *command_file);
/**
+ * Run a docker command passing the command file as an argument with terminal.
+ */
+int run_docker_with_pty(const char *command_file);
+
+/**
* Run a docker command without a command file
*/
int exec_docker_command(char *docker_command, char **argv, int argc);
@@ -295,4 +300,4 @@ char* flatten(char **args);
/**
* Remove docker container
*/
-int remove_docker_container(char **argv, int argc);
\ No newline at end of file
+int remove_docker_container(char **argv, int argc);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index 6aac1fe..372c17a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -68,7 +68,8 @@ enum errorcodes {
DOCKER_IMAGE_INVALID = 40,
// DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED)
ERROR_COMPILING_REGEX = 42,
- INVALID_CONTAINER_ID = 43
+ INVALID_CONTAINER_ID = 43,
+ DOCKER_EXEC_FAILED = 44
};
/* Macros for min/max. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 69f27ba..c0aa80b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -432,6 +432,8 @@ int get_docker_command(const char *command_file, const struct configuration *con
ret = get_docker_volume_command(command_file, conf, args);
} else if (strcmp(DOCKER_START_COMMAND, command) == 0) {
ret = get_docker_start_command(command_file, conf, args);
+ } else if (strcmp(DOCKER_EXEC_COMMAND, command) == 0) {
+ ret = get_docker_exec_command(command_file, conf, args);
} else {
ret = UNKNOWN_DOCKER_COMMAND;
}
@@ -820,6 +822,57 @@ free_and_exit:
return ret;
}
+int get_docker_exec_command(const char *command_file, const struct configuration *conf, args *args) {
+ int ret = 0, i = 0;
+ char *container_name = NULL;
+ char **launch_command = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_EXEC_COMMAND, &command_config);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, DOCKER_EXEC_COMMAND);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, "-it");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, container_name);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
+ ",");
+ if (launch_command != NULL) {
+ for (i = 0; launch_command[i] != NULL; ++i) {
+ ret = add_to_args(args, launch_command[i]);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ }
+ } else {
+ ret = INVALID_COMMAND_FILE;
+ }
+free_and_exit:
+ free(container_name);
+ free_configuration(&command_config);
+ free_values(launch_command);
+ return ret;
+}
+
static int detach_container(const struct configuration *command_config, args *args) {
return add_param_to_command(command_config, "detach", "-d", 0, args);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
index 7b7322d..fc7b4cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -34,6 +34,7 @@
#define DOCKER_KILL_COMMAND "kill"
#define DOCKER_VOLUME_COMMAND "volume"
#define DOCKER_START_COMMAND "start"
+#define DOCKER_EXEC_COMMAND "exec"
#define DOCKER_ARG_MAX 1024
#define ARGS_INITIAL_VALUE { 0 };
@@ -176,6 +177,15 @@ int get_docker_volume_command(const char *command_file, const struct configurati
int get_docker_start_command(const char* command_file, const struct configuration* conf, args *args);
/**
+ * Get the Docker exec command line string. The function will verify that the params file is meant for the exec command.
+ * @param command_file File containing the params for the Docker start command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param args Buffer to construct argv
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_exec_command(const char* command_file, const struct configuration* conf, args *args);
+
+/**
* Give an error message for the supplied error code
* @param error_code the error code
* @return const string containing the error message
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index b289857..2817e0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -1762,4 +1762,33 @@ namespace ContainerExecutor {
free_configuration(&container_executor_cfg);
}
}
+
+ TEST_F(TestDockerUtil, test_docker_exec) {
+ std::string container_executor_contents = "[docker]\n"
+ " docker.allowed.devices=/dev/test\n docker.trusted.registries=hadoop\n";
+ write_file(container_executor_cfg_file, container_executor_contents);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=exec\n name=container_e1_12312_11111_02_000001\n launch-command=bash",
+ "exec -it container_e1_12312_11111_02_000001 bash"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=exec\n image=hadoop/docker-image\n user=nobody",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_exec_command);
+ free_configuration(&container_executor_cfg);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDDS-601. On restart,
SCM throws 'No such datanode' exception.
Posted by sh...@apache.org.
HDDS-601. On restart, SCM throws 'No such datanode' exception.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb34b5f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb34b5f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb34b5f8
Branch: refs/heads/HDFS-12943
Commit: eb34b5f8aff538e90c7820c0114322d2dd75f6e6
Parents: 2addebb
Author: Hanisha Koneru <ha...@apache.org>
Authored: Thu Oct 11 13:20:32 2018 -0700
Committer: Hanisha Koneru <ha...@apache.org>
Committed: Thu Oct 11 13:20:32 2018 -0700
----------------------------------------------------------------------
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../protocol/StorageContainerNodeProtocol.java | 8 ++
.../hadoop/hdds/scm/node/NodeManager.java | 7 ++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 26 +++--
.../server/SCMDatanodeHeartbeatDispatcher.java | 107 ++++++++++++-------
.../hdds/scm/container/MockNodeManager.java | 11 ++
.../hadoop/hdds/scm/node/TestNodeManager.java | 53 ---------
.../TestSCMDatanodeHeartbeatDispatcher.java | 41 ++++++-
.../testutils/ReplicationNodeManagerMock.java | 11 ++
9 files changed, 164 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index 690aa01..acd4af9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -42,7 +42,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.Future;
/**
- * Register a container with SCM.
+ * Register a datanode with SCM.
*/
public final class RegisterEndpointTask implements
Callable<EndpointStateMachine.EndPointStates> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
index b3c3eb3..b5d75ef 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
@@ -67,4 +67,12 @@ public interface StorageContainerNodeProtocol {
*/
List<SCMCommand> processHeartbeat(DatanodeDetails datanodeDetails);
+ /**
+ * Check if node is registered or not.
+ * Return true if Node is registered and false otherwise.
+ * @param datanodeDetails - Datanode ID.
+ * @return true if Node is registered, false otherwise
+ */
+ Boolean isNodeRegistered(DatanodeDetails datanodeDetails);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 585d545..5f6a2e4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -196,4 +196,11 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* @param dnUuid datanode uuid.
*/
void processDeadNode(UUID dnUuid);
+
+ /**
+ * Get list of SCMCommands in the Command Queue for a particular Datanode.
+ * @param dnID - Datanode uuid.
+ * @return list of commands
+ */
+ List<SCMCommand> getCommandQueue(UUID dnID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 4a34136..acec6aa 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
-import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.slf4j.Logger;
@@ -196,7 +195,7 @@ public class SCMNodeManager
try {
stat = nodeStateManager.getNodeStat(dnId);
} catch (NodeNotFoundException e) {
- LOG.debug("SCM updateNodeStat based on heartbeat from previous" +
+ LOG.debug("SCM updateNodeStat based on heartbeat from previous " +
"dead datanode {}", dnId);
stat = new SCMNodeStat();
}
@@ -277,7 +276,7 @@ public class SCMNodeManager
nodeStateManager.setNodeStat(dnId, new SCMNodeStat());
// Updating Node Report, as registration is successful
updateNodeStat(datanodeDetails.getUuid(), nodeReport);
- LOG.info("Data node with ID: {} Registered.", datanodeDetails.getUuid());
+ LOG.info("Registered Data node : {}", datanodeDetails);
} catch (NodeAlreadyExistsException e) {
LOG.trace("Datanode is already registered. Datanode: {}",
datanodeDetails.toString());
@@ -304,14 +303,22 @@ public class SCMNodeManager
try {
nodeStateManager.updateLastHeartbeatTime(datanodeDetails);
} catch (NodeNotFoundException e) {
- LOG.warn("SCM receive heartbeat from unregistered datanode {}",
- datanodeDetails);
- commandQueue.addCommand(datanodeDetails.getUuid(),
- new ReregisterCommand());
+ LOG.error("SCM trying to process heartbeat from an " +
+ "unregistered node {}. Ignoring the heartbeat.", datanodeDetails);
}
return commandQueue.getCommand(datanodeDetails.getUuid());
}
+ @Override
+ public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) {
+ try {
+ nodeStateManager.getNode(datanodeDetails);
+ return true;
+ } catch (NodeNotFoundException e) {
+ return false;
+ }
+ }
+
/**
* Process node report.
*
@@ -487,4 +494,9 @@ public class SCMNodeManager
+ " doesn't exist or decommissioned already.", dnUuid);
}
}
+
+ @Override
+ public List<SCMCommand> getCommandQueue(UUID dnID) {
+ return commandQueue.getCommand(dnID);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index d9a0875..7c7df27 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -35,13 +35,15 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.UUID;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_ACTIONS;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
@@ -82,51 +84,78 @@ public final class SCMDatanodeHeartbeatDispatcher {
public List<SCMCommand> dispatch(SCMHeartbeatRequestProto heartbeat) {
DatanodeDetails datanodeDetails =
DatanodeDetails.getFromProtoBuf(heartbeat.getDatanodeDetails());
- // should we dispatch heartbeat through eventPublisher?
- List<SCMCommand> commands = nodeManager.processHeartbeat(datanodeDetails);
- if (heartbeat.hasNodeReport()) {
- LOG.debug("Dispatching Node Report.");
- eventPublisher.fireEvent(NODE_REPORT,
- new NodeReportFromDatanode(datanodeDetails,
- heartbeat.getNodeReport()));
- }
+ List<SCMCommand> commands;
+
+ // If node is not registered, ask the node to re-register. Do not process
+ // Heartbeat for unregistered nodes.
+ if (!nodeManager.isNodeRegistered(datanodeDetails)) {
+ LOG.info("SCM received heartbeat from an unregistered datanode {}. " +
+ "Asking datanode to re-register.", datanodeDetails);
+ UUID dnID = datanodeDetails.getUuid();
+ nodeManager.addDatanodeCommand(dnID, new ReregisterCommand());
+
+ commands = nodeManager.getCommandQueue(dnID);
+
+ } else {
+
+ // should we dispatch heartbeat through eventPublisher?
+ commands = nodeManager.processHeartbeat(datanodeDetails);
+ if (heartbeat.hasNodeReport()) {
+ LOG.debug("Dispatching Node Report.");
+ eventPublisher.fireEvent(
+ NODE_REPORT,
+ new NodeReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getNodeReport()));
+ }
- if (heartbeat.hasContainerReport()) {
- LOG.debug("Dispatching Container Report.");
- eventPublisher.fireEvent(CONTAINER_REPORT,
- new ContainerReportFromDatanode(datanodeDetails,
- heartbeat.getContainerReport()));
+ if (heartbeat.hasContainerReport()) {
+ LOG.debug("Dispatching Container Report.");
+ eventPublisher.fireEvent(
+ CONTAINER_REPORT,
+ new ContainerReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getContainerReport()));
- }
+ }
- if (heartbeat.hasContainerActions()) {
- LOG.debug("Dispatching Container Actions.");
- eventPublisher.fireEvent(CONTAINER_ACTIONS,
- new ContainerActionsFromDatanode(datanodeDetails,
- heartbeat.getContainerActions()));
- }
+ if (heartbeat.hasContainerActions()) {
+ LOG.debug("Dispatching Container Actions.");
+ eventPublisher.fireEvent(
+ CONTAINER_ACTIONS,
+ new ContainerActionsFromDatanode(
+ datanodeDetails,
+ heartbeat.getContainerActions()));
+ }
- if (heartbeat.hasPipelineReports()) {
- LOG.debug("Dispatching Pipeline Report.");
- eventPublisher.fireEvent(PIPELINE_REPORT,
- new PipelineReportFromDatanode(datanodeDetails,
- heartbeat.getPipelineReports()));
+ if (heartbeat.hasPipelineReports()) {
+ LOG.debug("Dispatching Pipeline Report.");
+ eventPublisher.fireEvent(
+ PIPELINE_REPORT,
+ new PipelineReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getPipelineReports()));
- }
+ }
- if (heartbeat.hasPipelineActions()) {
- LOG.debug("Dispatching Pipeline Actions.");
- eventPublisher.fireEvent(PIPELINE_ACTIONS,
- new PipelineActionsFromDatanode(datanodeDetails,
- heartbeat.getPipelineActions()));
- }
+ if (heartbeat.hasPipelineActions()) {
+ LOG.debug("Dispatching Pipeline Actions.");
+ eventPublisher.fireEvent(
+ PIPELINE_ACTIONS,
+ new PipelineActionsFromDatanode(
+ datanodeDetails,
+ heartbeat.getPipelineActions()));
+ }
- if (heartbeat.getCommandStatusReportsCount() != 0) {
- for (CommandStatusReportsProto commandStatusReport : heartbeat
- .getCommandStatusReportsList()) {
- eventPublisher.fireEvent(CMD_STATUS_REPORT,
- new CommandStatusReportFromDatanode(datanodeDetails,
- commandStatusReport));
+ if (heartbeat.getCommandStatusReportsCount() != 0) {
+ for (CommandStatusReportsProto commandStatusReport : heartbeat
+ .getCommandStatusReportsList()) {
+ eventPublisher.fireEvent(
+ CMD_STATUS_REPORT,
+ new CommandStatusReportFromDatanode(
+ datanodeDetails,
+ commandStatusReport));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index e32f316..f4ce102 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -408,6 +408,12 @@ public class MockNodeManager implements NodeManager {
}
@Override
+ public Boolean isNodeRegistered(
+ DatanodeDetails datanodeDetails) {
+ return null;
+ }
+
+ @Override
public Map<String, Integer> getNodeCount() {
Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) {
@@ -470,6 +476,11 @@ public class MockNodeManager implements NodeManager {
}
}
+ @Override
+ public List<SCMCommand> getCommandQueue(UUID dnID) {
+ return null;
+ }
+
/**
* A class to declare some values for the nodes so that our tests
* won't fail.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index 32ee371..ed95709 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -17,11 +17,8 @@
*/
package org.apache.hadoop.hdds.scm.node;
-import com.google.common.base.Supplier;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
@@ -214,56 +211,6 @@ public class TestNodeManager {
}
/**
- * Asserts scm informs datanodes to re-register with the nodemanager
- * on a restart.
- *
- * @throws Exception
- */
- @Test
- public void testScmHeartbeatAfterRestart() throws Exception {
- OzoneConfiguration conf = getConf();
- conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
- 100, TimeUnit.MILLISECONDS);
- DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- UUID dnId = datanodeDetails.getUuid();
- String storagePath = testDir.getAbsolutePath() + "/" + dnId;
- StorageReportProto report =
- TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
- try (SCMNodeManager nodemanager = createNodeManager(conf)) {
- nodemanager.register(datanodeDetails,
- TestUtils.createNodeReport(report),
- TestUtils.getRandomPipelineReports());
- List<SCMCommand> command = nodemanager.processHeartbeat(datanodeDetails);
- Assert.assertTrue(nodemanager.getAllNodes().contains(datanodeDetails));
- Assert.assertTrue("On regular HB calls, SCM responses a "
- + "datanode with an empty command list", command.isEmpty());
- }
-
- // Sends heartbeat without registering to SCM.
- // This happens when SCM restarts.
- try (SCMNodeManager nodemanager = createNodeManager(conf)) {
- Assert.assertFalse(nodemanager
- .getAllNodes().contains(datanodeDetails));
- try {
- // SCM handles heartbeat asynchronously.
- // It may need more than one heartbeat processing to
- // send the notification.
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- List<SCMCommand> command =
- nodemanager.processHeartbeat(datanodeDetails);
- return command.size() == 1 && command.get(0).getType()
- .equals(SCMCommandProto.Type.reregisterCommand);
- }
- }, 100, 3 * 1000);
- } catch (TimeoutException e) {
- Assert.fail("Times out to verify that scm informs "
- + "datanode to re-register itself.");
- }
- }
- }
-
- /**
* Asserts that we detect as many healthy nodes as we have generated heartbeat
* for.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
index f3cd4ea..eac8c90 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.server;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.UUID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.
@@ -39,6 +40,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.NodeReportFromDatanode;
import org.apache.hadoop.hdds.server.events.Event;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.junit.Assert;
import org.junit.Test;
@@ -61,8 +63,12 @@ public class TestSCMDatanodeHeartbeatDispatcher {
NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
+ NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
+ Mockito.when(mockNodeManager.isNodeRegistered(Mockito.any()))
+ .thenReturn(true);
+
SCMDatanodeHeartbeatDispatcher dispatcher =
- new SCMDatanodeHeartbeatDispatcher(Mockito.mock(NodeManager.class),
+ new SCMDatanodeHeartbeatDispatcher(mockNodeManager,
new EventPublisher() {
@Override
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
@@ -99,8 +105,13 @@ public class TestSCMDatanodeHeartbeatDispatcher {
CommandStatusReportsProto commandStatusReport =
CommandStatusReportsProto.getDefaultInstance();
+ NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
+ Mockito.when(mockNodeManager.isNodeRegistered(Mockito.any()))
+ .thenReturn(true);
+
SCMDatanodeHeartbeatDispatcher dispatcher =
- new SCMDatanodeHeartbeatDispatcher(Mockito.mock(NodeManager.class),
+ new SCMDatanodeHeartbeatDispatcher(
+ mockNodeManager,
new EventPublisher() {
@Override
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
@@ -135,4 +146,30 @@ public class TestSCMDatanodeHeartbeatDispatcher {
}
+ /**
+ * Asserts scm informs datanodes to re-register on a restart.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testScmHeartbeatAfterRestart() throws Exception {
+
+ NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
+ SCMDatanodeHeartbeatDispatcher dispatcher =
+ new SCMDatanodeHeartbeatDispatcher(
+ mockNodeManager, Mockito.mock(EventPublisher.class));
+
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
+
+ SCMHeartbeatRequestProto heartbeat =
+ SCMHeartbeatRequestProto.newBuilder()
+ .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
+ .build();
+
+ dispatcher.dispatch(heartbeat);
+ // If SCM receives heartbeat from a node after it restarts and the node
+ // is not registered, it should send a Re-Register command back to the node.
+ Mockito.verify(mockNodeManager, Mockito.times(1)).addDatanodeCommand(
+ Mockito.any(UUID.class), Mockito.any(ReregisterCommand.class));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 6584aec..069f1af 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -295,6 +295,12 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
+ @Override
+ public Boolean isNodeRegistered(
+ DatanodeDetails datanodeDetails) {
+ return null;
+ }
+
/**
* Clears all nodes from the node Manager.
*/
@@ -341,4 +347,9 @@ public class ReplicationNodeManagerMock implements NodeManager {
public void processDeadNode(UUID dnUuid) {
// do nothing.
}
+
+ @Override
+ public List<SCMCommand> getCommandQueue(UUID dnID) {
+ return null;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDDS-636. Turn off
ozone.metastore.rocksdb.statistics for now. Contributed by Arpit Agarwal.
Posted by sh...@apache.org.
HDDS-636. Turn off ozone.metastore.rocksdb.statistics for now. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50552479
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50552479
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50552479
Branch: refs/heads/HDFS-12943
Commit: 50552479c42d2e84cfeec83d4d67b050c5659b06
Parents: 0bc6d04
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Oct 11 17:21:27 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Oct 11 17:21:27 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +-
hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50552479/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index d6a6bf7..09d798f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -90,7 +90,7 @@ public final class OzoneConfigKeys {
"ozone.metastore.rocksdb.statistics";
public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT =
- "ALL";
+ "OFF";
public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF =
"OFF";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50552479/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0fde6bb..b2d47c9 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -514,7 +514,7 @@
<property>
<name>ozone.metastore.rocksdb.statistics</name>
- <value>ALL</value>
+ <value>OFF</value>
<tag>OZONE, OM, SCM, STORAGE, PERFORMANCE</tag>
<description>
The statistics level of the rocksdb store. If you use any value from
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-15785. [JDK10] Javadoc
build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Posted by sh...@apache.org.
HADOOP-15785. [JDK10] Javadoc build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b57f2f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b57f2f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b57f2f7
Branch: refs/heads/HDFS-12943
Commit: 7b57f2f71fbaa5af4897309597cca70a95b04edd
Parents: f261c31
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Thu Oct 11 13:51:51 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Thu Oct 11 13:51:51 2018 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configurable.java | 10 +++-
.../org/apache/hadoop/conf/Configuration.java | 45 +++++++++--------
.../hadoop/conf/ConfigurationWithLogging.java | 14 +++---
.../org/apache/hadoop/crypto/CryptoCodec.java | 7 +--
.../apache/hadoop/crypto/CryptoInputStream.java | 4 +-
.../hadoop/crypto/CryptoOutputStream.java | 4 +-
.../org/apache/hadoop/crypto/Decryptor.java | 14 +++---
.../org/apache/hadoop/crypto/Encryptor.java | 14 +++---
.../org/apache/hadoop/crypto/OpensslCipher.java | 18 +++----
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 15 +++---
.../apache/hadoop/crypto/key/KeyProvider.java | 6 +--
.../crypto/key/KeyProviderCryptoExtension.java | 9 ++--
.../KeyProviderDelegationTokenExtension.java | 2 +-
.../crypto/key/kms/KMSClientProvider.java | 6 +--
.../hadoop/crypto/key/kms/ValueQueue.java | 2 +-
.../crypto/random/OpensslSecureRandom.java | 10 ++--
.../apache/hadoop/fs/AbstractFileSystem.java | 45 +++++++++--------
.../apache/hadoop/fs/BufferedFSInputStream.java | 4 +-
.../apache/hadoop/fs/ByteBufferReadable.java | 8 +--
.../apache/hadoop/fs/ChecksumFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/ChecksumFs.java | 2 +-
.../hadoop/fs/CommonConfigurationKeys.java | 2 +-
.../java/org/apache/hadoop/fs/CreateFlag.java | 2 +-
.../org/apache/hadoop/fs/FSDataInputStream.java | 1 +
.../org/apache/hadoop/fs/FSInputChecker.java | 2 +-
.../java/org/apache/hadoop/fs/FileContext.java | 51 +++++++++++---------
.../java/org/apache/hadoop/fs/FileSystem.java | 11 ++---
.../java/org/apache/hadoop/fs/FileUtil.java | 4 +-
.../org/apache/hadoop/fs/HarFileSystem.java | 2 +-
.../hadoop/fs/HasEnhancedByteBufferAccess.java | 15 +++---
.../org/apache/hadoop/fs/LocalDirAllocator.java | 3 +-
.../org/apache/hadoop/fs/LocalFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/fs/Options.java | 2 +-
.../java/org/apache/hadoop/fs/QuotaUsage.java | 8 +--
.../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 6 +--
.../org/apache/hadoop/fs/ftp/FtpConfigKeys.java | 1 +
.../apache/hadoop/fs/local/LocalConfigKeys.java | 1 +
.../apache/hadoop/fs/permission/AclStatus.java | 2 +-
.../apache/hadoop/fs/permission/AclUtil.java | 10 ++--
.../hadoop/fs/permission/ScopedAclEntries.java | 14 +++---
.../org/apache/hadoop/fs/shell/Command.java | 24 ++++-----
.../apache/hadoop/fs/shell/CommandFormat.java | 2 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 ++---
.../apache/hadoop/ha/ActiveStandbyElector.java | 32 ++++++------
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 2 +-
.../java/org/apache/hadoop/ha/NodeFencer.java | 2 +-
.../org/apache/hadoop/ha/SshFenceByTcpPort.java | 2 +-
.../org/apache/hadoop/http/HttpServer2.java | 10 ++--
.../org/apache/hadoop/io/EnumSetWritable.java | 2 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 2 +-
.../org/apache/hadoop/io/ReadaheadPool.java | 2 +-
.../org/apache/hadoop/io/SecureIOUtils.java | 4 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 16 +++---
.../java/org/apache/hadoop/io/Writable.java | 4 +-
.../apache/hadoop/io/WritableComparable.java | 7 +--
.../org/apache/hadoop/io/WritableUtils.java | 6 ++-
.../io/compress/CompressionCodecFactory.java | 8 +--
.../org/apache/hadoop/io/compress/Lz4Codec.java | 4 +-
.../apache/hadoop/io/compress/SnappyCodec.java | 2 +-
.../io/compress/bzip2/Bzip2Compressor.java | 2 +-
.../io/compress/bzip2/Bzip2Decompressor.java | 6 +--
.../hadoop/io/compress/bzip2/Bzip2Factory.java | 4 +-
.../io/compress/bzip2/CBZip2InputStream.java | 26 +++++-----
.../io/compress/bzip2/CBZip2OutputStream.java | 7 +--
.../compress/zlib/BuiltInGzipDecompressor.java | 8 +--
.../hadoop/io/compress/zlib/ZlibCompressor.java | 2 +-
.../io/compress/zlib/ZlibDecompressor.java | 6 +--
.../hadoop/io/compress/zlib/ZlibFactory.java | 4 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 2 +-
.../erasurecode/rawcoder/util/GaloisField.java | 2 +-
.../org/apache/hadoop/io/file/tfile/TFile.java | 8 +--
.../org/apache/hadoop/io/file/tfile/Utils.java | 48 ++++++++++--------
.../org/apache/hadoop/io/retry/RetryProxy.java | 4 +-
.../hadoop/io/serializer/Deserializer.java | 2 +-
.../apache/hadoop/io/serializer/Serializer.java | 2 +-
.../org/apache/hadoop/ipc/CallerContext.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 9 ++--
.../java/org/apache/hadoop/ipc/ClientCache.java | 6 +--
.../apache/hadoop/ipc/DecayRpcScheduler.java | 3 +-
.../org/apache/hadoop/ipc/RefreshHandler.java | 1 -
.../org/apache/hadoop/ipc/RemoteException.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 2 +-
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +--
.../apache/hadoop/log/LogThrottlingHelper.java | 8 +--
.../metrics2/lib/MutableRollingAverages.java | 2 +-
.../apache/hadoop/metrics2/package-info.java | 3 ++
.../metrics2/sink/RollingFileSystemSink.java | 2 +-
.../apache/hadoop/metrics2/sink/StatsDSink.java | 4 +-
.../org/apache/hadoop/metrics2/util/MBeans.java | 10 ++--
.../hadoop/net/AbstractDNSToSwitchMapping.java | 8 +--
.../main/java/org/apache/hadoop/net/DNS.java | 2 +-
.../apache/hadoop/net/DNSToSwitchMapping.java | 2 +-
.../java/org/apache/hadoop/net/NetUtils.java | 14 +++---
.../org/apache/hadoop/net/NetworkTopology.java | 12 ++---
.../net/NetworkTopologyWithNodeGroup.java | 4 +-
.../apache/hadoop/net/ScriptBasedMapping.java | 8 +--
.../net/ScriptBasedMappingWithDependency.java | 7 ++-
.../apache/hadoop/net/SocketOutputStream.java | 6 +--
.../AuthenticationFilterInitializer.java | 6 +--
.../hadoop/security/HadoopKerberosName.java | 2 +-
.../security/IdMappingServiceProvider.java | 5 +-
.../apache/hadoop/security/SaslRpcClient.java | 10 ++--
.../apache/hadoop/security/SecurityUtil.java | 3 +-
.../hadoop/security/UserGroupInformation.java | 2 +-
.../security/alias/JavaKeyStoreProvider.java | 6 +--
.../alias/LocalJavaKeyStoreProvider.java | 3 +-
.../hadoop/security/authorize/ProxyUsers.java | 4 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 4 +-
.../apache/hadoop/security/ssl/SSLFactory.java | 4 +-
.../security/ssl/SSLHostnameVerifier.java | 20 ++++----
.../web/DelegationTokenAuthenticatedURL.java | 14 +++---
.../DelegationTokenAuthenticationFilter.java | 6 +--
.../DelegationTokenAuthenticationHandler.java | 4 +-
.../delegation/web/DelegationTokenManager.java | 2 +-
...rosDelegationTokenAuthenticationHandler.java | 2 +-
.../KerberosDelegationTokenAuthenticator.java | 2 +-
...emeDelegationTokenAuthenticationHandler.java | 3 +-
...udoDelegationTokenAuthenticationHandler.java | 2 +-
.../web/PseudoDelegationTokenAuthenticator.java | 2 +-
.../hadoop/service/ServiceOperations.java | 4 +-
.../service/launcher/ServiceLauncher.java | 2 +-
.../hadoop/service/launcher/package-info.java | 2 +-
.../java/org/apache/hadoop/util/ClassUtil.java | 1 -
.../apache/hadoop/util/ComparableVersion.java | 2 +-
.../java/org/apache/hadoop/util/FindClass.java | 4 +-
.../hadoop/util/GenericOptionsParser.java | 21 ++++----
.../apache/hadoop/util/HttpExceptionUtils.java | 6 +--
.../apache/hadoop/util/JsonSerialization.java | 2 +-
.../apache/hadoop/util/LightWeightCache.java | 10 ++--
.../java/org/apache/hadoop/util/LineReader.java | 3 --
.../org/apache/hadoop/util/MachineList.java | 2 +-
.../apache/hadoop/util/ShutdownHookManager.java | 2 +-
.../hadoop/util/ShutdownThreadsHelper.java | 2 -
.../org/apache/hadoop/util/StringUtils.java | 6 +--
.../main/java/org/apache/hadoop/util/Tool.java | 2 +-
.../java/org/apache/hadoop/util/ZKUtil.java | 4 +-
.../hadoop/util/bloom/DynamicBloomFilter.java | 5 +-
.../apache/hadoop/util/concurrent/AsyncGet.java | 2 +-
.../apache/hadoop/util/hash/JenkinsHash.java | 4 +-
.../org/apache/hadoop/cli/util/CLICommand.java | 6 +--
.../apache/hadoop/cli/util/CLICommandFS.java | 4 +-
.../apache/hadoop/cli/util/CLICommandTypes.java | 6 +--
.../rawcoder/RawErasureCoderBenchmark.java | 4 +-
.../hadoop/io/retry/TestDefaultRetryPolicy.java | 4 +-
.../org/apache/hadoop/net/StaticMapping.java | 4 +-
.../apache/hadoop/tracing/SetSpanReceiver.java | 2 +-
146 files changed, 518 insertions(+), 483 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
index d847f29..5816039 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
@@ -26,9 +26,15 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Stable
public interface Configurable {
- /** Set the configuration to be used by this object. */
+ /**
+ * Set the configuration to be used by this object.
+ * @param conf configuration to be used
+ */
void setConf(Configuration conf);
- /** Return the configuration used by this object. */
+ /**
+ * Return the configuration used by this object.
+ * @return Configuration
+ */
Configuration getConf();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index c004cb5..478e56e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -115,7 +115,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
/**
* Provides access to configuration parameters.
*
- * <h4 id="Resources">Resources</h4>
+ * <h3 id="Resources">Resources</h3>
*
* <p>Configurations are specified by resources. A resource contains a set of
* name/value pairs as XML data. Each resource is named by either a
@@ -141,12 +141,12 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* Once a resource declares a value final, no subsequently-loaded
* resource can alter that value.
* For example, one might define a final parameter with:
- * <tt><pre>
+ * <pre><code>
* <property>
* <name>dfs.hosts.include</name>
* <value>/etc/hadoop/conf/hosts.include</value>
* <b><final>true</final></b>
- * </property></pre></tt>
+ * </property></code></pre>
*
* Administrators typically define parameters as final in
* <tt>core-site.xml</tt> for values that user applications may not alter.
@@ -164,7 +164,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
*
* <p>For example, if a configuration resource contains the following property
* definitions:
- * <tt><pre>
+ * <pre><code>
* <property>
* <name>basedir</name>
* <value>/user/${<i>user.name</i>}</value>
@@ -179,7 +179,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* <name>otherdir</name>
* <value>${<i>env.BASE_DIR</i>}/other</value>
* </property>
- * </pre></tt>
+ * </code></pre>
*
* <p>When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
* will be resolved to another property in this Configuration, while
@@ -203,7 +203,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* can define there own custom tags in hadoop.tags.custom property.
*
* <p>For example, we can tag existing property as:
- * <tt><pre>
+ * <pre><code>
* <property>
* <name>dfs.replication</name>
* <value>3</value>
@@ -215,7 +215,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* <value>3</value>
* <tag>HDFS,SECURITY</tag>
* </property>
- * </pre></tt>
+ * </code></pre>
* <p> Properties marked with tags can be retrieved with <tt>conf
* .getAllPropertiesByTag("HDFS")</tt> or <tt>conf.getAllPropertiesByTags
* (Arrays.asList("YARN","SECURITY"))</tt>.</p>
@@ -581,9 +581,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* If you have multiple deprecation entries to add, it is more efficient to
* use #addDeprecations(DeprecationDelta[] deltas) instead.
*
- * @param key
- * @param newKeys
- * @param customMessage
+ * @param key to be deprecated
+ * @param newKeys list of keys that take up the values of deprecated key
+ * @param customMessage depcrication message
* @deprecated use {@link #addDeprecation(String key, String newKey,
String customMessage)} instead
*/
@@ -605,9 +605,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* If you have multiple deprecation entries to add, it is more efficient to
* use #addDeprecations(DeprecationDelta[] deltas) instead.
*
- * @param key
- * @param newKey
- * @param customMessage
+ * @param key to be deprecated
+ * @param newKey key that take up the values of deprecated key
+ * @param customMessage deprecation message
*/
public static void addDeprecation(String key, String newKey,
String customMessage) {
@@ -1428,6 +1428,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Unset a previously set property.
+ * @param name the property name
*/
public synchronized void unset(String name) {
String[] names = null;
@@ -1717,6 +1718,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* is equivalent to <code>set(<name>, value.toString())</code>.
* @param name property name
* @param value new value
+ * @param <T> enumeration type
*/
public <T extends Enum<T>> void setEnum(String name, T value) {
set(name, value.toString());
@@ -1727,8 +1729,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* Note that the returned value is trimmed by this method.
* @param name Property name
* @param defaultValue Value returned if no mapping exists
+ * @param <T> enumeration type
* @throws IllegalArgumentException If mapping is illegal for the type
* provided
+ * @return enumeration type
*/
public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
final String val = getTrimmed(name);
@@ -1807,6 +1811,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @param unit Unit to convert the stored property, if it exists.
* @throws NumberFormatException If the property stripped of its unit is not
* a number
+ * @return time duration in given time unit
*/
public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
String vStr = get(name);
@@ -2299,6 +2304,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* the CredentialProvider API and conditionally fallsback to config.
* @param name property name
* @return password
+ * @throws IOException when error in fetching password
*/
public char[] getPassword(String name) throws IOException {
char[] pass = null;
@@ -2358,7 +2364,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* alias.
* @param name alias of the provisioned credential
* @return password or null if not found
- * @throws IOException
+ * @throws IOException when error in fetching password
*/
public char[] getPasswordFromCredentialProviders(String name)
throws IOException {
@@ -3425,25 +3431,23 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Write out the non-default properties in this configuration to the
* given {@link Writer}.
- *
+ * <ul>
* <li>
* When property name is not empty and the property exists in the
* configuration, this method writes the property and its attributes
* to the {@link Writer}.
* </li>
- * <p>
*
* <li>
* When property name is null or empty, this method writes all the
* configuration properties and their attributes to the {@link Writer}.
* </li>
- * <p>
*
* <li>
* When property name is not empty but the property doesn't exist in
* the configuration, this method throws an {@link IllegalArgumentException}.
* </li>
- * <p>
+ * </ul>
* @param out the writer to write to.
*/
public void writeXml(String propertyName, Writer out)
@@ -3553,7 +3557,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Writes properties and their attributes (final and resource)
* to the given {@link Writer}.
- *
+ * <ul>
* <li>
* When propertyName is not empty, and the property exists
* in the configuration, the format of the output would be,
@@ -3593,6 +3597,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* found in the configuration, this method will throw an
* {@link IllegalArgumentException}.
* </li>
+ * </ul>
* <p>
* @param config the configuration
* @param propertyName property name
@@ -3791,7 +3796,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* get keys matching the the regex
* @param regex
- * @return Map<String,String> with matching keys
+ * @return {@literal Map<String,String>} with matching keys
*/
public Map<String,String> getValByRegex(String regex) {
Pattern p = Pattern.compile(regex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
index 8a5e054..68c5172 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
@@ -41,7 +41,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#get(String).
+ * See {@link Configuration#get(String)}.
*/
@Override
public String get(String name) {
@@ -51,7 +51,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#get(String, String).
+ * See {@link Configuration#get(String, String)}.
*/
@Override
public String get(String name, String defaultValue) {
@@ -62,7 +62,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getBoolean(String, boolean).
+ * See {@link Configuration#getBoolean(String, boolean)}.
*/
@Override
public boolean getBoolean(String name, boolean defaultValue) {
@@ -72,7 +72,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getFloat(String, float).
+ * See {@link Configuration#getFloat(String, float)}.
*/
@Override
public float getFloat(String name, float defaultValue) {
@@ -82,7 +82,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getInt(String, int).
+ * See {@link Configuration#getInt(String, int)}.
*/
@Override
public int getInt(String name, int defaultValue) {
@@ -92,7 +92,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getLong(String, long).
+ * See {@link Configuration#getLong(String, long)}.
*/
@Override
public long getLong(String name, long defaultValue) {
@@ -102,7 +102,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#set(String, String, String).
+ * See {@link Configuration#set(String, String, String)}.
*/
@Override
public void set(String name, String value, String source) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
index d9c16bb..bcf4a65 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
@@ -158,14 +158,15 @@ public abstract class CryptoCodec implements Configurable, Closeable {
* For example a {@link javax.crypto.Cipher} will maintain its encryption
* context internally when we do encryption/decryption using the
* Cipher#update interface.
- * <p/>
+ * <p>
* Encryption/Decryption is not always on the entire file. For example,
* in Hadoop, a node may only decrypt a portion of a file (i.e. a split).
* In these situations, the counter is derived from the file position.
- * <p/>
+ * <p>
* The IV can be calculated by combining the initial IV and the counter with
* a lossless operation (concatenation, addition, or XOR).
- * @see http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_.28CTR.29
+ * See http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_
+ * .28CTR.29
*
* @param initIV initial IV
* @param counter counter for input stream position
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index a2273bf..5c879ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -53,10 +53,10 @@ import org.apache.hadoop.util.StringUtils;
* required in order to ensure that the plain text and cipher text have a 1:1
* mapping. The decryption is buffer based. The key points of the decryption
* are (1) calculating the counter and (2) padding through stream position:
- * <p/>
+ * <p>
* counter = base + pos/(algorithm blocksize);
* padding = pos%(algorithm blocksize);
- * <p/>
+ * <p>
* The underlying stream offset is maintained as state.
*/
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
index 2f347c5..8d11043 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
@@ -36,10 +36,10 @@ import com.google.common.base.Preconditions;
* required in order to ensure that the plain text and cipher text have a 1:1
* mapping. The encryption is buffer based. The key points of the encryption are
* (1) calculating counter and (2) padding through stream position.
- * <p/>
+ * <p>
* counter = base + pos/(algorithm blocksize);
* padding = pos%(algorithm blocksize);
- * <p/>
+ * <p>
* The underlying stream offset is maintained as state.
*
* Note that while some of this class' methods are synchronized, this is just to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
index 9958415..7556f18 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
@@ -38,7 +38,7 @@ public interface Decryptor {
/**
* Indicate whether the decryption context is reset.
- * <p/>
+ * <p>
* Certain modes, like CTR, require a different IV depending on the
* position in the stream. Generally, the decryptor maintains any necessary
* context for calculating the IV and counter so that no reinit is necessary
@@ -49,22 +49,22 @@ public interface Decryptor {
/**
* This presents a direct interface decrypting with direct ByteBuffers.
- * <p/>
+ * <p>
* This function does not always decrypt the entire buffer and may potentially
* need to be called multiple times to process an entire buffer. The object
* may hold the decryption context internally.
- * <p/>
+ * <p>
* Some implementations may require sufficient space in the destination
* buffer to decrypt the entire input buffer.
- * <p/>
+ * <p>
* Upon return, inBuffer.position() will be advanced by the number of bytes
* read and outBuffer.position() by bytes written. Implementations should
* not modify inBuffer.limit() and outBuffer.limit().
- * <p/>
+ * <p>
* @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
- * not be null and inBuffer.remaining() must be > 0
+ * not be null and inBuffer.remaining() must be {@literal >} 0
* @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
- * not be null and outBuffer.remaining() must be > 0
+ * not be null and outBuffer.remaining() must be {@literal >} 0
* @throws IOException if decryption fails
*/
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
index 6dc3cfb..faeb176 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
@@ -37,7 +37,7 @@ public interface Encryptor {
/**
* Indicate whether the encryption context is reset.
- * <p/>
+ * <p>
* Certain modes, like CTR, require a different IV depending on the
* position in the stream. Generally, the encryptor maintains any necessary
* context for calculating the IV and counter so that no reinit is necessary
@@ -48,22 +48,22 @@ public interface Encryptor {
/**
* This presents a direct interface encrypting with direct ByteBuffers.
- * <p/>
+ * <p>
* This function does not always encrypt the entire buffer and may potentially
* need to be called multiple times to process an entire buffer. The object
* may hold the encryption context internally.
- * <p/>
+ * <p>
* Some implementations may require sufficient space in the destination
* buffer to encrypt the entire input buffer.
- * <p/>
+ * <p>
* Upon return, inBuffer.position() will be advanced by the number of bytes
* read and outBuffer.position() by bytes written. Implementations should
* not modify inBuffer.limit() and outBuffer.limit().
- * <p/>
+ * <p>
* @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
- * not be null and inBuffer.remaining() must be > 0
+ * not be null and inBuffer.remaining() must be > 0
* @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
- * not be null and outBuffer.remaining() must be > 0
+ * not be null and outBuffer.remaining() must be > 0
* @throws IOException if encryption fails
*/
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
index 133a9f9..0a2ba52 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
@@ -107,12 +107,12 @@ public final class OpensslCipher {
}
/**
- * Return an <code>OpensslCipher<code> object that implements the specified
+ * Return an <code>OpensslCipher</code> object that implements the specified
* transformation.
*
* @param transformation the name of the transformation, e.g.,
* AES/CTR/NoPadding.
- * @return OpensslCipher an <code>OpensslCipher<code> object
+ * @return OpensslCipher an <code>OpensslCipher</code> object
* @throws NoSuchAlgorithmException if <code>transformation</code> is null,
* empty, in an invalid format, or if Openssl doesn't implement the
* specified algorithm.
@@ -181,18 +181,18 @@ public final class OpensslCipher {
/**
* Continues a multiple-part encryption or decryption operation. The data
* is encrypted or decrypted, depending on how this cipher was initialized.
- * <p/>
+ * <p>
*
* All <code>input.remaining()</code> bytes starting at
* <code>input.position()</code> are processed. The result is stored in
* the output buffer.
- * <p/>
+ * <p>
*
* Upon return, the input buffer's position will be equal to its limit;
* its limit will not have changed. The output buffer's position will have
* advanced by n, when n is the value returned by this method; the output
* buffer's limit will not have changed.
- * <p/>
+ * <p>
*
* If <code>output.remaining()</code> bytes are insufficient to hold the
* result, a <code>ShortBufferException</code> is thrown.
@@ -218,21 +218,21 @@ public final class OpensslCipher {
/**
* Finishes a multiple-part operation. The data is encrypted or decrypted,
* depending on how this cipher was initialized.
- * <p/>
+ * <p>
*
* The result is stored in the output buffer. Upon return, the output buffer's
* position will have advanced by n, where n is the value returned by this
* method; the output buffer's limit will not have changed.
- * <p/>
+ * <p>
*
* If <code>output.remaining()</code> bytes are insufficient to hold the result,
* a <code>ShortBufferException</code> is thrown.
- * <p/>
+ * <p>
*
* Upon finishing, this method resets this cipher object to the state it was
* in when previously initialized. That is, the object is available to encrypt
* or decrypt more data.
- * <p/>
+ * <p>
*
* If any exception is thrown, this cipher object need to be reset before it
* can be used again.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index 5beda0d..7951af5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -62,23 +62,24 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* KeyProvider based on Java's KeyStore file format. The file may be stored in
* any Hadoop FileSystem using the following name mangling:
- * jks://hdfs@nn1.example.com/my/keys.jks -> hdfs://nn1.example.com/my/keys.jks
- * jks://file/home/owen/keys.jks -> file:///home/owen/keys.jks
- * <p/>
+ * jks://hdfs@nn1.example.com/my/keys.jks {@literal ->}
+ * hdfs://nn1.example.com/my/keys.jks
+ * jks://file/home/owen/keys.jks {@literal ->} file:///home/owen/keys.jks
+ * <p>
* If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is set,
* its value is used as the password for the keystore.
- * <p/>
+ * <p>
* If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is not set,
* the password for the keystore is read from file specified in the
* {@link #KEYSTORE_PASSWORD_FILE_KEY} configuration property. The password file
* is looked up in Hadoop's configuration directory via the classpath.
- * <p/>
+ * <p>
* <b>NOTE:</b> Make sure the password in the password file does not have an
* ENTER at the end, else it won't be valid for the Java KeyStore.
- * <p/>
+ * <p>
* If the environment variable, nor the property are not set, the password used
* is 'none'.
- * <p/>
+ * <p>
* It is expected for encrypted InputFormats and OutputFormats to copy the keys
* from the original provider into the job's Credentials object, which is
* accessed via the UserProvider. Therefore, this provider won't be used by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 286312c..2380f7a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -49,7 +49,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY
* abstraction to separate key storage from users of encryption. It
* is intended to support getting or storing keys in a variety of ways,
* including third party bindings.
- * <P/>
+ * <p>
* <code>KeyProvider</code> implementations must be thread safe.
*/
@InterfaceAudience.Public
@@ -549,7 +549,7 @@ public abstract class KeyProvider {
/**
* Create a new key generating the material for it.
* The given key must not already exist.
- * <p/>
+ * <p>
* This implementation generates the key material and calls the
* {@link #createKey(String, byte[], Options)} method.
*
@@ -593,7 +593,7 @@ public abstract class KeyProvider {
/**
* Roll a new version of the given key generating the material for it.
- * <p/>
+ * <p>
* This implementation generates the key material and calls the
* {@link #rollNewVersion(String, byte[])} method.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 3ee3bd7..00d7a7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -149,7 +149,7 @@ public class KeyProviderCryptoExtension extends
* Derive the initialization vector (IV) for the encryption key from the IV
* of the encrypted key. This derived IV is used with the encryption key to
* decrypt the encrypted key.
- * <p/>
+ * <p>
* The alternative to this is using the same IV for both the encryption key
* and the encrypted key. Even a simple symmetric transformation like this
* improves security by avoiding IV re-use. IVs will also be fairly unique
@@ -195,7 +195,7 @@ public class KeyProviderCryptoExtension extends
* The generated key material is of the same
* length as the <code>KeyVersion</code> material of the latest key version
* of the key and is encrypted using the same cipher.
- * <p/>
+ * <p>
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param encryptionKeyName
@@ -498,7 +498,7 @@ public class KeyProviderCryptoExtension extends
* and initialization vector. The generated key material is of the same
* length as the <code>KeyVersion</code> material and is encrypted using the
* same cipher.
- * <p/>
+ * <p>
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param encryptionKeyName The latest KeyVersion of this key's material will
@@ -576,7 +576,6 @@ public class KeyProviderCryptoExtension extends
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param ekvs List containing the EncryptedKeyVersion's
- * @return The re-encrypted EncryptedKeyVersion's, in the same order.
* @throws IOException If any EncryptedKeyVersion could not be re-encrypted
* @throws GeneralSecurityException If any EncryptedKeyVersion could not be
* re-encrypted because of a cryptographic issue.
@@ -589,7 +588,7 @@ public class KeyProviderCryptoExtension extends
/**
* Creates a <code>KeyProviderCryptoExtension</code> using a given
* {@link KeyProvider}.
- * <p/>
+ * <p>
* If the given <code>KeyProvider</code> implements the
* {@link CryptoExtension} interface the <code>KeyProvider</code> itself
* will provide the extension functionality.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
index 9212cbc..a63b7d5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
@@ -113,7 +113,7 @@ public class KeyProviderDelegationTokenExtension extends
/**
* Creates a <code>KeyProviderDelegationTokenExtension</code> using a given
* {@link KeyProvider}.
- * <p/>
+ * <p>
* If the given <code>KeyProvider</code> implements the
* {@link DelegationTokenExtension} interface the <code>KeyProvider</code>
* itself will provide the extension functionality, otherwise a default
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 8125510..26b528c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -246,12 +246,12 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
/**
* This provider expects URIs in the following form :
- * kms://<PROTO>@<AUTHORITY>/<PATH>
+ * {@literal kms://<PROTO>@<AUTHORITY>/<PATH>}
*
* where :
* - PROTO = http or https
- * - AUTHORITY = <HOSTS>[:<PORT>]
- * - HOSTS = <HOSTNAME>[;<HOSTS>]
+ * - AUTHORITY = {@literal <HOSTS>[:<PORT>]}
+ * - HOSTS = {@literal <HOSTNAME>[;<HOSTS>]}
* - HOSTNAME = string
* - PORT = integer
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index 1ddd8a3..b2ae084 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -344,7 +344,7 @@ public class ValueQueue <E> {
* <code>SyncGenerationPolicy</code> specified by the user.
* @param keyName String key name
* @param num Minimum number of values to return.
- * @return List<E> values returned
+ * @return {@literal List<E>} values returned
* @throws IOException
* @throws ExecutionException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
index 1219bf9..1863f5e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
@@ -30,16 +30,16 @@ import org.slf4j.LoggerFactory;
/**
* OpenSSL secure random using JNI.
* This implementation is thread-safe.
- * <p/>
+ * <p>
*
* If using an Intel chipset with RDRAND, the high-performance hardware
* random number generator will be used and it's much faster than
* {@link java.security.SecureRandom}. If RDRAND is unavailable, default
* OpenSSL secure random generator will be used. It's still faster
* and can generate strong random bytes.
- * <p/>
- * @see https://wiki.openssl.org/index.php/Random_Numbers
- * @see http://en.wikipedia.org/wiki/RdRand
+ * <p>
+ * See https://wiki.openssl.org/index.php/Random_Numbers
+ * See http://en.wikipedia.org/wiki/RdRand
*/
@InterfaceAudience.Private
public class OpensslSecureRandom extends Random {
@@ -97,7 +97,7 @@ public class OpensslSecureRandom extends Random {
* random bits (right justified, with leading zeros).
*
* @param numBits number of random bits to be generated, where
- * 0 <= <code>numBits</code> <= 32.
+ * 0 {@literal <=} <code>numBits</code> {@literal <=} 32.
*
* @return int an <code>int</code> containing the user-specified number
* of random bits (right justified, with leading zeros).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index cf484ca..721f009 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -336,7 +336,7 @@ public abstract class AbstractFileSystem {
* The default port of this file system.
*
* @return default port of this file system's Uri scheme
- * A uri with a port of -1 => default port;
+ * A uri with a port of -1 => default port;
*/
public abstract int getUriDefaultPort();
@@ -478,9 +478,11 @@ public abstract class AbstractFileSystem {
* through any internal symlinks or mount point
* @param p path to be resolved
* @return fully qualified path
- * @throws FileNotFoundException, AccessControlException, IOException
- * UnresolvedLinkException if symbolic link on path cannot be resolved
- * internally
+ * @throws FileNotFoundException
+ * @throws AccessControlException
+ * @throws IOException
+ * @throws UnresolvedLinkException if symbolic link on path cannot be
+ * resolved internally
*/
public Path resolvePath(final Path p) throws FileNotFoundException,
UnresolvedLinkException, AccessControlException, IOException {
@@ -1021,7 +1023,7 @@ public abstract class AbstractFileSystem {
* changes. (Modifications are merged into the current ACL.)
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications
+ * @param aclSpec List{@literal <AclEntry>} describing modifications
* @throws IOException if an ACL could not be modified
*/
public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
@@ -1035,7 +1037,7 @@ public abstract class AbstractFileSystem {
* retained.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing entries to remove
+ * @param aclSpec List{@literal <AclEntry>} describing entries to remove
* @throws IOException if an ACL could not be modified
*/
public void removeAclEntries(Path path, List<AclEntry> aclSpec)
@@ -1075,8 +1077,9 @@ public abstract class AbstractFileSystem {
* entries.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications, must include entries
- * for user, group, and others for compatibility with permission bits.
+ * @param aclSpec List{@literal <AclEntry>} describing modifications, must
+ * include entries for user, group, and others for compatibility with
+ * permission bits.
* @throws IOException if an ACL could not be modified
*/
public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
@@ -1088,7 +1091,7 @@ public abstract class AbstractFileSystem {
* Gets the ACLs of files and directories.
*
* @param path Path to get
- * @return RemoteIterator<AclStatus> which returns each AclStatus
+ * @return RemoteIterator{@literal <AclStatus>} which returns each AclStatus
* @throws IOException if an ACL could not be read
*/
public AclStatus getAclStatus(Path path) throws IOException {
@@ -1100,7 +1103,7 @@ public abstract class AbstractFileSystem {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -1118,7 +1121,7 @@ public abstract class AbstractFileSystem {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -1137,7 +1140,7 @@ public abstract class AbstractFileSystem {
* Get an xattr for a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
@@ -1154,11 +1157,13 @@ public abstract class AbstractFileSystem {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ *
+ * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ * or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
@@ -1170,12 +1175,13 @@ public abstract class AbstractFileSystem {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ * or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
@@ -1188,11 +1194,12 @@ public abstract class AbstractFileSystem {
* Get all of the xattr names for a file or directory.
* Only the xattr names for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ * or directory
* @throws IOException
*/
public List<String> listXAttrs(Path path)
@@ -1205,7 +1212,7 @@ public abstract class AbstractFileSystem {
* Remove an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
index 2eb8b95..973b136 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
- * A class that optimizes reading from FSInputStream by buffering
+ * A class that optimizes reading from FSInputStream by buffering.
*/
@InterfaceAudience.Private
@@ -44,7 +44,7 @@ implements Seekable, PositionedReadable, HasFileDescriptor {
*
* @param in the underlying input stream.
* @param size the buffer size.
- * @exception IllegalArgumentException if size <= 0.
+ * @exception IllegalArgumentException if size {@literal <=} 0.
*/
public BufferedFSInputStream(FSInputStream in, int size) {
super(in, size);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
index 20f7224..926b554 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
@@ -32,18 +32,18 @@ public interface ByteBufferReadable {
/**
* Reads up to buf.remaining() bytes into buf. Callers should use
* buf.limit(..) to control the size of the desired read.
- * <p/>
+ * <p>
* After a successful call, buf.position() will be advanced by the number
* of bytes read and buf.limit() should be unchanged.
- * <p/>
+ * <p>
* In the case of an exception, the values of buf.position() and buf.limit()
* are undefined, and callers should be prepared to recover from this
* eventuality.
- * <p/>
+ * <p>
* Many implementations will throw {@link UnsupportedOperationException}, so
* callers that are not confident in support for this method from the
* underlying filesystem should be prepared to handle that exception.
- * <p/>
+ * <p>
* Implementations should treat 0-length requests as legitimate, and must not
* signal an error upon their receipt.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 663c910..3db3173 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.util.Progressable;
* Abstract Checksumed FileSystem.
* It provide a basic implementation of a Checksumed FileSystem,
* which creates a checksum file for each raw file.
- * It generates & verifies checksums at the client side.
+ * It generates & verifies checksums at the client side.
*
*****************************************************************/
@InterfaceAudience.Public
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index c56f6e0..663b05d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -42,7 +42,7 @@ import org.slf4j.LoggerFactory;
* Abstract Checksumed Fs.
* It provide a basic implementation of a Checksumed Fs,
* which creates a checksum file for each raw file.
- * It generates & verifies checksums at the client side.
+ * It generates & verifies checksums at the client side.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 1eb27f8..72e5309 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -309,7 +309,7 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
"dr.who";
/**
- * User->groups static mapping to override the groups lookup
+ * User{@literal ->}groups static mapping to override the groups lookup
*/
public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES =
"hadoop.user.group.static.mapping.overrides";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index c3e088b..58b5f70 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* CreateFlag specifies the file create semantic. Users can combine flags like: <br>
* <code>
* EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND)
- * <code>
+ * </code>
* <p>
*
* Use the CreateFlag as follows:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
index 08d71f1..62c45f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs;
import java.io.DataInputStream;
+import java.io.EOFException;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
index 4f06e26..de66eab 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
@@ -101,7 +101,7 @@ abstract public class FSInputChecker extends FSInputStream {
* Implementors should simply pass through to the underlying data stream.
* or
* (b) needChecksum() will return true:
- * - len >= maxChunkSize
+ * - len {@literal >=} maxChunkSize
* - checksum.length is a multiple of CHECKSUM_SIZE
* Implementors should read an integer number of data chunks into
* buf. The amount read should be bounded by len or by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 0b3889b..084f8f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -765,7 +765,7 @@ public class FileContext {
* Make(create) a directory and all the non-existent parents.
*
* @param dir - the dir to make
- * @param permission - permissions is set permission&~umask
+ * @param permission - permissions is set permission{@literal &~}umask
* @param createParent - if true then missing parent dirs are created if false
* then parent must exist
*
@@ -979,7 +979,6 @@ public class FileContext {
/**
* Renames Path src to Path dst
* <ul>
- * <li
* <li>Fails if src is a file and dst is a directory.
* <li>Fails if src is a directory and dst is a file.
* <li>Fails if the parent of dst does not exist or is a file.
@@ -1001,7 +1000,7 @@ public class FileContext {
*
* @throws AccessControlException If access is denied
* @throws FileAlreadyExistsException If <code>dst</code> already exists and
- * <code>options</options> has {@link Options.Rename#OVERWRITE}
+ * <code>options</code> has {@link Options.Rename#OVERWRITE}
* option false.
* @throws FileNotFoundException If <code>src</code> does not exist
* @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
@@ -1250,7 +1249,7 @@ public class FileContext {
* checks to perform. If the requested permissions are granted, then the
* method returns normally. If access is denied, then the method throws an
* {@link AccessControlException}.
- * <p/>
+ * <p>
* The default implementation of this method calls {@link #getFileStatus(Path)}
* and checks the returned permissions against the requested permissions.
* Note that the getFileStatus call will be subject to authorization checks.
@@ -1497,9 +1496,9 @@ public class FileContext {
* <pre>
* Given a path referring to a symlink of form:
*
- * <---X--->
+ * {@literal <---}X{@literal --->}
* fs://host/A/B/link
- * <-----Y----->
+ * {@literal <-----}Y{@literal ----->}
*
* In this path X is the scheme and authority that identify the file system,
* and Y is the path leading up to the final path component "link". If Y is
@@ -1536,7 +1535,7 @@ public class FileContext {
*
*
* @throws AccessControlException If access is denied
- * @throws FileAlreadyExistsException If file <code>linkcode> already exists
+ * @throws FileAlreadyExistsException If file <code>link</code> already exists
* @throws FileNotFoundException If <code>target</code> does not exist
* @throws ParentNotDirectoryException If parent of <code>link</code> is not a
* directory.
@@ -2038,7 +2037,6 @@ public class FileContext {
* <dl>
* <dd>
* <dl>
- * <p>
* <dt> <tt> ? </tt>
* <dd> Matches any single character.
*
@@ -2400,7 +2398,8 @@ public class FileContext {
* changes. (Modifications are merged into the current ACL.)
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications
+ * @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ * modifications
* @throws IOException if an ACL could not be modified
*/
public void modifyAclEntries(final Path path, final List<AclEntry> aclSpec)
@@ -2421,7 +2420,8 @@ public class FileContext {
* retained.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing entries to remove
+ * @param aclSpec List{@literal <}AclEntry{@literal >} describing entries
+ * to remove
* @throws IOException if an ACL could not be modified
*/
public void removeAclEntries(final Path path, final List<AclEntry> aclSpec)
@@ -2481,8 +2481,9 @@ public class FileContext {
* entries.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications, must include entries
- * for user, group, and others for compatibility with permission bits.
+ * @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ * modifications, must include entries for user, group, and others for
+ * compatibility with permission bits.
* @throws IOException if an ACL could not be modified
*/
public void setAcl(Path path, final List<AclEntry> aclSpec)
@@ -2502,7 +2503,8 @@ public class FileContext {
* Gets the ACLs of files and directories.
*
* @param path Path to get
- * @return RemoteIterator<AclStatus> which returns each AclStatus
+ * @return RemoteIterator{@literal <}AclStatus{@literal >} which returns
+ * each AclStatus
* @throws IOException if an ACL could not be read
*/
public AclStatus getAclStatus(Path path) throws IOException {
@@ -2520,7 +2522,7 @@ public class FileContext {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -2538,7 +2540,7 @@ public class FileContext {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -2564,7 +2566,7 @@ public class FileContext {
* Get an xattr for a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
@@ -2587,11 +2589,12 @@ public class FileContext {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ * of the file or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
@@ -2609,12 +2612,13 @@ public class FileContext {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ * of the file or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path, final List<String> names)
@@ -2633,7 +2637,7 @@ public class FileContext {
* Remove an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
@@ -2656,11 +2660,12 @@ public class FileContext {
* Get all of the xattr names for a file or directory.
* Only those xattr names which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return List<String> of the XAttr names of the file or directory
+ * @return List{@literal <}String{@literal >} of the XAttr names of the
+ * file or directory
* @throws IOException
*/
public List<String> listXAttrs(Path path) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c309941..66b6d44 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -684,7 +684,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create a file with the provided permission.
*
* The permission of the file is set to be the provided permission as in
- * setPermission, not permission&~umask
+ * setPermission, not permission{@literal &~}umask
*
* The HDFS implementation is implemented using two RPCs.
* It is understood that it is inefficient,
@@ -709,7 +709,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Create a directory with the provided permission.
* The permission of the directory is set to be the provided permission as in
- * setPermission, not permission&~umask
+ * setPermission, not permission{@literal &~}umask
*
* @see #create(FileSystem, Path, FsPermission)
*
@@ -789,7 +789,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* <pre>
* if f == null :
* result = null
- * elif f.getLen() <= start:
+ * elif f.getLen() {@literal <=} start:
* result = []
* else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
* </pre>
@@ -2017,7 +2017,6 @@ public abstract class FileSystem extends Configured implements Closeable {
* <dl>
* <dd>
* <dl>
- * <p>
* <dt> <tt> ? </tt>
* <dd> Matches any single character.
*
@@ -2916,7 +2915,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* changes. (Modifications are merged into the current ACL.)
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications
+ * @param aclSpec List<AclEntry> describing modifications
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
@@ -3109,7 +3108,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return List<String> of the XAttr names of the file or directory
+ * @return List{@literal <String>} of the XAttr names of the file or directory
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index df89598..5f01d43 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -1474,8 +1474,8 @@ public class FileUtil {
* @param inputClassPath String input classpath to bundle into the jar manifest
* @param pwd Path to working directory to save jar
* @param targetDir path to where the jar execution will have its working dir
- * @param callerEnv Map<String, String> caller's environment variables to use
- * for expansion
+ * @param callerEnv Map {@literal <}String, String{@literal >} caller's
+ * environment variables to use for expansion
* @return String[] with absolute path to new jar in position 0 and
* unexpanded wild card entry path in position 1
* @throws IOException if there is an I/O error while writing the jar file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
index f6ffcb4..f7da819 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
@@ -83,7 +83,7 @@ public class HarFileSystem extends FileSystem {
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
+ * <p>
*
* @return <code>har</code>
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
index 982a0ef..8ceba7b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
@@ -52,18 +52,19 @@ public interface HasEnhancedByteBufferAccess {
* @return
* We will always return an empty buffer if maxLength was 0,
* whether or not we are at EOF.
- * If maxLength > 0, we will return null if the stream has
- * reached EOF.
+ * If maxLength > 0, we will return null if the stream
+ * has reached EOF.
* Otherwise, we will return a ByteBuffer containing at least one
* byte. You must free this ByteBuffer when you are done with it
* by calling releaseBuffer on it. The buffer will continue to be
* readable until it is released in this manner. However, the
* input stream's close method may warn about unclosed buffers.
- * @throws
- * IOException: if there was an error reading.
- * UnsupportedOperationException: if factory was null, and we
- * needed an external byte buffer. UnsupportedOperationException
- * will never be thrown unless the factory argument is null.
+ * @throws IOException if there was an error reading.
+ * @throws UnsupportedOperationException if factory was null,
+ * and we needed an external byte buffer.
+ * @throws UnsupportedOperationException will never be thrown
+ * unless the factory argument is null.
+ *
*/
public ByteBuffer read(ByteBufferPool factory, int maxLength,
EnumSet<ReadOption> opts)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index a4b158a..5f266a7b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -241,9 +241,8 @@ public class LocalDirAllocator {
* @param pathStr the requested file (this will be searched)
* @param conf the Configuration object
* @return true if files exist. false otherwise
- * @throws IOException
*/
- public boolean ifExists(String pathStr,Configuration conf) {
+ public boolean ifExists(String pathStr, Configuration conf) {
AllocatorPerContext context = obtainContext(contextCfgItemName);
return context.ifExists(pathStr, conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
index 538ccdf..c41190a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
@@ -54,7 +54,7 @@ public class LocalFileSystem extends ChecksumFileSystem {
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
+ * <p>
*
* @return <code>file</code>
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 5e93286..75bc12d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -290,7 +290,7 @@ public final class Options {
* @param defaultOpt Default checksum option
* @param userOpt User-specified checksum option. Ignored if null.
* @param userBytesPerChecksum User-specified bytesPerChecksum
- * Ignored if < 0.
+ * Ignored if {@literal <} 0.
*/
public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt,
ChecksumOpt userOpt, int userBytesPerChecksum) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
index 9ad6a28..3472362 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -229,8 +229,8 @@ public class QuotaUsage {
/**
* Output format:
- * <----12----> <----15----> <----15----> <----15----> <-------18------->
- * QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME
+ * |----12----| |----15----| |----15----| |----15----| |-------18-------|
+ * QUOTA REMAINING_QUOTA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME
*/
protected static final String QUOTA_STRING_FORMAT = "%12s %15s ";
protected static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";
@@ -244,9 +244,9 @@ public class QuotaUsage {
/**
* Output format:
- * <----12----> <------15-----> <------15-----> <------15----->
+ * |----12----| |------15-----| |------15-----| |------15-----|
* QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
- * <----12----> <----12----> <-------18------->
+ * |----12----| |----12----| |-------18-------|
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
private static final String STORAGE_TYPE_SUMMARY_FORMAT = "%13s %17s ";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index 644cf4e..676c207 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -76,7 +76,7 @@ public class FTPFileSystem extends FileSystem {
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
+ * <p>
*
* @return <code>ftp</code>
*/
@@ -162,7 +162,7 @@ public class FTPFileSystem extends FileSystem {
/**
* Set FTP's transfer mode based on configuration. Valid values are
* STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.
- * <p/>
+ * <p>
* Defaults to BLOCK_TRANSFER_MODE.
*
* @param conf
@@ -195,7 +195,7 @@ public class FTPFileSystem extends FileSystem {
* Set the FTPClient's data connection mode based on configuration. Valid
* values are ACTIVE_LOCAL_DATA_CONNECTION_MODE,
* PASSIVE_LOCAL_DATA_CONNECTION_MODE and PASSIVE_REMOTE_DATA_CONNECTION_MODE.
- * <p/>
+ * <p>
* Defaults to ACTIVE_LOCAL_DATA_CONNECTION_MODE.
*
* @param client
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
index e59efa5..b522102 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.ftp;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.util.DataChecksum;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
index 0b9e7455..e93858f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.util.DataChecksum;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
index 131aa19..385fed2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
@@ -69,7 +69,7 @@ public class AclStatus {
/**
* Returns the list of all ACL entries, ordered by their natural ordering.
*
- * @return List<AclEntry> unmodifiable ordered list of all ACL entries
+ * @return List<AclEntry> unmodifiable ordered list of all ACL entries
*/
public List<AclEntry> getEntries() {
return entries;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
index 2811a89..4249252 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
@@ -36,8 +36,8 @@ public final class AclUtil {
* Given permissions and extended ACL entries, returns the full logical ACL.
*
* @param perm FsPermission containing permissions
- * @param entries List<AclEntry> containing extended ACL entries
- * @return List<AclEntry> containing full logical ACL
+ * @param entries List<AclEntry> containing extended ACL entries
+ * @return List<AclEntry> containing full logical ACL
*/
public static List<AclEntry> getAclFromPermAndEntries(FsPermission perm,
List<AclEntry> entries) {
@@ -93,8 +93,8 @@ public final class AclUtil {
* Translates the given permission bits to the equivalent minimal ACL.
*
* @param perm FsPermission to translate
- * @return List<AclEntry> containing exactly 3 entries representing the owner,
- * group and other permissions
+ * @return List<AclEntry> containing exactly 3 entries representing the
+ * owner, group and other permissions
*/
public static List<AclEntry> getMinimalAcl(FsPermission perm) {
return Lists.newArrayList(
@@ -119,7 +119,7 @@ public final class AclUtil {
* Checks if the given entries represent a minimal ACL (contains exactly 3
* entries).
*
- * @param entries List<AclEntry> entries to check
+ * @param entries List<AclEntry> entries to check
* @return boolean true if the entries represent a minimal ACL
*/
public static boolean isMinimalAcl(List<AclEntry> entries) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDFS-11396.
TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out.
Contributed by Ayush Saxena.
Posted by sh...@apache.org.
HDFS-11396. TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/605622c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/605622c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/605622c8
Branch: refs/heads/HDFS-12943
Commit: 605622c87bc109f60ee1674be37a526e44723b67
Parents: 6a06bc3
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Oct 9 17:03:21 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Tue Oct 9 17:03:21 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/TestNameNodeMetadataConsistency.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605622c8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
index ff49ddf..2677781 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
import com.google.common.base.Supplier;
@@ -88,6 +89,7 @@ public class TestNameNodeMetadataConsistency {
final long genStamp = block.getGenerationStamp();
final int datanodeIndex = 0;
cluster.changeGenStampOfBlock(datanodeIndex, block, genStamp + 1);
+ DataNodeTestUtils.runDirectoryScanner(cluster.getDataNodes().get(0));
// stop the data node so that it won't remove block
final DataNodeProperties dnProps = cluster.stopDataNode(datanodeIndex);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDFS-13973. getErasureCodingPolicy
should log path in audit event. Contributed by Shweta.
Posted by sh...@apache.org.
HDFS-13973. getErasureCodingPolicy should log path in audit event. Contributed by Shweta.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0118105
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0118105
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0118105
Branch: refs/heads/HDFS-12943
Commit: c0118105b7486793bcaca48786d5f9d3e0ee2ff3
Parents: a3edfdd
Author: Shweta <sh...@cloudera.com>
Authored: Thu Oct 11 20:38:19 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Oct 11 20:39:41 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0118105/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 137dfb9..cc38036 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7728,7 +7728,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return ret;
} finally {
readUnlock(operationName);
- logAuditEvent(success, operationName, null);
+ logAuditEvent(success, operationName, src);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-6989 Ensure timeline service
v2 codebase gets UGI from HttpServletRequest in a consistent way. Contributed
by Abhishek Modi
Posted by sh...@apache.org.
YARN-6989 Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way. Contributed by Abhishek Modi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/045069ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/045069ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/045069ef
Branch: refs/heads/HDFS-12943
Commit: 045069efeca07674be1571252bc4c685aa57b440
Parents: bf3d591
Author: Vrushali C <vr...@apache.org>
Authored: Wed Oct 10 15:17:05 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Wed Oct 10 15:17:05 2018 -0700
----------------------------------------------------------------------
.../reader/TimelineReaderWebServicesUtils.java | 21 ++------------------
...elineReaderWhitelistAuthorizationFilter.java | 2 +-
2 files changed, 3 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/045069ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index 63529a4..bea81c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -269,25 +269,8 @@ public final class TimelineReaderWebServicesUtils {
* @return UGI.
*/
public static UserGroupInformation getUser(HttpServletRequest req) {
- return getCallerUserGroupInformation(req, false);
- }
-
- /**
- * Get UGI from the HTTP request.
- *
- * @param hsr HTTP request.
- * @param usePrincipal if true, use principal name else use remote user name
- * @return UGI.
- */
- public static UserGroupInformation getCallerUserGroupInformation(
- HttpServletRequest hsr, boolean usePrincipal) {
-
- String remoteUser = hsr.getRemoteUser();
- if (usePrincipal) {
- Principal princ = hsr.getUserPrincipal();
- remoteUser = princ == null ? null : princ.getName();
- }
-
+ Principal princ = req.getUserPrincipal();
+ String remoteUser = princ == null ? null : princ.getName();
UserGroupInformation callerUGI = null;
if (remoteUser != null) {
callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/045069ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
index dbe391c..b549778 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
@@ -67,7 +67,7 @@ public class TimelineReaderWhitelistAuthorizationFilter implements Filter {
if (isWhitelistReadAuthEnabled) {
UserGroupInformation callerUGI = TimelineReaderWebServicesUtils
- .getCallerUserGroupInformation(httpRequest, true);
+ .getUser(httpRequest);
if (callerUGI == null) {
String msg = "Unable to obtain user name, user not authenticated";
throw new AuthorizationException(msg);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-8753. [UI2] Lost nodes
representation missing from Nodemanagers Chart. Contributed by Yesha Vora.
Posted by sh...@apache.org.
YARN-8753. [UI2] Lost nodes representation missing from Nodemanagers Chart. Contributed by Yesha Vora.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a379839
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a379839
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a379839
Branch: refs/heads/HDFS-12943
Commit: 8a379839909197690af170afc13c49ef7aa54286
Parents: d3da741
Author: Sunil G <su...@apache.org>
Authored: Thu Oct 11 16:15:25 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Oct 11 16:15:25 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/models/cluster-metric.js | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a379839/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
index 764723a..4075aea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
@@ -97,8 +97,12 @@ export default DS.Model.extend({
label: "Decomissioned",
value: this.get("decommissionedNodes")
});
+ arr.push({
+ label: "Lost",
+ value: this.get("lostNodes")
+ });
return arr;
- }.property("activeNodes", "unhealthyNodes", "decommissioningNodes", "decommissionedNodes"),
+ }.property("activeNodes", "unhealthyNodes", "decommissioningNodes", "decommissionedNodes", "lostNodes"),
getMemoryDataForDonutChart: function() {
var type = "MB";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDFS-13949. Correct the
description of dfs.datanode.disk.check.timeout in hdfs-default.xml.
Contributed by Toshihiro Suzuki.
Posted by sh...@apache.org.
HDFS-13949. Correct the description of dfs.datanode.disk.check.timeout in hdfs-default.xml. Contributed by Toshihiro Suzuki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b39b8024
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b39b8024
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b39b8024
Branch: refs/heads/HDFS-12943
Commit: b39b802422b444629095bde4484b820f02fd6d1d
Parents: edce866
Author: Nanda kumar <na...@apache.org>
Authored: Wed Oct 10 12:29:07 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Oct 10 12:29:07 2018 +0530
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39b8024/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 2ee8399..503ba8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5134,11 +5134,10 @@
<name>dfs.datanode.disk.check.timeout</name>
<value>10m</value>
<description>
- Maximum allowed time for a disk check to complete during DataNode
- startup. If the check does not complete within this time interval
- then the disk is declared as failed. This setting supports
- multiple time unit suffixes as described in dfs.heartbeat.interval.
- If no suffix is specified then milliseconds is assumed.
+ Maximum allowed time for a disk check to complete. If the check does not
+ complete within this time interval then the disk is declared as failed.
+ This setting supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then milliseconds is assumed.
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org