You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2018/10/12 12:28:18 UTC
[01/50] [abbrv] hadoop git commit: HDDS-559. fs.default.name is
deprecated. Contributed by Dinesh Chitlangia. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDFS-13891 4875eea2a -> ebf6bf304 (forced update)
HDDS-559. fs.default.name is deprecated.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a06bc30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a06bc30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a06bc30
Branch: refs/heads/HDFS-13891
Commit: 6a06bc309d72c766694eb6296d5f3fb5c3c597c5
Parents: c1fe657
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 16:57:39 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 16:57:39 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/OzoneFS.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a06bc30/hadoop-ozone/docs/content/OzoneFS.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md
index d0621be..6853992 100644
--- a/hadoop-ozone/docs/content/OzoneFS.md
+++ b/hadoop-ozone/docs/content/OzoneFS.md
@@ -46,7 +46,7 @@ Please add the following entry to the core-site.xml.
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
</property>
<property>
- <name>fs.default.name</name>
+ <name>fs.defaultFS</name>
<value>o3://localhost:9864/volume/bucket</value>
</property>
{{< /highlight >}}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-8834 Provide Java client for
fetching Yarn specific entities from TimelineReader. Contributed by Abhishek
Modi
Posted by br...@apache.org.
YARN-8834 Provide Java client for fetching Yarn specific entities from TimelineReader. Contributed by Abhishek Modi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3edfddc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3edfddc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3edfddc
Branch: refs/heads/HDFS-13891
Commit: a3edfddcf7822ea13bdf4858672eb82cea5e0b5f
Parents: fb18cc5
Author: Vrushali C <vr...@apache.org>
Authored: Thu Oct 11 20:21:00 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Thu Oct 11 20:21:00 2018 -0700
----------------------------------------------------------------------
.../yarn/client/api/TimelineReaderClient.java | 120 ++++++++++
.../api/impl/TimelineReaderClientImpl.java | 239 +++++++++++++++++++
.../api/impl/TestTimelineReaderClientImpl.java | 157 ++++++++++++
3 files changed, 516 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3edfddc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
new file mode 100644
index 0000000..f73c2d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A client library that can be used to get Timeline Entities associated with
+ * application, application attempt or containers. This client library needs to
+ * be used along with time line v.2 server version.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class TimelineReaderClient extends CompositeService {
+
+ /**
+ * Create a new instance of Timeline Reader Client.
+ */
+ @InterfaceAudience.Public
+ public static TimelineReaderClient createTimelineReaderClient() {
+ return new TimelineReaderClientImpl();
+ }
+
+ @InterfaceAudience.Private
+ public TimelineReaderClient(String name) {
+ super(name);
+ }
+
+ /**
+ * Gets application entity.
+ * @param appId application id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @return entity of the application
+ * @throws IOException
+ */
+ public abstract TimelineEntity getApplicationEntity(
+ ApplicationId appId, String fields, Map<String, String> filters)
+ throws IOException;
+
+ /**
+ * Gets application attempt entity.
+ * @param appAttemptId application attempt id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @return entity associated with application attempt
+ * @throws IOException
+ */
+ public abstract TimelineEntity getApplicationAttemptEntity(
+ ApplicationAttemptId appAttemptId, String fields,
+ Map<String, String> filters) throws IOException;
+
+ /**
+ * Gets application attempt entities.
+ * @param appId application id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @param limit Number of entities to return.
+ * @param fromId Retrieve next set of generic ids from given fromId
+ * @return list of application attempt entities
+ * @throws IOException
+ */
+ public abstract List<TimelineEntity> getApplicationAttemptEntities(
+ ApplicationId appId, String fields, Map<String, String> filters,
+ long limit, String fromId) throws IOException;
+
+ /**
+ * Gets Timeline entity for the container.
+ * @param containerId container id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @return timeline entity for container
+ * @throws IOException
+ */
+ public abstract TimelineEntity getContainerEntity(
+ ContainerId containerId, String fields, Map<String, String> filters)
+ throws IOException;
+
+ /**
+ * Gets container entities for an application.
+ * @param appId application id
+ * @param fields Fields to be fetched. Defaults to INFO.
+ * @param filters Filters to be applied while fetching entities.
+ * @param limit Number of entities to return.
+ * @param fromId Retrieve next set of generic ids from given fromId
+ * @return list of entities
+ * @throws IOException
+ */
+ public abstract List<TimelineEntity> getContainerEntities(
+ ApplicationId appId, String fields,
+ Map<String, String> filters,
+ long limit, String fromId) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3edfddc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
new file mode 100644
index 0000000..b138a54
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.api.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.TimelineReaderClient;
+import com.sun.jersey.api.client.ClientResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_APPLICATION_ATTEMPT;
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_CONTAINER;
+import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
+
+/**
+ * Implementation of TimelineReaderClient interface.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TimelineReaderClientImpl extends TimelineReaderClient {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineReaderClientImpl.class);
+
+ private static final String RESOURCE_URI_STR_V2 = "/ws/v2/timeline/";
+
+ private TimelineConnector connector;
+ private URI baseUri;
+ private String clusterId;
+
+ public TimelineReaderClientImpl() {
+ super(TimelineReaderClientImpl.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ throw new IOException("Timeline V2 client is not properly configured. "
+ + "Either timeline service is not enabled or version is not set to"
+ + " 2");
+ }
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation realUgi = ugi.getRealUser();
+ String doAsUser;
+ UserGroupInformation authUgi;
+ if (realUgi != null) {
+ authUgi = realUgi;
+ doAsUser = ugi.getShortUserName();
+ } else {
+ authUgi = ugi;
+ doAsUser = null;
+ }
+ DelegationTokenAuthenticatedURL.Token token =
+ new DelegationTokenAuthenticatedURL.Token();
+ connector = new TimelineConnector(false, authUgi, doAsUser, token);
+ addIfService(connector);
+ String timelineReaderWebAppAddress =
+ WebAppUtils.getTimelineReaderWebAppURLWithoutScheme(conf);
+ baseUri = TimelineConnector.constructResURI(
+ conf, timelineReaderWebAppAddress, RESOURCE_URI_STR_V2);
+ clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public TimelineEntity getApplicationEntity(ApplicationId appId, String fields,
+ Map<String, String> filters)
+ throws IOException {
+ String path = PATH_JOINER.join("clusters", clusterId, "apps", appId);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity entity = response.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ @Override
+ public TimelineEntity getApplicationAttemptEntity(
+ ApplicationAttemptId appAttemptId,
+ String fields, Map<String, String> filters) throws IOException {
+ ApplicationId appId = appAttemptId.getApplicationId();
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_APPLICATION_ATTEMPT, appAttemptId);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity entity = response.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ @Override
+ public List<TimelineEntity> getApplicationAttemptEntities(
+ ApplicationId appId, String fields, Map<String, String> filters,
+ long limit, String fromId) throws IOException {
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_APPLICATION_ATTEMPT);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ if (limit > 0) {
+ params.add("limit", Long.toString(limit));
+ }
+ if (fromId != null && !fromId.isEmpty()) {
+ params.add("fromid", fromId);
+ }
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity[] entities = response.getEntity(TimelineEntity[].class);
+ return Arrays.asList(entities);
+ }
+
+ @Override
+ public TimelineEntity getContainerEntity(ContainerId containerId,
+ String fields, Map<String, String> filters) throws IOException {
+ ApplicationId appId = containerId.getApplicationAttemptId().
+ getApplicationId();
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_CONTAINER, containerId);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity entity = response.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ @Override
+ public List<TimelineEntity> getContainerEntities(
+ ApplicationId appId, String fields,
+ Map<String, String> filters,
+ long limit, String fromId) throws IOException {
+ String path = PATH_JOINER.join("clusters", clusterId, "apps",
+ appId, "entities", YARN_CONTAINER);
+
+ if (fields == null || fields.isEmpty()) {
+ fields = "INFO";
+ }
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", fields);
+ if (limit > 0) {
+ params.add("limit", Long.toString(limit));
+ }
+ if (fromId != null && !fromId.isEmpty()) {
+ params.add("fromid", fromId);
+ }
+ mergeFilters(params, filters);
+
+ ClientResponse response = doGetUri(baseUri, path, params);
+ TimelineEntity[] entity = response.getEntity(TimelineEntity[].class);
+ return Arrays.asList(entity);
+ }
+
+ private void mergeFilters(MultivaluedMap<String, String> defaults,
+ Map<String, String> filters) {
+ if (filters != null && !filters.isEmpty()) {
+ for (Map.Entry<String, String> entry : filters.entrySet()) {
+ if (!defaults.containsKey(entry.getKey())) {
+ defaults.add(entry.getKey(), filters.get(entry.getValue()));
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ protected ClientResponse doGetUri(URI base, String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ ClientResponse resp = connector.getClient().resource(base).path(path)
+ .queryParams(params).accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ if (resp == null ||
+ resp.getStatusInfo().getStatusCode() != ClientResponse.Status.OK
+ .getStatusCode()) {
+ String msg =
+ "Response from the timeline reader server is " +
+ ((resp == null) ? "null" : "not successful," +
+ " HTTP error code: " + resp.getStatus() +
+ ", Server response:\n" + resp.getEntity(String.class));
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ return resp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3edfddc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
new file mode 100644
index 0000000..f668472
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_APPLICATION_ATTEMPT;
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_CONTAINER;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.TimelineReaderClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.MultivaluedMap;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test class for Timeline Reader Client.
+ */
+public class TestTimelineReaderClientImpl {
+
+ private TimelineReaderClient client;
+
+ @Before
+ public void setup() {
+ client = new MockTimelineReaderClient();
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ client.init(conf);
+ client.start();
+ }
+
+ @Test
+ public void testGetApplication() throws Exception {
+ ApplicationId applicationId =
+ ApplicationId.fromString("application_1234_0001");
+ TimelineEntity entity = client.getApplicationEntity(applicationId,
+ null, null);
+ Assert.assertEquals("mockApp1", entity.getId());
+ }
+
+ @Test
+ public void getApplicationAttemptEntity() throws Exception {
+ ApplicationAttemptId attemptId =
+ ApplicationAttemptId.fromString("appattempt_1234_0001_000001");
+ TimelineEntity entity = client.getApplicationAttemptEntity(attemptId,
+ null, null);
+ Assert.assertEquals("mockAppAttempt1", entity.getId());
+ }
+
+ @Test
+ public void getApplicationAttemptEntities() throws Exception {
+ ApplicationId applicationId =
+ ApplicationId.fromString("application_1234_0001");
+ List<TimelineEntity> entities =
+ client.getApplicationAttemptEntities(applicationId, null,
+ null, 0, null);
+ Assert.assertEquals(2, entities.size());
+ Assert.assertEquals("mockAppAttempt2", entities.get(1).getId());
+ }
+
+ @Test
+ public void testGetContainer() throws Exception {
+ ContainerId containerId =
+ ContainerId.fromString("container_1234_0001_01_000001");
+ TimelineEntity entity = client.getContainerEntity(containerId,
+ null, null);
+ Assert.assertEquals("mockContainer1", entity.getId());
+ }
+
+ @Test
+ public void testGetContainers() throws Exception {
+ ApplicationId appId =
+ ApplicationId.fromString("application_1234_0001");
+ List<TimelineEntity> entities = client.getContainerEntities(appId,
+ null, null, 0, null);
+ Assert.assertEquals(2, entities.size());
+ Assert.assertEquals("mockContainer2", entities.get(1).getId());
+ }
+
+ @After
+ public void tearDown() {
+ if (client != null) {
+ client.stop();
+ }
+ }
+
+ private static TimelineEntity createTimelineEntity(String id) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ return entity;
+ }
+
+ private static TimelineEntity[] createTimelineEntities(String... ids) {
+ List<TimelineEntity> entities = new ArrayList<>();
+ for (String id : ids) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ entities.add(entity);
+ }
+ return entities.toArray(new TimelineEntity[entities.size()]);
+ }
+
+ private class MockTimelineReaderClient extends TimelineReaderClientImpl {
+ @Override
+ protected ClientResponse doGetUri(URI base, String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ ClientResponse mockClientResponse = mock(ClientResponse.class);
+ if (path.contains(YARN_CONTAINER.toString())) {
+ when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
+ createTimelineEntity("mockContainer1"));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockContainer1", "mockContainer2"));
+ } else if (path.contains(YARN_APPLICATION_ATTEMPT.toString())) {
+ when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
+ createTimelineEntity("mockAppAttempt1"));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockAppAttempt1", "mockAppAttempt2"));
+ } else {
+ when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
+ createTimelineEntity("mockApp1"));
+ when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+ createTimelineEntities("mockApp1", "mockApp2"));
+ }
+ return mockClientResponse;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-8710. Service AM should set a
finite limit on NM container max retries. Contributed by Suma Shivaprasad
Posted by br...@apache.org.
YARN-8710. Service AM should set a finite limit on NM container max retries. Contributed by Suma Shivaprasad
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aeeb0389
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aeeb0389
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aeeb0389
Branch: refs/heads/HDFS-13891
Commit: aeeb0389a58bf6bb4857bc0f246a63a4bd018d51
Parents: e787d65
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Oct 11 08:48:18 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Thu Oct 11 08:59:17 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/service/conf/YarnServiceConf.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aeeb0389/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 13ed1aa..b9568f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -34,12 +34,13 @@ public class YarnServiceConf {
// Retry settings for container failures
public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
- public static final int DEFAULT_CONTAINER_RETRY_MAX = -1;
+ public static final int DEFAULT_CONTAINER_RETRY_MAX = 10;
public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval-ms";
public static final int DEFAULT_CONTAINER_RETRY_INTERVAL = 30000;
public static final String CONTAINER_FAILURES_VALIDITY_INTERVAL =
"yarn.service.container-failure.validity-interval-ms";
- public static final long DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL = -1;
+ public static final long DEFAULT_CONTAINER_FAILURES_VALIDITY_INTERVAL =
+ 600000;
public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
public static final int DEFAULT_AM_RESTART_MAX = 20;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDFS-13951. HDFS
DelegationTokenFetcher can't print non-HDFS tokens in a tokenfile.
Contributed by Steve Loughran.
Posted by br...@apache.org.
HDFS-13951. HDFS DelegationTokenFetcher can't print non-HDFS tokens in a tokenfile.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2addebb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2addebb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2addebb9
Branch: refs/heads/HDFS-13891
Commit: 2addebb94f592e46b261e2edd9e95a82e83bd761
Parents: 735a36a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 20:03:56 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 20:03:56 2018 +0100
----------------------------------------------------------------------
.../hdfs/tools/DelegationTokenFetcher.java | 58 +++++++++++++++-----
1 file changed, 43 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2addebb9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 7d1e88f..0dc4bcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.tools;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
@@ -36,12 +34,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.GenericOptionsParser;
@@ -64,10 +62,29 @@ public class DelegationTokenFetcher {
private static final String RENEWER = "renewer";
private static final String VERBOSE = "verbose";
/**
- * Command-line interface
+ * Command-line interface.
+ * @param args argument list.
+ * @throws Exception on a failure.
+ * @throws org.apache.hadoop.util.ExitUtil.ExitException if the command
+ * failed and exiting was disabled.
*/
public static void main(final String[] args) throws Exception {
- final Configuration conf = new HdfsConfiguration();
+ main(new HdfsConfiguration(), args);
+ }
+
+ /**
+ * Command line interface with a specific configuration.
+ * Errors in this operation will call {@link ExitUtil#terminate(int)} to
+ * exit the process.
+ * @param conf configuration to create filesystems with.
+ * @param args argument list.
+ * @throws Exception on a failure.
+ * @throws org.apache.hadoop.util.ExitUtil.ExitException if the command
+ * failed and exiting was disabled.
+ */
+ @VisibleForTesting
+ public static void main(Configuration conf, final String[] args)
+ throws Exception {
Options fetcherOptions = new Options();
fetcherOptions
.addOption(WEBSERVICE, true, "HTTP url to reach the NameNode at")
@@ -96,17 +113,19 @@ public class DelegationTokenFetcher {
// check option validity
if (help) {
printUsage(System.out);
- System.exit(0);
+ return;
}
int commandCount = (cancel ? 1 : 0) + (renew ? 1 : 0) + (print ? 1 : 0);
if (commandCount > 1) {
System.err.println("ERROR: Only specify cancel, renew or print.");
printUsage(System.err);
+ return;
}
if (remaining.length != 1 || remaining[0].charAt(0) == '-') {
System.err.println("ERROR: Must specify exactly one token file");
printUsage(System.err);
+ return;
}
// default to using the local file system
FileSystem local = FileSystem.getLocal(conf);
@@ -195,19 +214,22 @@ public class DelegationTokenFetcher {
}
@VisibleForTesting
- static String printTokensToString(
+ public static String printTokensToString(
final Configuration conf,
final Path tokenFile,
final boolean verbose) throws IOException {
StringBuilder sbld = new StringBuilder();
final String nl = System.getProperty("line.separator");
- DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0,
- 0, null).createIdentifier();
for (Token<?> token : readTokens(tokenFile, conf)) {
- DataInputStream in = new DataInputStream(new ByteArrayInputStream(token
- .getIdentifier()));
- id.readFields(in);
- String idStr = (verbose? id.toString() : id.toStringStable());
+ TokenIdentifier tokenId = token.decodeIdentifier();
+
+ String idStr;
+ if (tokenId instanceof DelegationTokenIdentifier) {
+ DelegationTokenIdentifier id = (DelegationTokenIdentifier) tokenId;
+ idStr = (verbose? id.toString() : id.toStringStable());
+ } else {
+ idStr = tokenId.toString();
+ }
sbld
.append("Token (").append(idStr)
.append(") for ").append(token.getService()).append(nl);
@@ -222,13 +244,19 @@ public class DelegationTokenFetcher {
System.out.print(printTokensToString(conf, tokenFile, verbose));
}
+ /**
+ * Print usage to the error stream, then
+ * call {@link ExitUtil#terminate(int)} with status code 1.
+ * This will exit or raise an exception if that's been disabled.
+ * @param err stream for the messages.
+ */
private static void printUsage(PrintStream err) {
err.println("fetchdt retrieves delegation tokens from the NameNode");
err.println();
err.println("fetchdt <opts> <token file>");
err.println("Options:");
- err.println(" --webservice <url> Url to contact NN on (starts with " +
- "http:// or https://)");
+ err.println(" --webservice <url> URL to contact NN on (starts with " +
+ "http:// or https://), or other filesystem URL");
err.println(" --renewer <name> Name of the delegation token renewer");
err.println(" --cancel Cancel the delegation token");
err.println(" --renew Renew the delegation token. " +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDDS-601. On restart,
SCM throws 'No such datanode' exception.
Posted by br...@apache.org.
HDDS-601. On restart, SCM throws 'No such datanode' exception.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb34b5f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb34b5f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb34b5f8
Branch: refs/heads/HDFS-13891
Commit: eb34b5f8aff538e90c7820c0114322d2dd75f6e6
Parents: 2addebb
Author: Hanisha Koneru <ha...@apache.org>
Authored: Thu Oct 11 13:20:32 2018 -0700
Committer: Hanisha Koneru <ha...@apache.org>
Committed: Thu Oct 11 13:20:32 2018 -0700
----------------------------------------------------------------------
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../protocol/StorageContainerNodeProtocol.java | 8 ++
.../hadoop/hdds/scm/node/NodeManager.java | 7 ++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 26 +++--
.../server/SCMDatanodeHeartbeatDispatcher.java | 107 ++++++++++++-------
.../hdds/scm/container/MockNodeManager.java | 11 ++
.../hadoop/hdds/scm/node/TestNodeManager.java | 53 ---------
.../TestSCMDatanodeHeartbeatDispatcher.java | 41 ++++++-
.../testutils/ReplicationNodeManagerMock.java | 11 ++
9 files changed, 164 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index 690aa01..acd4af9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -42,7 +42,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.Future;
/**
- * Register a container with SCM.
+ * Register a datanode with SCM.
*/
public final class RegisterEndpointTask implements
Callable<EndpointStateMachine.EndPointStates> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
index b3c3eb3..b5d75ef 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
@@ -67,4 +67,12 @@ public interface StorageContainerNodeProtocol {
*/
List<SCMCommand> processHeartbeat(DatanodeDetails datanodeDetails);
+ /**
+ * Check if node is registered or not.
+ * Return true if Node is registered and false otherwise.
+ * @param datanodeDetails - Datanode ID.
+ * @return true if Node is registered, false otherwise
+ */
+ Boolean isNodeRegistered(DatanodeDetails datanodeDetails);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 585d545..5f6a2e4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -196,4 +196,11 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* @param dnUuid datanode uuid.
*/
void processDeadNode(UUID dnUuid);
+
+ /**
+ * Get list of SCMCommands in the Command Queue for a particular Datanode.
+ * @param dnID - Datanode uuid.
+ * @return list of commands
+ */
+ List<SCMCommand> getCommandQueue(UUID dnID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 4a34136..acec6aa 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
-import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.slf4j.Logger;
@@ -196,7 +195,7 @@ public class SCMNodeManager
try {
stat = nodeStateManager.getNodeStat(dnId);
} catch (NodeNotFoundException e) {
- LOG.debug("SCM updateNodeStat based on heartbeat from previous" +
+ LOG.debug("SCM updateNodeStat based on heartbeat from previous " +
"dead datanode {}", dnId);
stat = new SCMNodeStat();
}
@@ -277,7 +276,7 @@ public class SCMNodeManager
nodeStateManager.setNodeStat(dnId, new SCMNodeStat());
// Updating Node Report, as registration is successful
updateNodeStat(datanodeDetails.getUuid(), nodeReport);
- LOG.info("Data node with ID: {} Registered.", datanodeDetails.getUuid());
+ LOG.info("Registered Data node : {}", datanodeDetails);
} catch (NodeAlreadyExistsException e) {
LOG.trace("Datanode is already registered. Datanode: {}",
datanodeDetails.toString());
@@ -304,14 +303,22 @@ public class SCMNodeManager
try {
nodeStateManager.updateLastHeartbeatTime(datanodeDetails);
} catch (NodeNotFoundException e) {
- LOG.warn("SCM receive heartbeat from unregistered datanode {}",
- datanodeDetails);
- commandQueue.addCommand(datanodeDetails.getUuid(),
- new ReregisterCommand());
+ LOG.error("SCM trying to process heartbeat from an " +
+ "unregistered node {}. Ignoring the heartbeat.", datanodeDetails);
}
return commandQueue.getCommand(datanodeDetails.getUuid());
}
+ @Override
+ public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) {
+ try {
+ nodeStateManager.getNode(datanodeDetails);
+ return true;
+ } catch (NodeNotFoundException e) {
+ return false;
+ }
+ }
+
/**
* Process node report.
*
@@ -487,4 +494,9 @@ public class SCMNodeManager
+ " doesn't exist or decommissioned already.", dnUuid);
}
}
+
+ @Override
+ public List<SCMCommand> getCommandQueue(UUID dnID) {
+ return commandQueue.getCommand(dnID);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index d9a0875..7c7df27 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -35,13 +35,15 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.UUID;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_ACTIONS;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
@@ -82,51 +84,78 @@ public final class SCMDatanodeHeartbeatDispatcher {
public List<SCMCommand> dispatch(SCMHeartbeatRequestProto heartbeat) {
DatanodeDetails datanodeDetails =
DatanodeDetails.getFromProtoBuf(heartbeat.getDatanodeDetails());
- // should we dispatch heartbeat through eventPublisher?
- List<SCMCommand> commands = nodeManager.processHeartbeat(datanodeDetails);
- if (heartbeat.hasNodeReport()) {
- LOG.debug("Dispatching Node Report.");
- eventPublisher.fireEvent(NODE_REPORT,
- new NodeReportFromDatanode(datanodeDetails,
- heartbeat.getNodeReport()));
- }
+ List<SCMCommand> commands;
+
+ // If node is not registered, ask the node to re-register. Do not process
+ // Heartbeat for unregistered nodes.
+ if (!nodeManager.isNodeRegistered(datanodeDetails)) {
+ LOG.info("SCM received heartbeat from an unregistered datanode {}. " +
+ "Asking datanode to re-register.", datanodeDetails);
+ UUID dnID = datanodeDetails.getUuid();
+ nodeManager.addDatanodeCommand(dnID, new ReregisterCommand());
+
+ commands = nodeManager.getCommandQueue(dnID);
+
+ } else {
+
+ // should we dispatch heartbeat through eventPublisher?
+ commands = nodeManager.processHeartbeat(datanodeDetails);
+ if (heartbeat.hasNodeReport()) {
+ LOG.debug("Dispatching Node Report.");
+ eventPublisher.fireEvent(
+ NODE_REPORT,
+ new NodeReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getNodeReport()));
+ }
- if (heartbeat.hasContainerReport()) {
- LOG.debug("Dispatching Container Report.");
- eventPublisher.fireEvent(CONTAINER_REPORT,
- new ContainerReportFromDatanode(datanodeDetails,
- heartbeat.getContainerReport()));
+ if (heartbeat.hasContainerReport()) {
+ LOG.debug("Dispatching Container Report.");
+ eventPublisher.fireEvent(
+ CONTAINER_REPORT,
+ new ContainerReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getContainerReport()));
- }
+ }
- if (heartbeat.hasContainerActions()) {
- LOG.debug("Dispatching Container Actions.");
- eventPublisher.fireEvent(CONTAINER_ACTIONS,
- new ContainerActionsFromDatanode(datanodeDetails,
- heartbeat.getContainerActions()));
- }
+ if (heartbeat.hasContainerActions()) {
+ LOG.debug("Dispatching Container Actions.");
+ eventPublisher.fireEvent(
+ CONTAINER_ACTIONS,
+ new ContainerActionsFromDatanode(
+ datanodeDetails,
+ heartbeat.getContainerActions()));
+ }
- if (heartbeat.hasPipelineReports()) {
- LOG.debug("Dispatching Pipeline Report.");
- eventPublisher.fireEvent(PIPELINE_REPORT,
- new PipelineReportFromDatanode(datanodeDetails,
- heartbeat.getPipelineReports()));
+ if (heartbeat.hasPipelineReports()) {
+ LOG.debug("Dispatching Pipeline Report.");
+ eventPublisher.fireEvent(
+ PIPELINE_REPORT,
+ new PipelineReportFromDatanode(
+ datanodeDetails,
+ heartbeat.getPipelineReports()));
- }
+ }
- if (heartbeat.hasPipelineActions()) {
- LOG.debug("Dispatching Pipeline Actions.");
- eventPublisher.fireEvent(PIPELINE_ACTIONS,
- new PipelineActionsFromDatanode(datanodeDetails,
- heartbeat.getPipelineActions()));
- }
+ if (heartbeat.hasPipelineActions()) {
+ LOG.debug("Dispatching Pipeline Actions.");
+ eventPublisher.fireEvent(
+ PIPELINE_ACTIONS,
+ new PipelineActionsFromDatanode(
+ datanodeDetails,
+ heartbeat.getPipelineActions()));
+ }
- if (heartbeat.getCommandStatusReportsCount() != 0) {
- for (CommandStatusReportsProto commandStatusReport : heartbeat
- .getCommandStatusReportsList()) {
- eventPublisher.fireEvent(CMD_STATUS_REPORT,
- new CommandStatusReportFromDatanode(datanodeDetails,
- commandStatusReport));
+ if (heartbeat.getCommandStatusReportsCount() != 0) {
+ for (CommandStatusReportsProto commandStatusReport : heartbeat
+ .getCommandStatusReportsList()) {
+ eventPublisher.fireEvent(
+ CMD_STATUS_REPORT,
+ new CommandStatusReportFromDatanode(
+ datanodeDetails,
+ commandStatusReport));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index e32f316..f4ce102 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -408,6 +408,12 @@ public class MockNodeManager implements NodeManager {
}
@Override
+ public Boolean isNodeRegistered(
+ DatanodeDetails datanodeDetails) {
+ return null;
+ }
+
+ @Override
public Map<String, Integer> getNodeCount() {
Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) {
@@ -470,6 +476,11 @@ public class MockNodeManager implements NodeManager {
}
}
+ @Override
+ public List<SCMCommand> getCommandQueue(UUID dnID) {
+ return null;
+ }
+
/**
* A class to declare some values for the nodes so that our tests
* won't fail.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index 32ee371..ed95709 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -17,11 +17,8 @@
*/
package org.apache.hadoop.hdds.scm.node;
-import com.google.common.base.Supplier;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
@@ -214,56 +211,6 @@ public class TestNodeManager {
}
/**
- * Asserts scm informs datanodes to re-register with the nodemanager
- * on a restart.
- *
- * @throws Exception
- */
- @Test
- public void testScmHeartbeatAfterRestart() throws Exception {
- OzoneConfiguration conf = getConf();
- conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
- 100, TimeUnit.MILLISECONDS);
- DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- UUID dnId = datanodeDetails.getUuid();
- String storagePath = testDir.getAbsolutePath() + "/" + dnId;
- StorageReportProto report =
- TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
- try (SCMNodeManager nodemanager = createNodeManager(conf)) {
- nodemanager.register(datanodeDetails,
- TestUtils.createNodeReport(report),
- TestUtils.getRandomPipelineReports());
- List<SCMCommand> command = nodemanager.processHeartbeat(datanodeDetails);
- Assert.assertTrue(nodemanager.getAllNodes().contains(datanodeDetails));
- Assert.assertTrue("On regular HB calls, SCM responses a "
- + "datanode with an empty command list", command.isEmpty());
- }
-
- // Sends heartbeat without registering to SCM.
- // This happens when SCM restarts.
- try (SCMNodeManager nodemanager = createNodeManager(conf)) {
- Assert.assertFalse(nodemanager
- .getAllNodes().contains(datanodeDetails));
- try {
- // SCM handles heartbeat asynchronously.
- // It may need more than one heartbeat processing to
- // send the notification.
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- List<SCMCommand> command =
- nodemanager.processHeartbeat(datanodeDetails);
- return command.size() == 1 && command.get(0).getType()
- .equals(SCMCommandProto.Type.reregisterCommand);
- }
- }, 100, 3 * 1000);
- } catch (TimeoutException e) {
- Assert.fail("Times out to verify that scm informs "
- + "datanode to re-register itself.");
- }
- }
- }
-
- /**
* Asserts that we detect as many healthy nodes as we have generated heartbeat
* for.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
index f3cd4ea..eac8c90 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.server;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.UUID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.
@@ -39,6 +40,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.NodeReportFromDatanode;
import org.apache.hadoop.hdds.server.events.Event;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.junit.Assert;
import org.junit.Test;
@@ -61,8 +63,12 @@ public class TestSCMDatanodeHeartbeatDispatcher {
NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
+ NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
+ Mockito.when(mockNodeManager.isNodeRegistered(Mockito.any()))
+ .thenReturn(true);
+
SCMDatanodeHeartbeatDispatcher dispatcher =
- new SCMDatanodeHeartbeatDispatcher(Mockito.mock(NodeManager.class),
+ new SCMDatanodeHeartbeatDispatcher(mockNodeManager,
new EventPublisher() {
@Override
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
@@ -99,8 +105,13 @@ public class TestSCMDatanodeHeartbeatDispatcher {
CommandStatusReportsProto commandStatusReport =
CommandStatusReportsProto.getDefaultInstance();
+ NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
+ Mockito.when(mockNodeManager.isNodeRegistered(Mockito.any()))
+ .thenReturn(true);
+
SCMDatanodeHeartbeatDispatcher dispatcher =
- new SCMDatanodeHeartbeatDispatcher(Mockito.mock(NodeManager.class),
+ new SCMDatanodeHeartbeatDispatcher(
+ mockNodeManager,
new EventPublisher() {
@Override
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
@@ -135,4 +146,30 @@ public class TestSCMDatanodeHeartbeatDispatcher {
}
+ /**
+ * Asserts scm informs datanodes to re-register on a restart.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testScmHeartbeatAfterRestart() throws Exception {
+
+ NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
+ SCMDatanodeHeartbeatDispatcher dispatcher =
+ new SCMDatanodeHeartbeatDispatcher(
+ mockNodeManager, Mockito.mock(EventPublisher.class));
+
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
+
+ SCMHeartbeatRequestProto heartbeat =
+ SCMHeartbeatRequestProto.newBuilder()
+ .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
+ .build();
+
+ dispatcher.dispatch(heartbeat);
+ // If SCM receives heartbeat from a node after it restarts and the node
+ // is not registered, it should send a Re-Register command back to the node.
+ Mockito.verify(mockNodeManager, Mockito.times(1)).addDatanodeCommand(
+ Mockito.any(UUID.class), Mockito.any(ReregisterCommand.class));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb34b5f8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 6584aec..069f1af 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -295,6 +295,12 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
+ @Override
+ public Boolean isNodeRegistered(
+ DatanodeDetails datanodeDetails) {
+ return null;
+ }
+
/**
* Clears all nodes from the node Manager.
*/
@@ -341,4 +347,9 @@ public class ReplicationNodeManagerMock implements NodeManager {
public void processDeadNode(UUID dnUuid) {
// do nothing.
}
+
+ @Override
+ public List<SCMCommand> getCommandQueue(UUID dnID) {
+ return null;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-3879 [Storage implementation]
Create HDFS backing storage implementation for ATS reads. Contributed by
Abhishek Modi.
Posted by br...@apache.org.
YARN-3879 [Storage implementation] Create HDFS backing storage implementation for ATS reads. Contributed by Abhishek Modi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bca928d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bca928d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bca928d3
Branch: refs/heads/HDFS-13891
Commit: bca928d3c7b88f39e9bc1784889596f0b00964d4
Parents: 74db993
Author: Vrushali C <vr...@apache.org>
Authored: Thu Oct 11 21:13:52 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Thu Oct 11 21:14:06 2018 -0700
----------------------------------------------------------------------
.../storage/FileSystemTimelineReaderImpl.java | 87 +++++++++++---------
1 file changed, 50 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bca928d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index adb8821..ef08a9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -41,6 +40,11 @@ import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@@ -68,7 +72,9 @@ public class FileSystemTimelineReaderImpl extends AbstractService
private static final Logger LOG =
LoggerFactory.getLogger(FileSystemTimelineReaderImpl.class);
- private String rootPath;
+ private FileSystem fs;
+ private Path rootPath;
+ private Path entitiesPath;
private static final String ENTITIES_DIR = "entities";
/** Default extension for output files. */
@@ -94,7 +100,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
@VisibleForTesting
String getRootPath() {
- return rootPath;
+ return rootPath.toString();
}
private static ObjectMapper mapper;
@@ -162,12 +168,12 @@ public class FileSystemTimelineReaderImpl extends AbstractService
if (clusterId == null || appId == null) {
throw new IOException("Unable to get flow info");
}
- String appFlowMappingFile = rootPath + File.separator + ENTITIES_DIR +
- File.separator + clusterId + File.separator + APP_FLOW_MAPPING_FILE;
+ Path clusterIdPath = new Path(entitiesPath, clusterId);
+ Path appFlowMappingFilePath = new Path(clusterIdPath,
+ APP_FLOW_MAPPING_FILE);
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(
- new FileInputStream(
- appFlowMappingFile), Charset.forName("UTF-8")));
+ fs.open(appFlowMappingFilePath), Charset.forName("UTF-8")));
CSVParser parser = new CSVParser(reader, csvFormat)) {
for (CSVRecord record : parser.getRecords()) {
if (record.size() < 4) {
@@ -266,7 +272,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
return entity;
}
- private Set<TimelineEntity> getEntities(File dir, String entityType,
+ private Set<TimelineEntity> getEntities(Path dir, String entityType,
TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
throws IOException {
// First sort the selected entities based on created/start time.
@@ -280,15 +286,18 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
);
if (dir != null) {
- File[] files = dir.listFiles();
- if (files != null) {
- for (File entityFile : files) {
+ RemoteIterator<LocatedFileStatus> fileStatuses = fs.listFiles(dir,
+ false);
+ if (fileStatuses != null) {
+ while (fileStatuses.hasNext()) {
+ LocatedFileStatus locatedFileStatus = fileStatuses.next();
+ Path entityFile = locatedFileStatus.getPath();
if (!entityFile.getName()
.contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
continue;
}
try (BufferedReader reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(entityFile),
+ new InputStreamReader(fs.open(entityFile),
Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
if (!entity.getType().equals(entityType)) {
@@ -366,25 +375,30 @@ public class FileSystemTimelineReaderImpl extends AbstractService
@Override
public void serviceInit(Configuration conf) throws Exception {
- rootPath = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ String outputRoot = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
conf.get("hadoop.tmp.dir") + File.separator + STORAGE_DIR_ROOT);
+ rootPath = new Path(outputRoot);
+ entitiesPath = new Path(rootPath, ENTITIES_DIR);
+ fs = rootPath.getFileSystem(conf);
super.serviceInit(conf);
}
@Override
public TimelineEntity getEntity(TimelineReaderContext context,
TimelineDataToRetrieve dataToRetrieve) throws IOException {
- String flowRunPath = getFlowRunPath(context.getUserId(),
+ String flowRunPathStr = getFlowRunPath(context.getUserId(),
context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId());
- File dir = new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + File.separator + flowRunPath + File.separator +
- context.getAppId() + File.separator + context.getEntityType());
- File entityFile = new File(
- dir, context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
+ Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+ Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+ Path appIdPath = new Path(flowRunPath, context.getAppId());
+ Path entityTypePath = new Path(appIdPath, context.getEntityType());
+ Path entityFilePath = new Path(entityTypePath,
+ context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
+
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(
- new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
+ fs.open(entityFilePath), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
return createEntityToBeReturned(
entity, dataToRetrieve.getFieldsToRetrieve());
@@ -399,32 +413,31 @@ public class FileSystemTimelineReaderImpl extends AbstractService
public Set<TimelineEntity> getEntities(TimelineReaderContext context,
TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
throws IOException {
- String flowRunPath = getFlowRunPath(context.getUserId(),
+ String flowRunPathStr = getFlowRunPath(context.getUserId(),
context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId());
- File dir =
- new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + File.separator + flowRunPath +
- File.separator + context.getAppId() + File.separator +
- context.getEntityType());
- return getEntities(dir, context.getEntityType(), filters, dataToRetrieve);
+ Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+ Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+ Path appIdPath = new Path(flowRunPath, context.getAppId());
+ Path entityTypePath = new Path(appIdPath, context.getEntityType());
+
+ return getEntities(entityTypePath, context.getEntityType(), filters,
+ dataToRetrieve);
}
@Override public Set<String> getEntityTypes(TimelineReaderContext context)
throws IOException {
Set<String> result = new TreeSet<>();
- String flowRunPath = getFlowRunPath(context.getUserId(),
+ String flowRunPathStr = getFlowRunPath(context.getUserId(),
context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId());
- File dir = new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + File.separator + flowRunPath
- + File.separator + context.getAppId());
- File[] fileList = dir.listFiles();
- if (fileList != null) {
- for (File f : fileList) {
- if (f.isDirectory()) {
- result.add(f.getName());
- }
+ Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+ Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+ Path appIdPath = new Path(flowRunPath, context.getAppId());
+ FileStatus[] fileStatuses = fs.listStatus(appIdPath);
+ for (FileStatus fileStatus : fileStatuses) {
+ if (fileStatus.isDirectory()) {
+ result.add(fileStatus.getPath().getName());
}
}
return result;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-13882. Set a maximum delay
for retrying locateFollowingBlock. Contributed by Kitti Nanasi.
Posted by br...@apache.org.
HDFS-13882. Set a maximum delay for retrying locateFollowingBlock. Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10185d9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10185d9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10185d9a
Branch: refs/heads/HDFS-13891
Commit: 10185d9a77ce07080588f3c77399a07cd7ccf427
Parents: 90552b1
Author: Kitti Nanasi <kn...@cloudera.com>
Authored: Wed Oct 10 08:54:50 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Oct 10 08:55:32 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 21 ++++++++-
.../hdfs/client/HdfsClientConfigKeys.java | 3 ++
.../hadoop/hdfs/client/impl/DfsClientConf.java | 8 ++++
.../src/main/resources/hdfs-default.xml | 13 +++++-
.../hadoop/hdfs/TestDFSClientRetries.java | 47 +++++++++++++++-----
.../hadoop/tools/TestHdfsConfigFields.java | 1 +
6 files changed, 80 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index e977054..b8aae97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -942,6 +942,7 @@ public class DFSOutputStream extends FSOutputSummer
long localstart = Time.monotonicNow();
final DfsClientConf conf = dfsClient.getConf();
long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs();
+ long maxSleepTime = conf.getBlockWriteLocateFollowingMaxDelayMs();
boolean fileComplete = false;
int retries = conf.getNumBlockWriteLocateFollowingRetry();
while (!fileComplete) {
@@ -965,7 +966,7 @@ public class DFSOutputStream extends FSOutputSummer
}
retries--;
Thread.sleep(sleeptime);
- sleeptime *= 2;
+ sleeptime = calculateDelayForNextRetry(sleeptime, maxSleepTime);
if (Time.monotonicNow() - localstart > 5000) {
DFSClient.LOG.info("Could not complete " + src + " retrying...");
}
@@ -1075,6 +1076,7 @@ public class DFSOutputStream extends FSOutputSummer
final DfsClientConf conf = dfsClient.getConf();
int retries = conf.getNumBlockWriteLocateFollowingRetry();
long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs();
+ long maxSleepTime = conf.getBlockWriteLocateFollowingMaxDelayMs();
long localstart = Time.monotonicNow();
while (true) {
try {
@@ -1106,7 +1108,7 @@ public class DFSOutputStream extends FSOutputSummer
LOG.warn("NotReplicatedYetException sleeping " + src
+ " retries left " + retries);
Thread.sleep(sleeptime);
- sleeptime *= 2;
+ sleeptime = calculateDelayForNextRetry(sleeptime, maxSleepTime);
} catch (InterruptedException ie) {
LOG.warn("Caught exception", ie);
}
@@ -1117,4 +1119,19 @@ public class DFSOutputStream extends FSOutputSummer
}
}
}
+
+ /**
+ * Calculates the delay for the next retry.
+ *
+ * The delay is increased exponentially until the maximum delay is reached.
+ *
+ * @param previousDelay delay for the previous retry
+ * @param maxDelay maximum delay
+ * @return the minimum of the double of the previous delay
+ * and the maximum delay
+ */
+ private static long calculateDelayForNextRetry(long previousDelay,
+ long maxDelay) {
+ return Math.min(previousDelay * 2, maxDelay);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index a812670..b1ce78d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -322,6 +322,9 @@ public interface HdfsClientConfigKeys {
String LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY =
PREFIX + "locateFollowingBlock.initial.delay.ms";
int LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_DEFAULT = 400;
+ String LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_KEY =
+ PREFIX + "locateFollowingBlock.max.delay.ms";
+ int LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_DEFAULT = 60000;
interface ReplaceDatanodeOnFailure {
String PREFIX = BlockWrite.PREFIX + "replace-datanode-on-failure.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index e63e3f5..4110f94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -122,6 +122,7 @@ public class DfsClientConf {
private final int numBlockWriteRetry;
private final int numBlockWriteLocateFollowingRetry;
private final int blockWriteLocateFollowingInitialDelayMs;
+ private final int blockWriteLocateFollowingMaxDelayMs;
private final long defaultBlockSize;
private final long prefetchSize;
private final short defaultReplication;
@@ -237,6 +238,9 @@ public class DfsClientConf {
blockWriteLocateFollowingInitialDelayMs = conf.getInt(
BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY,
BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_DEFAULT);
+ blockWriteLocateFollowingMaxDelayMs = conf.getInt(
+ BlockWrite.LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_KEY,
+ BlockWrite.LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_DEFAULT);
uMask = FsPermission.getUMask(conf);
connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
@@ -349,6 +353,10 @@ public class DfsClientConf {
return blockWriteLocateFollowingInitialDelayMs;
}
+ public int getBlockWriteLocateFollowingMaxDelayMs() {
+ return blockWriteLocateFollowingMaxDelayMs;
+ }
+
/**
* @return the hdfsTimeout
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 503ba8f..cb770f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3135,11 +3135,22 @@
<name>dfs.client.block.write.locateFollowingBlock.initial.delay.ms</name>
<value>400</value>
<description>The initial delay (unit is ms) for locateFollowingBlock,
- the delay time will increase exponentially(double) for each retry.
+ the delay time will increase exponentially(double) for each retry
+ until dfs.client.block.write.locateFollowingBlock.max.delay.ms is reached,
+ after that the delay for each retry will be
+ dfs.client.block.write.locateFollowingBlock.max.delay.ms.
</description>
</property>
<property>
+ <name>dfs.client.block.write.locateFollowingBlock.max.delay.ms</name>
+ <value>60000</value>
+ <description>
+ The maximum delay (unit is ms) before retrying locateFollowingBlock.
+ </description>
+</property>
+
+<property>
<name>dfs.ha.zkfc.nn.http.timeout.ms</name>
<value>20000</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 14d0ee0..49de91d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -1229,27 +1229,54 @@ public class TestDFSClientRetries {
}
}
+ /**
+ * Tests default configuration values and configuration setting
+ * of locate following block delays and number of retries.
+ *
+ * Configuration values tested:
+ * - dfs.client.block.write.locateFollowingBlock.initial.delay.ms
+ * - dfs.client.block.write.locateFollowingBlock.max.delay.ms
+ * - dfs.client.block.write.locateFollowingBlock.retries
+ */
@Test
- public void testDFSClientConfigurationLocateFollowingBlockInitialDelay()
+ public void testDFSClientConfigurationLocateFollowingBlock()
throws Exception {
- // test if HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY
- // is not configured, verify DFSClient uses the default value 400.
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ final int initialDelayTestValue = 1000;
+ final int maxDelayTestValue = 35000;
+ final int retryTestValue = 7;
+
+ final int defaultInitialDelay = 400;
+ final int defaultMaxDelay = 60000;
+ final int defultRetry = 5;
+
try {
cluster.waitActive();
NamenodeProtocols nn = cluster.getNameNodeRpc();
DFSClient client = new DFSClient(null, nn, conf, null);
- assertEquals(client.getConf().
- getBlockWriteLocateFollowingInitialDelayMs(), 400);
+ assertEquals(defaultInitialDelay, client.getConf().
+ getBlockWriteLocateFollowingInitialDelayMs());
+ assertEquals(defaultMaxDelay, client.getConf().
+ getBlockWriteLocateFollowingMaxDelayMs());
+ assertEquals(defultRetry, client.getConf().
+ getNumBlockWriteLocateFollowingRetry());
- // change HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY,
- // verify DFSClient uses the configured value 1000.
conf.setInt(
HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY,
- 1000);
+ initialDelayTestValue);
+ conf.setInt(
+ HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_MAX_DELAY_MS_KEY,
+ maxDelayTestValue);
+ conf.setInt(
+ HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_RETRIES_KEY,
+ retryTestValue);
client = new DFSClient(null, nn, conf, null);
- assertEquals(client.getConf().
- getBlockWriteLocateFollowingInitialDelayMs(), 1000);
+ assertEquals(initialDelayTestValue, client.getConf().
+ getBlockWriteLocateFollowingInitialDelayMs());
+ assertEquals(maxDelayTestValue, client.getConf().
+ getBlockWriteLocateFollowingMaxDelayMs());
+ assertEquals(retryTestValue, client.getConf().
+ getNumBlockWriteLocateFollowingRetry());
} finally {
cluster.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10185d9a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 5cae2fc..9fd505f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -43,6 +43,7 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
configurationClasses = new Class[] { HdfsClientConfigKeys.class,
HdfsClientConfigKeys.Failover.class,
HdfsClientConfigKeys.StripedRead.class, DFSConfigKeys.class,
+ HdfsClientConfigKeys.BlockWrite.class,
HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.class };
// Set error modes
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDDS-628. Fix outdated names used
in HDDS documentations.
Posted by br...@apache.org.
HDDS-628. Fix outdated names used in HDDS documentations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5da04222
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5da04222
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5da04222
Branch: refs/heads/HDFS-13891
Commit: 5da042227cfce440eddc263b377a70ed5b3743fb
Parents: bca928d
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Oct 12 14:00:13 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Oct 12 14:00:13 2018 +0800
----------------------------------------------------------------------
hadoop-ozone/docs/content/Dozone.md | 4 ++--
hadoop-ozone/docs/content/JavaApi.md | 2 +-
hadoop-ozone/docs/content/OzoneManager.md | 2 +-
hadoop-ozone/docs/content/Settings.md | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5da04222/hadoop-ozone/docs/content/Dozone.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/Dozone.md b/hadoop-ozone/docs/content/Dozone.md
index 7906cf3..f6efb0f 100644
--- a/hadoop-ozone/docs/content/Dozone.md
+++ b/hadoop-ozone/docs/content/Dozone.md
@@ -63,14 +63,14 @@ Useful Docker & Ozone Commands
If you make any modifications to ozone, the simplest way to test it is to run freon and unit tests.
-Here are the instructions to run corona in a docker based cluster.
+Here are the instructions to run freon in a docker based cluster.
{{< highlight bash >}}
docker-compose exec datanode bash
{{< /highlight >}}
This will open a bash shell on the data node container.
-Now we can execute corona for load generation.
+Now we can execute freon for load generation.
{{< highlight bash >}}
ozone freon randomkeys --numOfVolumes=10 --numOfBuckets 10 --numOfKeys 10
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5da04222/hadoop-ozone/docs/content/JavaApi.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/JavaApi.md b/hadoop-ozone/docs/content/JavaApi.md
index 1d32bed..e538f4b 100644
--- a/hadoop-ozone/docs/content/JavaApi.md
+++ b/hadoop-ozone/docs/content/JavaApi.md
@@ -42,7 +42,7 @@ can use
OzoneClient ozClient = OzoneClientFactory.getRestClient();
{{< /highlight >}}
-And to get a a RPC client we can call
+And to get a RPC client we can call
{{< highlight java >}}
OzoneClient ozClient = OzoneClientFactory.getRpcClient();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5da04222/hadoop-ozone/docs/content/OzoneManager.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/OzoneManager.md b/hadoop-ozone/docs/content/OzoneManager.md
index 560f827..5eb8663 100644
--- a/hadoop-ozone/docs/content/OzoneManager.md
+++ b/hadoop-ozone/docs/content/OzoneManager.md
@@ -70,7 +70,7 @@ We are hopeful that this leads to a more straightforward way of discovering sett
OM and SCM
-------------------
[Storage container manager]({{< ref "Hdds.md" >}}) or (SCM) is the block manager
- for ozone. When a client requests OM for a set of data nodes to write data, OM talk to SCM and gets a block.
+ for ozone. When a client requests OM for a set of data nodes to write data, OM talks to SCM and gets a block.
A block returned by SCM contains a pipeline, which is a set of nodes that we participate in that block replication.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5da04222/hadoop-ozone/docs/content/Settings.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/Settings.md b/hadoop-ozone/docs/content/Settings.md
index 41ab04a..5c9bb41 100644
--- a/hadoop-ozone/docs/content/Settings.md
+++ b/hadoop-ozone/docs/content/Settings.md
@@ -43,7 +43,7 @@ requests blocks from SCM, to which clients can write data.
## Setting up an Ozone only cluster
-* Please untar the ozone-0.2.1-SNAPSHOT to the directory where you are going
+* Please untar the ozone-<version> to the directory where you are going
to run Ozone from. We need Ozone jars on all machines in the cluster. So you
need to do this on all machines in the cluster.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HDFS-13945.
TestDataNodeVolumeFailure is Flaky. Contributed by Ayush Saxena.
Posted by br...@apache.org.
HDFS-13945. TestDataNodeVolumeFailure is Flaky. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e0e6daa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e0e6daa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e0e6daa
Branch: refs/heads/HDFS-13891
Commit: 6e0e6daaf3215213c32cdffa79f3730d40e981ea
Parents: de44e10
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Oct 12 17:34:10 2018 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Fri Oct 12 17:34:10 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0e6daa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
index 16c0cfa..7d04942 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
@@ -424,10 +424,8 @@ public class TestDataNodeVolumeFailure {
@Override
public Boolean get() {
// underReplicatedBlocks are due to failed volumes
- int underReplicatedBlocks = BlockManagerTestUtil
- .checkHeartbeatAndGetUnderReplicatedBlocksCount(
- cluster.getNamesystem(), bm);
-
+ long underReplicatedBlocks = bm.getLowRedundancyBlocksCount()
+ + bm.getPendingReconstructionBlocksCount();
if (underReplicatedBlocks > 0) {
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-15676. Cleanup
TestSSLHttpServer. Contributed by Szilard Nemeth.
Posted by br...@apache.org.
HADOOP-15676. Cleanup TestSSLHttpServer. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64f2b32d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64f2b32d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64f2b32d
Branch: refs/heads/HDFS-13891
Commit: 64f2b32d57f35864b5c47b7e80f02e9c939f592a
Parents: c05b260
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Oct 11 15:08:22 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Oct 11 15:12:36 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/http/TestSSLHttpServer.java | 204 ++++++++++---------
1 file changed, 103 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64f2b32d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
index 2166464..3f6ee7b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
@@ -24,8 +24,9 @@ import java.io.InputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
-import java.net.UnknownHostException;
+import java.security.GeneralSecurityException;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLHandshakeException;
@@ -34,6 +35,7 @@ import javax.net.ssl.SSLSocketFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.http.TestHttpServer.EchoServlet;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
@@ -60,23 +62,34 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
LoggerFactory.getLogger(TestSSLHttpServer.class);
private static final String HTTPS_CIPHER_SUITES_KEY = "https.cipherSuites";
private static final String JAVAX_NET_DEBUG_KEY = "javax.net.debug";
- private static Configuration conf;
+ private static final String SSL_SERVER_KEYSTORE_PROP_PREFIX = "ssl.server" +
+ ".keystore";
+ private static final String SSL_SERVER_TRUSTSTORE_PROP_PREFIX = "ssl.server" +
+ ".truststore";
+
+ private static final String SERVLET_NAME_LONGHEADER = "longheader";
+ private static final String SERVLET_PATH_LONGHEADER =
+ "/" + SERVLET_NAME_LONGHEADER;
+ private static final String SERVLET_NAME_ECHO = "echo";
+ private static final String SERVLET_PATH_ECHO = "/" + SERVLET_NAME_ECHO;
+
private static HttpServer2 server;
- private static String keystoresDir;
+ private static String keystoreDir;
private static String sslConfDir;
private static SSLFactory clientSslFactory;
private static String cipherSuitesPropertyValue;
private static String sslDebugPropertyValue;
- private static final String excludeCiphers = "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ private static final String EXCLUDED_CIPHERS =
+ "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, \n"
+ "SSL_RSA_WITH_DES_CBC_SHA,"
+ "SSL_DHE_RSA_WITH_DES_CBC_SHA, "
+ "SSL_RSA_EXPORT_WITH_RC4_40_MD5,\t \n"
+ "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ "SSL_RSA_WITH_RC4_128_MD5 \t";
- private static final String oneEnabledCiphers = excludeCiphers
+ private static final String ONE_ENABLED_CIPHERS = EXCLUDED_CIPHERS
+ ",TLS_RSA_WITH_AES_128_CBC_SHA";
- private static final String exclusiveEnabledCiphers
+ private static final String EXCLUSIVE_ENABLED_CIPHERS
= "\tTLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, \n"
+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_RSA_WITH_AES_128_CBC_SHA,"
@@ -90,49 +103,54 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
turnOnSSLDebugLogging();
storeHttpsCipherSuites();
- conf = new Configuration();
+ Configuration conf = new Configuration();
conf.setInt(HttpServer2.HTTP_MAX_THREADS_KEY, 10);
File base = new File(BASEDIR);
FileUtil.fullyDelete(base);
base.mkdirs();
- keystoresDir = new File(BASEDIR).getAbsolutePath();
+ keystoreDir = new File(BASEDIR).getAbsolutePath();
sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class);
- KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false, true,
- excludeCiphers);
+ KeyStoreTestUtil.setupSSLConfig(keystoreDir, sslConfDir, conf, false, true,
+ EXCLUDED_CIPHERS);
Configuration sslConf = KeyStoreTestUtil.getSslConfig();
clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, sslConf);
clientSslFactory.init();
- server = new HttpServer2.Builder()
- .setName("test")
- .addEndpoint(new URI("https://localhost"))
- .setConf(conf)
- .keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
- .keyStore(sslConf.get("ssl.server.keystore.location"),
- sslConf.get("ssl.server.keystore.password"),
- sslConf.get("ssl.server.keystore.type", "jks"))
- .trustStore(sslConf.get("ssl.server.truststore.location"),
- sslConf.get("ssl.server.truststore.password"),
- sslConf.get("ssl.server.truststore.type", "jks"))
- .excludeCiphers(
- sslConf.get("ssl.server.exclude.cipher.list")).build();
- server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
- server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
- server.start();
+ setupServer(conf, sslConf);
baseUrl = new URL("https://"
+ NetUtils.getHostPortString(server.getConnectorAddress(0)));
LOG.info("HTTP server started: " + baseUrl);
}
+ private static void setupServer(Configuration conf, Configuration sslConf)
+ throws IOException, URISyntaxException {
+ server = new HttpServer2.Builder().setName("test")
+ .addEndpoint(new URI("https://localhost")).setConf(conf)
+ .keyPassword(
+ sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".keypassword"))
+ .keyStore(sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".location"),
+ sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".password"),
+ sslConf.get(SSL_SERVER_KEYSTORE_PROP_PREFIX + ".type", "jks"))
+ .trustStore(
+ sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".location"),
+ sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".password"),
+ sslConf.get(SSL_SERVER_TRUSTSTORE_PROP_PREFIX + ".type", "jks"))
+ .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")).build();
+ server.addServlet(SERVLET_NAME_ECHO, SERVLET_PATH_ECHO, EchoServlet.class);
+ server.addServlet(SERVLET_NAME_LONGHEADER, SERVLET_PATH_LONGHEADER,
+ LongHeaderServlet.class);
+ server.start();
+ }
+
@AfterClass
public static void cleanup() throws Exception {
server.stop();
FileUtil.fullyDelete(new File(BASEDIR));
- KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
+ KeyStoreTestUtil.cleanupSSLConfig(keystoreDir, sslConfDir);
clientSslFactory.destroy();
restoreHttpsCipherSuites();
restoreSSLDebugLogging();
@@ -181,11 +199,27 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
}
}
+ private HttpsURLConnection getConnectionWithSSLSocketFactory(URL url,
+ String ciphers) throws IOException, GeneralSecurityException {
+ HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
+ SSLSocketFactory sslSocketFactory = clientSslFactory
+ .createSSLSocketFactory();
+ LOG.info("Creating " +
+ PreferredCipherSSLSocketFactory.class.getCanonicalName() +
+ " with ciphers: " + ciphers);
+ PreferredCipherSSLSocketFactory cipherSSLSocketFactory
+ = new PreferredCipherSSLSocketFactory(sslSocketFactory,
+ StringUtils.getTrimmedStrings(ciphers));
+ conn.setSSLSocketFactory(cipherSSLSocketFactory);
+ return conn;
+ }
+
@Test
public void testEcho() throws Exception {
- assertEquals("a:b\nc:d\n", readOut(new URL(baseUrl, "/echo?a=b&c=d")));
- assertEquals("a:b\nc<:d\ne:>\n", readOut(new URL(baseUrl,
- "/echo?a=b&c<=d&e=>")));
+ assertEquals("a:b\nc:d\n",
+ readFromURL(new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d")));
+ assertEquals("a:b\nc<:d\ne:>\n",
+ readFromURL(new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c<=d&e=>")));
}
/**
@@ -195,15 +229,20 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testLongHeader() throws Exception {
- URL url = new URL(baseUrl, "/longheader");
+ URL url = new URL(baseUrl, SERVLET_PATH_LONGHEADER);
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());
testLongHeader(conn);
}
- private static String readOut(URL url) throws Exception {
+ private static String readFromURL(URL url) throws Exception {
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());
+ return readFromConnection(conn);
+ }
+
+ private static String readFromConnection(HttpsURLConnection conn)
+ throws IOException {
InputStream in = conn.getInputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copyBytes(in, out, 1024);
@@ -218,21 +257,15 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testExcludedCiphers() throws Exception {
- URL url = new URL(baseUrl, "/echo?a=b&c=d");
- HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
- SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
- PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
- = new PrefferedCipherSSLSocketFactory(sslSocketF,
- StringUtils.getTrimmedStrings(excludeCiphers));
- conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
- assertFalse("excludedCipher list is empty", excludeCiphers.isEmpty());
+ URL url = new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d");
+ HttpsURLConnection conn = getConnectionWithSSLSocketFactory(url,
+ EXCLUDED_CIPHERS);
+ assertFalse("excludedCipher list is empty", EXCLUDED_CIPHERS.isEmpty());
try {
- InputStream in = conn.getInputStream();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(in, out, 1024);
+ readFromConnection(conn);
fail("No Ciphers in common, SSLHandshake must fail.");
} catch (SSLHandshakeException ex) {
- LOG.info("No Ciphers in common, expected succesful test result.", ex);
+ LOG.info("No Ciphers in common, expected successful test result.", ex);
}
}
@@ -242,25 +275,7 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testOneEnabledCiphers() throws Exception {
- URL url = new URL(baseUrl, "/echo?a=b&c=d");
- HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
- SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
- PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
- = new PrefferedCipherSSLSocketFactory(sslSocketF,
- StringUtils.getTrimmedStrings(oneEnabledCiphers));
- conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
- assertFalse("excludedCipher list is empty", oneEnabledCiphers.isEmpty());
- try {
- InputStream in = conn.getInputStream();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(in, out, 1024);
- assertEquals(out.toString(), "a:b\nc:d\n");
- LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
- + " expected successful test result.");
- } catch (SSLHandshakeException ex) {
- fail("Atleast one additional cipher available for successful handshake."
- + " Unexpected test failure: " + ex);
- }
+ testEnabledCiphers(ONE_ENABLED_CIPHERS);
}
/** Test verifies that mutually exclusive server's disabled cipher suites and
@@ -268,33 +283,25 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
*/
@Test
public void testExclusiveEnabledCiphers() throws Exception {
- URL url = new URL(baseUrl, "/echo?a=b&c=d");
- HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
- SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
- PrefferedCipherSSLSocketFactory testPreferredCipherSSLSocketF
- = new PrefferedCipherSSLSocketFactory(sslSocketF,
- StringUtils.getTrimmedStrings(exclusiveEnabledCiphers));
- conn.setSSLSocketFactory(testPreferredCipherSSLSocketF);
- assertFalse("excludedCipher list is empty",
- exclusiveEnabledCiphers.isEmpty());
- try {
- InputStream in = conn.getInputStream();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(in, out, 1024);
- assertEquals(out.toString(), "a:b\nc:d\n");
- LOG.info("Atleast one additional enabled cipher than excluded ciphers,"
- + " expected successful test result.");
- } catch (SSLHandshakeException ex) {
- fail("Atleast one additional cipher available for successful handshake."
- + " Unexpected test failure: " + ex);
- }
+ testEnabledCiphers(EXCLUSIVE_ENABLED_CIPHERS);
}
- private class PrefferedCipherSSLSocketFactory extends SSLSocketFactory {
+ private void testEnabledCiphers(String ciphers) throws
+ IOException, GeneralSecurityException {
+ URL url = new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d");
+ HttpsURLConnection conn = getConnectionWithSSLSocketFactory(url, ciphers);
+ assertFalse("excludedCipher list is empty", ciphers.isEmpty());
+ String out = readFromConnection(conn);
+ assertEquals(out, "a:b\nc:d\n");
+ LOG.info("At least one additional enabled cipher than excluded ciphers,"
+ + " expected successful test result.");
+ }
+
+ private class PreferredCipherSSLSocketFactory extends SSLSocketFactory {
private final SSLSocketFactory delegateSocketFactory;
private final String[] enabledCipherSuites;
- public PrefferedCipherSSLSocketFactory(SSLSocketFactory sslSocketFactory,
+ PreferredCipherSSLSocketFactory(SSLSocketFactory sslSocketFactory,
String[] pEnabledCipherSuites) {
delegateSocketFactory = sslSocketFactory;
if (null != pEnabledCipherSuites && pEnabledCipherSuites.length > 0) {
@@ -319,31 +326,24 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
socket, string, i, bln);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@Override
- public Socket createSocket(String string, int i) throws IOException,
- UnknownHostException {
+ public Socket createSocket(String string, int i) throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
string, i);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@Override
public Socket createSocket(String string, int i, InetAddress ia, int i1)
- throws IOException, UnknownHostException {
+ throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(
string, i, ia, i1);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@@ -351,9 +351,7 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
public Socket createSocket(InetAddress ia, int i) throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
i);
- if (null != enabledCipherSuites) {
- sslSocket.setEnabledCipherSuites(enabledCipherSuites);
- }
+ setEnabledCipherSuites(sslSocket);
return sslSocket;
}
@@ -362,10 +360,14 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
throws IOException {
SSLSocket sslSocket = (SSLSocket) delegateSocketFactory.createSocket(ia,
i, ia1, i1);
+ setEnabledCipherSuites(sslSocket);
+ return sslSocket;
+ }
+
+ private void setEnabledCipherSuites(SSLSocket sslSocket) {
if (null != enabledCipherSuites) {
sslSocket.setEnabledCipherSuites(enabledCipherSuites);
}
- return sslSocket;
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HADOOP-15785. [JDK10] Javadoc
build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Posted by br...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
index a16f439..a67cafe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/ScopedAclEntries.java
@@ -42,7 +42,7 @@ public final class ScopedAclEntries {
* list is already sorted such that all access entries precede all default
* entries.
*
- * @param aclEntries List<AclEntry> to separate
+ * @param aclEntries List<AclEntry> to separate
*/
public ScopedAclEntries(List<AclEntry> aclEntries) {
int pivot = calculatePivotOnDefaultEntries(aclEntries);
@@ -59,8 +59,8 @@ public final class ScopedAclEntries {
/**
* Returns access entries.
*
- * @return List<AclEntry> containing just access entries, or an empty list if
- * there are no access entries
+ * @return List<AclEntry> containing just access entries, or an empty
+ * list if there are no access entries
*/
public List<AclEntry> getAccessEntries() {
return accessEntries;
@@ -69,8 +69,8 @@ public final class ScopedAclEntries {
/**
* Returns default entries.
*
- * @return List<AclEntry> containing just default entries, or an empty list if
- * there are no default entries
+ * @return List<AclEntry> containing just default entries, or an empty
+ * list if there are no default entries
*/
public List<AclEntry> getDefaultEntries() {
return defaultEntries;
@@ -78,8 +78,8 @@ public final class ScopedAclEntries {
/**
* Returns the pivot point in the list between the access entries and the
- * default entries. This is the index of the first element in the list that is
- * a default entry.
+ * default entries. This is the index of the first element in the list that
+ * is a default entry.
*
* @param aclBuilder ArrayList<AclEntry> containing entries to build
* @return int pivot point, or -1 if list contains no default entries
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index a4746cf..3eef278 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -148,16 +148,16 @@ abstract public class Command extends Configured {
* expand arguments, and then process each argument.
* <pre>
* run
- * |-> {@link #processOptions(LinkedList)}
- * \-> {@link #processRawArguments(LinkedList)}
- * |-> {@link #expandArguments(LinkedList)}
- * | \-> {@link #expandArgument(String)}*
- * \-> {@link #processArguments(LinkedList)}
- * |-> {@link #processArgument(PathData)}*
- * | |-> {@link #processPathArgument(PathData)}
- * | \-> {@link #processPaths(PathData, PathData...)}
- * | \-> {@link #processPath(PathData)}*
- * \-> {@link #processNonexistentPath(PathData)}
+ * |{@literal ->} {@link #processOptions(LinkedList)}
+ * \{@literal ->} {@link #processRawArguments(LinkedList)}
+ * |{@literal ->} {@link #expandArguments(LinkedList)}
+ * | \{@literal ->} {@link #expandArgument(String)}*
+ * \{@literal ->} {@link #processArguments(LinkedList)}
+ * |{@literal ->} {@link #processArgument(PathData)}*
+ * | |{@literal ->} {@link #processPathArgument(PathData)}
+ * | \{@literal ->} {@link #processPaths(PathData, PathData...)}
+ * | \{@literal ->} {@link #processPath(PathData)}*
+ * \{@literal ->} {@link #processNonexistentPath(PathData)}
* </pre>
* Most commands will chose to implement just
* {@link #processOptions(LinkedList)} and {@link #processPath(PathData)}
@@ -292,8 +292,8 @@ abstract public class Command extends Configured {
/**
* This is the last chance to modify an argument before going into the
* (possibly) recursive {@link #processPaths(PathData, PathData...)}
- * -> {@link #processPath(PathData)} loop. Ex. ls and du use this to
- * expand out directories.
+ * {@literal ->} {@link #processPath(PathData)} loop. Ex. ls and du use
+ * this to expand out directories.
* @param item a {@link PathData} representing a path which exists
* @throws IOException if anything goes wrong...
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
index bf30b22..4dd20d1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
@@ -162,7 +162,7 @@ public class CommandFormat {
/** Returns all the options that are set
*
- * @return Set<String> of the enabled options
+ * @return Set{@literal <}String{@literal >} of the enabled options
*/
public Set<String> getOpts() {
Set<String> optSet = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 463b14c..8366eac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -75,16 +75,16 @@ import org.apache.hadoop.util.Time;
* one or more individual file systems (a localFs or Hdfs, S3fs, etc).
* For example one could have a mount table that provides links such as
* <ul>
- * <li> /user -> hdfs://nnContainingUserDir/user
- * <li> /project/foo -> hdfs://nnProject1/projects/foo
- * <li> /project/bar -> hdfs://nnProject2/projects/bar
- * <li> /tmp -> hdfs://nnTmp/privateTmpForUserXXX
+ * <li> /user {@literal ->} hdfs://nnContainingUserDir/user
+ * <li> /project/foo {@literal ->} hdfs://nnProject1/projects/foo
+ * <li> /project/bar {@literal ->} hdfs://nnProject2/projects/bar
+ * <li> /tmp {@literal ->} hdfs://nnTmp/privateTmpForUserXXX
* </ul>
*
* ViewFs is specified with the following URI: <b>viewfs:///</b>
* <p>
* To use viewfs one would typically set the default file system in the
- * config (i.e. fs.defaultFS < = viewfs:///) along with the
+ * config (i.e. fs.defaultFS {@literal <} = viewfs:///) along with the
* mount table config variables as described below.
*
* <p>
@@ -132,7 +132,7 @@ import org.apache.hadoop.util.Time;
* (because they do not fit on one) then one could specify a mount
* entry such as following merges two dirs:
* <ul>
- * <li> /user -> hdfs://nnUser1/user,hdfs://nnUser2/user
+ * <li> /user {@literal ->} hdfs://nnUser1/user,hdfs://nnUser2/user
* </ul>
* Such a mergeLink can be specified with the following config var where ","
* is used as the separator for each of links to be merged:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index d099ca7..12de2ef 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -54,10 +54,10 @@ import org.slf4j.LoggerFactory;
* Apache Zookeeper. Using Zookeeper as a coordination service, leader election
* can be performed by atomically creating an ephemeral lock file (znode) on
* Zookeeper. The service instance that successfully creates the znode becomes
- * active and the rest become standbys. <br/>
+ * active and the rest become standbys. <br>
* This election mechanism is only efficient for small number of election
* candidates (order of 10's) because contention on single znode by a large
- * number of candidates can result in Zookeeper overload. <br/>
+ * number of candidates can result in Zookeeper overload. <br>
* The elector does not guarantee fencing (protection of shared resources) among
* service instances. After it has notified an instance about becoming a leader,
* then that instance must ensure that it meets the service consistency
@@ -70,10 +70,10 @@ import org.slf4j.LoggerFactory;
public class ActiveStandbyElector implements StatCallback, StringCallback {
/**
- * Callback interface to interact with the ActiveStandbyElector object. <br/>
+ * Callback interface to interact with the ActiveStandbyElector object. <br>
* The application will be notified with a callback only on state changes
* (i.e. there will never be successive calls to becomeActive without an
- * intermediate call to enterNeutralMode). <br/>
+ * intermediate call to enterNeutralMode). <br>
* The callbacks will be running on Zookeeper client library threads. The
* application should return from these callbacks quickly so as not to impede
* Zookeeper client library performance and notifications. The app will
@@ -105,7 +105,7 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
* interface. The service may choose to ignore this or stop doing state
* changing operations. Upon reconnection, the elector verifies the leader
* status and calls back on the becomeActive and becomeStandby app
- * interfaces. <br/>
+ * interfaces. <br>
* Zookeeper disconnects can happen due to network issues or loss of
* Zookeeper quorum. Thus enterNeutralMode can be used to guard against
* split-brain issues. In such situations it might be prudent to call
@@ -178,12 +178,12 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
private ZooKeeper monitorLockNodeClient;
/**
- * Create a new ActiveStandbyElector object <br/>
+ * Create a new ActiveStandbyElector object <br>
* The elector is created by providing to it the Zookeeper configuration, the
* parent znode under which to create the znode and a reference to the
- * callback interface. <br/>
+ * callback interface. <br>
* The parent znode name must be the same for all service instances and
- * different across services. <br/>
+ * different across services. <br>
* After the leader has been lost, a new leader will be elected after the
* session timeout expires. Hence, the app must set this parameter based on
* its needs for failure response time. The session timeout must be greater
@@ -217,12 +217,12 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
}
/**
- * Create a new ActiveStandbyElector object <br/>
+ * Create a new ActiveStandbyElector object <br>
* The elector is created by providing to it the Zookeeper configuration, the
* parent znode under which to create the znode and a reference to the
- * callback interface. <br/>
+ * callback interface. <br>
* The parent znode name must be the same for all service instances and
- * different across services. <br/>
+ * different across services. <br>
* After the leader has been lost, a new leader will be elected after the
* session timeout expires. Hence, the app must set this parameter based on
* its needs for failure response time. The session timeout must be greater
@@ -278,9 +278,9 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
/**
* To participate in election, the app will call joinElection. The result will
* be notified by a callback on either the becomeActive or becomeStandby app
- * interfaces. <br/>
+ * interfaces. <br>
* After this the elector will automatically monitor the leader status and
- * perform re-election if necessary<br/>
+ * perform re-election if necessary<br>
* The app could potentially start off in standby mode and ignore the
* becomeStandby call.
*
@@ -397,11 +397,11 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
/**
* Any service instance can drop out of the election by calling quitElection.
- * <br/>
+ * <br>
* This will lose any leader status, if held, and stop monitoring of the lock
- * node. <br/>
+ * node. <br>
* If the instance wants to participate in election again, then it needs to
- * call joinElection(). <br/>
+ * call joinElection(). <br>
* This allows service instances to take themselves out of rotation for known
* impending unavailable states (e.g. long GC pause or software upgrade).
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index e955979..2107bf1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -372,7 +372,7 @@ public abstract class HAAdmin extends Configured implements Tool {
/**
* Return the serviceId as is, we are assuming it was
- * given as a service address of form <host:ipcport>.
+ * given as a service address of form {@literal <}host:ipcport{@literal >}.
*/
protected String getServiceAddr(String serviceId) {
return serviceId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
index 2247a34..64e7315 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
@@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
* <code>com.example.foo.MyMethod</code>
* The class provided must implement the {@link FenceMethod} interface.
* The fencing methods that ship with Hadoop may also be referred to
- * by shortened names:<p>
+ * by shortened names:<br>
* <ul>
* <li><code>shell(/path/to/some/script.sh args...)</code></li>
* <li><code>sshfence(...)</code> (see {@link SshFenceByTcpPort})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
index 9ae113b..e0c2f4d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
@@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory;
* with ssh.
* <p>
* In order to achieve passwordless SSH, the operator must also configure
- * <code>dfs.ha.fencing.ssh.private-key-files<code> to point to an
+ * <code>dfs.ha.fencing.ssh.private-key-files</code> to point to an
* SSH key that has passphrase-less access to the given username and host.
*/
public class SshFenceByTcpPort extends Configured
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index d2ba469..a9c2319 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -107,9 +107,9 @@ import org.slf4j.LoggerFactory;
/**
* Create a Jetty embedded server to answer http requests. The primary goal is
* to serve up status information for the server. There are three contexts:
- * "/logs/" -> points to the log directory "/static/" -> points to common static
- * files (src/webapps/static) "/" -> the jsp server code from
- * (src/webapps/<name>)
+ * "/logs/" {@literal ->} points to the log directory "/static/" {@literal ->}
+ * points to common static files (src/webapps/static) "/" {@literal ->} the
+ * jsp server code from (src/webapps/{@literal <}name{@literal >})
*
* This class is a fork of the old HttpServer. HttpServer exists for
* compatibility reasons. See HBASE-10336 for more details.
@@ -1364,10 +1364,10 @@ public final class HttpServer2 implements FilterContainer {
/**
* Checks the user has privileges to access to instrumentation servlets.
- * <p/>
+ * <p>
* If <code>hadoop.security.instrumentation.requires.admin</code> is set to FALSE
* (default value) it always returns TRUE.
- * <p/>
+ * <p>
* If <code>hadoop.security.instrumentation.requires.admin</code> is set to TRUE
* it will check that if the current user is in the admin ACLS. If the user is
* in the admin ACLs it returns TRUE, otherwise it returns FALSE.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
index dc430cc..be86159 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
@@ -83,7 +83,7 @@ public class EnumSetWritable<E extends Enum<E>> extends AbstractCollection<E>
/**
* reset the EnumSetWritable with specified
- * <tt>value</value> and <tt>elementType</tt>. If the <tt>value</tt> argument
+ * <tt>value</tt> and <tt>elementType</tt>. If the <tt>value</tt> argument
* is null or its size is zero, the <tt>elementType</tt> argument must not be
* null. If the argument <tt>value</tt>'s size is bigger than zero, the
* argument <tt>elementType</tt> is not be used.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
index 3708a3b..5bbfba3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
@@ -370,7 +370,7 @@ public class IOUtils {
}
/**
- * Return the complete list of files in a directory as strings.<p/>
+ * Return the complete list of files in a directory as strings.<p>
*
* This is better than File#listDir because it does not ignore IOExceptions.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
index 7cd7f98..804d365 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
@@ -80,7 +80,7 @@ public class ReadaheadPool {
* @param readaheadLength the configured length to read ahead
* @param maxOffsetToRead the maximum offset that will be readahead
* (useful if, for example, only some segment of the file is
- * requested by the user). Pass {@link Long.MAX_VALUE} to allow
+ * requested by the user). Pass {@link Long#MAX_VALUE} to allow
* readahead to the end of the file.
* @param lastReadahead the result returned by the previous invocation
* of this function on this file descriptor, or null if this is
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
index 252ee4c..9d3c3c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
@@ -37,7 +37,7 @@ import com.google.common.annotations.VisibleForTesting;
/**
* This class provides secure APIs for opening and creating files on the local
* disk. The main issue this class tries to handle is that of symlink traversal.
- * <br/>
+ * <br>
* An example of such an attack is:
* <ol>
* <li> Malicious user removes his task's syslog file, and puts a link to the
@@ -50,7 +50,7 @@ import com.google.common.annotations.VisibleForTesting;
* </ol>
* A similar attack is possible involving task log truncation, but in that case
* due to an insecure write to a file.
- * <br/>
+ * <br>
*/
public class SecureIOUtils {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index f42848b..9afa621 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -79,7 +79,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* values.
* </li>
* <li>
- * <code>BlockCompressWriter</code> : Block-compressed files, both keys &
+ * <code>BlockCompressWriter</code> : Block-compressed files, both keys &
* values are collected in 'blocks'
* separately and compressed. The size of
* the 'block' is configurable.
@@ -94,13 +94,13 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* <p>The {@link SequenceFile.Reader} acts as the bridge and can read any of the
* above <code>SequenceFile</code> formats.</p>
*
- * <h4 id="Formats">SequenceFile Formats</h4>
+ * <h3 id="Formats">SequenceFile Formats</h3>
*
* <p>Essentially there are 3 different formats for <code>SequenceFile</code>s
* depending on the <code>CompressionType</code> specified. All of them share a
* <a href="#Header">common header</a> described below.
*
- * <h5 id="Header">SequenceFile Header</h5>
+ * <h4 id="Header">SequenceFile Header</h4>
* <ul>
* <li>
* version - 3 bytes of magic header <b>SEQ</b>, followed by 1 byte of actual
@@ -133,7 +133,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </li>
* </ul>
*
- * <h5 id="#UncompressedFormat">Uncompressed SequenceFile Format</h5>
+ * <h5>Uncompressed SequenceFile Format</h5>
* <ul>
* <li>
* <a href="#Header">Header</a>
@@ -152,7 +152,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </li>
* </ul>
*
- * <h5 id="#RecordCompressedFormat">Record-Compressed SequenceFile Format</h5>
+ * <h5>Record-Compressed SequenceFile Format</h5>
* <ul>
* <li>
* <a href="#Header">Header</a>
@@ -171,7 +171,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSU
* </li>
* </ul>
*
- * <h5 id="#BlockCompressedFormat">Block-Compressed SequenceFile Format</h5>
+ * <h5>Block-Compressed SequenceFile Format</h5>
* <ul>
* <li>
* <a href="#Header">Header</a>
@@ -1935,8 +1935,8 @@ public class SequenceFile {
* @param fs The file system used to open the file.
* @param file The file being read.
* @param bufferSize The buffer size used to read the file.
- * @param length The length being read if it is >= 0. Otherwise,
- * the length is not available.
+ * @param length The length being read if it is {@literal >=} 0.
+ * Otherwise, the length is not available.
* @return The opened stream.
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
index f0fe6fb..b94de6c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* and returns the instance.</p>
*
* <p>Example:</p>
- * <p><blockquote><pre>
+ * <blockquote><pre>
* public class MyWritable implements Writable {
* // Some data
* private int counter;
@@ -62,7 +62,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* return w;
* }
* }
- * </pre></blockquote></p>
+ * </pre></blockquote>
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
index b030481..c1208aa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparable.java
@@ -36,8 +36,9 @@ import org.apache.hadoop.classification.InterfaceStability;
* satisfy this property.</p>
*
* <p>Example:</p>
- * <p><blockquote><pre>
- * public class MyWritableComparable implements WritableComparable<MyWritableComparable> {
+ * <blockquote><pre>
+ * public class MyWritableComparable implements
+ * WritableComparable{@literal <MyWritableComparable>} {
* // Some data
* private int counter;
* private long timestamp;
@@ -66,7 +67,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* return result
* }
* }
- * </pre></blockquote></p>
+ * </pre></blockquote>
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
index e58e0e1..2062fb6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
@@ -236,7 +236,8 @@ public final class WritableUtils {
/**
* Serializes an integer to a binary stream with zero-compressed encoding.
- * For -112 <= i <= 127, only one byte is used with the actual value.
+ * For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ * actual value.
* For other values of i, the first byte value indicates whether the
* integer is positive or negative, and the number of bytes that follow.
* If the first byte value v is between -113 and -116, the following integer
@@ -255,7 +256,8 @@ public final class WritableUtils {
/**
* Serializes a long to a binary stream with zero-compressed encoding.
- * For -112 <= i <= 127, only one byte is used with the actual value.
+ * For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ * actual value.
* For other values of i, the first byte value indicates whether the
* long is positive or negative, and the number of bytes that follow.
* If the first byte value v is between -113 and -120, the following long
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
index 3701f20..e248120 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
@@ -227,9 +227,9 @@ public class CompressionCodecFactory {
/**
* Find the relevant compression codec for the codec's canonical class name
* or by codec alias.
- * <p/>
+ * <p>
* Codec aliases are case insensitive.
- * <p/>
+ * <p>
* The code alias is the short class name (without the package name).
* If the short class name ends with 'Codec', then there are two aliases for
* the codec, the complete short class name and the short class name without
@@ -255,9 +255,9 @@ public class CompressionCodecFactory {
/**
* Find the relevant compression codec for the codec's canonical class name
* or by codec alias and returns its implemetation class.
- * <p/>
+ * <p>
* Codec aliases are case insensitive.
- * <p/>
+ * <p>
* The code alias is the short class name (without the package name).
* If the short class name ends with 'Codec', then there are two aliases for
* the codec, the complete short class name and the short class name without
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
index 61462c0..45b5e9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
@@ -61,9 +61,9 @@ public class Lz4Codec implements Configurable, CompressionCodec {
}
/**
- * Are the native lz4 libraries loaded & initialized?
+ * Are the native lz4 libraries loaded & initialized?
*
- * @return true if loaded & initialized, otherwise false
+ * @return true if loaded & initialized, otherwise false
*/
public static boolean isNativeCodeLoaded() {
return NativeCodeLoader.isNativeCodeLoaded();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
index 20a4cd6..cd0c788 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
@@ -57,7 +57,7 @@ public class SnappyCodec implements Configurable, CompressionCodec, DirectDecomp
}
/**
- * Are the native snappy libraries loaded & initialized?
+ * Are the native snappy libraries loaded & initialized?
*/
public static void checkNativeCodeLoaded() {
if (!NativeCodeLoader.buildSupportsSnappy()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
index d4a9787..5713c56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
@@ -247,7 +247,7 @@ public class Bzip2Compressor implements Compressor {
}
/**
- * Returns the total number of uncompressed bytes input so far.</p>
+ * Returns the total number of uncompressed bytes input so far.
*
* @return the total (non-negative) number of uncompressed bytes input so far
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
index 96693ad..72ba976 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
@@ -183,7 +183,7 @@ public class Bzip2Decompressor implements Decompressor {
}
/**
- * Returns the total number of compressed bytes input so far.</p>
+ * Returns the total number of compressed bytes input so far.
*
* @return the total (non-negative) number of compressed bytes input so far
*/
@@ -195,7 +195,7 @@ public class Bzip2Decompressor implements Decompressor {
/**
* Returns the number of bytes remaining in the input buffers; normally
* called when finished() is true to determine amount of post-gzip-stream
- * data.</p>
+ * data.
*
* @return the total (non-negative) number of unprocessed bytes in input
*/
@@ -206,7 +206,7 @@ public class Bzip2Decompressor implements Decompressor {
}
/**
- * Resets everything including the input buffers (user and direct).</p>
+ * Resets everything including the input buffers (user and direct).
*/
@Override
public synchronized void reset() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
index d24b4bf..3af5309 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
@@ -37,11 +37,11 @@ public class Bzip2Factory {
private static boolean nativeBzip2Loaded;
/**
- * Check if native-bzip2 code is loaded & initialized correctly and
+ * Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
- * @return <code>true</code> if native-bzip2 is loaded & initialized
+ * @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static synchronized boolean isNativeBzip2Loaded(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
index bb02cf2..8426d25 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
@@ -200,20 +200,18 @@ public class CBZip2InputStream extends InputStream implements BZip2Constants {
}
/**
- * This method tries to find the marker (passed to it as the first parameter)
- * in the stream. It can find bit patterns of length <= 63 bits. Specifically
- * this method is used in CBZip2InputStream to find the end of block (EOB)
- * delimiter in the stream, starting from the current position of the stream.
- * If marker is found, the stream position will be at the byte containing
- * the starting bit of the marker.
- *
- * @param marker The bit pattern to be found in the stream
- * @param markerBitLength No of bits in the marker
- * @return true if the marker was found otherwise false
- *
- * @throws IOException
- * @throws IllegalArgumentException if marketBitLength is greater than 63
- */
+ * This method tries to find the marker (passed to it as the first parameter)
+ * in the stream. It can find bit patterns of length <= 63 bits.
+ * Specifically this method is used in CBZip2InputStream to find the end of
+ * block (EOB) delimiter in the stream, starting from the current position
+ * of the stream. If marker is found, the stream position will be at the
+ * byte containing the starting bit of the marker.
+ * @param marker The bit pattern to be found in the stream
+ * @param markerBitLength No of bits in the marker
+ * @return true if the marker was found otherwise false
+ * @throws IOException
+ * @throws IllegalArgumentException if marketBitLength is greater than 63
+ */
public boolean skipToNextMarker(long marker, int markerBitLength)
throws IOException, IllegalArgumentException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
index ca4e5cd..850fec7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
@@ -64,7 +64,8 @@ import org.apache.hadoop.io.IOUtils;
* </pre>
*
* <table width="100%" border="1">
- * <colgroup> <col width="33%" /> <col width="33%" /> <col width="33%" />
+ * <caption></caption>
+ * <colgroup> <col width="33%" > <col width="33%" > <col width="33%" >
* </colgroup>
* <tr>
* <th colspan="3">Memory usage by blocksize</th>
@@ -614,9 +615,9 @@ public class CBZip2OutputStream extends OutputStream implements BZip2Constants {
* @throws IOException
* if an I/O error occurs in the specified stream.
* @throws IllegalArgumentException
- * if <code>(blockSize < 1) || (blockSize > 9)</code>.
+ * if {@code (blockSize < 1) || (blockSize > 9)}
* @throws NullPointerException
- * if <code>out == null</code>.
+ * if {@code out == null}.
*
* @see #MIN_BLOCKSIZE
* @see #MAX_BLOCKSIZE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
index b4c6659..896d35e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
@@ -404,7 +404,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
/**
* Returns the total number of compressed bytes input so far, including
- * gzip header/trailer bytes.</p>
+ * gzip header/trailer bytes.
*
* @return the total (non-negative) number of compressed bytes read so far
*/
@@ -420,7 +420,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
* non-zero value unless called after {@link #setInput(byte[] b, int off,
* int len)} and before {@link #decompress(byte[] b, int off, int len)}.
* (That is, after {@link #decompress(byte[] b, int off, int len)} it
- * always returns zero, except in finished state with concatenated data.)</p>
+ * always returns zero, except in finished state with concatenated data.)
*
* @return the total (non-negative) number of unprocessed bytes in input
*/
@@ -441,7 +441,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
/**
* Returns true if the end of the gzip substream (single "member") has been
- * reached.</p>
+ * reached.
*/
@Override
public synchronized boolean finished() {
@@ -450,7 +450,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
/**
* Resets everything, including the input buffer, regardless of whether the
- * current gzip substream is finished.</p>
+ * current gzip substream is finished.
*/
@Override
public synchronized void reset() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
index 438c8be..da8a90b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
@@ -435,7 +435,7 @@ public class ZlibCompressor implements Compressor {
}
/**
- * Returns the total number of uncompressed bytes input so far.</p>
+ * Returns the total number of uncompressed bytes input so far.
*
* @return the total (non-negative) number of uncompressed bytes input so far
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
index dd550b9..f642d77 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
@@ -243,7 +243,7 @@ public class ZlibDecompressor implements Decompressor {
}
/**
- * Returns the total number of compressed bytes input so far.</p>
+ * Returns the total number of compressed bytes input so far.
*
* @return the total (non-negative) number of compressed bytes input so far
*/
@@ -255,7 +255,7 @@ public class ZlibDecompressor implements Decompressor {
/**
* Returns the number of bytes remaining in the input buffers; normally
* called when finished() is true to determine amount of post-gzip-stream
- * data.</p>
+ * data.
*
* @return the total (non-negative) number of unprocessed bytes in input
*/
@@ -266,7 +266,7 @@ public class ZlibDecompressor implements Decompressor {
}
/**
- * Resets everything including the input buffers (user and direct).</p>
+ * Resets everything including the input buffers (user and direct).
*/
@Override
public void reset() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
index 93b3b6d..07afbab 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
@@ -73,11 +73,11 @@ public class ZlibFactory {
ZlibFactory.nativeZlibLoaded = isLoaded;
}
/**
- * Check if native-zlib code is loaded & initialized correctly and
+ * Check if native-zlib code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
- * @return <code>true</code> if native-zlib is loaded & initialized
+ * @return <code>true</code> if native-zlib is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeZlibLoaded(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 8ec0e72..5ba6e9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -36,7 +36,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
/**
- * A codec & coder utility to help create coders conveniently.
+ * A codec & coder utility to help create coders conveniently.
*
* {@link CodecUtil} includes erasure coder configurations key and default
* values such as coder class name and erasure codec option values included
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
index fdb47be..f80fcec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
@@ -518,7 +518,7 @@ public class GaloisField {
/**
* Perform Gaussian elimination on the given matrix. This matrix has to be a
- * fat matrix (number of rows > number of columns).
+ * fat matrix (number of rows > number of columns).
*/
public void gaussianElimination(int[][] matrix) {
assert(matrix != null && matrix.length > 0 && matrix[0].length > 0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
index c63baa5..09cd282 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
@@ -1308,11 +1308,11 @@ public class TFile {
* @param reader
* The TFile reader object.
* @param beginKey
- * Begin key of the scan. If null, scan from the first <K,V>
- * entry of the TFile.
+ * Begin key of the scan. If null, scan from the first
+ * <K, V> entry of the TFile.
* @param endKey
- * End key of the scan. If null, scan up to the last <K, V> entry
- * of the TFile.
+ * End key of the scan. If null, scan up to the last <K, V>
+ * entry of the TFile.
* @throws IOException
*/
protected Scanner(Reader reader, RawComparable beginKey,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
index 8cb6e0d..17a27f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
@@ -62,27 +62,33 @@ public final class Utils {
* <li>if n in [-32, 127): encode in one byte with the actual value.
* Otherwise,
* <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52;
- * byte[1]=n&0xff. Otherwise,
+ * byte[1]=n&0xff. Otherwise,
* <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 -
- * 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
+ * 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
* <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112;
- * byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise:
+ * byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff;
+ * byte[3]=n&0xff.
+ * Otherwise:
* <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] =
- * (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
+ * (n>>24)&0xff; byte[2]=(n>>16)&0xff;
+ * byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
* <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] =
- * (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff;
- * byte[4]=(n>>8)&0xff; byte[5]=n&0xff
+ * (n>>32)&0xff; byte[2]=(n>>24)&0xff;
+ * byte[3]=(n>>16)&0xff; byte[4]=(n>>8)&0xff;
+ * byte[5]=n&0xff
* <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] =
- * (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff;
- * byte[4]=(n>>16)&0xff; byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
+ * (n>>40)&0xff; byte[2]=(n>>32)&0xff;
+ * byte[3]=(n>>24)&0xff; byte[4]=(n>>16)&0xff;
+ * byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
* <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] =
- * (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff;
- * byte[4]=(n>>24)&0xff; byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff;
- * byte[7]=n&0xff;
+ * (n>>48)&0xff; byte[2] = (n>>40)&0xff;
+ * byte[3]=(n>>32)&0xff; byte[4]=(n>>24)&0xff; byte[5]=
+ * (n>>16)&0xff; byte[6]=(n>>8)&0xff; byte[7]=n&0xff;
* <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] =
- * (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff;
- * byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff;
- * byte[7]=(n>>8)&0xff; byte[8]=n&0xff;
+ * (n>>54)&0xff; byte[2] = (n>>48)&0xff;
+ * byte[3] = (n>>40)&0xff; byte[4]=(n>>32)&0xff;
+ * byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; byte[7]=
+ * (n>>8)&0xff; byte[8]=n&0xff;
* </ul>
*
* @param out
@@ -181,15 +187,15 @@ public final class Utils {
* Decoding the variable-length integer. Suppose the value of the first byte
* is FB, and the following bytes are NB[*].
* <ul>
- * <li>if (FB >= -32), return (long)FB;
- * <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
- * <li>if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 +
- * NB[1]&0xff;
- * <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 +
- * (NB[1]&0xff)<<8 + NB[2]&0xff;
+ * <li>if (FB >= -32), return (long)FB;
+ * <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
+ * <li>if (FB in [-104, -73]), return (FB+88)<<16 +
+ * (NB[0]&0xff)<<8 + NB[1]&0xff;
+ * <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)
+ * <<16 + (NB[1]&0xff)<<8 + NB[2]&0xff;
* <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed
* big-endian integer.
- *
+ * </ul>
* @param in
* input stream
* @return the decoded long integer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
index 9875bcd..7fcd5fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
@@ -89,12 +89,12 @@ public class RetryProxy {
*
* @param iface the interface that the retry will implement
* @param proxyProvider provides implementation instances whose methods should be retried
- * @param methodNameToPolicyMapa map of method names to retry policies
+ * @param methodNameToPolicyMap map of method names to retry policies
* @return the retry proxy
*/
public static <T> Object create(Class<T> iface,
FailoverProxyProvider<T> proxyProvider,
- Map<String,RetryPolicy> methodNameToPolicyMap,
+ Map<String, RetryPolicy> methodNameToPolicyMap,
RetryPolicy defaultPolicy) {
return Proxy.newProxyInstance(
proxyProvider.getInterface().getClassLoader(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
index 3b727d9..3c8dfcc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* <p>
- * Provides a facility for deserializing objects of type <T> from an
+ * Provides a facility for deserializing objects of type {@literal <T>} from an
* {@link InputStream}.
* </p>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
index 63d3738..5ada541 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* <p>
- * Provides a facility for serializing objects of type <T> to an
+ * Provides a facility for serializing objects of type <T> to an
* {@link OutputStream}.
* </p>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index b156d1f..0be5939 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -131,7 +131,7 @@ public final class CallerContext {
/**
* The thread local current caller context.
- * <p/>
+ * <p>
* Internal class for defered singleton idiom.
* https://en.wikipedia.org/wiki/Initialization_on_demand_holder_idiom
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 07a2f13..49f1e49 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -1357,7 +1357,7 @@ public class Client implements AutoCloseable {
* @param remoteId - the target rpc server
* @param fallbackToSimpleAuth - set to true or false during this method to
* indicate if a secure client falls back to simple auth
- * @returns the rpc response
+ * @return the rpc response
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
*/
@@ -1392,7 +1392,7 @@ public class Client implements AutoCloseable {
* @param serviceClass - service class for RPC
* @param fallbackToSimpleAuth - set to true or false during this method to
* indicate if a secure client falls back to simple auth
- * @returns the rpc response
+ * @return the rpc response
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
*/
@@ -1461,7 +1461,7 @@ public class Client implements AutoCloseable {
/**
* Check if RPC is in asynchronous mode or not.
*
- * @returns true, if RPC is in asynchronous mode, otherwise false for
+ * @return true, if RPC is in asynchronous mode, otherwise false for
* synchronous mode.
*/
@Unstable
@@ -1575,7 +1575,8 @@ public class Client implements AutoCloseable {
/**
* This class holds the address and the user ticket. The client connections
- * to servers are uniquely identified by <remoteAddress, protocol, ticket>
+ * to servers are uniquely identified by {@literal <}remoteAddress, protocol,
+ * ticket{@literal >}
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
index d0fb8fd..00d9a79 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
@@ -37,7 +37,7 @@ public class ClientCache {
new HashMap<SocketFactory, Client>();
/**
- * Construct & cache an IPC client with the user-provided SocketFactory
+ * Construct & cache an IPC client with the user-provided SocketFactory
* if no cached client exists.
*
* @param conf Configuration
@@ -66,7 +66,7 @@ public class ClientCache {
}
/**
- * Construct & cache an IPC client with the default SocketFactory
+ * Construct & cache an IPC client with the default SocketFactory
* and default valueClass if no cached client exists.
*
* @param conf Configuration
@@ -77,7 +77,7 @@ public class ClientCache {
}
/**
- * Construct & cache an IPC client with the user-provided SocketFactory
+ * Construct & cache an IPC client with the user-provided SocketFactory
* if no cached client exists. Default response type is ObjectWritable.
*
* @param conf Configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index 8bb0ce4..512b0b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -76,7 +76,8 @@ public class DecayRpcScheduler implements RpcScheduler,
/**
* Decay factor controls how much each count is suppressed by on each sweep.
- * Valid numbers are > 0 and < 1. Decay factor works in tandem with period
+ * Valid numbers are > 0 and < 1. Decay factor works in tandem with
+ * period
* to control how long the scheduler remembers an identity.
*/
public static final String IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
index 3fe9eb7..3622d2c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshHandler.java
@@ -28,7 +28,6 @@ public interface RefreshHandler {
* Implement this method to accept refresh requests from the administrator.
* @param identifier is the identifier you registered earlier
* @param args contains a list of string args from the administrator
- * @throws Exception as a shorthand for a RefreshResponse(-1, message)
* @return a RefreshResponse
*/
RefreshResponse handleRefresh(String identifier, String[] args);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
index 620e100..36e280f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
@@ -102,7 +102,7 @@ public class RemoteException extends IOException {
* a <code>String</code> as a parameter.
* Otherwise it returns this.
*
- * @return <code>Throwable
+ * @return <code>Throwable</code>
*/
public IOException unwrapRemoteException() {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 76d9c40..d162d2f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -2747,7 +2747,7 @@ public abstract class Server {
/**
* Constructs a server listening on the named port and address. Parameters passed must
- * be of the named class. The <code>handlerCount</handlerCount> determines
+ * be of the named class. The <code>handlerCount</code> determines
* the number of handler threads that will be used to process calls.
* If queueSizePerHandler or numReaders are not -1 they will be used instead of parameters
* from configuration. Otherwise the configuration will be picked up.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
index 093d0af..c404ebe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
@@ -70,7 +70,7 @@ import java.util.Set;
* <p>
* The optional <code>get</code> parameter is used to query an specific
* attribute of a JMX bean. The format of the URL is
- * <code>http://.../jmx?get=MXBeanName::AttributeName<code>
+ * <code>http://.../jmx?get=MXBeanName::AttributeName</code>
* <p>
* For example
* <code>
@@ -85,7 +85,7 @@ import java.util.Set;
* <p>
* The return format is JSON and in the form
* <p>
- * <code><pre>
+ * <pre><code>
* {
* "beans" : [
* {
@@ -94,7 +94,7 @@ import java.util.Set;
* }
* ]
* }
- * </pre></code>
+ * </code></pre>
* <p>
* The servlet attempts to convert the the JMXBeans into JSON. Each
* bean's attributes will be converted to a JSON object member.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
index aa4e61c..41bee04 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
@@ -62,10 +62,10 @@ import org.apache.hadoop.util.Timer;
* still maintaining overall information about how many large requests were
* received.
*
- * <p/>This class can also be used to coordinate multiple logging points; see
+ * <p>This class can also be used to coordinate multiple logging points; see
* {@link #record(String, long, double...)} for more details.
*
- * <p/>This class is not thread-safe.
+ * <p>This class is not thread-safe.
*/
public class LogThrottlingHelper {
@@ -175,7 +175,7 @@ public class LogThrottlingHelper {
* about the values specified since the last time the caller was expected to
* write to its log.
*
- * <p/>Specifying multiple values will maintain separate summary statistics
+ * <p>Specifying multiple values will maintain separate summary statistics
* about each value. For example:
* <pre>{@code
* helper.record(1, 0);
@@ -230,7 +230,7 @@ public class LogThrottlingHelper {
* iteration as "pre", yet each one is able to maintain its own summary
* information.
*
- * <p/>Other behavior is the same as {@link #record(double...)}.
+ * <p>Other behavior is the same as {@link #record(double...)}.
*
* @param recorderName The name of the recorder. This is used to check if the
* current recorder is the primary. Other names are
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
index 22c288a..6803d11 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
@@ -251,7 +251,7 @@ public class MutableRollingAverages extends MutableMetric implements Closeable {
}
/**
- * Retrieve a map of metric name -> (aggregate).
+ * Retrieve a map of metric name {@literal ->} (aggregate).
* Filter out entries that don't have at least minSamples.
*
* @return a map of peer DataNode Id to the average latency to that
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
index ff7cd25..8fd3b33 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
@@ -87,6 +87,7 @@
<h2><a name="gettingstarted">Getting started</a></h2>
<h3>Implementing metrics sources</h3>
<table width="99%" border="1" cellspacing="0" cellpadding="4">
+ <caption></caption>
<tbody>
<tr>
<th>Using annotations</th><th>Using MetricsSource interface</th>
@@ -289,6 +290,7 @@
backend that can handle multiple contexts (file, gangalia etc.):
</p>
<table width="99%" border="1" cellspacing="0" cellpadding="4">
+ <caption></caption>
<tbody>
<tr>
<th width="40%">Before</th><th>After</th>
@@ -310,6 +312,7 @@
using the context option in the sink options like the following:
</p>
<table width="99%" border="1" cellspacing="0" cellpadding="4">
+ <caption></caption>
<tbody>
<tr>
<th width="40%">Before</th><th>After</th>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
index 92ac952..1d330c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
@@ -111,7 +111,7 @@ import org.apache.hadoop.security.UserGroupInformation;
* <i>unknown</i>.</p>
*
* <p>Instead of appending to an existing file, by default the sink
- * will create a new file with a suffix of ".<n>&quet;, where
+ * will create a new file with a suffix of ".<n>", where
* <i>n</i> is the next lowest integer that isn't already used in a file name,
* similar to the Hadoop daemon logs. NOTE: the file with the <b>highest</b>
* sequence number is the <b>newest</b> file, unlike the Hadoop daemon logs.</p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
index b2be0a2..c1dbf7e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
@@ -47,10 +47,10 @@ import org.slf4j.LoggerFactory;
* a daemon that is running on the localhost and will add the
* hostname to the metric (such as the
* <a href="https://collectd.org/">CollectD</a> StatsD plugin).
- * <br/>
+ * <br>
* To configure this plugin, you will need to add the following
* entries to your hadoop-metrics2.properties file:
- * <br/>
+ * <br>
* <pre>
* *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
* [prefix].sink.statsd.server.host=
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
index 916367f..1b50498 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
@@ -59,8 +59,9 @@ public final class MBeans {
/**
* Register the MBean using our standard MBeanName format
- * "hadoop:service=<serviceName>,name=<nameName>"
- * Where the <serviceName> and <nameName> are the supplied parameters.
+ * "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ * Where the {@literal <serviceName> and <nameName>} are the supplied
+ * parameters.
*
* @param serviceName
* @param nameName
@@ -75,8 +76,9 @@ public final class MBeans {
/**
* Register the MBean using our standard MBeanName format
- * "hadoop:service=<serviceName>,name=<nameName>"
- * Where the <serviceName> and <nameName> are the supplied parameters.
+ * "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ * Where the {@literal <serviceName> and <nameName>} are the supplied
+ * parameters.
*
* @param serviceName
* @param nameName
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
index b2d803c..97723c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
@@ -29,11 +29,11 @@ import java.util.Map;
import java.util.Set;
/**
- * This is a base class for DNS to Switch mappings. <p/> It is not mandatory to
+ * This is a base class for DNS to Switch mappings. <p> It is not mandatory to
* derive {@link DNSToSwitchMapping} implementations from it, but it is strongly
* recommended, as it makes it easy for the Hadoop developers to add new methods
* to this base class that are automatically picked up by all implementations.
- * <p/>
+ * <p>
*
* This class does not extend the <code>Configured</code>
* base class, and should not be changed to do so, as it causes problems
@@ -81,7 +81,7 @@ public abstract class AbstractDNSToSwitchMapping
* multi-rack. Subclasses may override this with methods that are more aware
* of their topologies.
*
- * <p/>
+ * <p>
*
* This method is used when parts of Hadoop need know whether to apply
* single rack vs multi-rack policies, such as during block placement.
@@ -140,7 +140,7 @@ public abstract class AbstractDNSToSwitchMapping
/**
* Query for a {@link DNSToSwitchMapping} instance being on a single
* switch.
- * <p/>
+ * <p>
* This predicate simply assumes that all mappings not derived from
* this class are multi-switch.
* @param mapping the mapping to query
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
index 2fb4d3e..061971c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
@@ -141,7 +141,7 @@ public class DNS {
}
/**
- * Like {@link DNS#getIPs(String, boolean), but returns all
+ * Like {@link DNS#getIPs(String, boolean)}, but returns all
* IPs associated with the given interface and its subinterfaces.
*/
public static String[] getIPs(String strInterface)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
index 7b1b332..1e6f5f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
@@ -40,7 +40,7 @@ public interface DNSToSwitchMapping {
* Note the hostname/ip-address is not part of the returned path.
* The network topology of the cluster would determine the number of
* components in the network path.
- * <p/>
+ * <p>
*
* If a name cannot be resolved to a rack, the implementation
* should return {@link NetworkTopology#DEFAULT_RACK}. This
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index 0f9cfc3..ceb8ec2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -146,8 +146,8 @@ public class NetUtils {
/**
* Util method to build socket addr from either:
- * <host>:<port>
- * <fs>://<host>:<port>/<path>
+ * {@literal <host>:<port>}
+ * {@literal <fs>://<host>:<port>/<path>}
*/
public static InetSocketAddress createSocketAddr(String target) {
return createSocketAddr(target, -1);
@@ -155,9 +155,9 @@ public class NetUtils {
/**
* Util method to build socket addr from either:
- * <host>
- * <host>:<port>
- * <fs>://<host>:<port>/<path>
+ * {@literal <host>}
+ * {@literal <host>:<port>}
+ * {@literal <fs>://<host>:<port>/<path>}
*/
public static InetSocketAddress createSocketAddr(String target,
int defaultPort) {
@@ -938,7 +938,7 @@ public class NetUtils {
* Return a free port number. There is no guarantee it will remain free, so
* it should be used immediately.
*
- * @returns A free port for binding a local socket
+ * @return A free port for binding a local socket
*/
public static int getFreeSocketPort() {
int port = 0;
@@ -959,7 +959,7 @@ public class NetUtils {
*
* @param localAddr
* @param bindWildCardAddress
- * @returns InetAddress
+ * @return InetAddress
*/
public static InetAddress bindToLocalAddress(InetAddress localAddr, boolean
bindWildCardAddress) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 1f077a7..4d4f2bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -114,7 +114,7 @@ public class NetworkTopology {
}
/** Add a leaf node
- * Update node counter & rack counter if necessary
+ * Update node counter & rack counter if necessary
* @param node node to be added; can be null
* @exception IllegalArgumentException if add a node to a leave
or node to be added is not a leaf
@@ -858,12 +858,12 @@ public class NetworkTopology {
/**
* Sort nodes array by network distance to <i>reader</i>.
- * <p/>
+ * <p>
* In a three-level topology, a node can be either local, on the same rack,
* or on a different rack from the reader. Sorting the nodes based on network
* distance from the reader reduces network traffic and improves
* performance.
- * <p/>
+ * <p>
* As an additional twist, we also randomize the nodes at each network
* distance. This helps with load balancing when there is data skew.
*
@@ -881,11 +881,11 @@ public class NetworkTopology {
/**
* Sort nodes array by network distance to <i>reader</i>.
- * <p/> using network location. This is used when the reader
+ * <p> using network location. This is used when the reader
* is not a datanode. Sorting the nodes based on network distance
* from the reader reduces network traffic and improves
* performance.
- * <p/>
+ * <p>
*
* @param reader Node where data will be read
* @param nodes Available replicas with the requested data
@@ -902,7 +902,7 @@ public class NetworkTopology {
/**
* Sort nodes array by network distance to <i>reader</i>.
- * <p/>
+ * <p>
* As an additional twist, we also randomize the nodes at each network
* distance. This helps with load balancing when there is data skew.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDDS-625. putKey hangs for a long
time after completion, sometimes forever. Contributed by Arpit Agarwal
Posted by br...@apache.org.
HDDS-625. putKey hangs for a long time after completion, sometimes forever. Contributed by Arpit Agarwal
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb18cc5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb18cc5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb18cc5e
Branch: refs/heads/HDFS-13891
Commit: fb18cc5eaa637772d5c8479cbae1bd19aa00e7fb
Parents: 604c27c
Author: Tsz Wo Nicholas Sze <sz...@apache.org>
Authored: Fri Oct 12 10:37:37 2018 +0800
Committer: Tsz Wo Nicholas Sze <sz...@apache.org>
Committed: Fri Oct 12 10:37:37 2018 +0800
----------------------------------------------------------------------
.../common/transport/server/ratis/XceiverServerRatis.java | 4 +++-
hadoop-project/pom.xml | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb18cc5e/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 9094217..0cd407a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -70,6 +70,7 @@ import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketAddress;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
@@ -240,7 +241,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
// Set the ratis storage directory
String storageDir = HddsServerUtil.getOzoneDatanodeRatisDirectory(conf);
- RaftServerConfigKeys.setStorageDir(properties, new File(storageDir));
+ RaftServerConfigKeys.setStorageDirs(properties,
+ Collections.singletonList(new File(storageDir)));
// For grpc set the maximum message size
GrpcConfigKeys.setMessageSizeMax(properties,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb18cc5e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7472a54..b850c7c 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -103,7 +103,7 @@
<ldap-api.version>1.0.0-M33</ldap-api.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-9b84d79-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-9b2d7b6-SNAPSHOT</ratis.version>
<jcache.version>1.0-alpha-1</jcache.version>
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HADOOP-15708. Reading values from
Configuration before adding deprecations make it impossible to read value
with deprecated key (zsiegl via rkanter)
Posted by br...@apache.org.
HADOOP-15708. Reading values from Configuration before adding deprecations make it impossible to read value with deprecated key (zsiegl via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f261c319
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f261c319
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f261c319
Branch: refs/heads/HDFS-13891
Commit: f261c319375c5a8c298338752ee77214c22f4e29
Parents: 2bd000c
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Oct 10 18:51:37 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Oct 10 18:51:37 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 26 +++++-
.../conf/TestConfigurationDeprecation.java | 88 ++++++++++++++------
2 files changed, 88 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f261c319/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index a78e311..c004cb5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -709,6 +709,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* deprecated key, the value of the deprecated key is set as the value for
* the provided property name.
*
+ * Also updates properties and overlays with deprecated keys, if the new
+ * key does not already exist.
+ *
* @param deprecations deprecation context
* @param name the property name
* @return the first property in the list of properties mapping
@@ -730,6 +733,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
// Override return value for deprecated keys
names = keyInfo.newKeys;
}
+
+ // Update properties with deprecated key if already loaded and new
+ // deprecation has been added
+ updatePropertiesWithDeprecatedKeys(deprecations, names);
+
// If there are no overlay values we can return early
Properties overlayProperties = getOverlay();
if (overlayProperties.isEmpty()) {
@@ -748,6 +756,19 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
return names;
}
+
+ private void updatePropertiesWithDeprecatedKeys(
+ DeprecationContext deprecations, String[] newNames) {
+ for (String newName : newNames) {
+ String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(newName);
+ if (deprecatedKey != null && !getProps().containsKey(newName)) {
+ String deprecatedValue = getProps().getProperty(deprecatedKey);
+ if (deprecatedValue != null) {
+ getProps().setProperty(newName, deprecatedValue);
+ }
+ }
+ }
+ }
private void handleDeprecation() {
LOG.debug("Handling deprecation for all properties in config...");
@@ -1187,7 +1208,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* the first key which replaces the deprecated key and is not null.
*
* Values are processed for <a href="#VariableExpansion">variable expansion</a>
- * before being returned.
+ * before being returned.
+ *
+ * As a side effect get loads the properties from the sources if called for
+ * the first time as a lazy init.
*
* @param name the property name, will be trimmed before get value.
* @return the value of the <code>name</code> or its replacing property,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f261c319/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
index 93a235a..4014b60 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
@@ -38,6 +38,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.junit.Assert;
import org.apache.hadoop.fs.Path;
@@ -52,9 +53,14 @@ import com.google.common.util.concurrent.Uninterruptibles;
public class TestConfigurationDeprecation {
private Configuration conf;
- final static String CONFIG = new File("./test-config-TestConfigurationDeprecation.xml").getAbsolutePath();
- final static String CONFIG2 = new File("./test-config2-TestConfigurationDeprecation.xml").getAbsolutePath();
- final static String CONFIG3 = new File("./test-config3-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG = new File("./test-config" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG2 = new File("./test-config2" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG3 = new File("./test-config3" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
+ final static String CONFIG4 = new File("./test-config4" +
+ "-TestConfigurationDeprecation.xml").getAbsolutePath();
BufferedWriter out;
static {
@@ -288,14 +294,14 @@ public class TestConfigurationDeprecation {
@Test
public void testIteratorWithDeprecatedKeys() {
Configuration conf = new Configuration();
- Configuration.addDeprecation("dK", new String[]{"nK"});
+ Configuration.addDeprecation("dK_iterator", new String[]{"nK_iterator"});
conf.set("k", "v");
- conf.set("dK", "V");
- assertEquals("V", conf.get("dK"));
- assertEquals("V", conf.get("nK"));
- conf.set("nK", "VV");
- assertEquals("VV", conf.get("dK"));
- assertEquals("VV", conf.get("nK"));
+ conf.set("dK_iterator", "V");
+ assertEquals("V", conf.get("dK_iterator"));
+ assertEquals("V", conf.get("nK_iterator"));
+ conf.set("nK_iterator", "VV");
+ assertEquals("VV", conf.get("dK_iterator"));
+ assertEquals("VV", conf.get("nK_iterator"));
boolean kFound = false;
boolean dKFound = false;
boolean nKFound = false;
@@ -304,11 +310,11 @@ public class TestConfigurationDeprecation {
assertEquals("v", entry.getValue());
kFound = true;
}
- if (entry.getKey().equals("dK")) {
+ if (entry.getKey().equals("dK_iterator")) {
assertEquals("VV", entry.getValue());
dKFound = true;
}
- if (entry.getKey().equals("nK")) {
+ if (entry.getKey().equals("nK_iterator")) {
assertEquals("VV", entry.getValue());
nKFound = true;
}
@@ -321,19 +327,19 @@ public class TestConfigurationDeprecation {
@Test
public void testUnsetWithDeprecatedKeys() {
Configuration conf = new Configuration();
- Configuration.addDeprecation("dK", new String[]{"nK"});
- conf.set("nK", "VV");
- assertEquals("VV", conf.get("dK"));
- assertEquals("VV", conf.get("nK"));
- conf.unset("dK");
- assertNull(conf.get("dK"));
- assertNull(conf.get("nK"));
- conf.set("nK", "VV");
- assertEquals("VV", conf.get("dK"));
- assertEquals("VV", conf.get("nK"));
- conf.unset("nK");
- assertNull(conf.get("dK"));
- assertNull(conf.get("nK"));
+ Configuration.addDeprecation("dK_unset", new String[]{"nK_unset"});
+ conf.set("nK_unset", "VV");
+ assertEquals("VV", conf.get("dK_unset"));
+ assertEquals("VV", conf.get("nK_unset"));
+ conf.unset("dK_unset");
+ assertNull(conf.get("dK_unset"));
+ assertNull(conf.get("nK_unset"));
+ conf.set("nK_unset", "VV");
+ assertEquals("VV", conf.get("dK_unset"));
+ assertEquals("VV", conf.get("nK_unset"));
+ conf.unset("nK_unset");
+ assertNull(conf.get("dK_unset"));
+ assertNull(conf.get("nK_unset"));
}
private static String getTestKeyName(int threadIndex, int testIndex) {
@@ -426,4 +432,36 @@ public class TestConfigurationDeprecation {
assertEquals(null, conf.get("Z"));
assertEquals(null, conf.get("X"));
}
+
+ @Test
+ public void testGetPropertyBeforeDeprecetionsAreSet() throws Exception {
+ // SETUP
+ final String oldZkAddressKey = "yarn.resourcemanager.zk-address";
+ final String newZkAddressKey = CommonConfigurationKeys.ZK_ADDRESS;
+ final String zkAddressValue = "dummyZkAddress";
+
+ try{
+ out = new BufferedWriter(new FileWriter(CONFIG4));
+ startConfig();
+ appendProperty(oldZkAddressKey, zkAddressValue);
+ endConfig();
+
+ Path fileResource = new Path(CONFIG4);
+ conf.addResource(fileResource);
+ } finally {
+ out.close();
+ }
+
+ // ACT
+ conf.get(oldZkAddressKey);
+ Configuration.addDeprecations(new Configuration.DeprecationDelta[] {
+ new Configuration.DeprecationDelta(oldZkAddressKey, newZkAddressKey)});
+
+ // ASSERT
+ assertEquals("Property should be accessible through deprecated key",
+ zkAddressValue, conf.get(oldZkAddressKey));
+ assertEquals("Property should be accessible through new key",
+ zkAddressValue, conf.get(newZkAddressKey));
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HDDS-604. Correct Ozone
getOzoneConf description. Contributed by Dinesh Chitlangia.
Posted by br...@apache.org.
HDDS-604. Correct Ozone getOzoneConf description.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/794c0451
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/794c0451
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/794c0451
Branch: refs/heads/HDFS-13891
Commit: 794c0451cffbe147234a2417943709c121d06620
Parents: 4de2dc2
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:52:06 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 17:52:06 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/common/src/main/bin/ozone | 4 ++--
hadoop-ozone/common/src/main/bin/start-ozone.sh | 10 +++++-----
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 10 +++++-----
hadoop-ozone/docs/content/CommandShell.md | 2 +-
4 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 4b50771..2ba9ea7 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -39,7 +39,7 @@ function hadoop_usage
hadoop_add_subcommand "fs" client "run a filesystem command on Ozone file system. Equivalent to 'hadoop fs'"
hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
- hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration"
+ hadoop_add_subcommand "getconf" client "get ozone config values from configuration"
hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
hadoop_add_subcommand "om" daemon "Ozone Manager"
@@ -94,7 +94,7 @@ function ozonecmd_case
HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
- getozoneconf)
+ getconf)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf;
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index cfb54e0..4c022fb 100755
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -67,8 +67,8 @@ fi
#Add other possible options
nameStartOpt="$nameStartOpt $*"
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
echo "Ozone is not supported in a security enabled cluster."
@@ -77,7 +77,7 @@ fi
#---------------------------------------------------------
# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ "${OZONE_ENABLED}" != "true" ]]; then
echo "Operation is not supported because ozone is not enabled."
exit -1
@@ -96,7 +96,7 @@ hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone ozonemanager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
echo "Starting Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
@@ -113,7 +113,7 @@ HADOOP_JUMBO_RETCOUNTER=$?
#---------------------------------------------------------
# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
echo "Starting storage container manager nodes [${SCM_NODES}]"
hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index 97e1df4..fe902eb 100755
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -47,8 +47,8 @@ else
exit 1
fi
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
echo "Ozone is not supported in a security enabled cluster."
@@ -57,7 +57,7 @@ fi
#---------------------------------------------------------
# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ "${OZONE_ENABLED}" != "true" ]]; then
echo "Operation is not supported because ozone is not enabled."
exit -1
@@ -76,7 +76,7 @@ hadoop_uservar_su ozone datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone Manager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
echo "Stopping Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
@@ -91,7 +91,7 @@ hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
echo "Stopping storage container manager nodes [${SCM_NODES}]"
hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/794c0451/hadoop-ozone/docs/content/CommandShell.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/CommandShell.md b/hadoop-ozone/docs/content/CommandShell.md
index 74072a5..fdd6c01 100644
--- a/hadoop-ozone/docs/content/CommandShell.md
+++ b/hadoop-ozone/docs/content/CommandShell.md
@@ -36,7 +36,7 @@ The commands supported by ozone are:
* **envvars** - Display computed Hadoop environment variables.
* **freon** - Runs the ozone load generator.
* **genesis** - Developer Only, Ozone micro-benchmark application.
- * **getozoneconf** - Reads ozone config values from configuration.
+ * **getconf** - Reads ozone config values from configuration.
* **jmxget** - Get JMX exported values from NameNode or DataNode.
* **om** - Ozone Manager, via daemon command can be started or stopped.
* **sh** - Primary command line interface for ozone.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDFS-13878. HttpFS: Implement
GETSNAPSHOTTABLEDIRECTORYLIST. Contributed by Siyao Meng.
Posted by br...@apache.org.
HDFS-13878. HttpFS: Implement GETSNAPSHOTTABLEDIRECTORYLIST. Contributed by Siyao Meng.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6dcfef79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6dcfef79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6dcfef79
Branch: refs/heads/HDFS-13891
Commit: 6dcfef79afe97eda6d09fe2567bb4d4074223141
Parents: 3532aa3
Author: Siyao Meng <sm...@cloudera.com>
Authored: Thu Oct 11 15:01:50 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Oct 11 15:01:50 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/http/client/HttpFSFileSystem.java | 16 +++++-
.../hadoop/fs/http/server/FSOperations.java | 35 +++++++++++++
.../http/server/HttpFSParametersProvider.java | 1 +
.../hadoop/fs/http/server/HttpFSServer.java | 8 +++
.../fs/http/client/BaseTestHttpFSWith.java | 54 +++++++++++++++++++-
.../hadoop/fs/http/server/TestHttpFSServer.java | 53 +++++++++++++++++++
6 files changed, 165 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 3904a87..d80c276 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.web.JsonUtilClient;
import org.apache.hadoop.lib.wsrs.EnumSetParam;
import org.apache.hadoop.security.UserGroupInformation;
@@ -234,7 +235,8 @@ public class HttpFSFileSystem extends FileSystem
SETSTORAGEPOLICY(HTTP_PUT), UNSETSTORAGEPOLICY(HTTP_POST),
ALLOWSNAPSHOT(HTTP_PUT), DISALLOWSNAPSHOT(HTTP_PUT),
CREATESNAPSHOT(HTTP_PUT), DELETESNAPSHOT(HTTP_DELETE),
- RENAMESNAPSHOT(HTTP_PUT), GETSNAPSHOTDIFF(HTTP_GET);
+ RENAMESNAPSHOT(HTTP_PUT), GETSNAPSHOTDIFF(HTTP_GET),
+ GETSNAPSHOTTABLEDIRECTORYLIST(HTTP_GET);
private String httpMethod;
@@ -1482,4 +1484,16 @@ public class HttpFSFileSystem extends FileSystem
return JsonUtilClient.toSnapshotDiffReport(json);
}
+ public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList()
+ throws IOException {
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(OP_PARAM, Operation.GETSNAPSHOTTABLEDIRECTORYLIST.toString());
+ HttpURLConnection conn = getConnection(
+ Operation.GETSNAPSHOTTABLEDIRECTORYLIST.getMethod(),
+ params, new Path(getUri().toString(), "/"), true);
+ HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+ return JsonUtilClient.toSnapshottableDirectoryList(json);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index ed7628f..7989895 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.lib.service.FileSystemAccess;
@@ -1701,4 +1702,38 @@ public class FSOperations {
}
}
}
+
+ /**
+ * Executor that performs a getSnapshottableDirListing operation.
+ */
+ @InterfaceAudience.Private
+ public static class FSGetSnapshottableDirListing implements
+ FileSystemAccess.FileSystemExecutor<String> {
+
+ /**
+ * Creates a getSnapshottableDirListing executor.
+ */
+ public FSGetSnapshottableDirListing() {
+ }
+
+ /**
+ * Executes the filesystem operation.
+ * @param fs filesystem instance to use.
+ * @return A JSON string of all snapshottable directories.
+ * @throws IOException thrown if an IO error occurred.
+ */
+ @Override
+ public String execute(FileSystem fs) throws IOException {
+ SnapshottableDirectoryStatus[] sds = null;
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem dfs = (DistributedFileSystem) fs;
+ sds = dfs.getSnapshottableDirListing();
+ } else {
+ throw new UnsupportedOperationException("getSnapshottableDirListing is "
+ + "not supported for HttpFs on " + fs.getClass()
+ + ". Please check your fs.defaultFS configuration");
+ }
+ return JsonUtil.toJsonString(sds);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
index 5301527..754ae2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
@@ -113,6 +113,7 @@ public class HttpFSParametersProvider extends ParametersProvider {
PARAMS_DEF.put(Operation.GETSNAPSHOTDIFF,
new Class[] {OldSnapshotNameParam.class,
SnapshotNameParam.class});
+ PARAMS_DEF.put(Operation.GETSNAPSHOTTABLEDIRECTORYLIST, new Class[] {});
}
public HttpFSParametersProvider() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index f510061..2895210 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -379,6 +379,14 @@ public class HttpFSServer {
response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
break;
}
+ case GETSNAPSHOTTABLEDIRECTORYLIST: {
+ FSOperations.FSGetSnapshottableDirListing command =
+ new FSOperations.FSGetSnapshottableDirListing();
+ String js = fsExecute(user, command);
+ AUDIT_LOG.info("[{}]", "/");
+ response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+ break;
+ }
default: {
throw new IOException(
MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index cd9d3b9..a23ca7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -41,6 +41,8 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -1071,7 +1073,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING,
CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT,
ALLOW_SNAPSHOT, DISALLOW_SNAPSHOT, DISALLOW_SNAPSHOT_EXCEPTION,
- FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF
+ FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST
}
private void operation(Operation op) throws Exception {
@@ -1185,6 +1187,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
testGetSnapshotDiff();
testGetSnapshotDiffIllegalParam();
break;
+ case GET_SNAPSHOTTABLE_DIRECTORY_LIST:
+ testGetSnapshottableDirListing();
+ break;
}
}
@@ -1529,4 +1534,51 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
fs.delete(path, true);
}
}
+
+ private void verifyGetSnapshottableDirListing(
+ FileSystem fs, DistributedFileSystem dfs) throws Exception {
+ // Get snapshottable directory list
+ SnapshottableDirectoryStatus[] sds = null;
+ if (fs instanceof HttpFSFileSystem) {
+ HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
+ sds = httpFS.getSnapshottableDirectoryList();
+ } else if (fs instanceof WebHdfsFileSystem) {
+ WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
+ sds = webHdfsFileSystem.getSnapshottableDirectoryList();
+ } else {
+ Assert.fail(fs.getClass().getSimpleName() +
+ " doesn't support getSnapshottableDirListing");
+ }
+ // Verify result with DFS
+ SnapshottableDirectoryStatus[] dfssds = dfs.getSnapshottableDirListing();
+ Assert.assertEquals(JsonUtil.toJsonString(sds),
+ JsonUtil.toJsonString(dfssds));
+ }
+
+ private void testGetSnapshottableDirListing() throws Exception {
+ if (!this.isLocalFS()) {
+ FileSystem fs = this.getHttpFSFileSystem();
+ // Create directories with snapshot allowed
+ Path path1 = new Path("/tmp/tmp-snap-dirlist-test-1");
+ DistributedFileSystem dfs = (DistributedFileSystem)
+ FileSystem.get(path1.toUri(), this.getProxiedFSConf());
+ // Verify response when there is no snapshottable directory
+ verifyGetSnapshottableDirListing(fs, dfs);
+ createSnapshotTestsPreconditions(path1);
+ Assert.assertTrue(fs.getFileStatus(path1).isSnapshotEnabled());
+ // Verify response when there is one snapshottable directory
+ verifyGetSnapshottableDirListing(fs, dfs);
+ Path path2 = new Path("/tmp/tmp-snap-dirlist-test-2");
+ createSnapshotTestsPreconditions(path2);
+ Assert.assertTrue(fs.getFileStatus(path2).isSnapshotEnabled());
+ // Verify response when there are two snapshottable directories
+ verifyGetSnapshottableDirListing(fs, dfs);
+
+ // Clean up and verify
+ fs.delete(path2, true);
+ verifyGetSnapshottableDirListing(fs, dfs);
+ fs.delete(path1, true);
+ verifyGetSnapshottableDirListing(fs, dfs);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dcfef79/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index f024c95..8f7662f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.http.server;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
@@ -1400,4 +1401,56 @@ public class TestHttpFSServer extends HFSTestCase {
// Clean up
dfs.delete(path, true);
}
+
+ private void verifyGetSnapshottableDirectoryList(DistributedFileSystem dfs)
+ throws Exception {
+ // Send a request
+ HttpURLConnection conn = sendRequestToHttpFSServer("/",
+ "GETSNAPSHOTTABLEDIRECTORYLIST", "");
+ // Should return HTTP_OK
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
+ // Verify the response
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(conn.getInputStream()));
+ // The response should be a one-line JSON string.
+ String dirLst = reader.readLine();
+ // Verify the content of diff with DFS API.
+ SnapshottableDirectoryStatus[] dfsDirLst = dfs.getSnapshottableDirListing();
+ Assert.assertEquals(dirLst, JsonUtil.toJsonString(dfsDirLst));
+ }
+
+ @Test
+ @TestDir
+ @TestJetty
+ @TestHdfs
+ public void testGetSnapshottableDirectoryList() throws Exception {
+ createHttpFSServer(false, false);
+ // Create test directories
+ String pathStr1 = "/tmp/tmp-snap-dirlist-test-1";
+ createDirWithHttp(pathStr1, "700", null);
+ Path path1 = new Path(pathStr1);
+ String pathStr2 = "/tmp/tmp-snap-dirlist-test-2";
+ createDirWithHttp(pathStr2, "700", null);
+ Path path2 = new Path(pathStr2);
+ DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+ path1.toUri(), TestHdfsHelper.getHdfsConf());
+ // Verify response when there is no snapshottable directory
+ verifyGetSnapshottableDirectoryList(dfs);
+ // Enable snapshot for path1
+ dfs.allowSnapshot(path1);
+ Assert.assertTrue(dfs.getFileStatus(path1).isSnapshotEnabled());
+ // Verify response when there is one snapshottable directory
+ verifyGetSnapshottableDirectoryList(dfs);
+ // Enable snapshot for path2
+ dfs.allowSnapshot(path2);
+ Assert.assertTrue(dfs.getFileStatus(path2).isSnapshotEnabled());
+ // Verify response when there are two snapshottable directories
+ verifyGetSnapshottableDirectoryList(dfs);
+
+ // Clean up and verify
+ dfs.delete(path2, true);
+ verifyGetSnapshottableDirectoryList(dfs);
+ dfs.delete(path1, true);
+ verifyGetSnapshottableDirectoryList(dfs);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HDFS-13906. RBF: Add multiple
paths for dfsrouteradmin 'rm' and 'clrquota' commands. Contributed by Ayush
Saxena.
Posted by br...@apache.org.
HDFS-13906. RBF: Add multiple paths for dfsrouteradmin 'rm' and 'clrquota' commands. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebf6bf30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebf6bf30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebf6bf30
Branch: refs/heads/HDFS-13891
Commit: ebf6bf30487cbdd78819a0cd3f8f04e22cd8e307
Parents: 6e0e6da
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Oct 12 17:19:55 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Oct 12 17:54:39 2018 +0530
----------------------------------------------------------------------
.../hdfs/tools/federation/RouterAdmin.java | 102 ++++++++++---------
.../federation/router/TestRouterAdminCLI.java | 82 ++++++++++++---
2 files changed, 122 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebf6bf30/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 1aefe4f..4a9cc7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -151,17 +151,7 @@ public class RouterAdmin extends Configured implements Tool {
* @param arg List of of command line parameters.
*/
private void validateMax(String[] arg) {
- if (arg[0].equals("-rm")) {
- if (arg.length > 2) {
- throw new IllegalArgumentException(
- "Too many arguments, Max=1 argument allowed");
- }
- } else if (arg[0].equals("-ls")) {
- if (arg.length > 2) {
- throw new IllegalArgumentException(
- "Too many arguments, Max=1 argument allowed");
- }
- } else if (arg[0].equals("-clrQuota")) {
+ if (arg[0].equals("-ls")) {
if (arg.length > 2) {
throw new IllegalArgumentException(
"Too many arguments, Max=1 argument allowed");
@@ -183,63 +173,63 @@ public class RouterAdmin extends Configured implements Tool {
}
}
- @Override
- public int run(String[] argv) throws Exception {
- if (argv.length < 1) {
- System.err.println("Not enough parameters specified");
- printUsage();
- return -1;
- }
-
- int exitCode = -1;
- int i = 0;
- String cmd = argv[i++];
-
- // Verify that we have enough command line parameters
+ /**
+ * Usage: validates the minimum number of arguments for a command.
+ * @param argv List of of command line parameters.
+ * @return true if number of arguments are valid for the command else false.
+ */
+ private boolean validateMin(String[] argv) {
+ String cmd = argv[0];
if ("-add".equals(cmd)) {
if (argv.length < 4) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-update".equals(cmd)) {
if (argv.length < 4) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-rm".equals(cmd)) {
if (argv.length < 2) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-setQuota".equals(cmd)) {
if (argv.length < 4) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-clrQuota".equals(cmd)) {
if (argv.length < 2) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-safemode".equals(cmd)) {
if (argv.length < 2) {
- System.err.println("Not enough parameters specified for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
} else if ("-nameservice".equals(cmd)) {
if (argv.length < 3) {
- System.err.println("Not enough parameters specificed for cmd " + cmd);
- printUsage(cmd);
- return exitCode;
+ return false;
}
}
+ return true;
+ }
+
+ @Override
+ public int run(String[] argv) throws Exception {
+ if (argv.length < 1) {
+ System.err.println("Not enough parameters specified");
+ printUsage();
+ return -1;
+ }
+
+ int exitCode = -1;
+ int i = 0;
+ String cmd = argv[i++];
+ // Verify that we have enough command line parameters
+ if (!validateMin(argv)) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage(cmd);
+ return exitCode;
+ }
// Initialize RouterClient
try {
String address = getConf().getTrimmed(
@@ -273,8 +263,17 @@ public class RouterAdmin extends Configured implements Tool {
exitCode = -1;
}
} else if ("-rm".equals(cmd)) {
- if (removeMount(argv[i])) {
- System.out.println("Successfully removed mount point " + argv[i]);
+ while (i < argv.length) {
+ try {
+ if (removeMount(argv[i])) {
+ System.out.println("Successfully removed mount point " + argv[i]);
+ }
+ } catch (IOException e) {
+ exitCode = -1;
+ System.err
+ .println(cmd.substring(1) + ": " + e.getLocalizedMessage());
+ }
+ i++;
}
} else if ("-ls".equals(cmd)) {
if (argv.length > 1) {
@@ -288,9 +287,12 @@ public class RouterAdmin extends Configured implements Tool {
"Successfully set quota for mount point " + argv[i]);
}
} else if ("-clrQuota".equals(cmd)) {
- if (clrQuota(argv[i])) {
- System.out.println(
- "Successfully clear quota for mount point " + argv[i]);
+ while (i < argv.length) {
+ if (clrQuota(argv[i])) {
+ System.out
+ .println("Successfully clear quota for mount point " + argv[i]);
+ i++;
+ }
}
} else if ("-safemode".equals(cmd)) {
manageSafeMode(argv[i]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebf6bf30/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index d968c60..65fe054 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -342,13 +342,43 @@ public class TestRouterAdminCLI {
assertEquals(0, ToolRunner.run(admin, argv));
assertTrue(out.toString().contains(
"Cannot remove mount point " + invalidPath));
+ }
- // test wrong number of arguments
- System.setErr(new PrintStream(err));
- argv = new String[] {"-rm", src, "check" };
- ToolRunner.run(admin, argv);
- assertTrue(err.toString()
- .contains("Too many arguments, Max=1 argument allowed"));
+ @Test
+ public void testMultiArgsRemoveMountTable() throws Exception {
+ String nsId = "ns0";
+ String src1 = "/test-rmmounttable1";
+ String src2 = "/test-rmmounttable2";
+ String dest1 = "/rmmounttable1";
+ String dest2 = "/rmmounttable2";
+ // Adding mount table entries
+ String[] argv = new String[] {"-add", src1, nsId, dest1};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ argv = new String[] {"-add", src2, nsId, dest2};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // Ensure mount table entries added successfully
+ GetMountTableEntriesRequest getRequest =
+ GetMountTableEntriesRequest.newInstance(src1);
+ GetMountTableEntriesResponse getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+ MountTable mountTable = getResponse.getEntries().get(0);
+ getRequest = GetMountTableEntriesRequest.newInstance(src2);
+ getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+ assertEquals(src1, mountTable.getSourcePath());
+ mountTable = getResponse.getEntries().get(0);
+ assertEquals(src2, mountTable.getSourcePath());
+ // Remove multiple mount table entries
+ argv = new String[] {"-rm", src1, src2};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // Verify successful deletion of mount table entries
+ getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+ assertEquals(0, getResponse.getEntries().size());
}
@Test
@@ -540,6 +570,7 @@ public class TestRouterAdminCLI {
public void testSetAndClearQuota() throws Exception {
String nsId = "ns0";
String src = "/test-QuotaMounttable";
+ String src1 = "/test-QuotaMounttable1";
String dest = "/QuotaMounttable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
@@ -605,15 +636,42 @@ public class TestRouterAdminCLI {
assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
+ // verify multi args ClrQuota
+ String dest1 = "/QuotaMounttable1";
+ // Add mount table entries.
+ argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ argv = new String[] {"-add", src1, nsId, dest1};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // SetQuota for the added entries
+ argv = new String[] {"-setQuota", src, "-nsQuota", String.valueOf(nsQuota),
+ "-ssQuota", String.valueOf(ssQuota)};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ argv = new String[] {"-setQuota", src1, "-nsQuota",
+ String.valueOf(nsQuota), "-ssQuota", String.valueOf(ssQuota)};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // Clear quota for the added entries
+ argv = new String[] {"-clrQuota", src, src1};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse =
+ client.getMountTableManager().getMountTableEntries(getRequest);
+
+ // Verify clear quota for the entries
+ for (int i = 0; i < 2; i++) {
+ mountTable = getResponse.getEntries().get(i);
+ quotaUsage = mountTable.getQuota();
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getQuota());
+ assertEquals(HdfsConstants.QUOTA_RESET, quotaUsage.getSpaceQuota());
+ }
+
// verify wrong arguments
System.setErr(new PrintStream(err));
- argv = new String[] {"-clrQuota", src, "check"};
- ToolRunner.run(admin, argv);
- assertTrue(err.toString(),
- err.toString().contains("Too many arguments, Max=1 argument allowed"));
-
argv = new String[] {"-setQuota", src, "check", "check2"};
- err.reset();
ToolRunner.run(admin, argv);
assertTrue(err.toString().contains("Invalid argument : check"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-7644. NM gets backed up
deleting docker containers. Contributed by Chandni Singh
Posted by br...@apache.org.
YARN-7644. NM gets backed up deleting docker containers. Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ce70e12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ce70e12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ce70e12
Branch: refs/heads/HDFS-13891
Commit: 5ce70e1211e624d58e8bb1181aec00729ebdc085
Parents: cd28051
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Oct 10 09:52:19 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Oct 10 09:52:19 2018 -0500
----------------------------------------------------------------------
.../launcher/ContainerCleanup.java | 229 +++++++++++++++++++
.../launcher/ContainerLaunch.java | 226 +++++-------------
.../launcher/ContainersLauncher.java | 14 +-
.../launcher/TestContainerCleanup.java | 108 +++++++++
.../launcher/TestContainersLauncher.java | 12 +-
5 files changed, 401 insertions(+), 188 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
new file mode 100644
index 0000000..963d28b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.EXIT_CODE_FILE_SUFFIX;
+
+/**
+ * Cleanup the container.
+ * Cancels the launch if launch has not started yet or signals
+ * the executor to not execute the process if not already done so.
+ * Also, sends a SIGTERM followed by a SIGKILL to the process if
+ * the process id is available.
+ */
+public class ContainerCleanup implements Runnable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerCleanup.class);
+
+ private final Context context;
+ private final Configuration conf;
+ private final Dispatcher dispatcher;
+ private final ContainerExecutor exec;
+ private final Container container;
+ private final ContainerLaunch launch;
+ private final long sleepDelayBeforeSigKill;
+
+
+ public ContainerCleanup(Context context, Configuration configuration,
+ Dispatcher dispatcher, ContainerExecutor exec,
+ Container container,
+ ContainerLaunch containerLaunch) {
+
+ this.context = Preconditions.checkNotNull(context, "context");
+ this.conf = Preconditions.checkNotNull(configuration, "config");
+ this.dispatcher = Preconditions.checkNotNull(dispatcher, "dispatcher");
+ this.exec = Preconditions.checkNotNull(exec, "exec");
+ this.container = Preconditions.checkNotNull(container, "container");
+ this.launch = Preconditions.checkNotNull(containerLaunch, "launch");
+ this.sleepDelayBeforeSigKill = conf.getLong(
+ YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
+ YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
+ }
+
+ @Override
+ public void run() {
+ ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
+ LOG.info("Cleaning up container " + containerIdStr);
+
+ try {
+ context.getNMStateStore().storeContainerKilled(containerId);
+ } catch (IOException e) {
+ LOG.error("Unable to mark container " + containerId
+ + " killed in store", e);
+ }
+
+ // launch flag will be set to true if process already launched
+ boolean alreadyLaunched = !launch.markLaunched();
+ if (!alreadyLaunched) {
+ LOG.info("Container " + containerIdStr + " not launched."
+ + " No cleanup needed to be done");
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Marking container " + containerIdStr + " as inactive");
+ }
+ // this should ensure that if the container process has not launched
+ // by this time, it will never be launched
+ exec.deactivateContainer(containerId);
+ Path pidFilePath = launch.getPidFilePath();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Getting pid for container {} to kill"
+ + " from pid file {}", containerIdStr, pidFilePath != null ?
+ pidFilePath : "null");
+ }
+
+ // however the container process may have already started
+ try {
+
+ // get process id from pid file if available
+ // else if shell is still active, get it from the shell
+ String processId = launch.getContainerPid();
+
+ // kill process
+ String user = container.getUser();
+ if (processId != null) {
+ signalProcess(processId, user, containerIdStr);
+ } else {
+ // Normally this means that the process was notified about
+ // deactivateContainer above and did not start.
+ // Since we already set the state to RUNNING or REINITIALIZING
+ // we have to send a killed event to continue.
+ if (!launch.isLaunchCompleted()) {
+ LOG.warn("Container clean up before pid file created "
+ + containerIdStr);
+ dispatcher.getEventHandler().handle(
+ new ContainerExitEvent(container.getContainerId(),
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ Shell.WINDOWS ?
+ ContainerExecutor.ExitCode.FORCE_KILLED.getExitCode() :
+ ContainerExecutor.ExitCode.TERMINATED.getExitCode(),
+ "Container terminated before pid file created."));
+ // There is a possibility that the launch grabbed the file name before
+ // the deactivateContainer above but it was slow enough to avoid
+ // getContainerPid.
+ // Increasing YarnConfiguration.NM_PROCESS_KILL_WAIT_MS
+ // reduces the likelihood of this race condition and process leak.
+ }
+ // The Docker container may not have fully started, reap the container.
+ if (DockerLinuxContainerRuntime.isDockerContainerRequested(conf,
+ container.getLaunchContext().getEnvironment())) {
+ reapDockerContainerNoPid(user);
+ }
+ }
+ } catch (Exception e) {
+ String message =
+ "Exception when trying to cleanup container " + containerIdStr
+ + ": " + StringUtils.stringifyException(e);
+ LOG.warn(message);
+ dispatcher.getEventHandler().handle(
+ new ContainerDiagnosticsUpdateEvent(containerId, message));
+ } finally {
+ // cleanup pid file if present
+ if (pidFilePath != null) {
+ try {
+ FileContext lfs = FileContext.getLocalFSFileContext();
+ lfs.delete(pidFilePath, false);
+ lfs.delete(pidFilePath.suffix(EXIT_CODE_FILE_SUFFIX), false);
+ } catch (IOException ioe) {
+ LOG.warn("{} exception trying to delete pid file {}. Ignoring.",
+ containerId, pidFilePath, ioe);
+ }
+ }
+ }
+
+ try {
+ // Reap the container
+ launch.reapContainer();
+ } catch (IOException ioe) {
+ LOG.warn("{} exception trying to reap container. Ignoring.", containerId,
+ ioe);
+ }
+ }
+
+ private void signalProcess(String processId, String user,
+ String containerIdStr) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending signal to pid " + processId + " as user " + user
+ + " for container " + containerIdStr);
+ }
+ final ContainerExecutor.Signal signal =
+ sleepDelayBeforeSigKill > 0 ? ContainerExecutor.Signal.TERM :
+ ContainerExecutor.Signal.KILL;
+
+ boolean result = sendSignal(user, processId, signal);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sent signal " + signal + " to pid " + processId + " as user "
+ + user + " for container " + containerIdStr + ", result="
+ + (result ? "success" : "failed"));
+ }
+ if (sleepDelayBeforeSigKill > 0) {
+ new ContainerExecutor.DelayedProcessKiller(container, user, processId,
+ sleepDelayBeforeSigKill, ContainerExecutor.Signal.KILL, exec).start();
+ }
+ }
+
+ private boolean sendSignal(String user, String processId,
+ ContainerExecutor.Signal signal)
+ throws IOException {
+ return exec.signalContainer(
+ new ContainerSignalContext.Builder().setContainer(container)
+ .setUser(user).setPid(processId).setSignal(signal).build());
+ }
+
+ private void reapDockerContainerNoPid(String user) throws IOException {
+ String containerIdStr =
+ container.getContainerTokenIdentifier().getContainerID().toString();
+ LOG.info("Unable to obtain pid, but docker container request detected. "
+ + "Attempting to reap container " + containerIdStr);
+ boolean result = exec.reapContainer(
+ new ContainerReapContext.Builder()
+ .setContainer(container)
+ .setUser(container.getUser())
+ .build());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sent signal to docker container " + containerIdStr
+ + " as user " + user + ", result=" + (result ? "success" : "failed"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 3fa73ec..6776836 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -85,7 +84,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
@@ -115,7 +113,7 @@ public class ContainerLaunch implements Callable<Integer> {
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
private static final String PID_FILE_NAME_FMT = "%s.pid";
- private static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";
+ static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";
protected final Dispatcher dispatcher;
protected final ContainerExecutor exec;
@@ -131,7 +129,6 @@ public class ContainerLaunch implements Callable<Integer> {
protected AtomicBoolean completed = new AtomicBoolean(false);
private volatile boolean killedBeforeStart = false;
- private long sleepDelayBeforeSigKill = 250;
private long maxKillWaitTime = 2000;
protected Path pidFilePath = null;
@@ -152,9 +149,6 @@ public class ContainerLaunch implements Callable<Integer> {
this.dispatcher = dispatcher;
this.dirsHandler = dirsHandler;
this.containerManager = containerManager;
- this.sleepDelayBeforeSigKill =
- conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
- YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
this.maxKillWaitTime =
conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
@@ -515,6 +509,25 @@ public class ContainerLaunch implements Callable<Integer> {
return launchPrep;
}
+ void reapContainer() throws IOException {
+ containerExecLock.lock();
+ try {
+ // Reap the container
+ boolean result = exec.reapContainer(
+ new ContainerReapContext.Builder()
+ .setContainer(container)
+ .setUser(container.getUser())
+ .build());
+ if (!result) {
+ throw new IOException("Reap container failed for container " +
+ container.getContainerId());
+ }
+ cleanupContainerFiles(getContainerWorkDir());
+ } finally {
+ containerExecLock.unlock();
+ }
+ }
+
protected int prepareForLaunch(ContainerStartContext ctx) throws IOException {
ContainerId containerId = container.getContainerId();
if (container.isMarkedForKilling()) {
@@ -721,121 +734,6 @@ public class ContainerLaunch implements Callable<Integer> {
return getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ String.format(ContainerLaunch.PID_FILE_NAME_FMT, containerIdStr);
}
-
- /**
- * Cleanup the container.
- * Cancels the launch if launch has not started yet or signals
- * the executor to not execute the process if not already done so.
- * Also, sends a SIGTERM followed by a SIGKILL to the process if
- * the process id is available.
- * @throws IOException
- */
- public void cleanupContainer() throws IOException {
- ContainerId containerId = container.getContainerId();
- String containerIdStr = containerId.toString();
- LOG.info("Cleaning up container " + containerIdStr);
-
- try {
- context.getNMStateStore().storeContainerKilled(containerId);
- } catch (IOException e) {
- LOG.error("Unable to mark container " + containerId
- + " killed in store", e);
- }
-
- // launch flag will be set to true if process already launched
- boolean alreadyLaunched =
- !containerAlreadyLaunched.compareAndSet(false, true);
- if (!alreadyLaunched) {
- LOG.info("Container " + containerIdStr + " not launched."
- + " No cleanup needed to be done");
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Marking container " + containerIdStr + " as inactive");
- }
- // this should ensure that if the container process has not launched
- // by this time, it will never be launched
- exec.deactivateContainer(containerId);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Getting pid for container " + containerIdStr + " to kill"
- + " from pid file "
- + (pidFilePath != null ? pidFilePath.toString() : "null"));
- }
-
- // however the container process may have already started
- try {
-
- // get process id from pid file if available
- // else if shell is still active, get it from the shell
- String processId = null;
- if (pidFilePath != null) {
- processId = getContainerPid(pidFilePath);
- }
-
- // kill process
- String user = container.getUser();
- if (processId != null) {
- signalProcess(processId, user, containerIdStr);
- } else {
- // Normally this means that the process was notified about
- // deactivateContainer above and did not start.
- // Since we already set the state to RUNNING or REINITIALIZING
- // we have to send a killed event to continue.
- if (!completed.get()) {
- LOG.warn("Container clean up before pid file created "
- + containerIdStr);
- dispatcher.getEventHandler().handle(
- new ContainerExitEvent(container.getContainerId(),
- ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
- Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
- ExitCode.TERMINATED.getExitCode(),
- "Container terminated before pid file created."));
- // There is a possibility that the launch grabbed the file name before
- // the deactivateContainer above but it was slow enough to avoid
- // getContainerPid.
- // Increasing YarnConfiguration.NM_PROCESS_KILL_WAIT_MS
- // reduces the likelihood of this race condition and process leak.
- }
- // The Docker container may not have fully started, reap the container.
- if (DockerLinuxContainerRuntime.isDockerContainerRequested(
- conf,
- container.getLaunchContext().getEnvironment())) {
- reapDockerContainerNoPid(user);
- }
- }
- } catch (Exception e) {
- String message =
- "Exception when trying to cleanup container " + containerIdStr
- + ": " + StringUtils.stringifyException(e);
- LOG.warn(message);
- dispatcher.getEventHandler().handle(
- new ContainerDiagnosticsUpdateEvent(containerId, message));
- } finally {
- // cleanup pid file if present
- if (pidFilePath != null) {
- FileContext lfs = FileContext.getLocalFSFileContext();
- lfs.delete(pidFilePath, false);
- lfs.delete(pidFilePath.suffix(EXIT_CODE_FILE_SUFFIX), false);
- }
- }
- containerExecLock.lock();
- try {
- // Reap the container
- boolean result = exec.reapContainer(
- new ContainerReapContext.Builder()
- .setContainer(container)
- .setUser(container.getUser())
- .build());
- if (!result) {
- throw new IOException("Reap container failed for container "
- + containerIdStr);
- }
- cleanupContainerFiles(getContainerWorkDir());
- } finally {
- containerExecLock.unlock();
- }
- }
/**
* Send a signal to the container.
@@ -874,11 +772,7 @@ public class ContainerLaunch implements Callable<Integer> {
try {
// get process id from pid file if available
// else if shell is still active, get it from the shell
- String processId = null;
- if (pidFilePath != null) {
- processId = getContainerPid(pidFilePath);
- }
-
+ String processId = getContainerPid();
if (processId != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Sending signal to pid " + processId
@@ -912,50 +806,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
}
- private boolean sendSignal(String user, String processId, Signal signal)
- throws IOException {
- return exec.signalContainer(
- new ContainerSignalContext.Builder().setContainer(container)
- .setUser(user).setPid(processId).setSignal(signal).build());
- }
-
- private void signalProcess(String processId, String user,
- String containerIdStr) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sending signal to pid " + processId + " as user " + user
- + " for container " + containerIdStr);
- }
- final Signal signal =
- sleepDelayBeforeSigKill > 0 ? Signal.TERM : Signal.KILL;
-
- boolean result = sendSignal(user, processId, signal);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sent signal " + signal + " to pid " + processId + " as user "
- + user + " for container " + containerIdStr + ", result="
- + (result ? "success" : "failed"));
- }
- if (sleepDelayBeforeSigKill > 0) {
- new DelayedProcessKiller(container, user, processId,
- sleepDelayBeforeSigKill, Signal.KILL, exec).start();
- }
- }
-
- private void reapDockerContainerNoPid(String user) throws IOException {
- String containerIdStr =
- container.getContainerTokenIdentifier().getContainerID().toString();
- LOG.info("Unable to obtain pid, but docker container request detected. "
- + "Attempting to reap container " + containerIdStr);
- boolean result = exec.reapContainer(
- new ContainerReapContext.Builder()
- .setContainer(container)
- .setUser(container.getUser())
- .build());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sent signal to docker container " + containerIdStr
- + " as user " + user + ", result=" + (result ? "success" : "failed"));
- }
- }
-
@VisibleForTesting
public static Signal translateCommandToSignal(
SignalContainerCommand command) {
@@ -1076,14 +926,16 @@ public class ContainerLaunch implements Callable<Integer> {
/**
* Loop through for a time-bounded interval waiting to
* read the process id from a file generated by a running process.
- * @param pidFilePath File from which to read the process id
- * @return Process ID
+ * @return Process ID; null when pidFilePath is null
* @throws Exception
*/
- private String getContainerPid(Path pidFilePath) throws Exception {
+ String getContainerPid() throws Exception {
+ if (pidFilePath == null) {
+ return null;
+ }
String containerIdStr =
container.getContainerId().toString();
- String processId = null;
+ String processId;
if (LOG.isDebugEnabled()) {
LOG.debug("Accessing pid for container " + containerIdStr
+ " from pid file " + pidFilePath);
@@ -1889,4 +1741,28 @@ public class ContainerLaunch implements Callable<Integer> {
LOG.warn("Failed to delete " + path, e);
}
}
+
+ /**
+ * Returns the PID File Path.
+ */
+ Path getPidFilePath() {
+ return pidFilePath;
+ }
+
+ /**
+ * Marks the container to be launched only if it was not launched.
+ *
+ * @return true if successful; false otherwise.
+ */
+ boolean markLaunched() {
+ return containerAlreadyLaunched.compareAndSet(false, true);
+ }
+
+ /**
+ * Returns if the launch is completed or not.
+ */
+ boolean isLaunchCompleted() {
+ return completed.get();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 7870f86..fdfe5b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.slf4j.Logger;
@@ -154,8 +153,8 @@ public class ContainersLauncher extends AbstractService
break;
case CLEANUP_CONTAINER:
case CLEANUP_CONTAINER_FOR_REINIT:
- ContainerLaunch launcher = running.remove(containerId);
- if (launcher == null) {
+ ContainerLaunch existingLaunch = running.remove(containerId);
+ if (existingLaunch == null) {
// Container not launched.
// triggering KILLING to CONTAINER_CLEANEDUP_AFTER_KILL transition.
dispatcher.getEventHandler().handle(
@@ -169,12 +168,9 @@ public class ContainersLauncher extends AbstractService
// Cleanup a container whether it is running/killed/completed, so that
// no sub-processes are alive.
- try {
- launcher.cleanupContainer();
- } catch (IOException e) {
- LOG.warn("Got exception while cleaning container " + containerId
- + ". Ignoring.");
- }
+ ContainerCleanup cleanup = new ContainerCleanup(context, getConfig(),
+ dispatcher, exec, event.getContainer(), existingLaunch);
+ containerLauncher.submit(cleanup);
break;
case SIGNAL_CONTAINER:
SignalContainersLauncherEvent signalEvent =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java
new file mode 100644
index 0000000..6c99379
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerCleanup.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.InlineDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link ContainerCleanup}.
+ */
+public class TestContainerCleanup {
+
+ private YarnConfiguration conf;
+ private ContainerId containerId;
+ private ContainerExecutor executor;
+ private ContainerLaunch launch;
+ private ContainerCleanup cleanup;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new YarnConfiguration();
+ conf.setLong(NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, 60000);
+ Context context = mock(Context.class);
+ NMStateStoreService storeService = mock(NMStateStoreService.class);
+ when(context.getNMStateStore()).thenReturn(storeService);
+
+ Dispatcher dispatcher = new InlineDispatcher();
+ executor = mock(ContainerExecutor.class);
+ when(executor.signalContainer(Mockito.any(
+ ContainerSignalContext.class))).thenReturn(true);
+
+ ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
+ 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ containerId = ContainerId.newContainerId(attemptId, 1);
+ Container container = mock(Container.class);
+
+ when(container.getContainerId()).thenReturn(containerId);
+
+ launch = mock(ContainerLaunch.class);
+ launch.containerAlreadyLaunched = new AtomicBoolean(false);
+
+ launch.pidFilePath = new Path("target/" + containerId.toString() + ".pid");
+ when(launch.getContainerPid()).thenReturn(containerId.toString());
+
+ cleanup = new ContainerCleanup(context, conf, dispatcher, executor,
+ container, launch);
+ }
+
+ @Test
+ public void testNoCleanupWhenContainerNotLaunched() throws IOException {
+ cleanup.run();
+ verify(launch, Mockito.times(0)).signalContainer(
+ Mockito.any(SignalContainerCommand.class));
+ }
+
+ @Test
+ public void testCleanup() throws Exception {
+ launch.containerAlreadyLaunched.set(true);
+ cleanup.run();
+ ArgumentCaptor<ContainerSignalContext> captor =
+ ArgumentCaptor.forClass(ContainerSignalContext.class);
+
+ verify(executor, Mockito.times(1)).signalContainer(captor.capture());
+ Assert.assertEquals("signal", ContainerExecutor.Signal.TERM,
+ captor.getValue().getSignal());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ce70e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
index 0234ac2..f2fafd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
@@ -192,12 +192,14 @@ public class TestContainersLauncher {
.synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
dummyMap.put(containerId, containerLaunch);
Whitebox.setInternalState(spy, "running", dummyMap);
+
when(event.getType())
.thenReturn(ContainersLauncherEventType.CLEANUP_CONTAINER);
- doNothing().when(containerLaunch).cleanupContainer();
+ assertEquals(1, dummyMap.size());
spy.handle(event);
assertEquals(0, dummyMap.size());
- Mockito.verify(containerLaunch, Mockito.times(1)).cleanupContainer();
+ Mockito.verify(containerLauncher, Mockito.times(1))
+ .submit(Mockito.any(ContainerCleanup.class));
}
@Test
@@ -207,12 +209,14 @@ public class TestContainersLauncher {
.synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
dummyMap.put(containerId, containerLaunch);
Whitebox.setInternalState(spy, "running", dummyMap);
+
when(event.getType())
.thenReturn(ContainersLauncherEventType.CLEANUP_CONTAINER_FOR_REINIT);
- doNothing().when(containerLaunch).cleanupContainer();
+ assertEquals(1, dummyMap.size());
spy.handle(event);
assertEquals(0, dummyMap.size());
- Mockito.verify(containerLaunch, Mockito.times(1)).cleanupContainer();
+ Mockito.verify(containerLauncher, Mockito.times(1))
+ .submit(Mockito.any(ContainerCleanup.class));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: MAPREDUCE-7149. Javadocs for
FileInputFormat and OutputFormat to mention DT collection. Contributed by
Steve Loughran.
Posted by br...@apache.org.
MAPREDUCE-7149. Javadocs for FileInputFormat and OutputFormat to mention DT collection.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/735a36af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/735a36af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/735a36af
Branch: refs/heads/HDFS-13891
Commit: 735a36afd528862d7690bee2d71b2f60dd282d21
Parents: 96d28b4
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 20:01:42 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 20:01:42 2018 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapred/FileInputFormat.java | 8 ++++++--
.../main/java/org/apache/hadoop/mapred/OutputFormat.java | 3 +++
.../java/org/apache/hadoop/mapreduce/OutputFormat.java | 3 +++
.../hadoop/mapreduce/lib/input/FileInputFormat.java | 10 +++++++---
4 files changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index fe43991..afdc0ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -198,11 +198,15 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
}
}
- /** List input directories.
+ /**
+ * List input directories.
* Subclasses may override to, e.g., select only files matching a regular
* expression.
*
- * @param job the job to list input paths for
+ * If security is enabled, this method collects
+ * delegation tokens from the input paths and adds them to the job's
+ * credentials.
+ * @param job the job to list input paths for and attach tokens to.
* @return array of FileStatus objects
* @throws IOException if zero items.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
index 6ea6a71..adfda0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
@@ -71,6 +71,9 @@ public interface OutputFormat<K, V> {
* throwing an exception when it already exists, so that output is not
* overwritten.</p>
*
+ * Implementations which write to filesystems which support delegation
+ * tokens usually collect the tokens for the destination path(s)
+ * and attach them to the job configuration.
* @param ignored
* @param job job configuration.
* @throws IOException when output should not be attempted
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
index 6f9e525..901d5ee 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
@@ -66,6 +66,9 @@ public abstract class OutputFormat<K, V> {
* throwing an exception when it already exists, so that output is not
* overwritten.</p>
*
+ * Implementations which write to filesystems which support delegation
+ * tokens usually collect the tokens for the destination path(s)
+ * and attach them to the job context's JobConf.
* @param context information about the job
* @throws IOException when output should not be attempted
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index e2d8e6f..e2658ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -233,11 +233,15 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
(PathFilter) ReflectionUtils.newInstance(filterClass, conf) : null;
}
- /** List input directories.
+ /**
+ * List input directories.
* Subclasses may override to, e.g., select only files matching a regular
* expression.
- *
- * @param job the job to list input paths for
+ *
+ * If security is enabled, this method collects
+ * delegation tokens from the input paths and adds them to the job's
+ * credentials.
+ * @param job the job to list input paths for and attach tokens to.
* @return array of FileStatus objects
* @throws IOException if zero items.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-15837. DynamoDB table
Update can fail S3A FS init. Contributed by Steve Loughran.
Posted by br...@apache.org.
HADOOP-15837. DynamoDB table Update can fail S3A FS init.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee816f1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee816f1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee816f1f
Branch: refs/heads/HDFS-13891
Commit: ee816f1fd78b029b9efa567e8b1b62391563de14
Parents: 8a37983
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 14:57:38 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 14:57:38 2018 +0100
----------------------------------------------------------------------
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 89 +++++++++++++++---
.../s3a/s3guard/TestDynamoDBMiscOperations.java | 94 ++++++++++++++++++++
2 files changed, 169 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee816f1f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index f34afb5..6f892e5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -42,6 +42,7 @@ import java.util.stream.Collectors;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkBaseException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome;
@@ -66,6 +67,7 @@ import com.amazonaws.services.dynamodbv2.model.TableDescription;
import com.amazonaws.services.dynamodbv2.model.Tag;
import com.amazonaws.services.dynamodbv2.model.TagResourceRequest;
import com.amazonaws.services.dynamodbv2.model.WriteRequest;
+import com.amazonaws.waiters.WaiterTimedOutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
@@ -78,6 +80,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AWSClientIOException;
import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
import org.apache.hadoop.fs.s3a.AWSServiceThrottledException;
import org.apache.hadoop.fs.s3a.Constants;
@@ -1129,6 +1132,9 @@ public class DynamoDBMetadataStore implements MetadataStore {
* overall, this method is synchronous, and the table is guaranteed to exist
* after this method returns successfully.
*
+ * The wait for a table becoming active is Retry+Translated; it can fail
+ * while a table is not yet ready.
+ *
* @throws IOException if table does not exist and auto-creation is disabled;
* or table is being deleted, or any other I/O exception occurred.
*/
@@ -1144,7 +1150,6 @@ public class DynamoDBMetadataStore implements MetadataStore {
final String status = description.getTableStatus();
switch (status) {
case "CREATING":
- case "UPDATING":
LOG.debug("Table {} in region {} is being created/updated. This may"
+ " indicate that the table is being operated by another "
+ "concurrent thread or process. Waiting for active...",
@@ -1155,6 +1160,10 @@ public class DynamoDBMetadataStore implements MetadataStore {
throw new FileNotFoundException("DynamoDB table "
+ "'" + tableName + "' is being "
+ "deleted in region " + region);
+ case "UPDATING":
+ // table being updated; it can still be used.
+ LOG.debug("Table is being updated.");
+ break;
case "ACTIVE":
break;
default:
@@ -1285,24 +1294,34 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @throws InterruptedIOException if the wait was interrupted
* @throws IllegalArgumentException if an exception was raised in the waiter
*/
- @Retries.OnceRaw
- private void waitForTableActive(Table t) throws InterruptedIOException {
- try {
- t.waitForActive();
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting for table {} in region {} active",
- tableName, region, e);
- Thread.currentThread().interrupt();
- throw (InterruptedIOException)
- new InterruptedIOException("DynamoDB table '"
- + tableName + "' is not active yet in region " + region)
- .initCause(e);
- }
+ @Retries.RetryTranslated
+ private void waitForTableActive(Table t) throws IOException {
+ invoker.retry("Waiting for active state of table " + tableName,
+ null,
+ true,
+ () -> {
+ try {
+ t.waitForActive();
+ } catch (IllegalArgumentException ex) {
+ throw translateTableWaitFailure(tableName, ex);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for table {} in region {}"
+ + " active",
+ tableName, region, e);
+ Thread.currentThread().interrupt();
+ throw (InterruptedIOException)
+ new InterruptedIOException("DynamoDB table '"
+ + tableName + "' is not active yet in region " + region)
+ .initCause(e);
+ }
+ });
}
/**
* Create a table, wait for it to become active, then add the version
* marker.
+ * Creating an setting up the table isn't wrapped by any retry operations;
+ * the wait for a table to become available is RetryTranslated.
* @param capacity capacity to provision
* @throws IOException on any failure.
* @throws InterruptedIOException if the wait was interrupted
@@ -1604,4 +1623,46 @@ public class DynamoDBMetadataStore implements MetadataStore {
public Invoker getInvoker() {
return invoker;
}
+
+ /**
+ * Take an {@code IllegalArgumentException} raised by a DDB operation
+ * and if it contains an inner SDK exception, unwrap it.
+ * @param ex exception.
+ * @return the inner AWS exception or null.
+ */
+ public static SdkBaseException extractInnerException(
+ IllegalArgumentException ex) {
+ if (ex.getCause() instanceof SdkBaseException) {
+ return (SdkBaseException) ex.getCause();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Handle a table wait failure by extracting any inner cause and
+ * converting it, or, if unconvertable by wrapping
+ * the IllegalArgumentException in an IOE.
+ *
+ * @param name name of the table
+ * @param e exception
+ * @return an IOE to raise.
+ */
+ @VisibleForTesting
+ static IOException translateTableWaitFailure(
+ final String name, IllegalArgumentException e) {
+ final SdkBaseException ex = extractInnerException(e);
+ if (ex != null) {
+ if (ex instanceof WaiterTimedOutException) {
+ // a timeout waiting for state change: extract the
+ // message from the outer exception, but translate
+ // the inner one for the throttle policy.
+ return new AWSClientIOException(e.getMessage(), ex);
+ } else {
+ return translateException(e.getMessage(), name, ex);
+ }
+ } else {
+ return new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee816f1f/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
new file mode 100644
index 0000000..03be39f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
+import com.amazonaws.waiters.WaiterTimedOutException;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.s3a.AWSClientIOException;
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.translateTableWaitFailure;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Unit test suite for misc dynamoDB metastore operations.
+ */
+public class TestDynamoDBMiscOperations extends HadoopTestBase {
+
+ private static final String TIMEOUT_ERROR_MESSAGE
+ = "Table table-name did not transition into ACTIVE state.";
+
+ @Test
+ public void testUnwrapTableWaitTimeout() throws Throwable {
+ final Exception waiterTimedOut =
+ new WaiterTimedOutException("waiter timed out");
+ final AWSClientIOException ex = intercept(AWSClientIOException.class,
+ TIMEOUT_ERROR_MESSAGE,
+ () -> {
+ throw translateTableWaitFailure("example",
+ new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE,
+ waiterTimedOut));
+ });
+ assertEquals(waiterTimedOut, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateIllegalArgumentException() throws Throwable {
+ final IllegalArgumentException e =
+ new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE);
+ final IOException ex = intercept(IOException.class,
+ TIMEOUT_ERROR_MESSAGE,
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(e, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateWrappedDDBException() throws Throwable {
+ final Exception inner = new ResourceNotFoundException("ddb");
+ final IllegalArgumentException e =
+ new IllegalArgumentException("outer", inner);
+ final FileNotFoundException ex = intercept(FileNotFoundException.class,
+ "outer",
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(inner, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateWrappedOtherException() throws Throwable {
+ final Exception inner = new NullPointerException("npe");
+ final IllegalArgumentException e =
+ new IllegalArgumentException("outer", inner);
+ final IOException ex = intercept(IOException.class,
+ "outer",
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(e, ex.getCause());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDDS-627. OzoneFS read from an MR
Job throws java.lang.ArrayIndexOutOfBoundsException. Contributed by Mukul
Kumar Singh.
Posted by br...@apache.org.
HDDS-627. OzoneFS read from an MR Job throws java.lang.ArrayIndexOutOfBoundsException. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c05b2600
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c05b2600
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c05b2600
Branch: refs/heads/HDFS-13891
Commit: c05b260069511f2993a67683555ed2e1aea49a0a
Parents: 6dcfef7
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Oct 11 15:02:18 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Oct 11 15:08:22 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b2600/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
index 2b10578..2f17035 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
@@ -129,7 +129,8 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
totalReadLen += numBytesRead;
off += numBytesRead;
len -= numBytesRead;
- if (current.getRemaining() <= 0) {
+ if (current.getRemaining() <= 0 &&
+ ((currentStreamIndex + 1) < streamEntries.size())) {
currentStreamIndex += 1;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDDS-608. Access denied exception
for user hdfs while making getContainerWithPipeline call. Contributed by
Mukul Kumar Singh.
Posted by br...@apache.org.
HDDS-608. Access denied exception for user hdfs while making getContainerWithPipeline call. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90552b1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90552b1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90552b1e
Branch: refs/heads/HDFS-13891
Commit: 90552b1ea8380c1155f7000bd9e4c23e709e6970
Parents: 5ce70e1
Author: Nanda kumar <na...@apache.org>
Authored: Wed Oct 10 20:50:49 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Oct 10 20:50:49 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90552b1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 4eedf1f..3523499 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -187,7 +187,7 @@ public class SCMClientProtocolServer implements
}
}
String remoteUser = getRpcRemoteUsername();
- getScm().checkAdminAccess(remoteUser);
+ getScm().checkAdminAccess(null);
return scm.getContainerManager()
.getContainerWithPipeline(containerID);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-6989 Ensure timeline service
v2 codebase gets UGI from HttpServletRequest in a consistent way. Contributed
by Abhishek Modi
Posted by br...@apache.org.
YARN-6989 Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way. Contributed by Abhishek Modi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/045069ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/045069ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/045069ef
Branch: refs/heads/HDFS-13891
Commit: 045069efeca07674be1571252bc4c685aa57b440
Parents: bf3d591
Author: Vrushali C <vr...@apache.org>
Authored: Wed Oct 10 15:17:05 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Wed Oct 10 15:17:05 2018 -0700
----------------------------------------------------------------------
.../reader/TimelineReaderWebServicesUtils.java | 21 ++------------------
...elineReaderWhitelistAuthorizationFilter.java | 2 +-
2 files changed, 3 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/045069ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index 63529a4..bea81c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -269,25 +269,8 @@ public final class TimelineReaderWebServicesUtils {
* @return UGI.
*/
public static UserGroupInformation getUser(HttpServletRequest req) {
- return getCallerUserGroupInformation(req, false);
- }
-
- /**
- * Get UGI from the HTTP request.
- *
- * @param hsr HTTP request.
- * @param usePrincipal if true, use principal name else use remote user name
- * @return UGI.
- */
- public static UserGroupInformation getCallerUserGroupInformation(
- HttpServletRequest hsr, boolean usePrincipal) {
-
- String remoteUser = hsr.getRemoteUser();
- if (usePrincipal) {
- Principal princ = hsr.getUserPrincipal();
- remoteUser = princ == null ? null : princ.getName();
- }
-
+ Principal princ = req.getUserPrincipal();
+ String remoteUser = princ == null ? null : princ.getName();
UserGroupInformation callerUGI = null;
if (remoteUser != null) {
callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/045069ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
index dbe391c..b549778 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
@@ -67,7 +67,7 @@ public class TimelineReaderWhitelistAuthorizationFilter implements Filter {
if (isWhitelistReadAuthEnabled) {
UserGroupInformation callerUGI = TimelineReaderWebServicesUtils
- .getCallerUserGroupInformation(httpRequest, true);
+ .getUser(httpRequest);
if (callerUGI == null) {
String msg = "Unable to obtain user name, user not authenticated";
throw new AuthorizationException(msg);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HADOOP-15717. TGT renewal thread
does not log IOException (snemeth via rkanter)
Posted by br...@apache.org.
HADOOP-15717. TGT renewal thread does not log IOException (snemeth via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d91d47bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d91d47bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d91d47bc
Branch: refs/heads/HDFS-13891
Commit: d91d47bc739f23ca22a7e44fc83d449db57ab130
Parents: 64f2b32
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Oct 11 15:34:41 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Oct 11 15:35:44 2018 -0700
----------------------------------------------------------------------
.../hadoop/security/UserGroupInformation.java | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d91d47bc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index db88106..915d6df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -920,8 +920,8 @@ public class UserGroupInformation {
final long now = Time.now();
if (tgt.isDestroyed()) {
- LOG.error("TGT is destroyed. Aborting renew thread for {}.",
- getUserName());
+ LOG.error(String.format("TGT is destroyed. " +
+ "Aborting renew thread for %s.", getUserName()), ie);
return;
}
@@ -933,16 +933,18 @@ public class UserGroupInformation {
try {
tgtEndTime = tgt.getEndTime().getTime();
} catch (NullPointerException npe) {
- LOG.error("NPE thrown while getting KerberosTicket endTime. "
- + "Aborting renew thread for {}.", getUserName());
+ LOG.error(String.format("NPE thrown while getting " +
+ "KerberosTicket endTime. Aborting renew thread for %s.",
+ getUserName()), ie);
return;
}
- LOG.warn("Exception encountered while running the renewal "
- + "command for {}. (TGT end time:{}, renewalFailures: {},"
- + "renewalFailuresTotal: {})", getUserName(), tgtEndTime,
+ LOG.warn(String.format("Exception encountered while running the " +
+ "renewal command for %s. " +
+ "(TGT end time:%d, renewalFailures: %d, " +
+ "renewalFailuresTotal: %d)", getUserName(), tgtEndTime,
metrics.renewalFailures.value(),
- metrics.renewalFailuresTotal.value(), ie);
+ metrics.renewalFailuresTotal.value()), ie);
if (rp == null) {
// Use a dummy maxRetries to create the policy. The policy will
// only be used to get next retry time with exponential back-off.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-568. ozone sh volume info,
update,
delete operations fail when volume name is not prefixed by /. Contributed by
Dinesh Chitlangia.
Posted by br...@apache.org.
HDDS-568. ozone sh volume info, update, delete operations fail when volume name is not prefixed by /.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4de2dc26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4de2dc26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4de2dc26
Branch: refs/heads/HDFS-13891
Commit: 4de2dc2699fc371b2de83ba55ecbcecef1f0423b
Parents: 605622c
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:16:52 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 17:32:04 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/ozShell/TestOzoneShell.java | 48 ++++++++++++++++++++
.../hadoop/ozone/web/ozShell/Handler.java | 28 ++++++++++++
.../web/ozShell/volume/DeleteVolumeHandler.java | 12 +----
.../web/ozShell/volume/InfoVolumeHandler.java | 22 +--------
.../web/ozShell/volume/UpdateVolumeHandler.java | 12 +----
5 files changed, 79 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index 6e73b8c..d5f2554 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -283,6 +283,33 @@ public class TestOzoneShell {
GenericTestUtils.assertExceptionContains(
"Info Volume failed, error:VOLUME_NOT_FOUND", e);
}
+
+
+ volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ volumeArgs = VolumeArgs.newBuilder()
+ .setOwner("bilbo")
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
+ volume = client.getVolumeDetails(volumeName);
+ assertNotNull(volume);
+
+ //volumeName prefixed with /
+ String volumeNameWithSlashPrefix = "/" + volumeName;
+ args = new String[] {"volume", "delete",
+ url + "/" + volumeNameWithSlashPrefix};
+ execute(shell, args);
+ output = out.toString();
+ assertTrue(output.contains("Volume " + volumeName + " is deleted"));
+
+ // verify if volume has been deleted
+ try {
+ client.getVolumeDetails(volumeName);
+ fail("Get volume call should have thrown.");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Info Volume failed, error:VOLUME_NOT_FOUND", e);
+ }
}
@Test
@@ -295,6 +322,7 @@ public class TestOzoneShell {
.build();
client.createVolume(volumeName, volumeArgs);
+ //volumeName supplied as-is
String[] args = new String[] {"volume", "info", url + "/" + volumeName};
execute(shell, args);
@@ -303,6 +331,17 @@ public class TestOzoneShell {
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
+ //volumeName prefixed with /
+ String volumeNameWithSlashPrefix = "/" + volumeName;
+ args = new String[] {"volume", "info",
+ url + "/" + volumeNameWithSlashPrefix};
+ execute(shell, args);
+
+ output = out.toString();
+ assertTrue(output.contains(volumeName));
+ assertTrue(output.contains("createdOn")
+ && output.contains(OzoneConsts.OZONE_TIME_ZONE));
+
// test infoVolume with invalid volume name
args = new String[] {"volume", "info",
url + "/" + volumeName + "/invalid-name"};
@@ -365,6 +404,15 @@ public class TestOzoneShell {
vol = client.getVolumeDetails(volumeName);
assertEquals(newUser, vol.getOwner());
+ //volume with / prefix
+ String volumeWithPrefix = "/" + volumeName;
+ String newUser2 = "new-user2";
+ args = new String[] {"volume", "update", url + "/" + volumeWithPrefix,
+ "--user", newUser2};
+ execute(shell, args);
+ vol = client.getVolumeDetails(volumeName);
+ assertEquals(newUser2, vol.getOwner());
+
// test error conditions
args = new String[] {"volume", "update", url + "/invalid-volume",
"--user", newUser};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
index a9550c2..c8549c9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.ozone.web.ozShell;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
@@ -153,6 +155,32 @@ public abstract class Handler implements Callable<Void> {
}
}
+ /**
+ *
+ * @param uri
+ * @return volumeName
+ * @throws Exception
+ * @throws OzoneClientException when uri is null or invalid volume name
+ */
+ protected String parseVolumeName(String uri) throws Exception{
+ URI ozoneURI = verifyURI(uri);
+ Path path = Paths.get(ozoneURI.getPath());
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 1) {
+ String errorMessage;
+ if (pathNameCount < 1) {
+ errorMessage = "Volume name is required to perform volume " +
+ "operations like info, update, create and delete. ";
+ } else {
+ errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
+ "volume name";
+ }
+ throw new OzoneClientException(errorMessage);
+ }
+
+ return ozoneURI.getPath().replaceAll("^/+", "");
+ }
+
public boolean isVerbose() {
return parent.isVerbose();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
index d1e96fc..d757e20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
-import java.net.URI;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
@@ -43,14 +40,7 @@ public class DeleteVolumeHandler extends Handler {
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to delete a volume");
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
if (isVerbose()) {
System.out.printf("Volume name : %s%n", volumeName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
index 16c0342..48ed9f6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
@@ -18,11 +18,6 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
-import java.net.URI;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.web.ozShell.Handler;
@@ -48,22 +43,7 @@ public class InfoVolumeHandler extends Handler{
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- Path path = Paths.get(ozoneURI.getPath());
- int pathNameCount = path.getNameCount();
- if (pathNameCount != 1) {
- String errorMessage;
- if (pathNameCount < 1) {
- errorMessage = "Volume name is required to get info of a volume";
- } else {
- errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
- "volume name";
- }
- throw new OzoneClientException(errorMessage);
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de2dc26/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
index 0336fc2..46803d8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
@@ -25,13 +25,10 @@ import picocli.CommandLine.Parameters;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import java.net.URI;
-
/**
* Executes update volume calls.
*/
@@ -57,14 +54,7 @@ public class UpdateVolumeHandler extends Handler {
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to update a volume");
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
if (quota != null && !quota.isEmpty()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDFS-13968. BlockReceiver
Array-Based Queue. Contributed by BELUGA BEHR.
Posted by br...@apache.org.
HDFS-13968. BlockReceiver Array-Based Queue. Contributed by BELUGA BEHR.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3532aa38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3532aa38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3532aa38
Branch: refs/heads/HDFS-13891
Commit: 3532aa38867dc928b415c6c5eb47cca4fac349fe
Parents: eb34b5f
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Oct 11 13:49:39 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Oct 11 13:49:39 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockReceiver.java | 43 +++++++++-----------
1 file changed, 19 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3532aa38/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index afb9e2a..cb1f73d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -28,8 +28,9 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
import java.util.Arrays;
-import java.util.LinkedList;
+import java.util.Queue;
import java.util.zip.Checksum;
import org.apache.commons.logging.Log;
@@ -1208,9 +1209,9 @@ class BlockReceiver implements Closeable {
* Processes responses from downstream datanodes in the pipeline
* and sends back replies to the originator.
*/
- class PacketResponder implements Runnable, Closeable {
+ class PacketResponder implements Runnable, Closeable {
/** queue for packets waiting for ack - synchronization using monitor lock */
- private final LinkedList<Packet> ackQueue = new LinkedList<Packet>();
+ private final Queue<Packet> ackQueue = new ArrayDeque<>();
/** the thread that spawns this responder */
private final Thread receiverThread = Thread.currentThread();
/** is this responder running? - synchronization using monitor lock */
@@ -1264,12 +1265,10 @@ class BlockReceiver implements Closeable {
final long offsetInBlock, final Status ackStatus) {
final Packet p = new Packet(seqno, lastPacketInBlock, offsetInBlock,
System.nanoTime(), ackStatus);
- if(LOG.isDebugEnabled()) {
- LOG.debug(myString + ": enqueue " + p);
- }
- synchronized(ackQueue) {
+ LOG.debug("{}: enqueue {}", this, p);
+ synchronized (ackQueue) {
if (running) {
- ackQueue.addLast(p);
+ ackQueue.add(p);
ackQueue.notifyAll();
}
}
@@ -1321,15 +1320,13 @@ class BlockReceiver implements Closeable {
/** Wait for a packet with given {@code seqno} to be enqueued to ackQueue */
Packet waitForAckHead(long seqno) throws InterruptedException {
- synchronized(ackQueue) {
- while (isRunning() && ackQueue.size() == 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(myString + ": seqno=" + seqno +
- " waiting for local datanode to finish write.");
- }
+ synchronized (ackQueue) {
+ while (isRunning() && ackQueue.isEmpty()) {
+ LOG.debug("{}: seqno={} waiting for local datanode to finish write.",
+ myString, seqno);
ackQueue.wait();
}
- return isRunning() ? ackQueue.getFirst() : null;
+ return isRunning() ? ackQueue.element() : null;
}
}
@@ -1338,8 +1335,8 @@ class BlockReceiver implements Closeable {
*/
@Override
public void close() {
- synchronized(ackQueue) {
- while (isRunning() && ackQueue.size() != 0) {
+ synchronized (ackQueue) {
+ while (isRunning() && !ackQueue.isEmpty()) {
try {
ackQueue.wait();
} catch (InterruptedException e) {
@@ -1347,14 +1344,12 @@ class BlockReceiver implements Closeable {
Thread.currentThread().interrupt();
}
}
- if(LOG.isDebugEnabled()) {
- LOG.debug(myString + ": closing");
- }
+ LOG.debug("{}: closing", this);
running = false;
ackQueue.notifyAll();
}
- synchronized(this) {
+ synchronized (this) {
running = false;
notifyAll();
}
@@ -1657,12 +1652,12 @@ class BlockReceiver implements Closeable {
/**
* Remove a packet from the head of the ack queue
- *
+ *
* This should be called only when the ack queue is not empty
*/
private void removeAckHead() {
- synchronized(ackQueue) {
- ackQueue.removeFirst();
+ synchronized (ackQueue) {
+ ackQueue.remove();
ackQueue.notifyAll();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDDS-443. Create reusable
ProgressBar utility for freon tests. Contributed by Zsolt Horvath.
Posted by br...@apache.org.
HDDS-443. Create reusable ProgressBar utility for freon tests.
Contributed by Zsolt Horvath.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f068296f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f068296f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f068296f
Branch: refs/heads/HDFS-13891
Commit: f068296f8a88fc2a4c7b1680bc190c5fa7fc2469
Parents: 6fa3feb
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 18:18:19 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 9 18:19:00 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/tools/pom.xml | 6 +
.../apache/hadoop/ozone/freon/ProgressBar.java | 210 +++++++++++++++++++
.../hadoop/ozone/freon/TestProgressBar.java | 112 ++++++++++
3 files changed, 328 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f068296f/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index ac819b5..2d273d1 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -77,6 +77,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.15.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f068296f/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
new file mode 100644
index 0000000..a8d7e73
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+/**
+ * Run an arbitrary code and print progress on the provided stream. The
+ * progressbar stops when: - the provided currentvalue is less the the maxvalue
+ * - exception thrown
+ */
+public class ProgressBar {
+
+ private static final long REFRESH_INTERVAL = 1000L;
+
+ private PrintStream stream;
+ private AtomicLong currentValue;
+ private long maxValue;
+ private Thread progressBar;
+ private volatile boolean exception = false;
+ private long startTime;
+
+ /**
+ * @param stream Used to display the progress
+ * @param maxValue Maximum value of the progress
+ */
+ ProgressBar(PrintStream stream, long maxValue) {
+ this.stream = stream;
+ this.maxValue = maxValue;
+ this.currentValue = new AtomicLong(0);
+ this.progressBar = new Thread(new ProgressBarThread());
+ }
+
+ /**
+ * Start a task with a progessbar without any in/out parameters Runnable used
+ * just a task wrapper.
+ *
+ * @param task Runnable
+ */
+ public void start(Runnable task) {
+
+ startTime = System.nanoTime();
+
+ try {
+
+ progressBar.start();
+ task.run();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * Start a task with only out parameters.
+ *
+ * @param task Supplier that represents the task
+ * @param <T> Generic return type
+ * @return Whatever the supllier produces
+ */
+ public <T> T start(Supplier<T> task) {
+
+ startTime = System.nanoTime();
+ T result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.get();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return result;
+ }
+ }
+
+ /**
+ * Start a task with in/out parameters.
+ *
+ * @param input Input of the function
+ * @param task A Function that does the task
+ * @param <T> type of the input
+ * @param <R> return type
+ * @return Whatever the Function returns
+ */
+ public <T, R> R start(T input, Function<T, R> task) {
+
+ startTime = System.nanoTime();
+ R result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.apply(input);
+
+ } catch (Exception e) {
+ exception = true;
+ throw e;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return result;
+ }
+ }
+
+ /**
+ * Increment the progress with one step.
+ */
+ public void incrementProgress() {
+ currentValue.incrementAndGet();
+ }
+
+ private class ProgressBarThread implements Runnable {
+
+ @Override
+ public void run() {
+ try {
+
+ stream.println();
+ long value;
+
+ while ((value = currentValue.get()) < maxValue) {
+ print(value);
+
+ if (exception) {
+ break;
+ }
+ Thread.sleep(REFRESH_INTERVAL);
+ }
+
+ if (exception) {
+ stream.println();
+ stream.println("Incomplete termination, " + "check log for " +
+ "exception.");
+ } else {
+ print(maxValue);
+ }
+ stream.println();
+ } catch (InterruptedException e) {
+ stream.println(e);
+ }
+ }
+
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('â–ˆ');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
+ }
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f068296f/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
new file mode 100644
index 0000000..cea7be8
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.PrintStream;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+import static org.mockito.Mockito.*;
+
+@RunWith(MockitoJUnitRunner.class)
+/**
+ * Tests for the Progressbar class for Freon.
+ */
+public class TestProgressBar {
+
+ private PrintStream stream;
+
+ @Before
+ public void setupMock() {
+ stream = mock(PrintStream.class);
+ }
+
+ @Test
+ public void testWithRunnable() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Runnable task = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+ };
+
+ progressbar.start(task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithSupplier() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Supplier<Long> tasks = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return 1L; //return the result of the dummy task
+ };
+
+ progressbar.start(tasks);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithFunction() {
+
+ int maxValue = 10;
+ Long result;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Function<Long, String> task = (Long l) -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return "dummy result"; //return the result of the dummy task
+ };
+
+ progressbar.start(1L, task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HADOOP-15785. [JDK10] Javadoc
build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Posted by br...@apache.org.
HADOOP-15785. [JDK10] Javadoc build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b57f2f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b57f2f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b57f2f7
Branch: refs/heads/HDFS-13891
Commit: 7b57f2f71fbaa5af4897309597cca70a95b04edd
Parents: f261c31
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Thu Oct 11 13:51:51 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Thu Oct 11 13:51:51 2018 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configurable.java | 10 +++-
.../org/apache/hadoop/conf/Configuration.java | 45 +++++++++--------
.../hadoop/conf/ConfigurationWithLogging.java | 14 +++---
.../org/apache/hadoop/crypto/CryptoCodec.java | 7 +--
.../apache/hadoop/crypto/CryptoInputStream.java | 4 +-
.../hadoop/crypto/CryptoOutputStream.java | 4 +-
.../org/apache/hadoop/crypto/Decryptor.java | 14 +++---
.../org/apache/hadoop/crypto/Encryptor.java | 14 +++---
.../org/apache/hadoop/crypto/OpensslCipher.java | 18 +++----
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 15 +++---
.../apache/hadoop/crypto/key/KeyProvider.java | 6 +--
.../crypto/key/KeyProviderCryptoExtension.java | 9 ++--
.../KeyProviderDelegationTokenExtension.java | 2 +-
.../crypto/key/kms/KMSClientProvider.java | 6 +--
.../hadoop/crypto/key/kms/ValueQueue.java | 2 +-
.../crypto/random/OpensslSecureRandom.java | 10 ++--
.../apache/hadoop/fs/AbstractFileSystem.java | 45 +++++++++--------
.../apache/hadoop/fs/BufferedFSInputStream.java | 4 +-
.../apache/hadoop/fs/ByteBufferReadable.java | 8 +--
.../apache/hadoop/fs/ChecksumFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/ChecksumFs.java | 2 +-
.../hadoop/fs/CommonConfigurationKeys.java | 2 +-
.../java/org/apache/hadoop/fs/CreateFlag.java | 2 +-
.../org/apache/hadoop/fs/FSDataInputStream.java | 1 +
.../org/apache/hadoop/fs/FSInputChecker.java | 2 +-
.../java/org/apache/hadoop/fs/FileContext.java | 51 +++++++++++---------
.../java/org/apache/hadoop/fs/FileSystem.java | 11 ++---
.../java/org/apache/hadoop/fs/FileUtil.java | 4 +-
.../org/apache/hadoop/fs/HarFileSystem.java | 2 +-
.../hadoop/fs/HasEnhancedByteBufferAccess.java | 15 +++---
.../org/apache/hadoop/fs/LocalDirAllocator.java | 3 +-
.../org/apache/hadoop/fs/LocalFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/fs/Options.java | 2 +-
.../java/org/apache/hadoop/fs/QuotaUsage.java | 8 +--
.../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 6 +--
.../org/apache/hadoop/fs/ftp/FtpConfigKeys.java | 1 +
.../apache/hadoop/fs/local/LocalConfigKeys.java | 1 +
.../apache/hadoop/fs/permission/AclStatus.java | 2 +-
.../apache/hadoop/fs/permission/AclUtil.java | 10 ++--
.../hadoop/fs/permission/ScopedAclEntries.java | 14 +++---
.../org/apache/hadoop/fs/shell/Command.java | 24 ++++-----
.../apache/hadoop/fs/shell/CommandFormat.java | 2 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 ++---
.../apache/hadoop/ha/ActiveStandbyElector.java | 32 ++++++------
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 2 +-
.../java/org/apache/hadoop/ha/NodeFencer.java | 2 +-
.../org/apache/hadoop/ha/SshFenceByTcpPort.java | 2 +-
.../org/apache/hadoop/http/HttpServer2.java | 10 ++--
.../org/apache/hadoop/io/EnumSetWritable.java | 2 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 2 +-
.../org/apache/hadoop/io/ReadaheadPool.java | 2 +-
.../org/apache/hadoop/io/SecureIOUtils.java | 4 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 16 +++---
.../java/org/apache/hadoop/io/Writable.java | 4 +-
.../apache/hadoop/io/WritableComparable.java | 7 +--
.../org/apache/hadoop/io/WritableUtils.java | 6 ++-
.../io/compress/CompressionCodecFactory.java | 8 +--
.../org/apache/hadoop/io/compress/Lz4Codec.java | 4 +-
.../apache/hadoop/io/compress/SnappyCodec.java | 2 +-
.../io/compress/bzip2/Bzip2Compressor.java | 2 +-
.../io/compress/bzip2/Bzip2Decompressor.java | 6 +--
.../hadoop/io/compress/bzip2/Bzip2Factory.java | 4 +-
.../io/compress/bzip2/CBZip2InputStream.java | 26 +++++-----
.../io/compress/bzip2/CBZip2OutputStream.java | 7 +--
.../compress/zlib/BuiltInGzipDecompressor.java | 8 +--
.../hadoop/io/compress/zlib/ZlibCompressor.java | 2 +-
.../io/compress/zlib/ZlibDecompressor.java | 6 +--
.../hadoop/io/compress/zlib/ZlibFactory.java | 4 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 2 +-
.../erasurecode/rawcoder/util/GaloisField.java | 2 +-
.../org/apache/hadoop/io/file/tfile/TFile.java | 8 +--
.../org/apache/hadoop/io/file/tfile/Utils.java | 48 ++++++++++--------
.../org/apache/hadoop/io/retry/RetryProxy.java | 4 +-
.../hadoop/io/serializer/Deserializer.java | 2 +-
.../apache/hadoop/io/serializer/Serializer.java | 2 +-
.../org/apache/hadoop/ipc/CallerContext.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 9 ++--
.../java/org/apache/hadoop/ipc/ClientCache.java | 6 +--
.../apache/hadoop/ipc/DecayRpcScheduler.java | 3 +-
.../org/apache/hadoop/ipc/RefreshHandler.java | 1 -
.../org/apache/hadoop/ipc/RemoteException.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 2 +-
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +--
.../apache/hadoop/log/LogThrottlingHelper.java | 8 +--
.../metrics2/lib/MutableRollingAverages.java | 2 +-
.../apache/hadoop/metrics2/package-info.java | 3 ++
.../metrics2/sink/RollingFileSystemSink.java | 2 +-
.../apache/hadoop/metrics2/sink/StatsDSink.java | 4 +-
.../org/apache/hadoop/metrics2/util/MBeans.java | 10 ++--
.../hadoop/net/AbstractDNSToSwitchMapping.java | 8 +--
.../main/java/org/apache/hadoop/net/DNS.java | 2 +-
.../apache/hadoop/net/DNSToSwitchMapping.java | 2 +-
.../java/org/apache/hadoop/net/NetUtils.java | 14 +++---
.../org/apache/hadoop/net/NetworkTopology.java | 12 ++---
.../net/NetworkTopologyWithNodeGroup.java | 4 +-
.../apache/hadoop/net/ScriptBasedMapping.java | 8 +--
.../net/ScriptBasedMappingWithDependency.java | 7 ++-
.../apache/hadoop/net/SocketOutputStream.java | 6 +--
.../AuthenticationFilterInitializer.java | 6 +--
.../hadoop/security/HadoopKerberosName.java | 2 +-
.../security/IdMappingServiceProvider.java | 5 +-
.../apache/hadoop/security/SaslRpcClient.java | 10 ++--
.../apache/hadoop/security/SecurityUtil.java | 3 +-
.../hadoop/security/UserGroupInformation.java | 2 +-
.../security/alias/JavaKeyStoreProvider.java | 6 +--
.../alias/LocalJavaKeyStoreProvider.java | 3 +-
.../hadoop/security/authorize/ProxyUsers.java | 4 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 4 +-
.../apache/hadoop/security/ssl/SSLFactory.java | 4 +-
.../security/ssl/SSLHostnameVerifier.java | 20 ++++----
.../web/DelegationTokenAuthenticatedURL.java | 14 +++---
.../DelegationTokenAuthenticationFilter.java | 6 +--
.../DelegationTokenAuthenticationHandler.java | 4 +-
.../delegation/web/DelegationTokenManager.java | 2 +-
...rosDelegationTokenAuthenticationHandler.java | 2 +-
.../KerberosDelegationTokenAuthenticator.java | 2 +-
...emeDelegationTokenAuthenticationHandler.java | 3 +-
...udoDelegationTokenAuthenticationHandler.java | 2 +-
.../web/PseudoDelegationTokenAuthenticator.java | 2 +-
.../hadoop/service/ServiceOperations.java | 4 +-
.../service/launcher/ServiceLauncher.java | 2 +-
.../hadoop/service/launcher/package-info.java | 2 +-
.../java/org/apache/hadoop/util/ClassUtil.java | 1 -
.../apache/hadoop/util/ComparableVersion.java | 2 +-
.../java/org/apache/hadoop/util/FindClass.java | 4 +-
.../hadoop/util/GenericOptionsParser.java | 21 ++++----
.../apache/hadoop/util/HttpExceptionUtils.java | 6 +--
.../apache/hadoop/util/JsonSerialization.java | 2 +-
.../apache/hadoop/util/LightWeightCache.java | 10 ++--
.../java/org/apache/hadoop/util/LineReader.java | 3 --
.../org/apache/hadoop/util/MachineList.java | 2 +-
.../apache/hadoop/util/ShutdownHookManager.java | 2 +-
.../hadoop/util/ShutdownThreadsHelper.java | 2 -
.../org/apache/hadoop/util/StringUtils.java | 6 +--
.../main/java/org/apache/hadoop/util/Tool.java | 2 +-
.../java/org/apache/hadoop/util/ZKUtil.java | 4 +-
.../hadoop/util/bloom/DynamicBloomFilter.java | 5 +-
.../apache/hadoop/util/concurrent/AsyncGet.java | 2 +-
.../apache/hadoop/util/hash/JenkinsHash.java | 4 +-
.../org/apache/hadoop/cli/util/CLICommand.java | 6 +--
.../apache/hadoop/cli/util/CLICommandFS.java | 4 +-
.../apache/hadoop/cli/util/CLICommandTypes.java | 6 +--
.../rawcoder/RawErasureCoderBenchmark.java | 4 +-
.../hadoop/io/retry/TestDefaultRetryPolicy.java | 4 +-
.../org/apache/hadoop/net/StaticMapping.java | 4 +-
.../apache/hadoop/tracing/SetSpanReceiver.java | 2 +-
146 files changed, 518 insertions(+), 483 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
index d847f29..5816039 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configurable.java
@@ -26,9 +26,15 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Stable
public interface Configurable {
- /** Set the configuration to be used by this object. */
+ /**
+ * Set the configuration to be used by this object.
+ * @param conf configuration to be used
+ */
void setConf(Configuration conf);
- /** Return the configuration used by this object. */
+ /**
+ * Return the configuration used by this object.
+ * @return Configuration
+ */
Configuration getConf();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index c004cb5..478e56e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -115,7 +115,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
/**
* Provides access to configuration parameters.
*
- * <h4 id="Resources">Resources</h4>
+ * <h3 id="Resources">Resources</h3>
*
* <p>Configurations are specified by resources. A resource contains a set of
* name/value pairs as XML data. Each resource is named by either a
@@ -141,12 +141,12 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* Once a resource declares a value final, no subsequently-loaded
* resource can alter that value.
* For example, one might define a final parameter with:
- * <tt><pre>
+ * <pre><code>
* <property>
* <name>dfs.hosts.include</name>
* <value>/etc/hadoop/conf/hosts.include</value>
* <b><final>true</final></b>
- * </property></pre></tt>
+ * </property></code></pre>
*
* Administrators typically define parameters as final in
* <tt>core-site.xml</tt> for values that user applications may not alter.
@@ -164,7 +164,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
*
* <p>For example, if a configuration resource contains the following property
* definitions:
- * <tt><pre>
+ * <pre><code>
* <property>
* <name>basedir</name>
* <value>/user/${<i>user.name</i>}</value>
@@ -179,7 +179,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* <name>otherdir</name>
* <value>${<i>env.BASE_DIR</i>}/other</value>
* </property>
- * </pre></tt>
+ * </code></pre>
*
* <p>When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
* will be resolved to another property in this Configuration, while
@@ -203,7 +203,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* can define there own custom tags in hadoop.tags.custom property.
*
* <p>For example, we can tag existing property as:
- * <tt><pre>
+ * <pre><code>
* <property>
* <name>dfs.replication</name>
* <value>3</value>
@@ -215,7 +215,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
* <value>3</value>
* <tag>HDFS,SECURITY</tag>
* </property>
- * </pre></tt>
+ * </code></pre>
* <p> Properties marked with tags can be retrieved with <tt>conf
* .getAllPropertiesByTag("HDFS")</tt> or <tt>conf.getAllPropertiesByTags
* (Arrays.asList("YARN","SECURITY"))</tt>.</p>
@@ -581,9 +581,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* If you have multiple deprecation entries to add, it is more efficient to
* use #addDeprecations(DeprecationDelta[] deltas) instead.
*
- * @param key
- * @param newKeys
- * @param customMessage
+ * @param key to be deprecated
+ * @param newKeys list of keys that take up the values of deprecated key
+ * @param customMessage depcrication message
* @deprecated use {@link #addDeprecation(String key, String newKey,
String customMessage)} instead
*/
@@ -605,9 +605,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* If you have multiple deprecation entries to add, it is more efficient to
* use #addDeprecations(DeprecationDelta[] deltas) instead.
*
- * @param key
- * @param newKey
- * @param customMessage
+ * @param key to be deprecated
+ * @param newKey key that take up the values of deprecated key
+ * @param customMessage deprecation message
*/
public static void addDeprecation(String key, String newKey,
String customMessage) {
@@ -1428,6 +1428,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Unset a previously set property.
+ * @param name the property name
*/
public synchronized void unset(String name) {
String[] names = null;
@@ -1717,6 +1718,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* is equivalent to <code>set(<name>, value.toString())</code>.
* @param name property name
* @param value new value
+ * @param <T> enumeration type
*/
public <T extends Enum<T>> void setEnum(String name, T value) {
set(name, value.toString());
@@ -1727,8 +1729,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* Note that the returned value is trimmed by this method.
* @param name Property name
* @param defaultValue Value returned if no mapping exists
+ * @param <T> enumeration type
* @throws IllegalArgumentException If mapping is illegal for the type
* provided
+ * @return enumeration type
*/
public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
final String val = getTrimmed(name);
@@ -1807,6 +1811,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @param unit Unit to convert the stored property, if it exists.
* @throws NumberFormatException If the property stripped of its unit is not
* a number
+ * @return time duration in given time unit
*/
public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
String vStr = get(name);
@@ -2299,6 +2304,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* the CredentialProvider API and conditionally fallsback to config.
* @param name property name
* @return password
+ * @throws IOException when error in fetching password
*/
public char[] getPassword(String name) throws IOException {
char[] pass = null;
@@ -2358,7 +2364,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* alias.
* @param name alias of the provisioned credential
* @return password or null if not found
- * @throws IOException
+ * @throws IOException when error in fetching password
*/
public char[] getPasswordFromCredentialProviders(String name)
throws IOException {
@@ -3425,25 +3431,23 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Write out the non-default properties in this configuration to the
* given {@link Writer}.
- *
+ * <ul>
* <li>
* When property name is not empty and the property exists in the
* configuration, this method writes the property and its attributes
* to the {@link Writer}.
* </li>
- * <p>
*
* <li>
* When property name is null or empty, this method writes all the
* configuration properties and their attributes to the {@link Writer}.
* </li>
- * <p>
*
* <li>
* When property name is not empty but the property doesn't exist in
* the configuration, this method throws an {@link IllegalArgumentException}.
* </li>
- * <p>
+ * </ul>
* @param out the writer to write to.
*/
public void writeXml(String propertyName, Writer out)
@@ -3553,7 +3557,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Writes properties and their attributes (final and resource)
* to the given {@link Writer}.
- *
+ * <ul>
* <li>
* When propertyName is not empty, and the property exists
* in the configuration, the format of the output would be,
@@ -3593,6 +3597,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* found in the configuration, this method will throw an
* {@link IllegalArgumentException}.
* </li>
+ * </ul>
* <p>
* @param config the configuration
* @param propertyName property name
@@ -3791,7 +3796,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* get keys matching the the regex
* @param regex
- * @return Map<String,String> with matching keys
+ * @return {@literal Map<String,String>} with matching keys
*/
public Map<String,String> getValByRegex(String regex) {
Pattern p = Pattern.compile(regex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
index 8a5e054..68c5172 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigurationWithLogging.java
@@ -41,7 +41,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#get(String).
+ * See {@link Configuration#get(String)}.
*/
@Override
public String get(String name) {
@@ -51,7 +51,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#get(String, String).
+ * See {@link Configuration#get(String, String)}.
*/
@Override
public String get(String name, String defaultValue) {
@@ -62,7 +62,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getBoolean(String, boolean).
+ * See {@link Configuration#getBoolean(String, boolean)}.
*/
@Override
public boolean getBoolean(String name, boolean defaultValue) {
@@ -72,7 +72,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getFloat(String, float).
+ * See {@link Configuration#getFloat(String, float)}.
*/
@Override
public float getFloat(String name, float defaultValue) {
@@ -82,7 +82,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getInt(String, int).
+ * See {@link Configuration#getInt(String, int)}.
*/
@Override
public int getInt(String name, int defaultValue) {
@@ -92,7 +92,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#getLong(String, long).
+ * See {@link Configuration#getLong(String, long)}.
*/
@Override
public long getLong(String name, long defaultValue) {
@@ -102,7 +102,7 @@ public class ConfigurationWithLogging extends Configuration {
}
/**
- * @see Configuration#set(String, String, String).
+ * See {@link Configuration#set(String, String, String)}.
*/
@Override
public void set(String name, String value, String source) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
index d9c16bb..bcf4a65 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
@@ -158,14 +158,15 @@ public abstract class CryptoCodec implements Configurable, Closeable {
* For example a {@link javax.crypto.Cipher} will maintain its encryption
* context internally when we do encryption/decryption using the
* Cipher#update interface.
- * <p/>
+ * <p>
* Encryption/Decryption is not always on the entire file. For example,
* in Hadoop, a node may only decrypt a portion of a file (i.e. a split).
* In these situations, the counter is derived from the file position.
- * <p/>
+ * <p>
* The IV can be calculated by combining the initial IV and the counter with
* a lossless operation (concatenation, addition, or XOR).
- * @see http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_.28CTR.29
+ * See http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_
+ * .28CTR.29
*
* @param initIV initial IV
* @param counter counter for input stream position
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index a2273bf..5c879ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -53,10 +53,10 @@ import org.apache.hadoop.util.StringUtils;
* required in order to ensure that the plain text and cipher text have a 1:1
* mapping. The decryption is buffer based. The key points of the decryption
* are (1) calculating the counter and (2) padding through stream position:
- * <p/>
+ * <p>
* counter = base + pos/(algorithm blocksize);
* padding = pos%(algorithm blocksize);
- * <p/>
+ * <p>
* The underlying stream offset is maintained as state.
*/
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
index 2f347c5..8d11043 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
@@ -36,10 +36,10 @@ import com.google.common.base.Preconditions;
* required in order to ensure that the plain text and cipher text have a 1:1
* mapping. The encryption is buffer based. The key points of the encryption are
* (1) calculating counter and (2) padding through stream position.
- * <p/>
+ * <p>
* counter = base + pos/(algorithm blocksize);
* padding = pos%(algorithm blocksize);
- * <p/>
+ * <p>
* The underlying stream offset is maintained as state.
*
* Note that while some of this class' methods are synchronized, this is just to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
index 9958415..7556f18 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Decryptor.java
@@ -38,7 +38,7 @@ public interface Decryptor {
/**
* Indicate whether the decryption context is reset.
- * <p/>
+ * <p>
* Certain modes, like CTR, require a different IV depending on the
* position in the stream. Generally, the decryptor maintains any necessary
* context for calculating the IV and counter so that no reinit is necessary
@@ -49,22 +49,22 @@ public interface Decryptor {
/**
* This presents a direct interface decrypting with direct ByteBuffers.
- * <p/>
+ * <p>
* This function does not always decrypt the entire buffer and may potentially
* need to be called multiple times to process an entire buffer. The object
* may hold the decryption context internally.
- * <p/>
+ * <p>
* Some implementations may require sufficient space in the destination
* buffer to decrypt the entire input buffer.
- * <p/>
+ * <p>
* Upon return, inBuffer.position() will be advanced by the number of bytes
* read and outBuffer.position() by bytes written. Implementations should
* not modify inBuffer.limit() and outBuffer.limit().
- * <p/>
+ * <p>
* @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
- * not be null and inBuffer.remaining() must be > 0
+ * not be null and inBuffer.remaining() must be {@literal >} 0
* @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
- * not be null and outBuffer.remaining() must be > 0
+ * not be null and outBuffer.remaining() must be {@literal >} 0
* @throws IOException if decryption fails
*/
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
index 6dc3cfb..faeb176 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/Encryptor.java
@@ -37,7 +37,7 @@ public interface Encryptor {
/**
* Indicate whether the encryption context is reset.
- * <p/>
+ * <p>
* Certain modes, like CTR, require a different IV depending on the
* position in the stream. Generally, the encryptor maintains any necessary
* context for calculating the IV and counter so that no reinit is necessary
@@ -48,22 +48,22 @@ public interface Encryptor {
/**
* This presents a direct interface encrypting with direct ByteBuffers.
- * <p/>
+ * <p>
* This function does not always encrypt the entire buffer and may potentially
* need to be called multiple times to process an entire buffer. The object
* may hold the encryption context internally.
- * <p/>
+ * <p>
* Some implementations may require sufficient space in the destination
* buffer to encrypt the entire input buffer.
- * <p/>
+ * <p>
* Upon return, inBuffer.position() will be advanced by the number of bytes
* read and outBuffer.position() by bytes written. Implementations should
* not modify inBuffer.limit() and outBuffer.limit().
- * <p/>
+ * <p>
* @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
- * not be null and inBuffer.remaining() must be > 0
+ * not be null and inBuffer.remaining() must be > 0
* @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
- * not be null and outBuffer.remaining() must be > 0
+ * not be null and outBuffer.remaining() must be > 0
* @throws IOException if encryption fails
*/
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
index 133a9f9..0a2ba52 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
@@ -107,12 +107,12 @@ public final class OpensslCipher {
}
/**
- * Return an <code>OpensslCipher<code> object that implements the specified
+ * Return an <code>OpensslCipher</code> object that implements the specified
* transformation.
*
* @param transformation the name of the transformation, e.g.,
* AES/CTR/NoPadding.
- * @return OpensslCipher an <code>OpensslCipher<code> object
+ * @return OpensslCipher an <code>OpensslCipher</code> object
* @throws NoSuchAlgorithmException if <code>transformation</code> is null,
* empty, in an invalid format, or if Openssl doesn't implement the
* specified algorithm.
@@ -181,18 +181,18 @@ public final class OpensslCipher {
/**
* Continues a multiple-part encryption or decryption operation. The data
* is encrypted or decrypted, depending on how this cipher was initialized.
- * <p/>
+ * <p>
*
* All <code>input.remaining()</code> bytes starting at
* <code>input.position()</code> are processed. The result is stored in
* the output buffer.
- * <p/>
+ * <p>
*
* Upon return, the input buffer's position will be equal to its limit;
* its limit will not have changed. The output buffer's position will have
* advanced by n, when n is the value returned by this method; the output
* buffer's limit will not have changed.
- * <p/>
+ * <p>
*
* If <code>output.remaining()</code> bytes are insufficient to hold the
* result, a <code>ShortBufferException</code> is thrown.
@@ -218,21 +218,21 @@ public final class OpensslCipher {
/**
* Finishes a multiple-part operation. The data is encrypted or decrypted,
* depending on how this cipher was initialized.
- * <p/>
+ * <p>
*
* The result is stored in the output buffer. Upon return, the output buffer's
* position will have advanced by n, where n is the value returned by this
* method; the output buffer's limit will not have changed.
- * <p/>
+ * <p>
*
* If <code>output.remaining()</code> bytes are insufficient to hold the result,
* a <code>ShortBufferException</code> is thrown.
- * <p/>
+ * <p>
*
* Upon finishing, this method resets this cipher object to the state it was
* in when previously initialized. That is, the object is available to encrypt
* or decrypt more data.
- * <p/>
+ * <p>
*
* If any exception is thrown, this cipher object need to be reset before it
* can be used again.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index 5beda0d..7951af5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -62,23 +62,24 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* KeyProvider based on Java's KeyStore file format. The file may be stored in
* any Hadoop FileSystem using the following name mangling:
- * jks://hdfs@nn1.example.com/my/keys.jks -> hdfs://nn1.example.com/my/keys.jks
- * jks://file/home/owen/keys.jks -> file:///home/owen/keys.jks
- * <p/>
+ * jks://hdfs@nn1.example.com/my/keys.jks {@literal ->}
+ * hdfs://nn1.example.com/my/keys.jks
+ * jks://file/home/owen/keys.jks {@literal ->} file:///home/owen/keys.jks
+ * <p>
* If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is set,
* its value is used as the password for the keystore.
- * <p/>
+ * <p>
* If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is not set,
* the password for the keystore is read from file specified in the
* {@link #KEYSTORE_PASSWORD_FILE_KEY} configuration property. The password file
* is looked up in Hadoop's configuration directory via the classpath.
- * <p/>
+ * <p>
* <b>NOTE:</b> Make sure the password in the password file does not have an
* ENTER at the end, else it won't be valid for the Java KeyStore.
- * <p/>
+ * <p>
* If the environment variable, nor the property are not set, the password used
* is 'none'.
- * <p/>
+ * <p>
* It is expected for encrypted InputFormats and OutputFormats to copy the keys
* from the original provider into the job's Credentials object, which is
* accessed via the UserProvider. Therefore, this provider won't be used by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 286312c..2380f7a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -49,7 +49,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY
* abstraction to separate key storage from users of encryption. It
* is intended to support getting or storing keys in a variety of ways,
* including third party bindings.
- * <P/>
+ * <p>
* <code>KeyProvider</code> implementations must be thread safe.
*/
@InterfaceAudience.Public
@@ -549,7 +549,7 @@ public abstract class KeyProvider {
/**
* Create a new key generating the material for it.
* The given key must not already exist.
- * <p/>
+ * <p>
* This implementation generates the key material and calls the
* {@link #createKey(String, byte[], Options)} method.
*
@@ -593,7 +593,7 @@ public abstract class KeyProvider {
/**
* Roll a new version of the given key generating the material for it.
- * <p/>
+ * <p>
* This implementation generates the key material and calls the
* {@link #rollNewVersion(String, byte[])} method.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 3ee3bd7..00d7a7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -149,7 +149,7 @@ public class KeyProviderCryptoExtension extends
* Derive the initialization vector (IV) for the encryption key from the IV
* of the encrypted key. This derived IV is used with the encryption key to
* decrypt the encrypted key.
- * <p/>
+ * <p>
* The alternative to this is using the same IV for both the encryption key
* and the encrypted key. Even a simple symmetric transformation like this
* improves security by avoiding IV re-use. IVs will also be fairly unique
@@ -195,7 +195,7 @@ public class KeyProviderCryptoExtension extends
* The generated key material is of the same
* length as the <code>KeyVersion</code> material of the latest key version
* of the key and is encrypted using the same cipher.
- * <p/>
+ * <p>
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param encryptionKeyName
@@ -498,7 +498,7 @@ public class KeyProviderCryptoExtension extends
* and initialization vector. The generated key material is of the same
* length as the <code>KeyVersion</code> material and is encrypted using the
* same cipher.
- * <p/>
+ * <p>
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param encryptionKeyName The latest KeyVersion of this key's material will
@@ -576,7 +576,6 @@ public class KeyProviderCryptoExtension extends
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param ekvs List containing the EncryptedKeyVersion's
- * @return The re-encrypted EncryptedKeyVersion's, in the same order.
* @throws IOException If any EncryptedKeyVersion could not be re-encrypted
* @throws GeneralSecurityException If any EncryptedKeyVersion could not be
* re-encrypted because of a cryptographic issue.
@@ -589,7 +588,7 @@ public class KeyProviderCryptoExtension extends
/**
* Creates a <code>KeyProviderCryptoExtension</code> using a given
* {@link KeyProvider}.
- * <p/>
+ * <p>
* If the given <code>KeyProvider</code> implements the
* {@link CryptoExtension} interface the <code>KeyProvider</code> itself
* will provide the extension functionality.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
index 9212cbc..a63b7d5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
@@ -113,7 +113,7 @@ public class KeyProviderDelegationTokenExtension extends
/**
* Creates a <code>KeyProviderDelegationTokenExtension</code> using a given
* {@link KeyProvider}.
- * <p/>
+ * <p>
* If the given <code>KeyProvider</code> implements the
* {@link DelegationTokenExtension} interface the <code>KeyProvider</code>
* itself will provide the extension functionality, otherwise a default
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 8125510..26b528c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -246,12 +246,12 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
/**
* This provider expects URIs in the following form :
- * kms://<PROTO>@<AUTHORITY>/<PATH>
+ * {@literal kms://<PROTO>@<AUTHORITY>/<PATH>}
*
* where :
* - PROTO = http or https
- * - AUTHORITY = <HOSTS>[:<PORT>]
- * - HOSTS = <HOSTNAME>[;<HOSTS>]
+ * - AUTHORITY = {@literal <HOSTS>[:<PORT>]}
+ * - HOSTS = {@literal <HOSTNAME>[;<HOSTS>]}
* - HOSTNAME = string
* - PORT = integer
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index 1ddd8a3..b2ae084 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -344,7 +344,7 @@ public class ValueQueue <E> {
* <code>SyncGenerationPolicy</code> specified by the user.
* @param keyName String key name
* @param num Minimum number of values to return.
- * @return List<E> values returned
+ * @return {@literal List<E>} values returned
* @throws IOException
* @throws ExecutionException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
index 1219bf9..1863f5e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
@@ -30,16 +30,16 @@ import org.slf4j.LoggerFactory;
/**
* OpenSSL secure random using JNI.
* This implementation is thread-safe.
- * <p/>
+ * <p>
*
* If using an Intel chipset with RDRAND, the high-performance hardware
* random number generator will be used and it's much faster than
* {@link java.security.SecureRandom}. If RDRAND is unavailable, default
* OpenSSL secure random generator will be used. It's still faster
* and can generate strong random bytes.
- * <p/>
- * @see https://wiki.openssl.org/index.php/Random_Numbers
- * @see http://en.wikipedia.org/wiki/RdRand
+ * <p>
+ * See https://wiki.openssl.org/index.php/Random_Numbers
+ * See http://en.wikipedia.org/wiki/RdRand
*/
@InterfaceAudience.Private
public class OpensslSecureRandom extends Random {
@@ -97,7 +97,7 @@ public class OpensslSecureRandom extends Random {
* random bits (right justified, with leading zeros).
*
* @param numBits number of random bits to be generated, where
- * 0 <= <code>numBits</code> <= 32.
+ * 0 {@literal <=} <code>numBits</code> {@literal <=} 32.
*
* @return int an <code>int</code> containing the user-specified number
* of random bits (right justified, with leading zeros).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index cf484ca..721f009 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -336,7 +336,7 @@ public abstract class AbstractFileSystem {
* The default port of this file system.
*
* @return default port of this file system's Uri scheme
- * A uri with a port of -1 => default port;
+ * A uri with a port of -1 => default port;
*/
public abstract int getUriDefaultPort();
@@ -478,9 +478,11 @@ public abstract class AbstractFileSystem {
* through any internal symlinks or mount point
* @param p path to be resolved
* @return fully qualified path
- * @throws FileNotFoundException, AccessControlException, IOException
- * UnresolvedLinkException if symbolic link on path cannot be resolved
- * internally
+ * @throws FileNotFoundException
+ * @throws AccessControlException
+ * @throws IOException
+ * @throws UnresolvedLinkException if symbolic link on path cannot be
+ * resolved internally
*/
public Path resolvePath(final Path p) throws FileNotFoundException,
UnresolvedLinkException, AccessControlException, IOException {
@@ -1021,7 +1023,7 @@ public abstract class AbstractFileSystem {
* changes. (Modifications are merged into the current ACL.)
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications
+ * @param aclSpec List{@literal <AclEntry>} describing modifications
* @throws IOException if an ACL could not be modified
*/
public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
@@ -1035,7 +1037,7 @@ public abstract class AbstractFileSystem {
* retained.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing entries to remove
+ * @param aclSpec List{@literal <AclEntry>} describing entries to remove
* @throws IOException if an ACL could not be modified
*/
public void removeAclEntries(Path path, List<AclEntry> aclSpec)
@@ -1075,8 +1077,9 @@ public abstract class AbstractFileSystem {
* entries.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications, must include entries
- * for user, group, and others for compatibility with permission bits.
+ * @param aclSpec List{@literal <AclEntry>} describing modifications, must
+ * include entries for user, group, and others for compatibility with
+ * permission bits.
* @throws IOException if an ACL could not be modified
*/
public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
@@ -1088,7 +1091,7 @@ public abstract class AbstractFileSystem {
* Gets the ACLs of files and directories.
*
* @param path Path to get
- * @return RemoteIterator<AclStatus> which returns each AclStatus
+ * @return RemoteIterator{@literal <AclStatus>} which returns each AclStatus
* @throws IOException if an ACL could not be read
*/
public AclStatus getAclStatus(Path path) throws IOException {
@@ -1100,7 +1103,7 @@ public abstract class AbstractFileSystem {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -1118,7 +1121,7 @@ public abstract class AbstractFileSystem {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -1137,7 +1140,7 @@ public abstract class AbstractFileSystem {
* Get an xattr for a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
@@ -1154,11 +1157,13 @@ public abstract class AbstractFileSystem {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ *
+ * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ * or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
@@ -1170,12 +1175,13 @@ public abstract class AbstractFileSystem {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ * or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
@@ -1188,11 +1194,12 @@ public abstract class AbstractFileSystem {
* Get all of the xattr names for a file or directory.
* Only the xattr names for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ * or directory
* @throws IOException
*/
public List<String> listXAttrs(Path path)
@@ -1205,7 +1212,7 @@ public abstract class AbstractFileSystem {
* Remove an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
index 2eb8b95..973b136 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
- * A class that optimizes reading from FSInputStream by buffering
+ * A class that optimizes reading from FSInputStream by buffering.
*/
@InterfaceAudience.Private
@@ -44,7 +44,7 @@ implements Seekable, PositionedReadable, HasFileDescriptor {
*
* @param in the underlying input stream.
* @param size the buffer size.
- * @exception IllegalArgumentException if size <= 0.
+ * @exception IllegalArgumentException if size {@literal <=} 0.
*/
public BufferedFSInputStream(FSInputStream in, int size) {
super(in, size);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
index 20f7224..926b554 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
@@ -32,18 +32,18 @@ public interface ByteBufferReadable {
/**
* Reads up to buf.remaining() bytes into buf. Callers should use
* buf.limit(..) to control the size of the desired read.
- * <p/>
+ * <p>
* After a successful call, buf.position() will be advanced by the number
* of bytes read and buf.limit() should be unchanged.
- * <p/>
+ * <p>
* In the case of an exception, the values of buf.position() and buf.limit()
* are undefined, and callers should be prepared to recover from this
* eventuality.
- * <p/>
+ * <p>
* Many implementations will throw {@link UnsupportedOperationException}, so
* callers that are not confident in support for this method from the
* underlying filesystem should be prepared to handle that exception.
- * <p/>
+ * <p>
* Implementations should treat 0-length requests as legitimate, and must not
* signal an error upon their receipt.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 663c910..3db3173 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.util.Progressable;
* Abstract Checksumed FileSystem.
* It provide a basic implementation of a Checksumed FileSystem,
* which creates a checksum file for each raw file.
- * It generates & verifies checksums at the client side.
+ * It generates & verifies checksums at the client side.
*
*****************************************************************/
@InterfaceAudience.Public
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index c56f6e0..663b05d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -42,7 +42,7 @@ import org.slf4j.LoggerFactory;
* Abstract Checksumed Fs.
* It provide a basic implementation of a Checksumed Fs,
* which creates a checksum file for each raw file.
- * It generates & verifies checksums at the client side.
+ * It generates & verifies checksums at the client side.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 1eb27f8..72e5309 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -309,7 +309,7 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
"dr.who";
/**
- * User->groups static mapping to override the groups lookup
+ * User{@literal ->}groups static mapping to override the groups lookup
*/
public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES =
"hadoop.user.group.static.mapping.overrides";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index c3e088b..58b5f70 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* CreateFlag specifies the file create semantic. Users can combine flags like: <br>
* <code>
* EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND)
- * <code>
+ * </code>
* <p>
*
* Use the CreateFlag as follows:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
index 08d71f1..62c45f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs;
import java.io.DataInputStream;
+import java.io.EOFException;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
index 4f06e26..de66eab 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
@@ -101,7 +101,7 @@ abstract public class FSInputChecker extends FSInputStream {
* Implementors should simply pass through to the underlying data stream.
* or
* (b) needChecksum() will return true:
- * - len >= maxChunkSize
+ * - len {@literal >=} maxChunkSize
* - checksum.length is a multiple of CHECKSUM_SIZE
* Implementors should read an integer number of data chunks into
* buf. The amount read should be bounded by len or by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 0b3889b..084f8f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -765,7 +765,7 @@ public class FileContext {
* Make(create) a directory and all the non-existent parents.
*
* @param dir - the dir to make
- * @param permission - permissions is set permission&~umask
+ * @param permission - permissions is set permission{@literal &~}umask
* @param createParent - if true then missing parent dirs are created if false
* then parent must exist
*
@@ -979,7 +979,6 @@ public class FileContext {
/**
* Renames Path src to Path dst
* <ul>
- * <li
* <li>Fails if src is a file and dst is a directory.
* <li>Fails if src is a directory and dst is a file.
* <li>Fails if the parent of dst does not exist or is a file.
@@ -1001,7 +1000,7 @@ public class FileContext {
*
* @throws AccessControlException If access is denied
* @throws FileAlreadyExistsException If <code>dst</code> already exists and
- * <code>options</options> has {@link Options.Rename#OVERWRITE}
+ * <code>options</code> has {@link Options.Rename#OVERWRITE}
* option false.
* @throws FileNotFoundException If <code>src</code> does not exist
* @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
@@ -1250,7 +1249,7 @@ public class FileContext {
* checks to perform. If the requested permissions are granted, then the
* method returns normally. If access is denied, then the method throws an
* {@link AccessControlException}.
- * <p/>
+ * <p>
* The default implementation of this method calls {@link #getFileStatus(Path)}
* and checks the returned permissions against the requested permissions.
* Note that the getFileStatus call will be subject to authorization checks.
@@ -1497,9 +1496,9 @@ public class FileContext {
* <pre>
* Given a path referring to a symlink of form:
*
- * <---X--->
+ * {@literal <---}X{@literal --->}
* fs://host/A/B/link
- * <-----Y----->
+ * {@literal <-----}Y{@literal ----->}
*
* In this path X is the scheme and authority that identify the file system,
* and Y is the path leading up to the final path component "link". If Y is
@@ -1536,7 +1535,7 @@ public class FileContext {
*
*
* @throws AccessControlException If access is denied
- * @throws FileAlreadyExistsException If file <code>linkcode> already exists
+ * @throws FileAlreadyExistsException If file <code>link</code> already exists
* @throws FileNotFoundException If <code>target</code> does not exist
* @throws ParentNotDirectoryException If parent of <code>link</code> is not a
* directory.
@@ -2038,7 +2037,6 @@ public class FileContext {
* <dl>
* <dd>
* <dl>
- * <p>
* <dt> <tt> ? </tt>
* <dd> Matches any single character.
*
@@ -2400,7 +2398,8 @@ public class FileContext {
* changes. (Modifications are merged into the current ACL.)
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications
+ * @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ * modifications
* @throws IOException if an ACL could not be modified
*/
public void modifyAclEntries(final Path path, final List<AclEntry> aclSpec)
@@ -2421,7 +2420,8 @@ public class FileContext {
* retained.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing entries to remove
+ * @param aclSpec List{@literal <}AclEntry{@literal >} describing entries
+ * to remove
* @throws IOException if an ACL could not be modified
*/
public void removeAclEntries(final Path path, final List<AclEntry> aclSpec)
@@ -2481,8 +2481,9 @@ public class FileContext {
* entries.
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications, must include entries
- * for user, group, and others for compatibility with permission bits.
+ * @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ * modifications, must include entries for user, group, and others for
+ * compatibility with permission bits.
* @throws IOException if an ACL could not be modified
*/
public void setAcl(Path path, final List<AclEntry> aclSpec)
@@ -2502,7 +2503,8 @@ public class FileContext {
* Gets the ACLs of files and directories.
*
* @param path Path to get
- * @return RemoteIterator<AclStatus> which returns each AclStatus
+ * @return RemoteIterator{@literal <}AclStatus{@literal >} which returns
+ * each AclStatus
* @throws IOException if an ACL could not be read
*/
public AclStatus getAclStatus(Path path) throws IOException {
@@ -2520,7 +2522,7 @@ public class FileContext {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -2538,7 +2540,7 @@ public class FileContext {
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
@@ -2564,7 +2566,7 @@ public class FileContext {
* Get an xattr for a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
@@ -2587,11 +2589,12 @@ public class FileContext {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ * of the file or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
@@ -2609,12 +2612,13 @@ public class FileContext {
* Get all of the xattrs for a file or directory.
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ * of the file or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path, final List<String> names)
@@ -2633,7 +2637,7 @@ public class FileContext {
* Remove an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
@@ -2656,11 +2660,12 @@ public class FileContext {
* Get all of the xattr names for a file or directory.
* Only those xattr names which the logged-in user has permissions to view
* are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return List<String> of the XAttr names of the file or directory
+ * @return List{@literal <}String{@literal >} of the XAttr names of the
+ * file or directory
* @throws IOException
*/
public List<String> listXAttrs(Path path) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c309941..66b6d44 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -684,7 +684,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create a file with the provided permission.
*
* The permission of the file is set to be the provided permission as in
- * setPermission, not permission&~umask
+ * setPermission, not permission{@literal &~}umask
*
* The HDFS implementation is implemented using two RPCs.
* It is understood that it is inefficient,
@@ -709,7 +709,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Create a directory with the provided permission.
* The permission of the directory is set to be the provided permission as in
- * setPermission, not permission&~umask
+ * setPermission, not permission{@literal &~}umask
*
* @see #create(FileSystem, Path, FsPermission)
*
@@ -789,7 +789,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* <pre>
* if f == null :
* result = null
- * elif f.getLen() <= start:
+ * elif f.getLen() {@literal <=} start:
* result = []
* else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
* </pre>
@@ -2017,7 +2017,6 @@ public abstract class FileSystem extends Configured implements Closeable {
* <dl>
* <dd>
* <dl>
- * <p>
* <dt> <tt> ? </tt>
* <dd> Matches any single character.
*
@@ -2916,7 +2915,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* changes. (Modifications are merged into the current ACL.)
*
* @param path Path to modify
- * @param aclSpec List<AclEntry> describing modifications
+ * @param aclSpec List<AclEntry> describing modifications
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
@@ -3109,7 +3108,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return List<String> of the XAttr names of the file or directory
+ * @return List{@literal <String>} of the XAttr names of the file or directory
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index df89598..5f01d43 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -1474,8 +1474,8 @@ public class FileUtil {
* @param inputClassPath String input classpath to bundle into the jar manifest
* @param pwd Path to working directory to save jar
* @param targetDir path to where the jar execution will have its working dir
- * @param callerEnv Map<String, String> caller's environment variables to use
- * for expansion
+ * @param callerEnv Map {@literal <}String, String{@literal >} caller's
+ * environment variables to use for expansion
* @return String[] with absolute path to new jar in position 0 and
* unexpanded wild card entry path in position 1
* @throws IOException if there is an I/O error while writing the jar file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
index f6ffcb4..f7da819 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
@@ -83,7 +83,7 @@ public class HarFileSystem extends FileSystem {
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
+ * <p>
*
* @return <code>har</code>
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
index 982a0ef..8ceba7b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasEnhancedByteBufferAccess.java
@@ -52,18 +52,19 @@ public interface HasEnhancedByteBufferAccess {
* @return
* We will always return an empty buffer if maxLength was 0,
* whether or not we are at EOF.
- * If maxLength > 0, we will return null if the stream has
- * reached EOF.
+ * If maxLength > 0, we will return null if the stream
+ * has reached EOF.
* Otherwise, we will return a ByteBuffer containing at least one
* byte. You must free this ByteBuffer when you are done with it
* by calling releaseBuffer on it. The buffer will continue to be
* readable until it is released in this manner. However, the
* input stream's close method may warn about unclosed buffers.
- * @throws
- * IOException: if there was an error reading.
- * UnsupportedOperationException: if factory was null, and we
- * needed an external byte buffer. UnsupportedOperationException
- * will never be thrown unless the factory argument is null.
+ * @throws IOException if there was an error reading.
+ * @throws UnsupportedOperationException if factory was null,
+ * and we needed an external byte buffer.
+ * @throws UnsupportedOperationException will never be thrown
+ * unless the factory argument is null.
+ *
*/
public ByteBuffer read(ByteBufferPool factory, int maxLength,
EnumSet<ReadOption> opts)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index a4b158a..5f266a7b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -241,9 +241,8 @@ public class LocalDirAllocator {
* @param pathStr the requested file (this will be searched)
* @param conf the Configuration object
* @return true if files exist. false otherwise
- * @throws IOException
*/
- public boolean ifExists(String pathStr,Configuration conf) {
+ public boolean ifExists(String pathStr, Configuration conf) {
AllocatorPerContext context = obtainContext(contextCfgItemName);
return context.ifExists(pathStr, conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
index 538ccdf..c41190a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
@@ -54,7 +54,7 @@ public class LocalFileSystem extends ChecksumFileSystem {
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
+ * <p>
*
* @return <code>file</code>
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 5e93286..75bc12d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -290,7 +290,7 @@ public final class Options {
* @param defaultOpt Default checksum option
* @param userOpt User-specified checksum option. Ignored if null.
* @param userBytesPerChecksum User-specified bytesPerChecksum
- * Ignored if < 0.
+ * Ignored if {@literal <} 0.
*/
public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt,
ChecksumOpt userOpt, int userBytesPerChecksum) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
index 9ad6a28..3472362 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -229,8 +229,8 @@ public class QuotaUsage {
/**
* Output format:
- * <----12----> <----15----> <----15----> <----15----> <-------18------->
- * QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME
+ * |----12----| |----15----| |----15----| |----15----| |-------18-------|
+ * QUOTA REMAINING_QUOTA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME
*/
protected static final String QUOTA_STRING_FORMAT = "%12s %15s ";
protected static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";
@@ -244,9 +244,9 @@ public class QuotaUsage {
/**
* Output format:
- * <----12----> <------15-----> <------15-----> <------15----->
+ * |----12----| |------15-----| |------15-----| |------15-----|
* QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
- * <----12----> <----12----> <-------18------->
+ * |----12----| |----12----| |-------18-------|
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
private static final String STORAGE_TYPE_SUMMARY_FORMAT = "%13s %17s ";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index 644cf4e..676c207 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -76,7 +76,7 @@ public class FTPFileSystem extends FileSystem {
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
+ * <p>
*
* @return <code>ftp</code>
*/
@@ -162,7 +162,7 @@ public class FTPFileSystem extends FileSystem {
/**
* Set FTP's transfer mode based on configuration. Valid values are
* STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.
- * <p/>
+ * <p>
* Defaults to BLOCK_TRANSFER_MODE.
*
* @param conf
@@ -195,7 +195,7 @@ public class FTPFileSystem extends FileSystem {
* Set the FTPClient's data connection mode based on configuration. Valid
* values are ACTIVE_LOCAL_DATA_CONNECTION_MODE,
* PASSIVE_LOCAL_DATA_CONNECTION_MODE and PASSIVE_REMOTE_DATA_CONNECTION_MODE.
- * <p/>
+ * <p>
* Defaults to ACTIVE_LOCAL_DATA_CONNECTION_MODE.
*
* @param client
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
index e59efa5..b522102 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.ftp;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.util.DataChecksum;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
index 0b9e7455..e93858f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.util.DataChecksum;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
index 131aa19..385fed2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
@@ -69,7 +69,7 @@ public class AclStatus {
/**
* Returns the list of all ACL entries, ordered by their natural ordering.
*
- * @return List<AclEntry> unmodifiable ordered list of all ACL entries
+ * @return List<AclEntry> unmodifiable ordered list of all ACL entries
*/
public List<AclEntry> getEntries() {
return entries;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
index 2811a89..4249252 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclUtil.java
@@ -36,8 +36,8 @@ public final class AclUtil {
* Given permissions and extended ACL entries, returns the full logical ACL.
*
* @param perm FsPermission containing permissions
- * @param entries List<AclEntry> containing extended ACL entries
- * @return List<AclEntry> containing full logical ACL
+ * @param entries List<AclEntry> containing extended ACL entries
+ * @return List<AclEntry> containing full logical ACL
*/
public static List<AclEntry> getAclFromPermAndEntries(FsPermission perm,
List<AclEntry> entries) {
@@ -93,8 +93,8 @@ public final class AclUtil {
* Translates the given permission bits to the equivalent minimal ACL.
*
* @param perm FsPermission to translate
- * @return List<AclEntry> containing exactly 3 entries representing the owner,
- * group and other permissions
+ * @return List<AclEntry> containing exactly 3 entries representing the
+ * owner, group and other permissions
*/
public static List<AclEntry> getMinimalAcl(FsPermission perm) {
return Lists.newArrayList(
@@ -119,7 +119,7 @@ public final class AclUtil {
* Checks if the given entries represent a minimal ACL (contains exactly 3
* entries).
*
- * @param entries List<AclEntry> entries to check
+ * @param entries List<AclEntry> entries to check
* @return boolean true if the entries represent a minimal ACL
*/
public static boolean isMinimalAcl(List<AclEntry> entries) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-8858. CapacityScheduler
should respect maximum node resource when per-queue maximum-allocation is
being used. Contributed by Wangda Tan.
Posted by br...@apache.org.
YARN-8858. CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edce8664
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edce8664
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edce8664
Branch: refs/heads/HDFS-13891
Commit: edce866489d83744f3f47a3b884b0c6136885e4a
Parents: f068296
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Oct 10 09:32:17 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Oct 10 09:32:27 2018 +0800
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 11 +++++
.../scheduler/capacity/CapacityScheduler.java | 12 ++++-
.../capacity/TestContainerAllocation.java | 52 ++++++++++++++++++++
3 files changed, 74 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edce8664/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 8c7e447..0f72c76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -258,6 +259,16 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
}
+ @VisibleForTesting
+ public void setForceConfiguredMaxAllocation(boolean flag) {
+ writeLock.lock();
+ try {
+ forceConfiguredMaxAllocation = flag;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
private void updateMaxResources(SchedulerNode node, boolean add) {
Resource totalResource = node.getTotalResource();
ResourceInformation[] totalResources;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edce8664/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 75d6144..fddd361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2581,7 +2581,17 @@ public class CapacityScheduler extends
LOG.error("queue " + queueName + " is not an leaf queue");
return getMaximumResourceCapability();
}
- return ((LeafQueue)queue).getMaximumAllocation();
+
+ // queue.getMaxAllocation returns *configured* maximum allocation.
+ // getMaximumResourceCapability() returns maximum allocation considers
+ // per-node maximum resources. So return (component-wise) min of the two.
+
+ Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation();
+ Resource clusterMaxAllocationConsiderNodeMax =
+ getMaximumResourceCapability();
+
+ return Resources.componentwiseMin(queueMaxAllocation,
+ clusterMaxAllocationConsiderNodeMax);
}
private String handleMoveToPlanQueue(String targetQueueName) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edce8664/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index f1b4444..d5f6ab1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@@ -68,6 +69,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
public class TestContainerAllocation {
@@ -1130,4 +1132,54 @@ public class TestContainerAllocation {
rm1.close();
}
+
+ @Test(timeout = 60000)
+ public void testContainerRejectionWhenAskBeyondDynamicMax()
+ throws Exception {
+ CapacitySchedulerConfiguration newConf =
+ (CapacitySchedulerConfiguration) TestUtils
+ .getConfigurationWithMultipleQueues(conf);
+ newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+ newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+ + MAXIMUM_ALLOCATION_MB, "4096");
+
+ MockRM rm1 = new MockRM(newConf);
+ rm1.start();
+
+ // before any node registered or before registration timeout,
+ // submit an app beyond queue max leads to failure.
+ boolean submitFailed = false;
+ MockNM nm1 = rm1.registerNode("h1:1234", 2 * GB, 1);
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ try {
+ am1.allocate("*", 5 * GB, 1, null);
+ } catch (InvalidResourceRequestException e) {
+ submitFailed = true;
+ }
+ Assert.assertTrue(submitFailed);
+
+ // Ask 4GB succeeded.
+ am1.allocate("*", 4 * GB, 1, null);
+
+ // Add a new node, now the cluster maximum should be refreshed to 3GB.
+ CapacityScheduler cs = (CapacityScheduler)rm1.getResourceScheduler();
+ cs.getNodeTracker().setForceConfiguredMaxAllocation(false);
+ rm1.registerNode("h2:1234", 3 * GB, 1);
+
+ // Now ask 4 GB will fail
+ submitFailed = false;
+ try {
+ am1.allocate("*", 4 * GB, 1, null);
+ } catch (InvalidResourceRequestException e) {
+ submitFailed = true;
+ }
+ Assert.assertTrue(submitFailed);
+
+ // But ask 3 GB succeeded.
+ am1.allocate("*", 3 * GB, 1, null);
+
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-609. On restart,
SCM does not exit chill mode as it expects DNs to report containers
in ALLOCATED state. Contributed by Hanisha Koneru.
Posted by br...@apache.org.
HDDS-609. On restart, SCM does not exit chill mode as it expects DNs to report containers in ALLOCATED state. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74db993a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74db993a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74db993a
Branch: refs/heads/HDFS-13891
Commit: 74db993a61e201d2e8d33bb42f787e0a71388d0d
Parents: c011810
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Thu Oct 11 21:03:01 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Thu Oct 11 21:03:01 2018 -0700
----------------------------------------------------------------------
.../hdds/scm/server/SCMChillModeManager.java | 12 +++--
.../scm/server/TestSCMChillModeManager.java | 53 +++++++++++++++++++-
2 files changed, 61 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74db993a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
index c11a60f..44744f0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -162,7 +163,7 @@ public class SCMChillModeManager implements
// Required cutoff % for containers with at least 1 reported replica.
private double chillModeCutoff;
- // Containers read from scm db.
+ // Containers read from scm db (excluding containers in ALLOCATED state).
private Map<Long, ContainerInfo> containerMap;
private double maxContainer;
@@ -174,11 +175,16 @@ public class SCMChillModeManager implements
containerMap = new ConcurrentHashMap<>();
if(containers != null) {
containers.forEach(c -> {
- if (c != null) {
+ // Containers in ALLOCATED state should not be included while
+ // calculating the total number of containers here. They are not
+ // reported by DNs and hence should not affect the chill mode exit
+ // rule.
+ if (c != null && c.getState() != null &&
+ !c.getState().equals(HddsProtos.LifeCycleState.ALLOCATED)) {
containerMap.put(c.getContainerID(), c);
}
});
- maxContainer = containers.size();
+ maxContainer = containerMap.size();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74db993a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
index 53d76e6..41708a4 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -70,6 +71,11 @@ public class TestSCMChillModeManager {
private void testChillMode(int numContainers) throws Exception {
containers = new ArrayList<>();
containers.addAll(HddsTestUtils.getContainerInfo(numContainers));
+ // Assign open state to containers to be included in the chill mode
+ // container list
+ for (ContainerInfo container : containers) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
scmChillModeManager = new SCMChillModeManager(config, containers, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
@@ -85,6 +91,11 @@ public class TestSCMChillModeManager {
public void testChillModeExitRule() throws Exception {
containers = new ArrayList<>();
containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ // Assign open state to containers to be included in the chill mode
+ // container list
+ for (ContainerInfo container : containers) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
scmChillModeManager = new SCMChillModeManager(config, containers, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
@@ -119,6 +130,46 @@ public class TestSCMChillModeManager {
testChillModeDataNodes(5);
}
+ /**
+ * Check that containers in Allocated state are not considered while
+ * computing percentage of containers with at least 1 reported replica in
+ * chill mode exit rule.
+ */
+ @Test
+ public void testContainerChillModeRule() throws Exception {
+ containers = new ArrayList<>();
+ // Add 100 containers to the list of containers in SCM
+ containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ // Assign OPEN state to first 25 containers and ALLLOCATED state to rest
+ // of the containers
+ for (ContainerInfo container : containers.subList(0, 25)) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
+ for (ContainerInfo container : containers.subList(25, 100)) {
+ container.setState(HddsProtos.LifeCycleState.ALLOCATED);
+ }
+
+ scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // When 10 OPEN containers are reported by DNs, the computed container
+ // threshold should be 10/25 as there are only 25 open containers.
+ // Containers in ALLOCATED state should not contribute towards list of
+ // containers while calculating container threshold in SCMChillNodeManager
+ testContainerThreshold(containers.subList(0, 10), 0.4);
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // When remaining 15 OPEN containers are reported by DNs, the container
+ // threshold should be (10+15)/25.
+ testContainerThreshold(containers.subList(10, 25), 1.0);
+
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 100, 1000 * 5);
+ }
+
private void testChillModeDataNodes(int numOfDns) throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(config);
conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
@@ -146,7 +197,7 @@ public class TestSCMChillModeManager {
queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
HddsTestUtils.createNodeRegistrationContainerReport(containers));
GenericTestUtils.waitFor(() -> {
- return scmChillModeManager.getInChillMode();
+ return !scmChillModeManager.getInChillMode();
}, 10, 1000 * 10);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-8666. [UI2] Remove
application tab from YARN Queue Page. Contributed by Yesha Vora.
Posted by br...@apache.org.
YARN-8666. [UI2] Remove application tab from YARN Queue Page. Contributed by Yesha Vora.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3da741f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3da741f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3da741f
Branch: refs/heads/HDFS-13891
Commit: d3da741fa27c14f3d1391a76d6ade5d1a8365aa5
Parents: 7b57f2f
Author: Sunil G <su...@apache.org>
Authored: Thu Oct 11 16:05:53 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Oct 11 16:05:53 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/templates/yarn-queue.hbs | 11 -----------
1 file changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3da741f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
index 5cced1c..756ef69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue.hbs
@@ -60,17 +60,6 @@
</div>
</div>
</div>
- <div class="panel-heading">
- <div class="clearfix">
- <ul class="nav nav-pills">
- <ul class="nav nav-pills collapse in">
- {{#link-to 'yarn-queue.apps' tagName="li" class=(if (eq target.currentPath 'yarn-queue.apps') "active")}}
- {{#link-to 'yarn-queue.apps' appId (query-params service=model.serviceName)}}Applications{{/link-to}}
- {{/link-to}}
- </ul>
- </ul>
- </div>
- </div>
<div class="panel-body yarn-app-body">
{{outlet}}
</div>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-8777. Container Executor C
binary change to execute interactive docker command. Contributed by Eric Yang
Posted by br...@apache.org.
YARN-8777. Container Executor C binary change to execute interactive docker command. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96d28b47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96d28b47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96d28b47
Branch: refs/heads/HDFS-13891
Commit: 96d28b4750f1088f2a10c83991b5a8149c97ef86
Parents: 47eeb3e
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Oct 11 09:25:21 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Thu Oct 11 09:25:21 2018 -0700
----------------------------------------------------------------------
.../impl/container-executor.c | 164 ++++++++++++++++++-
.../impl/container-executor.h | 7 +-
.../main/native/container-executor/impl/util.h | 3 +-
.../container-executor/impl/utils/docker-util.c | 53 ++++++
.../container-executor/impl/utils/docker-util.h | 10 ++
.../test/utils/test_docker_util.cc | 29 ++++
6 files changed, 263 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 7765308..1f7ae3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -47,6 +47,7 @@
#include <sys/wait.h>
#include <getopt.h>
#include <sys/param.h>
+#include <termios.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -1357,8 +1358,25 @@ char **construct_docker_command(const char *command_file) {
}
int run_docker(const char *command_file) {
+ struct configuration command_config = {0, NULL};
+
+ int ret = read_config(command_file, &command_config);
+ if (ret != 0) {
+ free_configuration(&command_config);
+ return INVALID_COMMAND_FILE;
+ }
+ char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (value != NULL && strcasecmp(value, "exec") == 0) {
+ free(value);
+ free_configuration(&command_config);
+ return run_docker_with_pty(command_file);
+ }
+ free_configuration(&command_config);
+ free(value);
+
char **args = construct_docker_command(command_file);
char* docker_binary = get_docker_binary(&CFG);
+
int exit_code = -1;
if (execvp(docker_binary, args) != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
@@ -1375,6 +1393,150 @@ int run_docker(const char *command_file) {
return exit_code;
}
+int run_docker_with_pty(const char *command_file) {
+ int exit_code = -1;
+ char **args = construct_docker_command(command_file);
+ char* docker_binary = get_docker_binary(&CFG);
+ int fdm, fds, rc;
+ char input[4000];
+
+ fdm = posix_openpt(O_RDWR);
+ if (fdm < 0) {
+ fprintf(stderr, "Error %d on posix_openpt()\n", errno);
+ return DOCKER_EXEC_FAILED;
+ }
+
+ rc = grantpt(fdm);
+ if (rc != 0) {
+ fprintf(stderr, "Error %d on grantpt()\n", errno);
+ return DOCKER_EXEC_FAILED;
+ }
+
+ rc = unlockpt(fdm);
+ if (rc != 0) {
+ fprintf(stderr, "Error %d on unlockpt()\n", errno);
+ return DOCKER_EXEC_FAILED;
+ }
+
+ // Open the slave PTY
+ fds = open(ptsname(fdm), O_RDWR);
+
+ // Creation of a child process
+ if (fork()) {
+ fd_set fd_in;
+ // Parent
+
+ // Close the slave side of the PTY
+ close(fds);
+ while (1) {
+ // Wait for data from standard input and master side of PTY
+ FD_ZERO(&fd_in);
+ FD_SET(0, &fd_in);
+ FD_SET(fdm, &fd_in);
+ rc = select(fdm + 1, &fd_in, NULL, NULL, NULL);
+ switch(rc) {
+ case -1 : fprintf(stderr, "Error %d on select()\n", errno);
+ exit(1);
+ default :
+ {
+ // If data on standard input
+ if (FD_ISSET(0, &fd_in)) {
+ rc = read(0, input, sizeof(input));
+ if (rc > 0) {
+ // Send data on the master side of PTY
+ ssize_t written = write(fdm, input, rc);
+ if (written == -1) {
+ fprintf(stderr, "Error %d writing to container.\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ } else {
+ if (rc < 0) {
+ fprintf(stderr, "Error %d on read standard input\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ }
+ }
+
+ // If data on master side of PTY
+ if (FD_ISSET(fdm, &fd_in)) {
+ rc = read(fdm, input, sizeof(input));
+ if (rc > 0) {
+ // Send data on standard output
+ ssize_t written = write(1, input, rc);
+ if (written == -1) {
+ fprintf(stderr, "Error %d writing to terminal.\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ } else {
+ if (rc < 0) {
+ fprintf(stderr, "Error %d on read master PTY\n", errno);
+ exit(DOCKER_EXEC_FAILED);
+ }
+ }
+ }
+ }
+ } // End switch
+ } // End while
+ } else {
+ struct termios slave_orig_term_settings; // Saved terminal settings
+ struct termios new_term_settings; // Current terminal settings
+
+ // Child
+
+ // Close the master side of the PTY
+ close(fdm);
+
+ // Save the default parameters of the slave side of the PTY
+ rc = tcgetattr(fds, &slave_orig_term_settings);
+
+ // Set raw mode on the slave side of the PTY
+ new_term_settings = slave_orig_term_settings;
+ cfmakeraw (&new_term_settings);
+ tcsetattr (fds, TCSANOW, &new_term_settings);
+
+ // The slave side of the PTY becomes the standard input and outputs of the child process
+ close(0); // Close standard input (current terminal)
+ close(1); // Close standard output (current terminal)
+ close(2); // Close standard error (current terminal)
+
+ if (dup(fds) == -1) {
+ // PTY becomes standard input (0)
+ exit(DOCKER_EXEC_FAILED);
+ }
+ if (dup(fds) == -1) {
+ // PTY becomes standard output (1)
+ exit(DOCKER_EXEC_FAILED);
+ }
+ if (dup(fds) == -1) {
+ // PTY becomes standard error (2)
+ exit(DOCKER_EXEC_FAILED);
+ }
+
+ // Now the original file descriptor is useless
+ close(fds);
+
+ // Make the current process a new session leader
+ setsid();
+
+ // As the child is a session leader, set the controlling terminal to be the slave side of the PTY
+ // (Mandatory for programs like the shell to make them manage correctly their outputs)
+ ioctl(0, TIOCSCTTY, 1);
+
+ if (execvp(docker_binary, args) != 0) {
+ fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
+ docker_binary, strerror(errno));
+ free(docker_binary);
+ free_values(args);
+ exit_code = DOCKER_EXEC_FAILED;
+ } else {
+ free_values(args);
+ exit_code = 0;
+ }
+ }
+
+ return exit_code;
+}
+
int exec_docker_command(char *docker_command, char **argv, int argc) {
int i;
char* docker_binary = get_docker_binary(&CFG);
@@ -2708,4 +2870,4 @@ int remove_docker_container(char**argv, int argc) {
exit_code = clean_docker_cgroups(yarn_hierarchy, container_id);
}
return exit_code;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index cd09e1e..ce05c57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -269,6 +269,11 @@ int is_docker_support_enabled();
int run_docker(const char *command_file);
/**
+ * Run a docker command passing the command file as an argument with terminal.
+ */
+int run_docker_with_pty(const char *command_file);
+
+/**
* Run a docker command without a command file
*/
int exec_docker_command(char *docker_command, char **argv, int argc);
@@ -295,4 +300,4 @@ char* flatten(char **args);
/**
* Remove docker container
*/
-int remove_docker_container(char **argv, int argc);
\ No newline at end of file
+int remove_docker_container(char **argv, int argc);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index 6aac1fe..372c17a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -68,7 +68,8 @@ enum errorcodes {
DOCKER_IMAGE_INVALID = 40,
// DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED)
ERROR_COMPILING_REGEX = 42,
- INVALID_CONTAINER_ID = 43
+ INVALID_CONTAINER_ID = 43,
+ DOCKER_EXEC_FAILED = 44
};
/* Macros for min/max. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 69f27ba..c0aa80b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -432,6 +432,8 @@ int get_docker_command(const char *command_file, const struct configuration *con
ret = get_docker_volume_command(command_file, conf, args);
} else if (strcmp(DOCKER_START_COMMAND, command) == 0) {
ret = get_docker_start_command(command_file, conf, args);
+ } else if (strcmp(DOCKER_EXEC_COMMAND, command) == 0) {
+ ret = get_docker_exec_command(command_file, conf, args);
} else {
ret = UNKNOWN_DOCKER_COMMAND;
}
@@ -820,6 +822,57 @@ free_and_exit:
return ret;
}
+int get_docker_exec_command(const char *command_file, const struct configuration *conf, args *args) {
+ int ret = 0, i = 0;
+ char *container_name = NULL;
+ char **launch_command = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_EXEC_COMMAND, &command_config);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ ret = INVALID_DOCKER_CONTAINER_NAME;
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, DOCKER_EXEC_COMMAND);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, "-it");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, container_name);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+
+ launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
+ ",");
+ if (launch_command != NULL) {
+ for (i = 0; launch_command[i] != NULL; ++i) {
+ ret = add_to_args(args, launch_command[i]);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ }
+ } else {
+ ret = INVALID_COMMAND_FILE;
+ }
+free_and_exit:
+ free(container_name);
+ free_configuration(&command_config);
+ free_values(launch_command);
+ return ret;
+}
+
static int detach_container(const struct configuration *command_config, args *args) {
return add_param_to_command(command_config, "detach", "-d", 0, args);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
index 7b7322d..fc7b4cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -34,6 +34,7 @@
#define DOCKER_KILL_COMMAND "kill"
#define DOCKER_VOLUME_COMMAND "volume"
#define DOCKER_START_COMMAND "start"
+#define DOCKER_EXEC_COMMAND "exec"
#define DOCKER_ARG_MAX 1024
#define ARGS_INITIAL_VALUE { 0 };
@@ -176,6 +177,15 @@ int get_docker_volume_command(const char *command_file, const struct configurati
int get_docker_start_command(const char* command_file, const struct configuration* conf, args *args);
/**
+ * Get the Docker exec command line string. The function will verify that the params file is meant for the exec command.
+ * @param command_file File containing the params for the Docker start command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param args Buffer to construct argv
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_exec_command(const char* command_file, const struct configuration* conf, args *args);
+
+/**
* Give an error message for the supplied error code
* @param error_code the error code
* @return const string containing the error message
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d28b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index b289857..2817e0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -1762,4 +1762,33 @@ namespace ContainerExecutor {
free_configuration(&container_executor_cfg);
}
}
+
+ TEST_F(TestDockerUtil, test_docker_exec) {
+ std::string container_executor_contents = "[docker]\n"
+ " docker.allowed.devices=/dev/test\n docker.trusted.registries=hadoop\n";
+ write_file(container_executor_cfg_file, container_executor_contents);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=exec\n name=container_e1_12312_11111_02_000001\n launch-command=bash",
+ "exec -it container_e1_12312_11111_02_000001 bash"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=exec\n image=hadoop/docker-image\n user=nobody",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_exec_command);
+ free_configuration(&container_executor_cfg);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-5742 Serve aggregated logs of
historical apps from timeline service. Contributed by Rohith Sharma KS
Posted by br...@apache.org.
YARN-5742 Serve aggregated logs of historical apps from timeline service. Contributed by Rohith Sharma KS
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d198180
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d198180
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d198180
Branch: refs/heads/HDFS-13891
Commit: 8d1981806feb8278966c02a9eff42d72541bb35e
Parents: d91d47b
Author: Vrushali C <vr...@apache.org>
Authored: Thu Oct 11 16:26:07 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Thu Oct 11 16:26:07 2018 -0700
----------------------------------------------------------------------
.../webapp/AHSWebServices.java | 250 ++-------
.../webapp/TestAHSWebServices.java | 7 +-
.../yarn/server/webapp/LogWebService.java | 506 +++++++++++++++++++
.../yarn/server/webapp/LogWebServiceUtils.java | 258 ++++++++++
.../server/webapp/YarnWebServiceParams.java | 1 +
.../yarn/server/webapp/TestLogWebService.java | 126 +++++
.../reader/TimelineReaderServer.java | 4 +-
7 files changed, 945 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
index 9aa71a7..d94605f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
@@ -18,13 +18,7 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
@@ -35,15 +29,16 @@ import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
-import javax.ws.rs.core.GenericEntity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.Response.ResponseBuilder;
import javax.ws.rs.core.Response.Status;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -54,10 +49,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
-import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
-import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
-import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
+import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils;
import org.apache.hadoop.yarn.server.webapp.WebServices;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
@@ -65,21 +58,14 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.UniformInterfaceException;
+import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -271,15 +257,18 @@ public class AHSWebServices extends WebServices {
appInfo = super.getApp(req, res, appId.toString());
} catch (Exception ex) {
// directly find logs from HDFS.
- return getContainerLogMeta(appId, null, null, containerIdStr, false);
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
}
// if the application finishes, directly find logs
// from HDFS.
- if (isFinishedState(appInfo.getAppState())) {
- return getContainerLogMeta(appId, null, null,
- containerIdStr, false);
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
}
- if (isRunningState(appInfo.getAppState())) {
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
String appOwner = appInfo.getUser();
String nodeHttpAddress = null;
if (nmId != null && !nmId.isEmpty()) {
@@ -301,8 +290,9 @@ public class AHSWebServices extends WebServices {
} catch (Exception ex) {
// return log meta for the aggregated logs if exists.
// It will also return empty log meta for the local logs.
- return getContainerLogMeta(appId, appOwner, null,
- containerIdStr, true);
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
}
nodeHttpAddress = containerInfo.getNodeHttpAddress();
// make sure nodeHttpAddress is not null and not empty. Otherwise,
@@ -314,12 +304,14 @@ public class AHSWebServices extends WebServices {
// It will also return empty log meta for the local logs.
// If this is the redirect request from NM, we should not
// re-direct the request back. Simply output the aggregated log meta.
- return getContainerLogMeta(appId, appOwner, null,
- containerIdStr, true);
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
}
}
String uri = "/" + containerId.toString() + "/logs";
- String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress),
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(conf, nodeHttpAddress),
NM_DOWNLOAD_URI_STR, uri);
String query = req.getQueryString();
if (query != null && !query.isEmpty()) {
@@ -397,11 +389,11 @@ public class AHSWebServices extends WebServices {
try {
containerId = ContainerId.fromString(containerIdStr);
} catch (IllegalArgumentException ex) {
- return createBadResponse(Status.NOT_FOUND,
+ return LogWebServiceUtils.createBadResponse(Status.NOT_FOUND,
"Invalid ContainerId: " + containerIdStr);
}
- final long length = parseLongParam(size);
+ final long length = LogWebServiceUtils.parseLongParam(size);
ApplicationId appId = containerId.getApplicationAttemptId()
.getApplicationId();
@@ -410,17 +402,19 @@ public class AHSWebServices extends WebServices {
appInfo = super.getApp(req, res, appId.toString());
} catch (Exception ex) {
// directly find logs from HDFS.
- return sendStreamOutputResponse(appId, null, null, containerIdStr,
- filename, format, length, false);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, null, null, containerIdStr,
+ filename, format, length, false);
}
String appOwner = appInfo.getUser();
- if (isFinishedState(appInfo.getAppState())) {
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
// directly find logs from HDFS.
- return sendStreamOutputResponse(appId, appOwner, null, containerIdStr,
- filename, format, length, false);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, false);
}
- if (isRunningState(appInfo.getAppState())) {
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
String nodeHttpAddress = null;
if (nmId != null && !nmId.isEmpty()) {
try {
@@ -440,8 +434,9 @@ public class AHSWebServices extends WebServices {
containerId.toString());
} catch (Exception ex) {
// output the aggregated logs
- return sendStreamOutputResponse(appId, appOwner, null,
- containerIdStr, filename, format, length, true);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
}
nodeHttpAddress = containerInfo.getNodeHttpAddress();
// make sure nodeHttpAddress is not null and not empty. Otherwise,
@@ -452,12 +447,14 @@ public class AHSWebServices extends WebServices {
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()
|| redirected_from_node) {
// output the aggregated logs
- return sendStreamOutputResponse(appId, appOwner, null,
- containerIdStr, filename, format, length, true);
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
}
}
String uri = "/" + containerId.toString() + "/logs/" + filename;
- String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress),
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(conf, nodeHttpAddress),
NM_DOWNLOAD_URI_STR, uri);
String query = req.getQueryString();
if (query != null && !query.isEmpty()) {
@@ -468,170 +465,15 @@ public class AHSWebServices extends WebServices {
response.header("Location", resURI);
return response.build();
} else {
- return createBadResponse(Status.NOT_FOUND,
+ return LogWebServiceUtils.createBadResponse(Status.NOT_FOUND,
"The application is not at Running or Finished State.");
}
}
- private boolean isRunningState(YarnApplicationState appState) {
- return appState == YarnApplicationState.RUNNING;
- }
-
- private boolean isFinishedState(YarnApplicationState appState) {
- return appState == YarnApplicationState.FINISHED
- || appState == YarnApplicationState.FAILED
- || appState == YarnApplicationState.KILLED;
- }
-
- private Response createBadResponse(Status status, String errMessage) {
- Response response = Response.status(status)
- .entity(DOT_JOINER.join(status.toString(), errMessage)).build();
- return response;
- }
-
- private Response sendStreamOutputResponse(ApplicationId appId,
- String appOwner, String nodeId, String containerIdStr,
- String fileName, String format, long bytes,
- boolean printEmptyLocalContainerLog) {
- String contentType = WebAppUtils.getDefaultLogContentType();
- if (format != null && !format.isEmpty()) {
- contentType = WebAppUtils.getSupportedLogContentType(format);
- if (contentType == null) {
- String errorMessage = "The valid values for the parameter : format "
- + "are " + WebAppUtils.listSupportedLogContentType();
- return Response.status(Status.BAD_REQUEST).entity(errorMessage)
- .build();
- }
- }
- StreamingOutput stream = null;
- try {
- stream = getStreamingOutput(appId, appOwner, nodeId,
- containerIdStr, fileName, bytes, printEmptyLocalContainerLog);
- } catch (Exception ex) {
- return createBadResponse(Status.INTERNAL_SERVER_ERROR,
- ex.getMessage());
- }
- ResponseBuilder response = Response.ok(stream);
- response.header("Content-Type", contentType);
- // Sending the X-Content-Type-Options response header with the value
- // nosniff will prevent Internet Explorer from MIME-sniffing a response
- // away from the declared content-type.
- response.header("X-Content-Type-Options", "nosniff");
- return response.build();
- }
-
- private StreamingOutput getStreamingOutput(final ApplicationId appId,
- final String appOwner, final String nodeId, final String containerIdStr,
- final String logFile, final long bytes,
- final boolean printEmptyLocalContainerLog) throws IOException{
- StreamingOutput stream = new StreamingOutput() {
-
- @Override
- public void write(OutputStream os) throws IOException,
- WebApplicationException {
- ContainerLogsRequest request = new ContainerLogsRequest();
- request.setAppId(appId);
- request.setAppOwner(appOwner);
- request.setContainerId(containerIdStr);
- request.setBytes(bytes);
- request.setNodeId(nodeId);
- Set<String> logTypes = new HashSet<>();
- logTypes.add(logFile);
- request.setLogTypes(logTypes);
- boolean findLogs = factory.getFileControllerForRead(appId, appOwner)
- .readAggregatedLogs(request, os);
- if (!findLogs) {
- os.write(("Can not find logs for container:"
- + containerIdStr).getBytes(Charset.forName("UTF-8")));
- } else {
- if (printEmptyLocalContainerLog) {
- StringBuilder sb = new StringBuilder();
- sb.append(containerIdStr + "\n");
- sb.append("LogAggregationType: "
- + ContainerLogAggregationType.LOCAL + "\n");
- sb.append("LogContents:\n");
- sb.append(getNoRedirectWarning() + "\n");
- os.write(sb.toString().getBytes(Charset.forName("UTF-8")));
- }
- }
- }
- };
- return stream;
- }
-
- private long parseLongParam(String bytes) {
- if (bytes == null || bytes.isEmpty()) {
- return Long.MAX_VALUE;
- }
- return Long.parseLong(bytes);
- }
-
- private Response getContainerLogMeta(ApplicationId appId, String appOwner,
- final String nodeId, final String containerIdStr,
- boolean emptyLocalContainerLogMeta) {
- try {
- ContainerLogsRequest request = new ContainerLogsRequest();
- request.setAppId(appId);
- request.setAppOwner(appOwner);
- request.setContainerId(containerIdStr);
- request.setNodeId(nodeId);
- List<ContainerLogMeta> containerLogMeta = factory
- .getFileControllerForRead(appId, appOwner)
- .readAggregatedLogsMeta(request);
- if (containerLogMeta.isEmpty()) {
- throw new NotFoundException(
- "Can not get log meta for container: " + containerIdStr);
- }
- List<ContainerLogsInfo> containersLogsInfo = new ArrayList<>();
- for (ContainerLogMeta meta : containerLogMeta) {
- ContainerLogsInfo logInfo = new ContainerLogsInfo(meta,
- ContainerLogAggregationType.AGGREGATED);
- containersLogsInfo.add(logInfo);
- }
- if (emptyLocalContainerLogMeta) {
- ContainerLogMeta emptyMeta = new ContainerLogMeta(
- containerIdStr, "N/A");
- ContainerLogsInfo empty = new ContainerLogsInfo(emptyMeta,
- ContainerLogAggregationType.LOCAL);
- containersLogsInfo.add(empty);
- }
- GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<
- ContainerLogsInfo>>(containersLogsInfo){};
- ResponseBuilder response = Response.ok(meta);
- // Sending the X-Content-Type-Options response header with the value
- // nosniff will prevent Internet Explorer from MIME-sniffing a response
- // away from the declared content-type.
- response.header("X-Content-Type-Options", "nosniff");
- return response.build();
- } catch (Exception ex) {
- throw new WebApplicationException(ex);
- }
- }
-
- @Private
- @VisibleForTesting
- public static String getNoRedirectWarning() {
- return "We do not have NodeManager web address, so we can not "
- + "re-direct the request to related NodeManager "
- + "for local container logs.";
- }
-
- private String getAbsoluteNMWebAddress(String nmWebAddress) {
- if (nmWebAddress.contains(WebAppUtils.HTTP_PREFIX) ||
- nmWebAddress.contains(WebAppUtils.HTTPS_PREFIX)) {
- return nmWebAddress;
- }
- return WebAppUtils.getHttpSchemePrefix(conf) + nmWebAddress;
- }
-
- @VisibleForTesting
- @Private
+ @VisibleForTesting @InterfaceAudience.Private
public String getNMWebAddressFromRM(Configuration configuration,
- String nodeId) throws ClientHandlerException,
- UniformInterfaceException, JSONException {
- JSONObject nodeInfo = YarnWebServiceUtils.getNodeInfoFromRMWebService(
- configuration, nodeId).getJSONObject("node");
- return nodeInfo.has("nodeHTTPAddress") ?
- nodeInfo.getString("nodeHTTPAddress") : null;
+ String nodeId)
+ throws ClientHandlerException, UniformInterfaceException, JSONException {
+ return LogWebServiceUtils.getNMWebAddressFromRM(configuration, nodeId);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index a4f56ff..e72714b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
+import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
@@ -774,7 +775,8 @@ public class TestAHSWebServices extends JerseyTestBase {
// the warning message.
assertTrue(responseText.contains("LogAggregationType: "
+ ContainerLogAggregationType.LOCAL));
- assertTrue(responseText.contains(AHSWebServices.getNoRedirectWarning()));
+ assertTrue(
+ responseText.contains(LogWebServiceUtils.getNoRedirectWarning()));
// If we can not container information from ATS, and we specify the NM id,
// but we can not get nm web address, we would still try to
@@ -790,7 +792,8 @@ public class TestAHSWebServices extends JerseyTestBase {
assertTrue(responseText.contains(content));
assertTrue(responseText.contains("LogAggregationType: "
+ ContainerLogAggregationType.LOCAL));
- assertTrue(responseText.contains(AHSWebServices.getNoRedirectWarning()));
+ assertTrue(
+ responseText.contains(LogWebServiceUtils.getNoRedirectWarning()));
// If this is the redirect request, we would not re-direct the request
// back and get the aggregated logs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java
new file mode 100644
index 0000000..246ee9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java
@@ -0,0 +1,506 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.inject.Singleton;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * Support only ATSv2 client only.
+ */
+@Singleton @Path("/ws/v2/applicationlog") public class LogWebService {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LogWebService.class);
+ private static final String RESOURCE_URI_STR_V2 = "/ws/v2/timeline/";
+ private static final String NM_DOWNLOAD_URI_STR = "/ws/v1/node/containers";
+ private static final Joiner JOINER = Joiner.on("");
+ private static Configuration yarnConf = new YarnConfiguration();
+ private static LogAggregationFileControllerFactory factory;
+ private static String base;
+ private static String defaultClusterid;
+ private volatile Client webTimelineClient;
+
+ static {
+ init();
+ }
+
+ // initialize all the common resources - order is important
+ private static void init() {
+ factory = new LogAggregationFileControllerFactory(yarnConf);
+ base = JOINER.join(WebAppUtils.getHttpSchemePrefix(yarnConf),
+ WebAppUtils.getTimelineReaderWebAppURLWithoutScheme(yarnConf),
+ RESOURCE_URI_STR_V2);
+ defaultClusterid = yarnConf.get(YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
+ LOG.info("Initialized LogWeService with clusterid " + defaultClusterid
+ + " for URI: " + base);
+ }
+
+ private Client createTimelineWebClient() {
+ ClientConfig cfg = new DefaultClientConfig();
+ cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+ Client client = new Client(
+ new URLConnectionClientHandler(new HttpURLConnectionFactory() {
+ @Override public HttpURLConnection getHttpURLConnection(URL url)
+ throws IOException {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ HttpURLConnection conn = null;
+ try {
+ conn = new AuthenticatedURL().openConnection(url, token);
+ LOG.info("LogWeService:Connecetion created.");
+ } catch (AuthenticationException e) {
+ throw new IOException(e);
+ }
+ return conn;
+ }
+ }), cfg);
+
+ return client;
+ }
+
+ private void initForReadableEndpoints(HttpServletResponse response) {
+ // clear content type
+ response.setContentType(null);
+ }
+
+ /**
+ * Returns log file's name as well as current file size for a container.
+ *
+ * @param req HttpServletRequest
+ * @param res HttpServletResponse
+ * @param containerIdStr The container ID
+ * @param nmId The Node Manager NodeId
+ * @param redirectedFromNode Whether this is a redirected request from NM
+ * @return The log file's name and current file size
+ */
+ @GET @Path("/containers/{containerid}/logs")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response getContainerLogsInfo(@Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr,
+ @QueryParam(YarnWebServiceParams.NM_ID) String nmId,
+ @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE)
+ @DefaultValue("false") boolean redirectedFromNode,
+ @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) {
+ ContainerId containerId = null;
+ initForReadableEndpoints(res);
+ try {
+ containerId = ContainerId.fromString(containerIdStr);
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("invalid container id, " + containerIdStr);
+ }
+
+ ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+ AppInfo appInfo;
+ try {
+ appInfo = getApp(req, appId.toString(), clusterId);
+ } catch (Exception ex) {
+ // directly find logs from HDFS.
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
+ }
+ // if the application finishes, directly find logs
+ // from HDFS.
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, null, null, containerIdStr,
+ false);
+ }
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
+ String appOwner = appInfo.getUser();
+ String nodeHttpAddress = null;
+ if (nmId != null && !nmId.isEmpty()) {
+ try {
+ nodeHttpAddress =
+ LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
+ } catch (Exception ex) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(ex.getMessage());
+ }
+ }
+ }
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ ContainerInfo containerInfo;
+ try {
+ containerInfo =
+ getContainer(req, appId.toString(), containerId.toString(),
+ clusterId);
+ } catch (Exception ex) {
+ // return log meta for the aggregated logs if exists.
+ // It will also return empty log meta for the local logs.
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
+ }
+ nodeHttpAddress = containerInfo.getNodeHttpAddress();
+ // make sure nodeHttpAddress is not null and not empty. Otherwise,
+ // we would only get log meta for aggregated logs instead of
+ // re-directing the request
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()
+ || redirectedFromNode) {
+ // return log meta for the aggregated logs if exists.
+ // It will also return empty log meta for the local logs.
+ // If this is the redirect request from NM, we should not
+ // re-direct the request back. Simply output the aggregated log meta.
+ return LogWebServiceUtils
+ .getContainerLogMeta(factory, appId, appOwner, null,
+ containerIdStr, true);
+ }
+ }
+ String uri = "/" + containerId.toString() + "/logs";
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(yarnConf, nodeHttpAddress),
+ NM_DOWNLOAD_URI_STR, uri);
+ String query = req.getQueryString();
+ if (query != null && !query.isEmpty()) {
+ resURI += "?" + query;
+ }
+ Response.ResponseBuilder response =
+ Response.status(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ response.header("Location", resURI);
+ return response.build();
+ } else {
+ throw new NotFoundException(
+ "The application is not at Running or Finished State.");
+ }
+ }
+
+ protected ContainerInfo getContainer(HttpServletRequest req, String appId,
+ String containerId, String clusterId) {
+ UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req);
+ String cId = clusterId != null ? clusterId : defaultClusterid;
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", "INFO");
+ String path = JOINER.join("clusters/", cId, "/apps/", appId, "/entities/",
+ TimelineEntityType.YARN_CONTAINER.toString(), "/", containerId);
+ TimelineEntity conEntity = null;
+ try {
+ if (callerUGI == null) {
+ conEntity = getEntity(path, params);
+ } else {
+ setUserName(params, callerUGI.getShortUserName());
+ conEntity =
+ callerUGI.doAs(new PrivilegedExceptionAction<TimelineEntity>() {
+ @Override public TimelineEntity run() throws Exception {
+ return getEntity(path, params);
+ }
+ });
+ }
+ } catch (Exception e) {
+ LogWebServiceUtils.rewrapAndThrowException(e);
+ }
+ if (conEntity == null) {
+ return null;
+ }
+ String nodeHttpAddress = (String) conEntity.getInfo()
+ .get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO);
+
+ ContainerInfo info = new ContainerInfo(nodeHttpAddress);
+ return info;
+ }
+
+ protected AppInfo getApp(HttpServletRequest req, String appId,
+ String clusterId) {
+ UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req);
+
+ String cId = clusterId != null ? clusterId : defaultClusterid;
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("fields", "INFO");
+ String path = JOINER.join("clusters/", cId, "/apps/", appId);
+ TimelineEntity appEntity = null;
+
+ try {
+ if (callerUGI == null) {
+ appEntity = getEntity(path, params);
+ } else {
+ setUserName(params, callerUGI.getShortUserName());
+ appEntity =
+ callerUGI.doAs(new PrivilegedExceptionAction<TimelineEntity>() {
+ @Override public TimelineEntity run() throws Exception {
+ return getEntity(path, params);
+ }
+ });
+ }
+ } catch (Exception e) {
+ LogWebServiceUtils.rewrapAndThrowException(e);
+ }
+
+ if (appEntity == null) {
+ return null;
+ }
+ String appOwner = (String) appEntity.getInfo()
+ .get(ApplicationMetricsConstants.USER_ENTITY_INFO);
+ String state = (String) appEntity.getInfo()
+ .get(ApplicationMetricsConstants.STATE_EVENT_INFO);
+ YarnApplicationState appState = YarnApplicationState.valueOf(state);
+ AppInfo info = new AppInfo(appState, appOwner);
+ return info;
+ }
+
+ /**
+ * Returns the contents of a container's log file in plain text.
+ *
+ * @param req HttpServletRequest
+ * @param res HttpServletResponse
+ * @param containerIdStr The container ID
+ * @param filename The name of the log file
+ * @param format The content type
+ * @param size the size of the log file
+ * @param nmId The Node Manager NodeId
+ * @param redirectedFromNode Whether this is the redirect request from NM
+ * @return The contents of the container's log file
+ */
+ @GET @Path("/containers/{containerid}/logs/{filename}")
+ @Produces({ MediaType.TEXT_PLAIN }) @InterfaceAudience.Public
+ @InterfaceStability.Unstable public Response getContainerLogFile(
+ @Context HttpServletRequest req, @Context HttpServletResponse res,
+ @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr,
+ @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size,
+ @QueryParam(YarnWebServiceParams.NM_ID) String nmId,
+ @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE)
+ boolean redirectedFromNode,
+ @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) {
+ return getLogs(req, res, containerIdStr, filename, format, size, nmId,
+ redirectedFromNode, clusterId);
+ }
+
+ //TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and
+ // container log webservice introduced in AHS to minimize
+ // the duplication.
+ @GET @Path("/containerlogs/{containerid}/{filename}")
+ @Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 })
+ @InterfaceAudience.Public @InterfaceStability.Unstable
+ public Response getLogs(@Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr,
+ @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format,
+ @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size,
+ @QueryParam(YarnWebServiceParams.NM_ID) String nmId,
+ @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE)
+ @DefaultValue("false") boolean redirectedFromNode,
+ @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) {
+ initForReadableEndpoints(res);
+ ContainerId containerId;
+ try {
+ containerId = ContainerId.fromString(containerIdStr);
+ } catch (IllegalArgumentException ex) {
+ return LogWebServiceUtils.createBadResponse(Response.Status.NOT_FOUND,
+ "Invalid ContainerId: " + containerIdStr);
+ }
+
+ final long length = LogWebServiceUtils.parseLongParam(size);
+
+ ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+ AppInfo appInfo;
+ try {
+ appInfo = getApp(req, appId.toString(), clusterId);
+ } catch (Exception ex) {
+ // directly find logs from HDFS.
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, null, null, containerIdStr,
+ filename, format, length, false);
+ }
+ String appOwner = appInfo.getUser();
+ if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
+ // directly find logs from HDFS.
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, false);
+ }
+
+ if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) {
+ String nodeHttpAddress = null;
+ if (nmId != null && !nmId.isEmpty()) {
+ try {
+ nodeHttpAddress =
+ LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
+ } catch (Exception ex) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(ex.getMessage());
+ }
+ }
+ }
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ ContainerInfo containerInfo;
+ try {
+ containerInfo =
+ getContainer(req, appId.toString(), containerId.toString(),
+ clusterId);
+ } catch (Exception ex) {
+ // output the aggregated logs
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
+ }
+ nodeHttpAddress = containerInfo.getNodeHttpAddress();
+ // make sure nodeHttpAddress is not null and not empty. Otherwise,
+ // we would only get aggregated logs instead of re-directing the
+ // request.
+ // If this is the redirect request from NM, we should not re-direct the
+ // request back. Simply output the aggregated logs.
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()
+ || redirectedFromNode) {
+ // output the aggregated logs
+ return LogWebServiceUtils
+ .sendStreamOutputResponse(factory, appId, appOwner, null,
+ containerIdStr, filename, format, length, true);
+ }
+ }
+ String uri = "/" + containerId.toString() + "/logs/" + filename;
+ String resURI = JOINER.join(
+ LogWebServiceUtils.getAbsoluteNMWebAddress(yarnConf, nodeHttpAddress),
+ NM_DOWNLOAD_URI_STR, uri);
+ String query = req.getQueryString();
+ if (query != null && !query.isEmpty()) {
+ resURI += "?" + query;
+ }
+ Response.ResponseBuilder response =
+ Response.status(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ response.header("Location", resURI);
+ return response.build();
+ } else {
+ return LogWebServiceUtils.createBadResponse(Response.Status.NOT_FOUND,
+ "The application is not at Running or Finished State.");
+ }
+ }
+
+ protected static class AppInfo {
+ private YarnApplicationState appState;
+ private String user;
+
+ AppInfo(YarnApplicationState appState, String user) {
+ this.appState = appState;
+ this.user = user;
+ }
+
+ public YarnApplicationState getAppState() {
+ return this.appState;
+ }
+
+ public String getUser() {
+ return this.user;
+ }
+ }
+
+ protected static class ContainerInfo {
+ private String nodeHttpAddress;
+
+ ContainerInfo(String nodeHttpAddress) {
+ this.nodeHttpAddress = nodeHttpAddress;
+ }
+
+ public String getNodeHttpAddress() {
+ return nodeHttpAddress;
+ }
+ }
+
+ @VisibleForTesting protected TimelineEntity getEntity(String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ ClientResponse resp =
+ getClient().resource(base).path(path).queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ if (resp == null
+ || resp.getStatusInfo().getStatusCode() != ClientResponse.Status.OK
+ .getStatusCode()) {
+ String msg =
+ "Response from the timeline reader server is " + ((resp == null) ?
+ "null" :
+ "not successful," + " HTTP error code: " + resp.getStatus()
+ + ", Server response:\n" + resp.getEntity(String.class));
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ return entity;
+ }
+
+ private Client getClient() {
+ if (webTimelineClient == null) {
+ synchronized (LogWebService.class) {
+ if (webTimelineClient == null) {
+ webTimelineClient = createTimelineWebClient();
+ }
+ }
+ }
+ return webTimelineClient;
+ }
+
+ /**
+ * Set user.name in non-secure mode to delegate to next rest call.
+ */
+ private void setUserName(MultivaluedMap<String, String> params, String user) {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ params.add("user.name", user);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java
new file mode 100644
index 0000000..bc301bb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import com.google.common.base.Joiner;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
+import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
+import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.GenericEntity;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Log web service utils class.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class LogWebServiceUtils {
+
+ private LogWebServiceUtils() {
+ }
+
+ private static final Joiner DOT_JOINER = Joiner.on(". ");
+
+ public static Response getContainerLogMeta(
+ LogAggregationFileControllerFactory factory, ApplicationId appId,
+ String appOwner, final String nodeId, final String containerIdStr,
+ boolean emptyLocalContainerLogMeta) {
+ try {
+ ContainerLogsRequest request = new ContainerLogsRequest();
+ request.setAppId(appId);
+ request.setAppOwner(appOwner);
+ request.setContainerId(containerIdStr);
+ request.setNodeId(nodeId);
+ List<ContainerLogMeta> containerLogMeta =
+ factory.getFileControllerForRead(appId, appOwner)
+ .readAggregatedLogsMeta(request);
+ if (containerLogMeta.isEmpty()) {
+ throw new NotFoundException(
+ "Can not get log meta for container: " + containerIdStr);
+ }
+ List<ContainerLogsInfo> containersLogsInfo = new ArrayList<>();
+ for (ContainerLogMeta meta : containerLogMeta) {
+ ContainerLogsInfo logInfo =
+ new ContainerLogsInfo(meta, ContainerLogAggregationType.AGGREGATED);
+ containersLogsInfo.add(logInfo);
+ }
+ if (emptyLocalContainerLogMeta) {
+ ContainerLogMeta emptyMeta =
+ new ContainerLogMeta(containerIdStr, "N/A");
+ ContainerLogsInfo empty =
+ new ContainerLogsInfo(emptyMeta, ContainerLogAggregationType.LOCAL);
+ containersLogsInfo.add(empty);
+ }
+ GenericEntity<List<ContainerLogsInfo>> meta =
+ new GenericEntity<List<ContainerLogsInfo>>(containersLogsInfo) {
+ };
+ Response.ResponseBuilder response = Response.ok(meta);
+ // Sending the X-Content-Type-Options response header with the value
+ // nosniff will prevent Internet Explorer from MIME-sniffing a response
+ // away from the declared content-type.
+ response.header("X-Content-Type-Options", "nosniff");
+ return response.build();
+ } catch (Exception ex) {
+ throw new WebApplicationException(ex);
+ }
+ }
+
+ public static Response sendStreamOutputResponse(
+ LogAggregationFileControllerFactory factory, ApplicationId appId,
+ String appOwner, String nodeId, String containerIdStr, String fileName,
+ String format, long bytes, boolean printEmptyLocalContainerLog) {
+ String contentType = WebAppUtils.getDefaultLogContentType();
+ if (format != null && !format.isEmpty()) {
+ contentType = WebAppUtils.getSupportedLogContentType(format);
+ if (contentType == null) {
+ String errorMessage =
+ "The valid values for the parameter : format " + "are "
+ + WebAppUtils.listSupportedLogContentType();
+ return Response.status(Response.Status.BAD_REQUEST).entity(errorMessage)
+ .build();
+ }
+ }
+ StreamingOutput stream = null;
+ try {
+ stream =
+ getStreamingOutput(factory, appId, appOwner, nodeId, containerIdStr,
+ fileName, bytes, printEmptyLocalContainerLog);
+ } catch (Exception ex) {
+ return createBadResponse(Response.Status.INTERNAL_SERVER_ERROR,
+ ex.getMessage());
+ }
+ Response.ResponseBuilder response = Response.ok(stream);
+ response.header("Content-Type", contentType);
+ // Sending the X-Content-Type-Options response header with the value
+ // nosniff will prevent Internet Explorer from MIME-sniffing a response
+ // away from the declared content-type.
+ response.header("X-Content-Type-Options", "nosniff");
+ return response.build();
+ }
+
+ private static StreamingOutput getStreamingOutput(
+ final LogAggregationFileControllerFactory factory,
+ final ApplicationId appId, final String appOwner, final String nodeId,
+ final String containerIdStr, final String logFile, final long bytes,
+ final boolean printEmptyLocalContainerLog) throws IOException {
+ StreamingOutput stream = new StreamingOutput() {
+
+ @Override public void write(OutputStream os)
+ throws IOException, WebApplicationException {
+ ContainerLogsRequest request = new ContainerLogsRequest();
+ request.setAppId(appId);
+ request.setAppOwner(appOwner);
+ request.setContainerId(containerIdStr);
+ request.setBytes(bytes);
+ request.setNodeId(nodeId);
+ Set<String> logTypes = new HashSet<>();
+ logTypes.add(logFile);
+ request.setLogTypes(logTypes);
+ boolean findLogs = factory.getFileControllerForRead(appId, appOwner)
+ .readAggregatedLogs(request, os);
+ if (!findLogs) {
+ os.write(("Can not find logs for container:" + containerIdStr)
+ .getBytes(Charset.forName("UTF-8")));
+ } else {
+ if (printEmptyLocalContainerLog) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(containerIdStr + "\n");
+ sb.append("LogAggregationType: " + ContainerLogAggregationType.LOCAL
+ + "\n");
+ sb.append("LogContents:\n");
+ sb.append(getNoRedirectWarning() + "\n");
+ os.write(sb.toString().getBytes(Charset.forName("UTF-8")));
+ }
+ }
+ }
+ };
+ return stream;
+ }
+
+ public static String getNoRedirectWarning() {
+ return "We do not have NodeManager web address, so we can not "
+ + "re-direct the request to related NodeManager "
+ + "for local container logs.";
+ }
+
+ public static void rewrapAndThrowException(Exception e) {
+ if (e instanceof UndeclaredThrowableException) {
+ rewrapAndThrowThrowable(e.getCause());
+ } else {
+ rewrapAndThrowThrowable(e);
+ }
+ }
+
+ public static void rewrapAndThrowThrowable(Throwable t) {
+ if (t instanceof AuthorizationException) {
+ throw new ForbiddenException(t);
+ } else {
+ throw new WebApplicationException(t);
+ }
+ }
+
+ public static long parseLongParam(String bytes) {
+ if (bytes == null || bytes.isEmpty()) {
+ return Long.MAX_VALUE;
+ }
+ return Long.parseLong(bytes);
+ }
+
+ public static Response createBadResponse(Response.Status status,
+ String errMessage) {
+ Response response = Response.status(status)
+ .entity(DOT_JOINER.join(status.toString(), errMessage)).build();
+ return response;
+ }
+
+ public static boolean isRunningState(YarnApplicationState appState) {
+ return appState == YarnApplicationState.RUNNING;
+ }
+
+ public static boolean isFinishedState(YarnApplicationState appState) {
+ return appState == YarnApplicationState.FINISHED
+ || appState == YarnApplicationState.FAILED
+ || appState == YarnApplicationState.KILLED;
+ }
+
+ protected static UserGroupInformation getUser(HttpServletRequest req) {
+ String remoteUser = req.getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return callerUGI;
+ }
+
+ public static String getNMWebAddressFromRM(Configuration yarnConf,
+ String nodeId)
+ throws ClientHandlerException, UniformInterfaceException, JSONException {
+ JSONObject nodeInfo =
+ YarnWebServiceUtils.getNodeInfoFromRMWebService(yarnConf, nodeId)
+ .getJSONObject("node");
+ return nodeInfo.has("nodeHTTPAddress") ?
+ nodeInfo.getString("nodeHTTPAddress") : null;
+ }
+
+ public static String getAbsoluteNMWebAddress(Configuration yarnConf,
+ String nmWebAddress) {
+ if (nmWebAddress.contains(WebAppUtils.HTTP_PREFIX) || nmWebAddress
+ .contains(WebAppUtils.HTTPS_PREFIX)) {
+ return nmWebAddress;
+ }
+ return WebAppUtils.getHttpSchemePrefix(yarnConf) + nmWebAddress;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
index 479cc75..5f96f23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java
@@ -35,4 +35,5 @@ public interface YarnWebServiceParams {
String RESPONSE_CONTENT_SIZE = "size";
String NM_ID = "nm.id";
String REDIRECTED_FROM_NODE = "redirected_from_node";
+ String CLUSTER_ID = "clusterid";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java
new file mode 100644
index 0000000..9ceb629
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.MultivaluedMap;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * test class for log web service.
+ */
+public class TestLogWebService {
+
+ private HttpServletRequest request;
+ private LogWebServiceTest logWebService;
+ private static TimelineEntity entity;
+ private ApplicationId appId;
+ private ContainerId cId;
+ private String user = "user1";
+ private Map<String, TimelineEntity> entities;
+ private String nodeHttpAddress = "localhost:0";
+
+ @Before public void setup() throws Exception {
+ appId = ApplicationId.fromString("application_1518143905142_509690");
+ cId =
+ ContainerId.fromString("container_e138_1518143905142_509690_01_000001");
+ entities = new HashMap<>();
+ generateEntity();
+ request = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(request.getRemoteUser())
+ .thenReturn(System.getProperty("user.name"));
+ logWebService = new LogWebServiceTest();
+
+ }
+
+ @Test public void testGetApp() {
+
+ LogWebService.AppInfo app =
+ logWebService.getApp(request, appId.toString(), null);
+ Assert.assertEquals("RUNNING", app.getAppState().toString());
+ Assert.assertEquals(user, app.getUser());
+ }
+
+ @Test public void testGetContainer() {
+ LogWebService.ContainerInfo container = logWebService
+ .getContainer(request, appId.toString(), cId.toString(), null);
+ Assert.assertEquals(nodeHttpAddress, container.getNodeHttpAddress());
+ }
+
+ class LogWebServiceTest extends LogWebService {
+
+ @Override protected TimelineEntity getEntity(String path,
+ MultivaluedMap<String, String> params) throws IOException {
+ if (path.endsWith(cId.toString())) {
+ return entities.get(cId.toString());
+ } else if (path.endsWith(appId.toString())) {
+ return entities.get(appId.toString());
+ } else {
+ throw new IOException();
+ }
+ }
+ }
+
+ private void generateEntity() {
+ createAppEntities();
+ createContainerEntities();
+ }
+
+ private void createContainerEntities() {
+ TimelineEntity timelineEntity =
+ generateEntity(TimelineEntityType.YARN_APPLICATION.toString(),
+ appId.toString());
+ timelineEntity.addInfo(ApplicationMetricsConstants.USER_ENTITY_INFO, user);
+ timelineEntity
+ .addInfo(ApplicationMetricsConstants.STATE_EVENT_INFO, "RUNNING");
+ entities.put(appId.toString(), timelineEntity);
+ }
+
+ private void createAppEntities() {
+ TimelineEntity timelineEntity =
+ generateEntity(TimelineEntityType.YARN_CONTAINER.toString(),
+ cId.toString());
+ timelineEntity
+ .addInfo(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO,
+ nodeHttpAddress);
+ entities.put(cId.toString(), timelineEntity);
+ }
+
+ private TimelineEntity generateEntity(String entityType,
+ String entityId) {
+ TimelineEntity timelineEntity = new TimelineEntity();
+ timelineEntity.setId(entityId);
+ timelineEntity.setType(entityType);
+ timelineEntity.setCreatedTime(System.currentTimeMillis());
+ return timelineEntity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d198180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index bd2c428..8f1e7d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineRea
import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderWhitelistAuthorizationFilterInitializer;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils;
+import org.apache.hadoop.yarn.server.webapp.LogWebService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -201,7 +202,8 @@ public class TimelineReaderServer extends CompositeService {
readerWebServer.addJerseyResourcePackage(
TimelineReaderWebServices.class.getPackage().getName() + ";"
+ GenericExceptionHandler.class.getPackage().getName() + ";"
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName()+ ";"
+ + LogWebService.class.getPackage().getName(),
"/*");
readerWebServer.setAttribute(TIMELINE_READER_MANAGER_ATTR,
timelineReaderManager);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDDS-636. Turn off
ozone.metastore.rocksdb.statistics for now. Contributed by Arpit Agarwal.
Posted by br...@apache.org.
HDDS-636. Turn off ozone.metastore.rocksdb.statistics for now. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50552479
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50552479
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50552479
Branch: refs/heads/HDFS-13891
Commit: 50552479c42d2e84cfeec83d4d67b050c5659b06
Parents: 0bc6d04
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Oct 11 17:21:27 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Oct 11 17:21:27 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +-
hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50552479/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index d6a6bf7..09d798f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -90,7 +90,7 @@ public final class OzoneConfigKeys {
"ozone.metastore.rocksdb.statistics";
public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT =
- "ALL";
+ "OFF";
public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF =
"OFF";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50552479/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0fde6bb..b2d47c9 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -514,7 +514,7 @@
<property>
<name>ozone.metastore.rocksdb.statistics</name>
- <value>ALL</value>
+ <value>OFF</value>
<tag>OZONE, OM, SCM, STORAGE, PERFORMANCE</tag>
<description>
The statistics level of the rocksdb store. If you use any value from
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HADOOP-15785. [JDK10] Javadoc
build fails on JDK 10 in hadoop-common. Contributed by Dinesh Chitlangia.
Posted by br...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
index bec0fe1..487a457 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
@@ -168,7 +168,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
}
/** Add a leaf node
- * Update node counter & rack counter if necessary
+ * Update node counter & rack counter if necessary
* @param node node to be added; can be null
* @exception IllegalArgumentException if add a node to a leave
* or node to be added is not a leaf
@@ -272,7 +272,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
/**
* Sort nodes array by their distances to <i>reader</i>.
- * <p/>
+ * <p>
* This is the same as {@link NetworkTopology#sortByDistance(Node, Node[],
* int)} except with a four-level network topology which contains the
* additional network distance of a "node group" which is between local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
index 02b44a5..4db8155 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
@@ -33,13 +33,13 @@ import org.slf4j.LoggerFactory;
* This class implements the {@link DNSToSwitchMapping} interface using a
* script configured via the
* {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
- * <p/>
+ * <p>
* It contains a static class <code>RawScriptBasedMapping</code> that performs
* the work: reading the configuration parameters, executing any defined
* script, handling errors and such like. The outer
* class extends {@link CachedDNSToSwitchMapping} to cache the delegated
* queries.
- * <p/>
+ * <p>
* This DNS mapper's {@link #isSingleSwitch()} predicate returns
* true if and only if a script is defined.
*/
@@ -78,7 +78,7 @@ public class ScriptBasedMapping extends CachedDNSToSwitchMapping {
/**
* Create an instance with the default configuration.
- * </p>
+ * <p>
* Calling {@link #setConf(Configuration)} will trigger a
* re-evaluation of the configuration settings and so be used to
* set up the mapping script.
@@ -125,7 +125,7 @@ public class ScriptBasedMapping extends CachedDNSToSwitchMapping {
/**
* {@inheritDoc}
- * <p/>
+ * <p>
* This will get called in the superclass constructor, so a check is needed
* to ensure that the raw mapping is defined before trying to relaying a null
* configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
index 086650b..e05fae6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
@@ -32,10 +32,9 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
* the {@link DNSToSwitchMappingWithDependency} interface using
* a script configured via the
* {@link CommonConfigurationKeys#NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY} option.
- * <p/>
+ * <p>
* It contains a static class <code>RawScriptBasedMappingWithDependency</code>
* that performs the getDependency work.
- * <p/>
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -52,7 +51,7 @@ public class ScriptBasedMappingWithDependency extends ScriptBasedMapping
/**
* Create an instance with the default configuration.
- * </p>
+ * <p>
* Calling {@link #setConf(Configuration)} will trigger a
* re-evaluation of the configuration settings and so be used to
* set up the mapping script.
@@ -76,7 +75,7 @@ public class ScriptBasedMappingWithDependency extends ScriptBasedMapping
/**
* {@inheritDoc}
- * <p/>
+ * <p>
* This will get called in the superclass constructor, so a check is needed
* to ensure that the raw mapping is defined before trying to relaying a null
* configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
index ead1d7b2..93f4f56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
@@ -32,7 +32,6 @@ import java.nio.channels.WritableByteChannel;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.metrics2.lib.MutableRate;
/**
* This implements an output stream that can have a timeout while writing.
@@ -187,7 +186,7 @@ public class SocketOutputStream extends OutputStream
* @param count number of bytes to transfer.
* @param waitForWritableTime nanoseconds spent waiting for the socket
* to become writable
- * @param transferTime nanoseconds spent transferring data
+ * @param transferToTime nanoseconds spent transferring data
*
* @throws EOFException
* If end of input file is reached before requested number of
@@ -253,7 +252,8 @@ public class SocketOutputStream extends OutputStream
/**
* Call
- * {@link #transferToFully(FileChannel, long, int, MutableRate, MutableRate)}
+ * {@link #transferToFully(FileChannel, long, int, LongWritable, LongWritable)
+ * }
* with null <code>waitForWritableTime</code> and <code>transferToTime</code>
*/
public void transferToFully(FileChannel fileCh, long position, int count)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
index 7e6b3a8..b25d5d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
@@ -31,11 +31,11 @@ import java.util.Map;
/**
* Initializes hadoop-auth AuthenticationFilter which provides support for
* Kerberos HTTP SPNEGO authentication.
- * <p/>
+ * <p>
* It enables anonymous access, simple/speudo and Kerberos HTTP SPNEGO
* authentication for Hadoop JobTracker, NameNode, DataNodes and
* TaskTrackers.
- * <p/>
+ * <p>
* Refer to the <code>core-default.xml</code> file, after the comment
* 'HTTP Authentication' for details on the configuration options.
* All related configuration properties have 'hadoop.http.authentication.'
@@ -47,7 +47,7 @@ public class AuthenticationFilterInitializer extends FilterInitializer {
/**
* Initializes hadoop-auth AuthenticationFilter.
- * <p/>
+ * <p>
* Propagates to hadoop-auth AuthenticationFilter configuration all Hadoop
* configuration properties prefixed with "hadoop.http.authentication."
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
index 55b2786..d7f0e81 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
@@ -46,7 +46,7 @@ public class HadoopKerberosName extends KerberosName {
}
/**
* Set the static configuration to get the rules.
- * <p/>
+ * <p>
* IMPORTANT: This method does a NOP if the rules have been set already.
* If there is a need to reset the rules, the {@link KerberosName#setRules(String)}
* method should be invoked directly.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
index 4a1185e..86edab7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
@@ -25,8 +25,9 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/**
- * An interface for the implementation of <userId, userName> mapping
- * and <groupId, groupName> mapping
+ * An interface for the implementation of {@literal <}userId,
+ * userName{@literal >} mapping and {@literal <}groupId, groupName{@literal >}
+ * mapping.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index 11714b1..d236ab0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -343,13 +343,9 @@ public class SaslRpcClient {
}
/**
- * Do client side SASL authentication with server via the given InputStream
- * and OutputStream
- *
- * @param inS
- * InputStream to use
- * @param outS
- * OutputStream to use
+ * Do client side SASL authentication with server via the given IpcStreams.
+ *
+ * @param ipcStreams
* @return AuthMethod used to negotiate the connection
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 2313119..aa12b93 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -332,7 +332,8 @@ public final class SecurityUtil {
}
/**
- * Get the host name from the principal name of format <service>/host@realm.
+ * Get the host name from the principal name of format {@literal <}service
+ * {@literal >}/host@realm.
* @param principalName principal name of format as described above
* @return host name if the the string conforms to the above format, else null
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 6ce72edb..db88106 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -676,7 +676,7 @@ public class UserGroupInformation {
/**
* remove the login method that is followed by a space from the username
- * e.g. "jack (auth:SIMPLE)" -> "jack"
+ * e.g. "jack (auth:SIMPLE)" {@literal ->} "jack"
*
* @param userName
* @return userName without login method
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
index 52f39ef..5028482 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
@@ -33,9 +33,9 @@ import java.net.URI;
/**
* CredentialProvider based on Java's KeyStore file format. The file may be
* stored in any Hadoop FileSystem using the following name mangling:
- * jceks://hdfs@nn1.example.com/my/creds.jceks ->
- * hdfs://nn1.example.com/my/creds.jceks jceks://file/home/larry/creds.jceks ->
- * file:///home/larry/creds.jceks
+ * jceks://hdfs@nn1.example.com/my/creds.jceks {@literal ->}
+ * hdfs://nn1.example.com/my/creds.jceks jceks://file/home/larry/creds.jceks
+ * {@literal ->} file:///home/larry/creds.jceks
*/
@InterfaceAudience.Private
public class JavaKeyStoreProvider extends AbstractJavaKeyStoreProvider {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
index 9ea9a57..ce0eb7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
@@ -44,7 +44,8 @@ import java.util.EnumSet;
/**
* CredentialProvider based on Java's KeyStore file format. The file may be
* stored only on the local filesystem using the following name mangling:
- * localjceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
+ * localjceks://file/home/larry/creds.jceks {@literal ->}
+ * file:///home/larry/creds.jceks
*/
@InterfaceAudience.Private
public final class LocalJavaKeyStoreProvider extends
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
index 60d82cb..97a7f08 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
@@ -108,12 +108,12 @@ public class ProxyUsers {
* @param remoteAddress
* @param conf
* @throws AuthorizationException
- * @deprecated use {@link #authorize(UserGroupInformation, String) instead.
+ * @deprecated use {@link #authorize(UserGroupInformation, String)} instead.
*/
@Deprecated
public static void authorize(UserGroupInformation user,
String remoteAddress, Configuration conf) throws AuthorizationException {
- authorize(user,remoteAddress);
+ authorize(user, remoteAddress);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
index b0df8f0..e5bdab3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
@@ -38,7 +38,7 @@ import java.text.MessageFormat;
/**
* {@link KeyStoresFactory} implementation that reads the certificates from
* keystore files.
- * <p/>
+ * <p>
* if the trust certificates keystore file changes, the {@link TrustManager}
* is refreshed with the new trust certificate entries (using a
* {@link ReloadingX509TrustManager} trustmanager).
@@ -87,7 +87,7 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
/**
* Resolves a property name to its client/server version if applicable.
- * <p/>
+ * <p>
* NOTE: This method is public for testing purposes.
*
* @param mode client/server mode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index f05274a..10c1d7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -44,10 +44,10 @@ import java.util.List;
/**
* Factory that creates SSLEngine and SSLSocketFactory instances using
* Hadoop configuration information.
- * <p/>
+ * <p>
* This SSLFactory uses a {@link ReloadingX509TrustManager} instance,
* which reloads public keys if the truststore file changes.
- * <p/>
+ * <p>
* This factory is used to configure HTTPS in Hadoop HTTP based endpoints, both
* client and server.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
index 47546b3..f9236b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
@@ -58,15 +58,15 @@ import org.slf4j.LoggerFactory;
* or X509Certificate, or ultimately (they all end up calling this one),
* String. (It's easier to supply JUnit with Strings instead of mock
* SSLSession objects!)
- * </p><p>Our check() methods throw exceptions if the name is
+ * <p>Our check() methods throw exceptions if the name is
* invalid, whereas javax.net.ssl.HostnameVerifier just returns true/false.
- * <p/>
+ * <p>
* We provide the HostnameVerifier.DEFAULT, HostnameVerifier.STRICT, and
* HostnameVerifier.ALLOW_ALL implementations. We also provide the more
* specialized HostnameVerifier.DEFAULT_AND_LOCALHOST, as well as
* HostnameVerifier.STRICT_IE6. But feel free to define your own
* implementations!
- * <p/>
+ * <p>
* Inspired by Sebastian Hauer's original StrictSSLProtocolSocketFactory in the
* HttpClient "contrib" repository.
*/
@@ -109,10 +109,10 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
/**
* The DEFAULT HostnameVerifier works the same way as Curl and Firefox.
- * <p/>
+ * <p>
* The hostname must match either the first CN, or any of the subject-alts.
* A wildcard can occur in the CN, and in any of the subject-alts.
- * <p/>
+ * <p>
* The only difference between DEFAULT and STRICT is that a wildcard (such
* as "*.foo.com") with DEFAULT matches all subdomains, including
* "a.b.foo.com".
@@ -158,13 +158,13 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
* Java 1.4, Sun Java 5, Sun Java 6. It's also pretty close to IE6.
* This implementation appears to be compliant with RFC 2818 for dealing
* with wildcards.
- * <p/>
+ * <p>
* The hostname must match either the first CN, or any of the subject-alts.
* A wildcard can occur in the CN, and in any of the subject-alts. The
* one divergence from IE6 is how we only check the first CN. IE6 allows
* a match against any of the CNs present. We decided to follow in
* Sun Java 1.4's footsteps and only check the first CN.
- * <p/>
+ * <p>
* A wildcard such as "*.foo.com" matches only subdomains in the same
* level, for example "a.foo.com". It does not match deeper subdomains
* such as "a.b.foo.com".
@@ -229,7 +229,7 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
* This contains a list of 2nd-level domains that aren't allowed to
* have wildcards when combined with country-codes.
* For example: [*.co.uk].
- * <p/>
+ * <p>
* The [*.co.uk] problem is an interesting one. Should we just hope
* that CA's would never foolishly allow such a certificate to happen?
* Looks like we're the only implementation guarding against this.
@@ -564,11 +564,11 @@ public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier {
/**
* Extracts the array of SubjectAlt DNS names from an X509Certificate.
* Returns null if there aren't any.
- * <p/>
+ * <p>
* Note: Java doesn't appear able to extract international characters
* from the SubjectAlts. It can only extract international characters
* from the CN field.
- * <p/>
+ * <p>
* (Or maybe the version of OpenSSL I'm using to test isn't storing the
* international characters correctly in the SubjectAlts?).
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index 0b1fdf8..35589a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -44,19 +44,19 @@ import java.util.Map;
* The <code>DelegationTokenAuthenticatedURL</code> is a
* {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token
* functionality.
- * <p/>
+ * <p>
* The authentication mechanisms supported by default are Hadoop Simple
* authentication (also known as pseudo authentication) and Kerberos SPNEGO
* authentication.
- * <p/>
+ * <p>
* Additional authentication mechanisms can be supported via {@link
* DelegationTokenAuthenticator} implementations.
- * <p/>
+ * <p>
* The default {@link DelegationTokenAuthenticator} is the {@link
* KerberosDelegationTokenAuthenticator} class which supports
* automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via
* the {@link PseudoDelegationTokenAuthenticator} class.
- * <p/>
+ * <p>
* <code>AuthenticatedURL</code> instances are not thread-safe.
*/
@InterfaceAudience.Public
@@ -115,7 +115,7 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
* Returns the default {@link DelegationTokenAuthenticator} class to use when
* an {@link DelegationTokenAuthenticatedURL} instance is created without
* specifying one.
- * <p/>
+ * <p>
* The default class is {@link KerberosDelegationTokenAuthenticator}
*
* @return the delegation token authenticator class to use as default.
@@ -143,7 +143,7 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
/**
* Creates an <code>DelegationTokenAuthenticatedURL</code>.
- * <p/>
+ * <p>
* An instance of the default {@link DelegationTokenAuthenticator} will be
* used.
*/
@@ -191,7 +191,7 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
* Sets if delegation token should be transmitted in the URL query string.
* By default it is transmitted using the
* {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
- * <p/>
+ * <p>
* This method is provided to enable WebHDFS backwards compatibility.
*
* @param useQueryString <code>TRUE</code> if the token is transmitted in the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index f5e798e..5275526 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -60,7 +60,7 @@ import java.util.Properties;
/**
* The <code>DelegationTokenAuthenticationFilter</code> filter is a
* {@link AuthenticationFilter} with Hadoop Delegation Token support.
- * <p/>
+ * <p>
* By default it uses it own instance of the {@link
* AbstractDelegationTokenSecretManager}. For situations where an external
* <code>AbstractDelegationTokenSecretManager</code> is required (i.e. one that
@@ -86,7 +86,7 @@ public class DelegationTokenAuthenticationFilter
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
- * <p/>
+ * <p>
* This is useful for use cases where secrets must be shared across multiple
* services.
*/
@@ -148,7 +148,7 @@ public class DelegationTokenAuthenticationFilter
/**
* Returns the proxyuser configuration. All returned properties must start
* with <code>proxyuser.</code>'
- * <p/>
+ * <p>
* Subclasses may override this method if the proxyuser configuration is
* read from other place than the filter init parameters.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index 6ee59f1..284044f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -59,7 +59,7 @@ import com.google.common.annotations.VisibleForTesting;
/**
* An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism
* for HTTP and supports Delegation Token functionality.
- * <p/>
+ * <p>
* In addition to the wrapped {@link AuthenticationHandler} configuration
* properties, this handler supports the following properties prefixed
* with the type of the wrapped <code>AuthenticationHandler</code>:
@@ -135,7 +135,7 @@ public abstract class DelegationTokenAuthenticationHandler
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
- * <p/>
+ * <p>
* This is useful for use cases where secrets must be shared across multiple
* services.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
index fd19b67..e1445fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
@@ -127,7 +127,7 @@ public class DelegationTokenManager {
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
- * <p/>
+ * <p>
* This is useful for use cases where secrets must be shared across multiple
* services.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
index 395d2f2..28509e1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticationHandler.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHa
/**
* An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism
* for HTTP and supports Delegation Token functionality.
- * <p/>
+ * <p>
* In addition to the {@link KerberosAuthenticationHandler} configuration
* properties, this handler supports:
* <ul>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
index 7e0e266..8b8a4bd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/KerberosDelegationTokenAuthenticator.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
* The <code>KerberosDelegationTokenAuthenticator</code> provides support for
* Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation
* Token operations.
- * <p/>
+ * <p>
* It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP
* endpoint does not trigger a SPNEGO authentication
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
index fc32a19..0661fb2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java
@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
import org.apache.hadoop.security.authentication.server.AuthenticationHandlerUtil;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.hadoop.security.authentication.server.CompositeAuthenticationHandler;
@@ -52,7 +53,7 @@ import com.google.common.base.Splitter;
* required to ensure that only schemes with strongest level of security should
* be used for delegation token management.
*
- * <p/>
+ * <p>
* In addition to the wrapped {@link AuthenticationHandler} configuration
* properties, this handler supports the following properties prefixed with the
* type of the wrapped <code>AuthenticationHandler</code>:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
index 6846fdb..9a4527a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticationHandler.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHand
/**
* An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism
* for HTTP and supports Delegation Token functionality.
- * <p/>
+ * <p>
* In addition to the {@link KerberosAuthenticationHandler} configuration
* properties, this handler supports:
* <ul>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
index 8713aa4..3478f39 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/PseudoDelegationTokenAuthenticator.java
@@ -29,7 +29,7 @@ import java.io.IOException;
* Hadoop's pseudo authentication mechanism that accepts
* the user name specified as a query string parameter and support for Hadoop
* Delegation Token operations.
- * <p/>
+ * <p>
* This mimics the model of Hadoop Simple authentication trusting the
* {@link UserGroupInformation#getCurrentUser()} value.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
index d064ef9..726a83d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceOperations.java
@@ -42,9 +42,9 @@ public final class ServiceOperations {
/**
* Stop a service.
- * <p/>Do nothing if the service is null or not
+ * <p>Do nothing if the service is null or not
* in a state in which it can be/needs to be stopped.
- * <p/>
+ * <p>
* The service state is checked <i>before</i> the operation begins.
* This process is <i>not</i> thread safe.
* @param service a service or null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
index 6b0b4e8..da91a3d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
@@ -268,7 +268,7 @@ public class ServiceLauncher<S extends Service>
* <ol>
* <li>Parse the command line.</li>
* <li>Build the service configuration from it.</li>
- * <li>Start the service.</li>.
+ * <li>Start the service.</li>
* <li>If it is a {@link LaunchableService}: execute it</li>
* <li>Otherwise: wait for it to finish.</li>
* <li>Exit passing the status code to the {@link #exit(int, String)}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
index 8516357..f582fa2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
@@ -379,7 +379,7 @@
a new configuration is created:
<pre>
- public Configuration bindArgs(Configuration config, List<String> args)
+ public Configuration bindArgs(Configuration config, List<String> args)
throws Exception {
Configuration newConf = new YarnConfiguration(config);
return newConf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
index 6f94989..44c9466 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
@@ -34,7 +34,6 @@ public class ClassUtil {
*
* @param clazz the class to find.
* @return a jar file that contains the class, or null.
- * @throws IOException
*/
public static String findContainingJar(Class<?> clazz) {
ClassLoader loader = clazz.getClassLoader();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
index 1f34291..bcb17d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
@@ -64,7 +64,7 @@ import java.util.Stack;
* Unknown qualifiers are considered after known qualifiers, with lexical order (always case insensitive),
* </li>
* <li>a dash usually precedes a qualifier, and is always less important than something preceded with a dot.</li>
- * </ul></p>
+ * </ul><p>
*
* @see <a href="https://cwiki.apache.org/confluence/display/MAVENOLD/Versioning">"Versioning" on Maven Wiki</a>
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
index b7feb22..690d097 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
@@ -33,9 +33,9 @@ import java.security.CodeSource;
* This entry point exists for diagnosing classloader problems:
* is a class or resource present -and if so, where?
*
- * <p/>
+ * <p>
* Actions
- * <p/>
+ * <br>
* <ul>
* <li><pre>load</pre>: load a class but do not attempt to create it </li>
* <li><pre>create</pre>: load and create a class, print its string value</li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
index a8a513d..8ca7a90 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.util;import java.io.File;
+package org.apache.hadoop.util;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
@@ -54,9 +55,9 @@ import org.slf4j.LoggerFactory;
* line arguments, enabling applications to easily specify a namenode, a
* ResourceManager, additional configuration resources etc.
*
- * <h4 id="GenericOptions">Generic Options</h4>
+ * <h3 id="GenericOptions">Generic Options</h3>
*
- * <p>The supported generic options are:</p>
+ * <p>The supported generic options are:
* <p><blockquote><pre>
* -conf <configuration file> specify a configuration file
* -D <property=value> use value for given property
@@ -69,12 +70,12 @@ import org.slf4j.LoggerFactory;
* -archives <comma separated list of archives> specify comma
* separated archives to be unarchived on the compute machines.
- * </pre></blockquote></p>
+ * </pre></blockquote><p>
*
* <p>The general command line syntax is:</p>
- * <p><tt><pre>
+ * <p><pre><code>
* bin/hadoop command [genericOptions] [commandOptions]
- * </pre></tt></p>
+ * </code></pre><p>
*
* <p>Generic command line arguments <strong>might</strong> modify
* <code>Configuration </code> objects, given to constructors.</p>
@@ -104,7 +105,7 @@ import org.slf4j.LoggerFactory;
* $ bin/hadoop jar -libjars testlib.jar
* -archives test.tgz -files file.txt inputjar args
* job submission with libjars, files and archives
- * </pre></blockquote></p>
+ * </pre></blockquote><p>
*
* @see Tool
* @see ToolRunner
@@ -141,8 +142,8 @@ public class GenericOptionsParser {
}
/**
- * Create a <code>GenericOptionsParser<code> to parse only the generic Hadoop
- * arguments.
+ * Create a <code>GenericOptionsParser</code> to parse only the generic
+ * Hadoop arguments.
*
* The array of string arguments other than the generic arguments can be
* obtained by {@link #getRemainingArgs()}.
@@ -217,7 +218,7 @@ public class GenericOptionsParser {
/**
* Specify properties of each generic option.
- * <i>Important</i?: as {@link OptionBuilder} is not thread safe, subclasses
+ * <i>Important</i>: as {@link OptionBuilder} is not thread safe, subclasses
* must synchronize use on {@code OptionBuilder.class}
*/
@SuppressWarnings("static-access")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
index 50be1c3..366c8c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
@@ -34,10 +34,10 @@ import java.util.Map;
/**
* HTTP utility class to help propagate server side exception to the client
* over HTTP as a JSON payload.
- * <p/>
+ * <p>
* It creates HTTP Servlet and JAX-RPC error responses including details of the
* exception that allows a client to recreate the remote exception.
- * <p/>
+ * <p>
* It parses HTTP client connections and recreates the exception.
*/
@InterfaceAudience.Private
@@ -125,7 +125,7 @@ public class HttpExceptionUtils {
* expected HTTP status code. If the current status code is not the expected
* one it throws an exception with a detail message using Server side error
* messages if available.
- * <p/>
+ * <p>
* <b>NOTE:</b> this method will throw the deserialized exception even if not
* declared in the <code>throws</code> of the method signature.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
index cbc8560..7e09a61 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -185,7 +185,7 @@ public class JsonSerialization<T> {
* Save to a local file. Any existing file is overwritten unless
* the OS blocks that.
* @param file file
- * @param path path
+ * @param instance instance
* @throws IOException IO exception
*/
public void save(File file, T instance) throws
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
index d79aade..79de1ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
@@ -99,11 +99,11 @@ public class LightWeightCache<K, E extends K> extends LightWeightGSet<K, E> {
/**
* @param recommendedLength Recommended size of the internal array.
* @param sizeLimit the limit of the size of the cache.
- * The limit is disabled if it is <= 0.
- * @param creationExpirationPeriod the time period C > 0 in nanoseconds that
- * the creation of an entry is expired if it is added to the cache
- * longer than C.
- * @param accessExpirationPeriod the time period A >= 0 in nanoseconds that
+ * The limit is disabled if it is <= 0.
+ * @param creationExpirationPeriod the time period C > 0 in nanoseconds
+ * that the creation of an entry is expired if it is added to the
+ * cache longer than C.
+ * @param accessExpirationPeriod the time period A >= 0 in nanoseconds that
* the access of an entry is expired if it is not accessed
* longer than A.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
index a1cf709..e2cd304 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
@@ -62,7 +62,6 @@ public class LineReader implements Closeable {
* Create a line reader that reads from the given stream using the
* default buffer-size (64k).
* @param in The input stream
- * @throws IOException
*/
public LineReader(InputStream in) {
this(in, DEFAULT_BUFFER_SIZE);
@@ -73,7 +72,6 @@ public class LineReader implements Closeable {
* given buffer-size.
* @param in The input stream
* @param bufferSize Size of the read buffer
- * @throws IOException
*/
public LineReader(InputStream in, int bufferSize) {
this.in = in;
@@ -115,7 +113,6 @@ public class LineReader implements Closeable {
* @param in The input stream
* @param bufferSize Size of the read buffer
* @param recordDelimiterBytes The delimiter
- * @throws IOException
*/
public LineReader(InputStream in, int bufferSize,
byte[] recordDelimiterBytes) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
index b01330f..20931de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
@@ -187,7 +187,7 @@ public class MachineList {
}
/**
- * returns the contents of the MachineList as a Collection<String>
+ * returns the contents of the MachineList as a Collection<String>
* This can be used for testing
* @return contents of the MachineList
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
index 2ca8e55..556f4e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
@@ -46,7 +46,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOW
/**
* The <code>ShutdownHookManager</code> enables running shutdownHook
* in a deterministic order, higher priority first.
- * <p/>
+ * <p>
* The JVM runs ShutdownHooks in a non-deterministic order or in parallel.
* This class registers a single JVM shutdownHook and run all the
* shutdownHooks registered to it (to this class) in order based on their
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
index 5405d77..50a728e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
@@ -39,7 +39,6 @@ public class ShutdownThreadsHelper {
* @param thread {@link Thread to be shutdown}
* @return <tt>true</tt> if the thread is successfully interrupted,
* <tt>false</tt> otherwise
- * @throws InterruptedException
*/
public static boolean shutdownThread(Thread thread) {
return shutdownThread(thread, SHUTDOWN_WAIT_MS);
@@ -51,7 +50,6 @@ public class ShutdownThreadsHelper {
* interrupted
* @return <tt>true</tt> if the thread is successfully interrupted,
* <tt>false</tt> otherwise
- * @throws InterruptedException
*/
public static boolean shutdownThread(Thread thread,
long timeoutInMilliSeconds) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index f49698c..d19ced9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -428,7 +428,7 @@ public class StringUtils {
* Splits a comma separated value <code>String</code>, trimming leading and
* trailing whitespace on each value. Duplicate and empty values are removed.
*
- * @param str a comma separated <String> with values, may be null
+ * @param str a comma separated <code>String</code> with values, may be null
* @return a <code>Collection</code> of <code>String</code> values, empty
* Collection if null String input
*/
@@ -1006,8 +1006,8 @@ public class StringUtils {
* @param template String template to receive replacements
* @param pattern Pattern to match for identifying tokens, must use a capturing
* group
- * @param replacements Map<String, String> mapping tokens identified by the
- * capturing group to their replacement values
+ * @param replacements Map<String, String> mapping tokens identified by
+ * the capturing group to their replacement values
* @return String template with replacements
*/
public static String replaceTokens(String template, Pattern pattern,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
index b6e3d7d..a4fbce4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.conf.Configurable;
* System.exit(res);
* }
* }
- * </pre></blockquote></p>
+ * </pre></blockquote><p>
*
* @see GenericOptionsParser
* @see ToolRunner
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
index d23df79..fe64adb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
@@ -90,7 +90,7 @@ public class ZKUtil {
* <code>sasl:hdfs/host1@MY.DOMAIN:cdrwa,sasl:hdfs/host2@MY.DOMAIN:cdrwa</code>
*
* @return ACL list
- * @throws {@link BadAclFormatException} if an ACL is invalid
+ * @throws BadAclFormatException if an ACL is invalid
*/
public static List<ACL> parseACLs(String aclString) throws
BadAclFormatException {
@@ -128,7 +128,7 @@ public class ZKUtil {
*
* @param authString the comma-separated auth mechanisms
* @return a list of parsed authentications
- * @throws {@link BadAuthFormatException} if the auth format is invalid
+ * @throws BadAuthFormatException if the auth format is invalid
*/
public static List<ZKAuthInfo> parseAuth(String authString) throws
BadAuthFormatException{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
index be9a407..8a7ec69 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/DynamicBloomFilter.java
@@ -64,8 +64,9 @@ import org.apache.hadoop.classification.InterfaceStability;
* process of a DBF is iterative. At the start, the DBF is a <code>1 * m</code>
* bit matrix, i.e., it is composed of a single standard Bloom filter.
* It assumes that <code>n<sub>r</sub></code> elements are recorded in the
- * initial bit vector, where <code>n<sub>r</sub> <= n</code> (<code>n</code> is
- * the cardinality of the set <code>A</code> to record in the filter).
+ * initial bit vector, where <code>n<sub>r</sub> {@literal <=} n</code>
+ * (<code>n</code> is the cardinality of the set <code>A</code> to record in
+ * the filter).
* <p>
* As the size of <code>A</code> grows during the execution of the application,
* several keys must be inserted in the DBF. When inserting a key into the DBF,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
index f124890..9304b48 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
@@ -37,7 +37,7 @@ public interface AsyncGet<R, E extends Throwable> {
*
* @param timeout The maximum time period to wait.
* When timeout == 0, it does not wait at all.
- * When timeout < 0, it waits indefinitely.
+ * When timeout < 0, it waits indefinitely.
* @param unit The unit of the timeout value
* @return the result, which is possibly null.
* @throws E an exception thrown by the underlying implementation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
index f3895d0..8c3b9da 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
@@ -69,11 +69,11 @@ public class JenkinsHash extends Hash {
* <p>The best hash table sizes are powers of 2. There is no need to do mod
* a prime (mod is sooo slow!). If you need less than 32 bits, use a bitmask.
* For example, if you need only 10 bits, do
- * <code>h = (h & hashmask(10));</code>
+ * <code>h = (h & hashmask(10));</code>
* In which case, the hash table should have hashsize(10) elements.
*
* <p>If you are hashing n strings byte[][] k, do it like this:
- * for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h);
+ * for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h);
*
* <p>By Bob Jenkins, 2006. bob_jenkins@burtleburtle.net. You may use this
* code any way you wish, private, educational, or commercial. It's free.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
index 8823f5c..a7a8fa5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommand.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +20,7 @@ package org.apache.hadoop.cli.util;
import org.apache.hadoop.conf.Configuration;
/**
- * This interface is to generalize types of test command for upstream projects
+ * This interface is to generalize types of test command for upstream projects.
*/
public interface CLICommand {
public CommandExecutor getExecutor(String tag, Configuration conf)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
index eb96a06..e22c200 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandFS.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
index 44e0c30..8efe70c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CLICommandTypes.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,7 +18,7 @@
package org.apache.hadoop.cli.util;
/**
- * This interface is to provide command type for test commands enums
+ * This interface is to provide command type for test commands enums.
*/
public interface CLICommandTypes {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
index df8c54b..362bde9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
index 56dec3a..6b82077 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestDefaultRetryPolicy.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
index 493d860..eb51820 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
@@ -44,9 +44,9 @@ public class StaticMapping extends AbstractDNSToSwitchMapping {
/**
* Key to define the node mapping as a comma-delimited list of host=rack
* mappings, e.g. <code>host1=r1,host2=r1,host3=r2</code>.
- * <p/>
+ * <p>
* Value: {@value}
- * <p/>
+ * <p>
* <b>Important: </b>spaces not trimmed and are considered significant.
*/
public static final String KEY_HADOOP_CONFIGURED_NODE_MAPPING =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b57f2f7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
index 2bc68ce..d4599b0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/SetSpanReceiver.java
@@ -35,7 +35,7 @@ import org.junit.Assert;
/**
* Span receiver that puts all spans into a single set.
* This is useful for testing.
- * <p/>
+ * <p>
* We're not using HTrace's POJOReceiver here so as that doesn't
* push all the metrics to a static place, and would make testing
* SpanReceiverHost harder.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDDS-630. Rename KSM to OM in
Hdds.md. Contributed by Takanobu Asanuma.
Posted by br...@apache.org.
HDDS-630. Rename KSM to OM in Hdds.md. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47eeb3eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47eeb3eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47eeb3eb
Branch: refs/heads/HDFS-13891
Commit: 47eeb3eb7bffa20f2c3526a5cbb85cae9af667bd
Parents: aeeb038
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Oct 11 09:20:11 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Oct 11 09:20:11 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/Hdds.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47eeb3eb/hadoop-ozone/docs/content/Hdds.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/Hdds.md b/hadoop-ozone/docs/content/Hdds.md
index 9978c26..35654cd 100644
--- a/hadoop-ozone/docs/content/Hdds.md
+++ b/hadoop-ozone/docs/content/Hdds.md
@@ -33,7 +33,7 @@ to maintain the replication levels needed by the cluster.
It is easier to look at a putKey operation to understand the role that SCM plays.
-To put a key, a client makes a call to KSM with the following arguments.
+To put a key, a client makes a call to OM with the following arguments.
-- putKey(keyName, data, pipeline type, replication count)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-8861. executorLock is
misleading in ContainerLaunch. Contributed by Chandni Singh
Posted by br...@apache.org.
YARN-8861. executorLock is misleading in ContainerLaunch. Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e787d65a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e787d65a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e787d65a
Branch: refs/heads/HDFS-13891
Commit: e787d65a08f5d5245d2313fc34f2dde518bfaa5b
Parents: ee816f1
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Oct 11 10:54:57 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Oct 11 10:54:57 2018 -0500
----------------------------------------------------------------------
.../containermanager/launcher/ContainerLaunch.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e787d65a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6776836..f198e83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -135,7 +135,7 @@ public class ContainerLaunch implements Callable<Integer> {
protected final LocalDirsHandlerService dirsHandler;
- private final Lock containerExecLock = new ReentrantLock();
+ private final Lock launchLock = new ReentrantLock();
public ContainerLaunch(Context context, Configuration configuration,
Dispatcher dispatcher, ContainerExecutor exec, Application app,
@@ -485,11 +485,11 @@ public class ContainerLaunch implements Callable<Integer> {
throws IOException, ConfigurationException {
int launchPrep = prepareForLaunch(ctx);
if (launchPrep == 0) {
- containerExecLock.lock();
+ launchLock.lock();
try {
return exec.launchContainer(ctx);
} finally {
- containerExecLock.unlock();
+ launchLock.unlock();
}
}
return launchPrep;
@@ -499,18 +499,18 @@ public class ContainerLaunch implements Callable<Integer> {
throws IOException, ConfigurationException {
int launchPrep = prepareForLaunch(ctx);
if (launchPrep == 0) {
- containerExecLock.lock();
+ launchLock.lock();
try {
return exec.relaunchContainer(ctx);
} finally {
- containerExecLock.unlock();
+ launchLock.unlock();
}
}
return launchPrep;
}
void reapContainer() throws IOException {
- containerExecLock.lock();
+ launchLock.lock();
try {
// Reap the container
boolean result = exec.reapContainer(
@@ -524,7 +524,7 @@ public class ContainerLaunch implements Callable<Integer> {
}
cleanupContainerFiles(getContainerWorkDir());
} finally {
- containerExecLock.unlock();
+ launchLock.unlock();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDDS-634. OzoneFS: support basic
user group and permission for file/dir. Contributed by Xiaoyu Yao.
Posted by br...@apache.org.
HDDS-634. OzoneFS: support basic user group and permission for file/dir.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/604c27c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/604c27c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/604c27c3
Branch: refs/heads/HDFS-13891
Commit: 604c27c33a3bd30644debaf192cec3cf80dfa36d
Parents: 5055247
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Oct 11 17:19:38 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Oct 11 17:25:25 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 16 ++++++++++++++--
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 17 +++++++++++++++--
2 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/604c27c3/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 8fbd688..0b721fb 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -448,6 +448,14 @@ public class OzoneFileSystem extends FileSystem {
}
/**
+ * Get the username of the FS.
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
* Check whether the path is valid and then create directories.
* Directory is represented using a key with no value.
* All the non-existent parent directories are also created.
@@ -528,11 +536,15 @@ public class OzoneFileSystem extends FileSystem {
throw new FileNotFoundException(f + ": No such file or directory!");
} else if (isDirectory(meta)) {
return new FileStatus(0, true, 1, 0,
- meta.getModificationTime(), qualifiedPath);
+ meta.getModificationTime(), 0,
+ FsPermission.getDirDefault(), getUsername(), getUsername(),
+ qualifiedPath);
} else {
//TODO: Fetch replication count from ratis config
return new FileStatus(meta.getDataSize(), false, 1,
- getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
+ getDefaultBlockSize(f), meta.getModificationTime(), 0,
+ FsPermission.getFileDefault(), getUsername(), getUsername(),
+ qualifiedPath);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/604c27c3/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index a225702..7cf6e3d 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -23,6 +23,7 @@ import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.Before;
@@ -167,6 +168,10 @@ public class TestOzoneFileInterfaces {
assertTrue("Modification time has not been recorded: " + status,
status.getModificationTime() > currentTime);
+ assertEquals(false, status.isDirectory());
+ assertEquals(FsPermission.getFileDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
try (FSDataInputStream inputStream = fs.open(path)) {
byte[] buffer = new byte[stringLen];
inputStream.readFully(0, buffer);
@@ -175,6 +180,12 @@ public class TestOzoneFileInterfaces {
}
}
+ private void verifyOwnerGroup(FileStatus fileStatus) {
+ String owner = getCurrentUser();
+ assertEquals(owner, fileStatus.getOwner());
+ assertEquals(owner, fileStatus.getGroup());
+ }
+
@Test
public void testDirectory() throws IOException {
@@ -186,6 +197,10 @@ public class TestOzoneFileInterfaces {
FileStatus status = fs.getFileStatus(path);
assertTrue("The created path is not directory.", status.isDirectory());
+ assertEquals(true, status.isDirectory());
+ assertEquals(FsPermission.getDirDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
assertEquals(0, status.getLen());
FileStatus[] statusList = fs.listStatus(createPath("/"));
@@ -195,8 +210,6 @@ public class TestOzoneFileInterfaces {
FileStatus statusRoot = fs.getFileStatus(createPath("/"));
assertTrue("Root dir (/) is not a directory.", status.isDirectory());
assertEquals(0, status.getLen());
-
-
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-8753. [UI2] Lost nodes
representation missing from Nodemanagers Chart. Contributed by Yesha Vora.
Posted by br...@apache.org.
YARN-8753. [UI2] Lost nodes representation missing from Nodemanagers Chart. Contributed by Yesha Vora.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a379839
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a379839
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a379839
Branch: refs/heads/HDFS-13891
Commit: 8a379839909197690af170afc13c49ef7aa54286
Parents: d3da741
Author: Sunil G <su...@apache.org>
Authored: Thu Oct 11 16:15:25 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Oct 11 16:15:25 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/models/cluster-metric.js | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a379839/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
index 764723a..4075aea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
@@ -97,8 +97,12 @@ export default DS.Model.extend({
label: "Decomissioned",
value: this.get("decommissionedNodes")
});
+ arr.push({
+ label: "Lost",
+ value: this.get("lostNodes")
+ });
return arr;
- }.property("activeNodes", "unhealthyNodes", "decommissioningNodes", "decommissionedNodes"),
+ }.property("activeNodes", "unhealthyNodes", "decommissioningNodes", "decommissionedNodes", "lostNodes"),
getMemoryDataForDonutChart: function() {
var type = "MB";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDDS-619. hdds.db.profile should
not be tagged as a required setting & should default to DISK. Contributed by
Dinesh Chitlangia.
Posted by br...@apache.org.
HDDS-619. hdds.db.profile should not be tagged as a required setting & should default to DISK. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bd000c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bd000c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bd000c8
Branch: refs/heads/HDFS-13891
Commit: 2bd000c85195416b9bcd06a3abb5100aeeda9727
Parents: 045069e
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Oct 10 17:37:20 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Oct 10 17:37:20 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd000c8/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 13b3bb7..210b075 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -72,7 +72,7 @@ public final class HddsConfigKeys {
// DB Profiles used by ROCKDB instances.
public static final String HDDS_DB_PROFILE = "hdds.db.profile";
- public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.SSD;
+ public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.DISK;
// Once a container usage crosses this threshold, it is eligible for
// closing.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd000c8/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aee471e..ef54ed2 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1184,8 +1184,8 @@
<property>
<name>hdds.db.profile</name>
- <value>SSD</value>
- <tag>OZONE, OM, PERFORMANCE, REQUIRED</tag>
+ <value>DISK</value>
+ <tag>OZONE, OM, PERFORMANCE</tag>
<description>This property allows user to pick a configuration
that tunes the RocksDB settings for the hardware it is running
on. Right now, we have SSD and DISK as profile options.</description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDFS-13949. Correct the
description of dfs.datanode.disk.check.timeout in hdfs-default.xml.
Contributed by Toshihiro Suzuki.
Posted by br...@apache.org.
HDFS-13949. Correct the description of dfs.datanode.disk.check.timeout in hdfs-default.xml. Contributed by Toshihiro Suzuki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b39b8024
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b39b8024
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b39b8024
Branch: refs/heads/HDFS-13891
Commit: b39b802422b444629095bde4484b820f02fd6d1d
Parents: edce866
Author: Nanda kumar <na...@apache.org>
Authored: Wed Oct 10 12:29:07 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Oct 10 12:29:07 2018 +0530
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39b8024/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 2ee8399..503ba8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5134,11 +5134,10 @@
<name>dfs.datanode.disk.check.timeout</name>
<value>10m</value>
<description>
- Maximum allowed time for a disk check to complete during DataNode
- startup. If the check does not complete within this time interval
- then the disk is declared as failed. This setting supports
- multiple time unit suffixes as described in dfs.heartbeat.interval.
- If no suffix is specified then milliseconds is assumed.
+ Maximum allowed time for a disk check to complete. If the check does not
+ complete within this time interval then the disk is declared as failed.
+ This setting supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then milliseconds is assumed.
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDFS-12459. Fix revert: Add new op
GETFILEBLOCKLOCATIONS to WebHDFS REST API. Contributed by Weiwei Yang.
Posted by br...@apache.org.
HDFS-12459. Fix revert: Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API. Contributed by Weiwei Yang.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ead525c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ead525c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ead525c
Branch: refs/heads/HDFS-13891
Commit: 3ead525c71cba068e7abf1c76ad629bfeec10852
Parents: cdc4350
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Oct 10 10:07:22 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Oct 10 10:11:10 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/web/resources/GetOpParam.java | 12 +-
.../web/resources/NamenodeWebHdfsMethods.java | 14 ++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 32 ++++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 173 +++++++++++++++++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 148 ++++++++++++++++
5 files changed, 378 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index 85bb11d..3f4f146 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -33,8 +33,18 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
GETHOMEDIRECTORY(false, HttpURLConnection.HTTP_OK),
GETDELEGATIONTOKEN(false, HttpURLConnection.HTTP_OK, true),
- /** GET_BLOCK_LOCATIONS is a private unstable op. */
+ /**
+ * GET_BLOCK_LOCATIONS is a private/stable API op. It returns a
+ * {@link org.apache.hadoop.hdfs.protocol.LocatedBlocks}
+ * json object.
+ */
GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK),
+ /**
+ * GETFILEBLOCKLOCATIONS is the public op that complies with
+ * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations}
+ * interface.
+ */
+ GETFILEBLOCKLOCATIONS(false, HttpURLConnection.HTTP_OK),
GETACLSTATUS(false, HttpURLConnection.HTTP_OK),
GETXATTRS(false, HttpURLConnection.HTTP_OK),
GETTRASHROOT(false, HttpURLConnection.HTTP_OK),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 619d439..d73fd45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -59,6 +59,7 @@ import javax.ws.rs.core.Response.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
@@ -72,6 +73,7 @@ import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -1075,6 +1077,18 @@ public class NamenodeWebHdfsMethods {
.build();
}
}
+ case GETFILEBLOCKLOCATIONS:
+ {
+ final long offsetValue = offset.getValue();
+ final Long lengthValue = length.getValue();
+ LocatedBlocks locatedBlocks = getRpcClientProtocol()
+ .getBlockLocations(fullpath, offsetValue, lengthValue != null ?
+ lengthValue : Long.MAX_VALUE);
+ BlockLocation[] locations =
+ DFSUtilClient.locatedBlocks2Locations(locatedBlocks);
+ final String js = JsonUtil.toJsonString(locations);
+ return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+ }
case GET_BLOCK_LOCATIONS:
{
final long offsetValue = offset.getValue();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 5c810bf..f966640 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -572,4 +572,36 @@ public class JsonUtil {
m.put("dirStatus", toJsonMap(snapshottableDirectoryStatus.getDirStatus()));
return m;
}
+
+ private static Map<String, Object> toJsonMap(
+ final BlockLocation blockLocation) throws IOException {
+ if (blockLocation == null) {
+ return null;
+ }
+
+ final Map<String, Object> m = new HashMap<>();
+ m.put("length", blockLocation.getLength());
+ m.put("offset", blockLocation.getOffset());
+ m.put("corrupt", blockLocation.isCorrupt());
+ m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes()));
+ m.put("cachedHosts", blockLocation.getCachedHosts());
+ m.put("hosts", blockLocation.getHosts());
+ m.put("names", blockLocation.getNames());
+ m.put("topologyPaths", blockLocation.getTopologyPaths());
+ return m;
+ }
+
+ public static String toJsonString(BlockLocation[] locations)
+ throws IOException {
+ if (locations == null) {
+ return null;
+ }
+ final Map<String, Object> m = new HashMap<>();
+ Object[] blockLocations = new Object[locations.length];
+ for(int i=0; i<locations.length; i++) {
+ blockLocations[i] = toJsonMap(locations[i]);
+ }
+ m.put(BlockLocation.class.getSimpleName(), blockLocations);
+ return toJsonString("BlockLocations", m);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index eb11815..e29b932 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -52,6 +52,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`GETSTORAGEPOLICY`](#Get_Storage_Policy) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStoragePolicy)
* [`GETSNAPSHOTDIFF`](#Get_Snapshot_Diff)
* [`GETSNAPSHOTTABLEDIRECTORYLIST`](#Get_Snapshottable_Directory_List)
+ * [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations)
* HTTP PUT
* [`CREATE`](#Create_and_Write_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create)
* [`MKDIRS`](#Make_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs)
@@ -1087,6 +1088,49 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).unsetStor
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStoragePolicy
+### Get File Block Locations
+
+* Submit a HTTP GET request.
+
+ curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILEBLOCKLOCATIONS
+
+ The client receives a response with a [`BlockLocations` JSON Object](#Block_Locations_JSON_Schema):
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {
+ "BlockLocations" :
+ {
+ "BlockLocation":
+ [
+ {
+ "cachedHosts" : [],
+ "corrupt" : false,
+ "hosts" : ["host"],
+ "length" : 134217728, // length of this block
+ "names" : ["host:ip"],
+ "offset" : 0, // offset of the block in the file
+ "storageTypes" : ["DISK"], // enum {RAM_DISK, SSD, DISK, ARCHIVE}
+ "topologyPaths" : ["/default-rack/hostname:ip"]
+ }, {
+ "cachedHosts" : [],
+ "corrupt" : false,
+ "hosts" : ["host"],
+ "length" : 62599364,
+ "names" : ["host:ip"],
+ "offset" : 134217728,
+ "storageTypes" : ["DISK"],
+ "topologyPaths" : ["/default-rack/hostname:ip"]
+ },
+ ...
+ ]
+ }
+ }
+
+See also: [`offset`](#Offset), [`length`](#Length), [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations
+
Extended Attributes(XAttrs) Operations
--------------------------------------
@@ -2227,6 +2271,135 @@ var snapshottableDirectoryStatus =
}
```
+### BlockLocations JSON Schema
+
+A `BlockLocations` JSON object represents an array of `BlockLocation` JSON objects.
+
+```json
+{
+ "name" : "BlockLocations",
+ "properties":
+ {
+ "BlockLocations":
+ {
+ "type" : "object",
+ "properties":
+ {
+ "BlockLocation":
+ {
+ "description": "An array of BlockLocation",
+ "type" : "array",
+ "items" : blockLocationProperties //See BlockLocation Properties
+ }
+ }
+ }
+ }
+}
+```
+
+See also [`BlockLocation` Properties](#BlockLocation_Properties), [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations), [BlockLocation](../../api/org/apache/hadoop/fs/BlockLocation.html)
+
+### BlockLocation JSON Schema
+
+```json
+{
+ "name" : "BlockLocation",
+ "properties":
+ {
+ "BlockLocation": blockLocationProperties //See BlockLocation Properties
+ }
+}
+```
+
+See also [`BlockLocation` Properties](#BlockLocation_Properties), [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations), [BlockLocation](../../api/org/apache/hadoop/fs/BlockLocation.html)
+
+#### BlockLocation Properties
+
+JavaScript syntax is used to define `blockLocationProperties` so that it can be referred in both `BlockLocation` and `BlockLocations` JSON schemas.
+
+```javascript
+var blockLocationProperties =
+{
+ "type" : "object",
+ "properties":
+ {
+ "cachedHosts":
+ {
+ "description": "Datanode hostnames with a cached replica",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "A datanode hostname",
+ "type" : "string"
+ }
+ },
+ "corrupt":
+ {
+ "description": "True if the block is corrupted",
+ "type" : "boolean",
+ "required" : "true"
+ },
+ "hosts":
+ {
+ "description": "Datanode hostnames store the block",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "A datanode hostname",
+ "type" : "string"
+ }
+ },
+ "length":
+ {
+ "description": "Length of the block",
+ "type" : "integer",
+ "required" : "true"
+ },
+ "names":
+ {
+ "description": "Datanode IP:xferPort for accessing the block",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "DatanodeIP:xferPort",
+ "type" : "string"
+ }
+ },
+ "offset":
+ {
+ "description": "Offset of the block in the file",
+ "type" : "integer",
+ "required" : "true"
+ },
+ "storageTypes":
+ {
+ "description": "Storage type of each replica",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "Storage type",
+ "enum" : ["RAM_DISK", "SSD", "DISK", "ARCHIVE"]
+ }
+ },
+ "topologyPaths":
+ {
+ "description": "Datanode addresses in network topology",
+ "type" : "array",
+ "required" : "true",
+ "items" :
+ {
+ "description": "/rack/host:ip",
+ "type" : "string"
+ }
+ }
+ }
+};
+```
+
HTTP Query Parameter Dictionary
-------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ead525c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index d7ed46b..71948d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -47,6 +47,7 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
+import java.util.Map;
import java.util.Random;
import com.google.common.collect.ImmutableList;
@@ -118,6 +119,9 @@ import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.MapType;
+
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.doReturn;
@@ -1136,6 +1140,150 @@ public class TestWebHDFS {
}
}
+ @Test
+ public void testWebHdfsGetBlockLocations() throws Exception{
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ final int offset = 42;
+ final int length = 512;
+ final Path path = new Path("/foo");
+ byte[] contents = new byte[1024];
+ RANDOM.nextBytes(contents);
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsConstants.WEBHDFS_SCHEME);
+ try (OutputStream os = fs.create(path)) {
+ os.write(contents);
+ }
+ BlockLocation[] locations = fs.getFileBlockLocations(path, offset,
+ length);
+
+ // Query webhdfs REST API to get block locations
+ InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
+
+ // Case 1
+ // URL without length or offset parameters
+ URL url1 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS");
+
+ String response1 = getResponse(url1, "GET");
+ // Parse BlockLocation array from json output using object mapper
+ BlockLocation[] locationArray1 = toBlockLocationArray(response1);
+
+ // Verify the result from rest call is same as file system api
+ verifyEquals(locations, locationArray1);
+
+ // Case 2
+ // URL contains length and offset parameters
+ URL url2 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&length=" + length + "&offset=" + offset);
+
+ String response2 = getResponse(url2, "GET");
+ BlockLocation[] locationArray2 = toBlockLocationArray(response2);
+
+ verifyEquals(locations, locationArray2);
+
+ // Case 3
+ // URL contains length parameter but without offset parameters
+ URL url3 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&length=" + length);
+
+ String response3 = getResponse(url3, "GET");
+ BlockLocation[] locationArray3 = toBlockLocationArray(response3);
+
+ verifyEquals(locations, locationArray3);
+
+ // Case 4
+ // URL contains offset parameter but without length parameter
+ URL url4 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&offset=" + offset);
+
+ String response4 = getResponse(url4, "GET");
+ BlockLocation[] locationArray4 = toBlockLocationArray(response4);
+
+ verifyEquals(locations, locationArray4);
+
+ // Case 5
+ // URL specifies offset exceeds the file length
+ URL url5 = new URL("http", addr.getHostString(), addr.getPort(),
+ WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"
+ + "&offset=1200");
+
+ String response5 = getResponse(url5, "GET");
+ BlockLocation[] locationArray5 = toBlockLocationArray(response5);
+
+ // Expected an empty array of BlockLocation
+ verifyEquals(new BlockLocation[] {}, locationArray5);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private BlockLocation[] toBlockLocationArray(String json)
+ throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ MapType subType = mapper.getTypeFactory().constructMapType(
+ Map.class,
+ String.class,
+ BlockLocation[].class);
+ MapType rootType = mapper.getTypeFactory().constructMapType(
+ Map.class,
+ mapper.constructType(String.class),
+ mapper.constructType(subType));
+
+ Map<String, Map<String, BlockLocation[]>> jsonMap = mapper
+ .readValue(json, rootType);
+ Map<String, BlockLocation[]> locationMap = jsonMap
+ .get("BlockLocations");
+ BlockLocation[] locationArray = locationMap.get(
+ BlockLocation.class.getSimpleName());
+ return locationArray;
+ }
+
+ private void verifyEquals(BlockLocation[] locations1,
+ BlockLocation[] locations2) throws IOException {
+ for(int i=0; i<locations1.length; i++) {
+ BlockLocation location1 = locations1[i];
+ BlockLocation location2 = locations2[i];
+ Assert.assertEquals(location1.getLength(),
+ location2.getLength());
+ Assert.assertEquals(location1.getOffset(),
+ location2.getOffset());
+ Assert.assertArrayEquals(location1.getCachedHosts(),
+ location2.getCachedHosts());
+ Assert.assertArrayEquals(location1.getHosts(),
+ location2.getHosts());
+ Assert.assertArrayEquals(location1.getNames(),
+ location2.getNames());
+ Assert.assertArrayEquals(location1.getTopologyPaths(),
+ location2.getTopologyPaths());
+ Assert.assertArrayEquals(location1.getStorageTypes(),
+ location2.getStorageTypes());
+ }
+ }
+
+ private static String getResponse(URL url, String httpRequestType)
+ throws IOException {
+ HttpURLConnection conn = null;
+ try {
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod(httpRequestType);
+ conn.setInstanceFollowRedirects(false);
+ return IOUtils.toString(conn.getInputStream(),
+ StandardCharsets.UTF_8);
+ } finally {
+ if(conn != null) {
+ conn.disconnect();
+ }
+ }
+ }
+
private WebHdfsFileSystem createWebHDFSAsTestUser(final Configuration conf,
final URI uri, final String userName) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDFS-13156. HDFS Block Placement
Policy - Client Local Rack. Contributed by Ayush Saxena.
Posted by br...@apache.org.
HDFS-13156. HDFS Block Placement Policy - Client Local Rack. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de44e106
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de44e106
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de44e106
Branch: refs/heads/HDFS-13891
Commit: de44e1064f051248934ceffdd98a3cc13653d886
Parents: e36ae96
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Oct 12 17:27:23 2018 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Fri Oct 12 17:27:23 2018 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de44e106/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
index 471a27f..a0121e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
@@ -102,7 +102,7 @@ Large HDFS instances run on a cluster of computers that commonly spread across m
The NameNode determines the rack id each DataNode belongs to via the process outlined in [Hadoop Rack Awareness](../hadoop-common/RackAwareness.html).
A simple but non-optimal policy is to place replicas on unique racks. This prevents losing data when an entire rack fails and allows use of bandwidth from multiple racks when reading data. This policy evenly distributes replicas in the cluster which makes it easy to balance load on component failure. However, this policy increases the cost of writes because a write needs to transfer blocks to multiple racks.
-For the common case, when the replication factor is three, HDFS’s placement policy is to put one replica on the local machine if the writer is on a datanode, otherwise on a random datanode, another replica on a node in a different (remote) rack, and the last on a different node in the same remote rack. This policy cuts the inter-rack write traffic which generally improves write performance. The chance of rack failure is far less than that of node failure; this policy does not impact data reliability and availability guarantees. However, it does reduce the aggregate network bandwidth used when reading data since a block is placed in only two unique racks rather than three. With this policy, the replicas of a file do not evenly distribute across the racks. One third of replicas are on one node, two thirds of replicas are on one rack, and the other third are evenly distributed across the remaining racks. This policy improves write performance without compromising data reliability or
read performance.
+For the common case, when the replication factor is three, HDFS’s placement policy is to put one replica on the local machine if the writer is on a datanode, otherwise on a random datanode in the same rack as that of the writer, another replica on a node in a different (remote) rack, and the last on a different node in the same remote rack. This policy cuts the inter-rack write traffic which generally improves write performance. The chance of rack failure is far less than that of node failure; this policy does not impact data reliability and availability guarantees. However, it does reduce the aggregate network bandwidth used when reading data since a block is placed in only two unique racks rather than three. With this policy, the replicas of a file do not evenly distribute across the racks. One third of replicas are on one node, two thirds of replicas are on one rack, and the other third are evenly distributed across the remaining racks. This policy improves write performance wi
thout compromising data reliability or read performance.
If the replication factor is greater than 3,
the placement of the 4th and following replicas are determined randomly
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HADOOP-15831. Include
modificationTime in the toString method of CopyListingFileStatus. Contributed
by Ted Yu.
Posted by br...@apache.org.
HADOOP-15831. Include modificationTime in the toString method of CopyListingFileStatus.
Contributed by Ted Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e36ae963
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e36ae963
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e36ae963
Branch: refs/heads/HDFS-13891
Commit: e36ae9639fa7ac87b05a3860517c23082cf76791
Parents: 5da0422
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 12 09:59:19 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 12 09:59:19 2018 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/tools/CopyListingFileStatus.java | 3 ++-
.../apache/hadoop/tools/TestCopyListingFileStatus.java | 10 ++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e36ae963/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
index e01a6b1..e8e5ed8 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
@@ -405,10 +405,11 @@ public final class CopyListingFileStatus implements Writable {
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
sb.append('{');
- sb.append(this.getPath().toString());
+ sb.append(this.getPath() == null ? "" : this.getPath().toString());
sb.append(" length = ").append(this.getLen());
sb.append(" aclEntries = ").append(aclEntries);
sb.append(", xAttrs = ").append(xAttrs);
+ sb.append(", modTime = ").append(modificationTime);
if (isSplit()) {
sb.append(", chunkOffset = ").append(this.getChunkOffset());
sb.append(", chunkLength = ").append(this.getChunkLength());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e36ae963/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
index 8efc5cf..f665627 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java
@@ -32,6 +32,16 @@ import static org.junit.Assert.assertEquals;
public class TestCopyListingFileStatus {
@Test
+ public void testToString() {
+ CopyListingFileStatus src = new CopyListingFileStatus(
+ 4344L, false, 2, 512 << 20, 1234L, 5678L, new FsPermission((short)0512),
+ "dingo", "yaks", new Path("hdfs://localhost:4344"));
+ src.toString();
+ src = new CopyListingFileStatus();
+ src.toString();
+ }
+
+ @Test
public void testCopyListingFileStatusSerialization() throws Exception {
CopyListingFileStatus src = new CopyListingFileStatus(
4344L, false, 2, 512 << 20, 1234L, 5678L, new FsPermission((short)0512),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-13973. getErasureCodingPolicy
should log path in audit event. Contributed by Shweta.
Posted by br...@apache.org.
HDFS-13973. getErasureCodingPolicy should log path in audit event. Contributed by Shweta.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0118105
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0118105
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0118105
Branch: refs/heads/HDFS-13891
Commit: c0118105b7486793bcaca48786d5f9d3e0ee2ff3
Parents: a3edfdd
Author: Shweta <sh...@cloudera.com>
Authored: Thu Oct 11 20:38:19 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Oct 11 20:39:41 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0118105/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 137dfb9..cc38036 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7728,7 +7728,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return ret;
} finally {
readUnlock(operationName);
- logAuditEvent(success, operationName, null);
+ logAuditEvent(success, operationName, src);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDFS-13956. iNotify should include
information to identify a file as either replicated or erasure coded.
Contributed by Hrishikesh Gadre.
Posted by br...@apache.org.
HDFS-13956. iNotify should include information to identify a file as either replicated or erasure coded. Contributed by Hrishikesh Gadre.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf3d591f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf3d591f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf3d591f
Branch: refs/heads/HDFS-13891
Commit: bf3d591f0cb0fedeab5d89cc8d2270d3b9a70313
Parents: 3ead525
Author: Hrishikesh Gadre <hg...@apache.org>
Authored: Wed Oct 10 10:23:07 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Oct 10 10:23:07 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/inotify/Event.java | 14 +++
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 60 ++++++-----
.../src/main/proto/inotify.proto | 1 +
.../namenode/InotifyFSEditLogOpTranslator.java | 3 +
.../hdfs/TestDFSInotifyEventInputStream.java | 100 +++++++++++++++++++
5 files changed, 152 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index 78a85b0..2ae8f0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import java.util.List;
+import java.util.Optional;
/**
* Events sent by the inotify system. Note that no events are necessarily sent
@@ -112,6 +113,7 @@ public abstract class Event {
private String symlinkTarget;
private boolean overwrite;
private long defaultBlockSize;
+ private Optional<Boolean> erasureCoded;
public static class Builder {
private INodeType iNodeType;
@@ -124,6 +126,7 @@ public abstract class Event {
private String symlinkTarget;
private boolean overwrite;
private long defaultBlockSize = 0;
+ private Optional<Boolean> erasureCoded = Optional.empty();
public Builder iNodeType(INodeType type) {
this.iNodeType = type;
@@ -175,6 +178,11 @@ public abstract class Event {
return this;
}
+ public Builder erasureCoded(boolean ecCoded) {
+ this.erasureCoded = Optional.of(ecCoded);
+ return this;
+ }
+
public CreateEvent build() {
return new CreateEvent(this);
}
@@ -192,6 +200,7 @@ public abstract class Event {
this.symlinkTarget = b.symlinkTarget;
this.overwrite = b.overwrite;
this.defaultBlockSize = b.defaultBlockSize;
+ this.erasureCoded = b.erasureCoded;
}
public INodeType getiNodeType() {
@@ -243,6 +252,10 @@ public abstract class Event {
return defaultBlockSize;
}
+ public Optional<Boolean> isErasureCoded() {
+ return erasureCoded;
+ }
+
@Override
@InterfaceStability.Unstable
public String toString() {
@@ -261,6 +274,7 @@ public abstract class Event {
content.append("overwrite=").append(overwrite)
.append(", defaultBlockSize=").append(defaultBlockSize)
+ .append(", erasureCoded=").append(erasureCoded)
.append("]");
return content.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 4a5a493..4b966bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -908,18 +908,22 @@ public class PBHelperClient {
case EVENT_CREATE:
InotifyProtos.CreateEventProto create =
InotifyProtos.CreateEventProto.parseFrom(p.getContents());
- events.add(new Event.CreateEvent.Builder()
- .iNodeType(createTypeConvert(create.getType()))
- .path(create.getPath())
- .ctime(create.getCtime())
- .ownerName(create.getOwnerName())
- .groupName(create.getGroupName())
- .perms(convert(create.getPerms()))
- .replication(create.getReplication())
- .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
- create.getSymlinkTarget())
- .defaultBlockSize(create.getDefaultBlockSize())
- .overwrite(create.getOverwrite()).build());
+ Event.CreateEvent.Builder builder = new Event.CreateEvent.Builder()
+ .iNodeType(createTypeConvert(create.getType()))
+ .path(create.getPath())
+ .ctime(create.getCtime())
+ .ownerName(create.getOwnerName())
+ .groupName(create.getGroupName())
+ .perms(convert(create.getPerms()))
+ .replication(create.getReplication())
+ .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
+ create.getSymlinkTarget())
+ .defaultBlockSize(create.getDefaultBlockSize())
+ .overwrite(create.getOverwrite());
+ if (create.hasErasureCoded()) {
+ builder.erasureCoded(create.getErasureCoded());
+ }
+ events.add(builder.build());
break;
case EVENT_METADATA:
InotifyProtos.MetadataUpdateEventProto meta =
@@ -2909,22 +2913,26 @@ public class PBHelperClient {
break;
case CREATE:
Event.CreateEvent ce2 = (Event.CreateEvent) e;
+ InotifyProtos.CreateEventProto.Builder pB =
+ (InotifyProtos.CreateEventProto.newBuilder());
+ pB.setType(createTypeConvert(ce2.getiNodeType()))
+ .setPath(ce2.getPath())
+ .setCtime(ce2.getCtime())
+ .setOwnerName(ce2.getOwnerName())
+ .setGroupName(ce2.getGroupName())
+ .setPerms(convert(ce2.getPerms()))
+ .setReplication(ce2.getReplication())
+ .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
+ "" : ce2.getSymlinkTarget())
+ .setDefaultBlockSize(ce2.getDefaultBlockSize())
+ .setOverwrite(ce2.getOverwrite());
+ if (ce2.isErasureCoded().isPresent()) {
+ pB.setErasureCoded(ce2.isErasureCoded().get());
+ }
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_CREATE)
- .setContents(
- InotifyProtos.CreateEventProto.newBuilder()
- .setType(createTypeConvert(ce2.getiNodeType()))
- .setPath(ce2.getPath())
- .setCtime(ce2.getCtime())
- .setOwnerName(ce2.getOwnerName())
- .setGroupName(ce2.getGroupName())
- .setPerms(convert(ce2.getPerms()))
- .setReplication(ce2.getReplication())
- .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
- "" : ce2.getSymlinkTarget())
- .setDefaultBlockSize(ce2.getDefaultBlockSize())
- .setOverwrite(ce2.getOverwrite()).build().toByteString()
- ).build());
+ .setContents(pB.build().toByteString())
+ .build());
break;
case METADATA:
Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
index 5339902..f193408 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
@@ -80,6 +80,7 @@ message CreateEventProto {
optional string symlinkTarget = 8;
optional bool overwrite = 9;
optional int64 defaultBlockSize = 10 [default=0];
+ optional bool erasureCoded = 11;
}
message CloseEventProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
index 0918107..8a54c8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.inotify.Event;
import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import java.util.List;
@@ -54,6 +55,8 @@ public class InotifyFSEditLogOpTranslator {
.perms(addOp.permissions.getPermission())
.overwrite(addOp.overwrite)
.defaultBlockSize(addOp.blockSize)
+ .erasureCoded(addOp.erasureCodingPolicyId
+ != ErasureCodeConstants.REPLICATION_POLICY_ID)
.iNodeType(Event.CreateEvent.INodeType.FILE).build() });
} else { // append
return new EventBatch(op.txid,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf3d591f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index b0b85e7..05d3c63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -140,6 +141,7 @@ public class TestDFSInotifyEventInputStream {
client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent
//TruncateOp -> TruncateEvent
client.truncate("/truncate_file", BLOCK_SIZE);
+ client.create("/file_ec_test1", false);
EventBatch batch = null;
// RenameOp
@@ -180,6 +182,8 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(ce.getSymlinkTarget() == null);
Assert.assertTrue(ce.getOverwrite());
Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize());
+ Assert.assertTrue(ce.isErasureCoded().isPresent());
+ Assert.assertFalse(ce.isErasureCoded().get());
LOG.info(ce.toString());
Assert.assertTrue(ce.toString().startsWith("CreateEvent [INodeType="));
@@ -395,6 +399,25 @@ public class TestDFSInotifyEventInputStream {
LOG.info(et.toString());
Assert.assertTrue(et.toString().startsWith("TruncateEvent [path="));
+ // CreateEvent without overwrite
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType()
+ == Event.EventType.CREATE);
+ ce = (Event.CreateEvent) batch.getEvents()[0];
+ Assert.assertTrue(ce.getiNodeType() == Event.CreateEvent.INodeType.FILE);
+ Assert.assertTrue(ce.getPath().equals("/file_ec_test1"));
+ Assert.assertTrue(ce.getCtime() > 0);
+ Assert.assertTrue(ce.getReplication() > 0);
+ Assert.assertTrue(ce.getSymlinkTarget() == null);
+ Assert.assertFalse(ce.getOverwrite());
+ Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize());
+ Assert.assertTrue(ce.isErasureCoded().isPresent());
+ Assert.assertFalse(ce.isErasureCoded().get());
+ LOG.info(ce.toString());
+ Assert.assertTrue(ce.toString().startsWith("CreateEvent [INodeType="));
+
// Returns null when there are no further events
Assert.assertTrue(eis.poll() == null);
@@ -411,6 +434,83 @@ public class TestDFSInotifyEventInputStream {
}
@Test(timeout = 120000)
+ public void testErasureCodedFiles() throws Exception {
+ ErasureCodingPolicy ecPolicy = StripedFileTestUtil.getDefaultECPolicy();
+ final int dataUnits = ecPolicy.getNumDataUnits();
+ final int parityUnits = ecPolicy.getNumParityUnits();
+
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, ecPolicy.getCellSize());
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ // so that we can get an atime change
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 1);
+
+ MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
+ builder.getDfsBuilder().numDataNodes(dataUnits + parityUnits);
+ MiniQJMHACluster cluster = builder.build();
+
+ try {
+ cluster.getDfsCluster().waitActive();
+ cluster.getDfsCluster().transitionToActive(0);
+ DFSClient client = new DFSClient(cluster.getDfsCluster().getNameNode(0)
+ .getNameNodeAddress(), conf);
+ DistributedFileSystem fs =
+ (DistributedFileSystem)cluster.getDfsCluster().getFileSystem(0);
+
+ Path ecDir = new Path("/ecdir");
+ fs.mkdirs(ecDir);
+ fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
+
+ DFSInotifyEventInputStream eis = client.getInotifyEventStream();
+
+ int sz = ecPolicy.getNumDataUnits() * ecPolicy.getCellSize();
+ byte[] contents = new byte[sz];
+ DFSTestUtil.writeFile(fs, new Path("/ecdir/file_ec_test2"), contents);
+
+ EventBatch batch = null;
+
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ long txid = batch.getTxid();
+ long eventsBehind = eis.getTxidsBehindEstimate();
+ Assert.assertTrue(batch.getEvents()[0].getEventType()
+ == Event.EventType.CREATE);
+ Event.CreateEvent ce = (Event.CreateEvent) batch.getEvents()[0];
+ Assert.assertTrue(ce.getiNodeType() == Event.CreateEvent.INodeType.FILE);
+ Assert.assertTrue(ce.getPath().equals("/ecdir/file_ec_test2"));
+ Assert.assertTrue(ce.getCtime() > 0);
+ Assert.assertEquals(1, ce.getReplication());
+ Assert.assertTrue(ce.getSymlinkTarget() == null);
+ Assert.assertTrue(ce.getOverwrite());
+ Assert.assertEquals(ecPolicy.getCellSize(), ce.getDefaultBlockSize());
+ Assert.assertTrue(ce.isErasureCoded().isPresent());
+ Assert.assertTrue(ce.isErasureCoded().get());
+ LOG.info(ce.toString());
+ Assert.assertTrue(ce.toString().startsWith("CreateEvent [INodeType="));
+
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType()
+ == Event.EventType.CLOSE);
+ Assert.assertTrue(((Event.CloseEvent) batch.getEvents()[0]).getPath()
+ .equals("/ecdir/file_ec_test2"));
+
+ // Returns null when there are no further events
+ Assert.assertTrue(eis.poll() == null);
+
+ // make sure the estimate hasn't changed since the above assertion
+ // tells us that we are fully caught up to the current namesystem state
+ // and we should not have been behind at all when eventsBehind was set
+ // either, since there were few enough events that they should have all
+ // been read to the client during the first poll() call
+ Assert.assertTrue(eis.getTxidsBehindEstimate() == eventsBehind);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 120000)
public void testNNFailover() throws IOException, URISyntaxException,
MissingEventsException {
Configuration conf = new HdfsConfiguration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDFS-11396.
TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out.
Contributed by Ayush Saxena.
Posted by br...@apache.org.
HDFS-11396. TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/605622c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/605622c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/605622c8
Branch: refs/heads/HDFS-13891
Commit: 605622c87bc109f60ee1674be37a526e44723b67
Parents: 6a06bc3
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Oct 9 17:03:21 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Tue Oct 9 17:03:21 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/TestNameNodeMetadataConsistency.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605622c8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
index ff49ddf..2677781 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetadataConsistency.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
import com.google.common.base.Supplier;
@@ -88,6 +89,7 @@ public class TestNameNodeMetadataConsistency {
final long genStamp = block.getGenerationStamp();
final int datanodeIndex = 0;
cluster.changeGenStampOfBlock(datanodeIndex, block, genStamp + 1);
+ DataNodeTestUtils.runDirectoryScanner(cluster.getDataNodes().get(0));
// stop the data node so that it won't remove block
final DataNodeProperties dnProps = cluster.stopDataNode(datanodeIndex);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDDS-517. Implement HeadObject
REST endpoint. Contributed by LiXin Ge.
Posted by br...@apache.org.
HDDS-517. Implement HeadObject REST endpoint. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd280514
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd280514
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd280514
Branch: refs/heads/HDFS-13891
Commit: cd280514b84e8a2915f274883a07f152b772ebad
Parents: b39b802
Author: Márton Elek <el...@apache.org>
Authored: Wed Oct 10 15:23:00 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Wed Oct 10 15:28:41 2018 +0200
----------------------------------------------------------------------
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +-
.../hadoop/ozone/s3/object/HeadObject.java | 38 +++++---
.../hadoop/ozone/client/OzoneBucketStub.java | 6 +-
.../hadoop/ozone/s3/object/TestHeadObject.java | 95 ++++++++++++++++++++
4 files changed, 131 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
index 3fee7be..99ae7a0 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@ -42,6 +42,8 @@ public final class S3ErrorTable {
"BucketNotEmpty", "The bucket you tried to delete is not empty.",
HTTP_CONFLICT);
+ public static final OS3Exception NO_SUCH_OBJECT = new OS3Exception(
+ "NoSuchObject", "The specified object does not exist", HTTP_NOT_FOUND);
/**
* Create a new instance of Error.
@@ -60,7 +62,8 @@ public final class S3ErrorTable {
* Resources, which can be defined in OS3Exception.
*/
public enum Resource {
- BUCKET("Bucket");
+ BUCKET("Bucket"),
+ OBJECT("Object");
private final String resource;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
index e284100..8bbdf76 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java
@@ -18,41 +18,57 @@
package org.apache.hadoop.ozone.s3.object;
import javax.ws.rs.HEAD;
-import javax.ws.rs.HeaderParam;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.IOException;
-import java.io.InputStream;
-import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.s3.EndpointBase;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+
+import org.apache.http.HttpStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Get object info rest endpoint.
*/
@Path("/{volume}/{bucket}/{path:.+}")
public class HeadObject extends EndpointBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HeadObject.class);
@HEAD
@Produces(MediaType.APPLICATION_XML)
public Response head(
@PathParam("volume") String volumeName,
@PathParam("bucket") String bucketName,
- @PathParam("path") String keyPath,
- @HeaderParam("Content-Length") long length,
- InputStream body) throws IOException {
+ @PathParam("path") String keyPath) throws Exception {
+ OzoneKeyDetails key;
- OzoneBucket bucket = getBucket(volumeName, bucketName);
- OzoneKeyDetails key = bucket.getKey(keyPath);
+ try {
+ key = getVolume(volumeName).getBucket(bucketName).getKey(keyPath);
+ // TODO: return the specified range bytes of this object.
+ } catch (IOException ex) {
+ LOG.error("Exception occurred in HeadObject", ex);
+ if (ex.getMessage().contains("KEY_NOT_FOUND")) {
+ OS3Exception os3Exception = S3ErrorTable.newError(S3ErrorTable
+ .NO_SUCH_OBJECT, S3ErrorTable.Resource.OBJECT);
+ throw os3Exception;
+ } else {
+ throw ex;
+ }
+ }
- return Response.
- ok()
+ return Response.ok().status(HttpStatus.SC_OK)
+ .header("Last-Modified", key.getModificationTime())
+ .header("ETag", "" + key.getModificationTime())
.header("Content-Length", key.getDataSize())
+ .header("Content-Type", "binary/octet-stream")
.build();
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 18dff71..ff0f706 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -107,7 +107,11 @@ public class OzoneBucketStub extends OzoneBucket {
@Override
public OzoneKeyDetails getKey(String key) throws IOException {
- return keyDetails.get(key);
+ if (keyDetails.containsKey(key)) {
+ return keyDetails.get(key);
+ } else {
+ throw new IOException("Lookup key failed, error:KEY_NOT_FOUND");
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd280514/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
new file mode 100644
index 0000000..5b65dac
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.hadoop.ozone.s3.object;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.Response;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import java.io.IOException;
+
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+
+/**
+ * Test head object.
+ */
+public class TestHeadObject {
+ private String volName = "vol1";
+ private String bucketName = "b1";
+ private OzoneClientStub clientStub;
+ private ObjectStore objectStoreStub;
+ private HeadObject headObject;
+ private OzoneBucket bucket;
+
+ @Before
+ public void setup() throws IOException {
+ //Create client stub and object store stub.
+ clientStub = new OzoneClientStub();
+ objectStoreStub = clientStub.getObjectStore();
+
+ // Create volume and bucket
+ objectStoreStub.createVolume(volName);
+ OzoneVolume volumeStub = objectStoreStub.getVolume(volName);
+ volumeStub.createBucket(bucketName);
+ bucket = objectStoreStub.getVolume(volName).getBucket(bucketName);
+
+ // Create HeadBucket and setClient to OzoneClientStub
+ headObject = new HeadObject();
+ headObject.setClient(clientStub);
+ }
+
+ @Test
+ public void testHeadObject() throws Exception {
+ //GIVEN
+ String value = RandomStringUtils.randomAlphanumeric(32);
+ OzoneOutputStream out = bucket.createKey("key1",
+ value.getBytes().length, ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE);
+ out.write(value.getBytes());
+ out.close();
+
+ //WHEN
+ Response response = headObject.head(volName, bucketName, "key1");
+
+ //THEN
+ Assert.assertEquals(200, response.getStatus());
+ Assert.assertEquals(value.getBytes().length,
+ Long.parseLong(response.getHeaderString("Content-Length")));
+ }
+
+ @Test
+ public void testHeadFailByBadName() throws Exception {
+ //Head an object that doesn't exist.
+ try {
+ headObject.head(volName, bucketName, "badKeyName");
+ } catch (OS3Exception ex) {
+ Assert.assertTrue(ex.getCode().contains("NoSuchObject"));
+ Assert.assertTrue(ex.getErrorMessage().contains("object does not exist"));
+ Assert.assertEquals(HTTP_NOT_FOUND, ex.getHttpCode());
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-15839. Review + update
cloud store sensitive keys in hadoop.security.sensitive-config-keys.
Contributed by Steve Loughran.
Posted by br...@apache.org.
HADOOP-15839. Review + update cloud store sensitive keys in hadoop.security.sensitive-config-keys.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdc43507
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdc43507
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdc43507
Branch: refs/heads/HDFS-13891
Commit: cdc4350718055189fef8c70e31314607001d4009
Parents: 10185d9
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 10 17:00:37 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 10 17:00:37 2018 +0100
----------------------------------------------------------------------
.../hadoop-common/src/main/resources/core-default.xml | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdc43507/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 3ea2797..32dd622 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -599,9 +599,17 @@
secret$
password$
ssl.keystore.pass$
- fs.s3.*[Ss]ecret.?[Kk]ey
+ fs.s3a.server-side-encryption.key
fs.s3a.*.server-side-encryption.key
+ fs.s3a.secret.key
+ fs.s3a.*.secret.key
+ fs.s3a.session.key
+ fs.s3a.*.session.key
+ fs.s3a.session.token
+ fs.s3a.*.session.token
fs.azure.account.key.*
+ fs.azure.oauth2.*
+ fs.adl.oauth2.*
credential$
oauth.*secret
oauth.*password
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-15832. Upgrade BouncyCastle
to 1.60. Contributed by Robert Kanter.
Posted by br...@apache.org.
HADOOP-15832. Upgrade BouncyCastle to 1.60. Contributed by Robert Kanter.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fa3feb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fa3feb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fa3feb5
Branch: refs/heads/HDFS-13891
Commit: 6fa3feb577d05d73a2eb1bc8e39800326f678c31
Parents: 794c045
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 10 10:16:57 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 10 10:16:57 2018 +0900
----------------------------------------------------------------------
.../hadoop-client-check-invariants/pom.xml | 2 ++
.../hadoop-client-check-test-invariants/pom.xml | 2 ++
.../hadoop-client-minicluster/pom.xml | 2 ++
.../hadoop-client-runtime/pom.xml | 2 ++
hadoop-common-project/hadoop-common/pom.xml | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 2 +-
hadoop-hdds/server-scm/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 +-
.../hadoop-mapreduce-client-app/pom.xml | 20 ++++++++++++++++++++
.../hadoop-mapreduce-client-jobclient/pom.xml | 7 ++++++-
hadoop-ozone/ozone-manager/pom.xml | 2 +-
hadoop-project/pom.xml | 12 +++++++++---
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../pom.xml | 2 +-
.../hadoop-yarn-server-tests/pom.xml | 2 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 8 ++++++++
18 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
index 4319a8b..f062986 100644
--- a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
@@ -90,6 +90,8 @@
<exclude>log4j:log4j</exclude>
<!-- Leave javax annotations we need exposed -->
<exclude>com.google.code.findbugs:jsr305</exclude>
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
index 70a987f..c02ce56 100644
--- a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
@@ -98,6 +98,8 @@
<exclude> org.hamcrest:hamcrest-core</exclude>
<!-- Leave javax annotations we need exposed -->
<exclude>com.google.code.findbugs:jsr305</exclude>
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index dac5e69..70af1ed 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -671,6 +671,8 @@
<exclude>javax.annotation:javax.annotation-api</exclude>
<exclude>org.eclipse.jetty:jetty-jndi</exclude>
<!-- We need a filter that matches just those things that are included in the above artiacts -->
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</artifactSet>
<filters>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-client-modules/hadoop-client-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml
index 8afc89e..1e98a61 100644
--- a/hadoop-client-modules/hadoop-client-runtime/pom.xml
+++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml
@@ -162,6 +162,8 @@
<exclude>org.eclipse.jetty:jetty-servlet</exclude>
<exclude>org.eclipse.jetty:jetty-security</exclude>
<exclude>org.ow2.asm:*</exclude>
+ <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
+ <exclude>org.bouncycastle:*</exclude>
</excludes>
</artifactSet>
<filters>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index a9e0fb1..3036b64 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -303,7 +303,7 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 9b1936f..3a6ed54 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -171,7 +171,7 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdds/server-scm/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
index 92563e7..5fcd2bb 100644
--- a/hadoop-hdds/server-scm/pom.xml
+++ b/hadoop-hdds/server-scm/pom.xml
@@ -93,7 +93,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index c0078da..c3d112e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -204,7 +204,7 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 8d796ab..d99e495 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -165,7 +165,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index a0ea04f..f97ca17 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -190,7 +190,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 0a354c8..fab6466 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -46,6 +46,16 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -88,6 +98,16 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index b961cc7..b956fd4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -108,7 +108,12 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-ozone/ozone-manager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
index e2062ef..76e3afb 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -42,7 +42,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0f54a55..5f88371 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -96,6 +96,8 @@
<guice.version>4.0</guice.version>
<joda-time.version>2.9.9</joda-time.version>
+ <bouncycastle.version>1.60</bouncycastle.version>
+
<!-- Required for testing LDAP integration -->
<apacheds.version>2.0.0-M21</apacheds.version>
<ldap-api.version>1.0.0-M33</ldap-api.version>
@@ -1424,10 +1426,14 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
- <version>1.46</version>
- <scope>test</scope>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <version>${bouncycastle.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <version>${bouncycastle.version}</version>
+ </dependency>
<dependency>
<groupId>joda-time</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index de0d78b..133003a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -139,7 +139,7 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 5ea8c9e..bfac673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -177,7 +177,7 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 8c559b9..a05cdc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -127,7 +127,7 @@
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk16</artifactId>
+ <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa3feb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index aa9f753..d08c25f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -115,6 +115,14 @@
<artifactId>jersey-test-framework-grizzly2</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ </dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDDS-550. Serialize
ApplyTransaction calls per Container in ContainerStateMachine. Contributed by
Shashikant Banerjee.
Posted by br...@apache.org.
HDDS-550. Serialize ApplyTransaction calls per Container in ContainerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bc6d048
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bc6d048
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bc6d048
Branch: refs/heads/HDFS-13891
Commit: 0bc6d0484a1c3652234b76e4f99e2ef99173796d
Parents: 8d19818
Author: Jitendra Pandey <ji...@apache.org>
Authored: Thu Oct 11 16:59:59 2018 -0700
Committer: Jitendra Pandey <ji...@apache.org>
Committed: Thu Oct 11 17:12:47 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 6 +-
.../hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hdds/scm/storage/ChunkInputStream.java | 2 +-
.../hdds/scm/storage/ChunkOutputStream.java | 3 +-
hadoop-hdds/common/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-
.../scm/storage/ContainerProtocolCalls.java | 2 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 6 +-
.../org/apache/hadoop/utils/RocksDBStore.java | 3 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 3 +-
.../main/java/org/apache/ratis/RatisHelper.java | 14 +-
.../main/proto/DatanodeContainerProtocol.proto | 1 -
.../common/src/main/resources/ozone-default.xml | 8 +
.../transport/server/GrpcXceiverService.java | 2 +-
.../transport/server/XceiverServerGrpc.java | 8 +-
.../server/ratis/ContainerStateMachine.java | 440 +++++--------------
.../server/ratis/XceiverServerRatis.java | 15 +-
.../container/keyvalue/helpers/ChunkUtils.java | 2 +-
.../keyvalue/helpers/SmallFileUtils.java | 2 +-
.../background/BlockDeletingService.java | 2 +-
.../replication/GrpcReplicationClient.java | 6 +-
.../replication/GrpcReplicationService.java | 4 +-
.../common/impl/TestHddsDispatcher.java | 2 +-
.../ozone/client/io/ChunkGroupOutputStream.java | 5 +
.../rpc/TestCloseContainerHandlingByClient.java | 1 -
.../ozone/container/ContainerTestHelper.java | 3 +-
.../common/impl/TestCloseContainerHandler.java | 2 +-
.../container/server/TestContainerServer.java | 1 -
.../server/TestContainerStateMachine.java | 201 ---------
.../genesis/BenchMarkDatanodeDispatcher.java | 2 +-
hadoop-project/pom.xml | 2 +-
31 files changed, 192 insertions(+), 570 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index d353e7a..2f11872 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -31,9 +31,9 @@ import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.util.Time;
-import org.apache.ratis.shaded.io.grpc.ManagedChannel;
-import org.apache.ratis.shaded.io.grpc.netty.NettyChannelBuilder;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
+import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 0d301d9..4efe7ba 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.ratis.retry.RetryPolicy;
-import org.apache.ratis.shaded.com.google.protobuf
+import org.apache.ratis.thirdparty.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -39,7 +39,7 @@ import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.util.CheckedBiConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
index a483197..21b8974 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdds.scm.storage;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
index cc1ea8d..4547163 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
@@ -18,8 +18,9 @@
package org.apache.hadoop.hdds.scm.storage;
+
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index eea2264..bf2a6b9 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -172,10 +172,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<phase>generate-sources</phase>
<configuration>
<tasks>
- <replace token="com.google.protobuf" value="org.apache.ratis.shaded.com.google.protobuf"
+ <replace token="com.google.protobuf" value="org.apache.ratis.thirdparty.com.google.protobuf"
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
</replace>
- <replace token="io.grpc" value="org.apache.ratis.shaded.io.grpc"
+ <replace token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc"
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
</replace>
</tasks>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 63f5916..71d6e07 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.ratis.shaded.proto.RaftProtos.ReplicationLevel;
+import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
import org.apache.ratis.util.TimeDuration;
import java.util.concurrent.TimeUnit;
@@ -62,6 +62,10 @@ public final class ScmConfigKeys {
= "dfs.container.ratis.replication.level";
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT = ReplicationLevel.MAJORITY;
+ public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
+ = "dfs.container.ratis.num.container.op.threads";
+ public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
+ = 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
index 1df50b1..278b129 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdds.scm.storage;
import org.apache.hadoop.hdds.scm.container.common.helpers
.BlockNotCommittedException;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index e8aa22c..d6a6bf7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.ratis.shaded.proto.RaftProtos.ReplicationLevel;
+import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
import org.apache.ratis.util.TimeDuration;
/**
@@ -220,6 +220,10 @@ public final class OzoneConfigKeys {
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT;
+ public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
+ = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY;
+ public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
+ = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
index 379d9e9..377153a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
@@ -22,7 +22,8 @@ import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.ratis.thirdparty.com.google.common.annotations.
+ VisibleForTesting;
import org.rocksdb.DbPath;
import org.rocksdb.Options;
import org.rocksdb.RocksDB;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
index 5078b3e..cdee10b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
@@ -23,7 +23,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.utils.RocksDBStoreMBean;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.ratis.thirdparty.com.google.common.annotations.
+ VisibleForTesting;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 04bfeb2..2dbe2e6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -32,8 +32,8 @@ import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.retry.RetryPolicies;
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.proto.RaftProtos;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.util.Preconditions;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
@@ -103,7 +103,7 @@ public interface RatisHelper {
RaftGroupId DUMMY_GROUP_ID =
RaftGroupId.valueOf(ByteString.copyFromUtf8("AOzoneRatisGroup"));
- RaftGroup EMPTY_GROUP = new RaftGroup(DUMMY_GROUP_ID,
+ RaftGroup EMPTY_GROUP = RaftGroup.valueOf(DUMMY_GROUP_ID,
Collections.emptyList());
static RaftGroup emptyRaftGroup() {
@@ -112,7 +112,7 @@ public interface RatisHelper {
static RaftGroup newRaftGroup(Collection<RaftPeer> peers) {
return peers.isEmpty()? emptyRaftGroup()
- : new RaftGroup(DUMMY_GROUP_ID, peers);
+ : RaftGroup.valueOf(DUMMY_GROUP_ID, peers);
}
static RaftGroup newRaftGroup(RaftGroupId groupId,
@@ -120,12 +120,12 @@ public interface RatisHelper {
final List<RaftPeer> newPeers = peers.stream()
.map(RatisHelper::toRaftPeer)
.collect(Collectors.toList());
- return peers.isEmpty() ? new RaftGroup(groupId, Collections.emptyList())
- : new RaftGroup(groupId, newPeers);
+ return peers.isEmpty() ? RaftGroup.valueOf(groupId, Collections.emptyList())
+ : RaftGroup.valueOf(groupId, newPeers);
}
static RaftGroup newRaftGroup(Pipeline pipeline) {
- return new RaftGroup(pipeline.getId().getRaftGroupID(),
+ return RaftGroup.valueOf(pipeline.getId().getRaftGroupID(),
toRaftPeers(pipeline));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 4567750..662df8f 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -197,7 +197,6 @@ message ContainerCommandRequestProto {
optional PutSmallFileRequestProto putSmallFile = 19;
optional GetSmallFileRequestProto getSmallFile = 20;
-
optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 21;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index ef54ed2..0fde6bb 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -136,6 +136,14 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.num.container.op.executors</name>
+ <value>10</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>Number of executors that will be used by Ratis to execute
+ container ops.(10 by default).
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.segment.size</name>
<value>1073741824</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
index db4a86a..5fc3661 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto
.XceiverClientProtocolServiceGrpc;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index c51da98..8ebfe49 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -33,10 +33,10 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.ratis.shaded.io.grpc.BindableService;
-import org.apache.ratis.shaded.io.grpc.Server;
-import org.apache.ratis.shaded.io.grpc.ServerBuilder;
-import org.apache.ratis.shaded.io.grpc.netty.NettyServerBuilder;
+import org.apache.ratis.thirdparty.io.grpc.BindableService;
+import org.apache.ratis.thirdparty.io.grpc.Server;
+import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
+import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index f07c95f..7b7be91 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -26,7 +25,7 @@ import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.shaded.com.google.protobuf
+import org.apache.ratis.thirdparty.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Stage;
@@ -44,10 +43,10 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.protocol.Message;
import org.apache.ratis.protocol.RaftClientRequest;
import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.proto.RaftProtos.RoleInfoProto;
-import org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto;
-import org.apache.ratis.shaded.proto.RaftProtos.SMLogEntryProto;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
+import org.apache.ratis.proto.RaftProtos.LogEntryProto;
+import org.apache.ratis.proto.RaftProtos.SMLogEntryProto;
import org.apache.ratis.statemachine.StateMachineStorage;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.statemachine.impl.BaseStateMachine;
@@ -57,12 +56,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
/** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
@@ -98,44 +97,43 @@ import java.util.stream.Collectors;
*
* 2) Write chunk commit operation is executed after write chunk state machine
* operation. This will ensure that commit operation is sync'd with the state
- * machine operation.
- *
- * Synchronization between {@link #writeStateMachineData} and
- * {@link #applyTransaction} need to be enforced in the StateMachine
- * implementation. For example, synchronization between writeChunk and
+ * machine operation.For example, synchronization between writeChunk and
* createContainer in {@link ContainerStateMachine}.
- *
- * PutBlock is synchronized with WriteChunk operations, PutBlock for a block is
- * executed only after all the WriteChunk preceding the PutBlock have finished.
- *
- * CloseContainer is synchronized with WriteChunk and PutBlock operations,
- * CloseContainer for a container is processed after all the preceding write
- * operations for the container have finished.
- * */
+ **/
+
public class ContainerStateMachine extends BaseStateMachine {
- static final Logger LOG = LoggerFactory.getLogger(
- ContainerStateMachine.class);
- private final SimpleStateMachineStorage storage
- = new SimpleStateMachineStorage();
+ static final Logger LOG =
+ LoggerFactory.getLogger(ContainerStateMachine.class);
+ private final SimpleStateMachineStorage storage =
+ new SimpleStateMachineStorage();
private final ContainerDispatcher dispatcher;
private ThreadPoolExecutor chunkExecutor;
private final XceiverServerRatis ratisServer;
private final ConcurrentHashMap<Long, CompletableFuture<Message>>
writeChunkFutureMap;
- private final ConcurrentHashMap<Long, StateMachineHelper> stateMachineMap;
+ private final ConcurrentHashMap<Long, CompletableFuture<Message>>
+ createContainerFutureMap;
+ private ExecutorService[] executors;
+ private final int numExecutors;
/**
* CSM metrics.
*/
private final CSMMetrics metrics;
public ContainerStateMachine(ContainerDispatcher dispatcher,
- ThreadPoolExecutor chunkExecutor, XceiverServerRatis ratisServer) {
+ ThreadPoolExecutor chunkExecutor, XceiverServerRatis ratisServer,
+ int numOfExecutors) {
this.dispatcher = dispatcher;
this.chunkExecutor = chunkExecutor;
this.ratisServer = ratisServer;
this.writeChunkFutureMap = new ConcurrentHashMap<>();
- this.stateMachineMap = new ConcurrentHashMap<>();
metrics = CSMMetrics.create();
+ this.createContainerFutureMap = new ConcurrentHashMap<>();
+ this.numExecutors = numOfExecutors;
+ executors = new ExecutorService[numExecutors];
+ for (int i = 0; i < numExecutors; i++) {
+ executors[i] = Executors.newSingleThreadExecutor();
+ }
}
@Override
@@ -229,6 +227,41 @@ public class ContainerStateMachine extends BaseStateMachine {
return dispatchCommand(requestProto)::toByteString;
}
+ private ExecutorService getCommandExecutor(
+ ContainerCommandRequestProto requestProto) {
+ int executorId = (int)(requestProto.getContainerID() % numExecutors);
+ return executors[executorId];
+ }
+
+ private CompletableFuture<Message> handleWriteChunk(
+ ContainerCommandRequestProto requestProto, long entryIndex) {
+ final WriteChunkRequestProto write = requestProto.getWriteChunk();
+ long containerID = write.getBlockID().getContainerID();
+ CompletableFuture<Message> future =
+ createContainerFutureMap.get(containerID);
+ CompletableFuture<Message> writeChunkFuture;
+ if (future != null) {
+ writeChunkFuture = future.thenApplyAsync(
+ v -> runCommand(requestProto), chunkExecutor);
+ } else {
+ writeChunkFuture = CompletableFuture.supplyAsync(
+ () -> runCommand(requestProto), chunkExecutor);
+ }
+ writeChunkFutureMap.put(entryIndex, writeChunkFuture);
+ // Remove the future once it finishes execution from the
+ // writeChunkFutureMap.
+ writeChunkFuture.thenApply(r -> writeChunkFutureMap.remove(entryIndex));
+ return writeChunkFuture;
+ }
+
+ private CompletableFuture<Message> handleCreateContainer(
+ ContainerCommandRequestProto requestProto) {
+ long containerID = requestProto.getContainerID();
+ createContainerFutureMap.
+ computeIfAbsent(containerID, k -> new CompletableFuture<>());
+ return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
+ }
+
/*
* writeStateMachineData calls are not synchronized with each other
* and also with applyTransaction.
@@ -240,17 +273,15 @@ public class ContainerStateMachine extends BaseStateMachine {
final ContainerCommandRequestProto requestProto =
getRequestProto(entry.getSmLogEntry().getStateMachineData());
Type cmdType = requestProto.getCmdType();
- long containerId = requestProto.getContainerID();
- stateMachineMap
- .computeIfAbsent(containerId, k -> new StateMachineHelper());
- CompletableFuture<Message> stateMachineFuture =
- stateMachineMap.get(containerId)
- .handleStateMachineData(requestProto, entry.getIndex());
- if (stateMachineFuture == null) {
- throw new IllegalStateException(
- "Cmd Type:" + cmdType + " should not have state machine data");
+ switch (cmdType) {
+ case CreateContainer:
+ return handleCreateContainer(requestProto);
+ case WriteChunk:
+ return handleWriteChunk(requestProto, entry.getIndex());
+ default:
+ throw new IllegalStateException("Cmd Type:" + cmdType
+ + " should not have state machine data");
}
- return stateMachineFuture;
} catch (IOException e) {
metrics.incNumWriteStateMachineFails();
return completeExceptionally(e);
@@ -270,14 +301,14 @@ public class ContainerStateMachine extends BaseStateMachine {
}
}
- private LogEntryProto readStateMachineData(LogEntryProto entry,
+ private ByteString readStateMachineData(LogEntryProto entry,
ContainerCommandRequestProto requestProto) {
WriteChunkRequestProto writeChunkRequestProto =
requestProto.getWriteChunk();
// Assert that store log entry is for COMMIT_DATA, the WRITE_DATA is
// written through writeStateMachineData.
- Preconditions.checkArgument(writeChunkRequestProto.getStage()
- == Stage.COMMIT_DATA);
+ Preconditions
+ .checkArgument(writeChunkRequestProto.getStage() == Stage.COMMIT_DATA);
// prepare the chunk to be read
ReadChunkRequestProto.Builder readChunkRequestProto =
@@ -286,8 +317,7 @@ public class ContainerStateMachine extends BaseStateMachine {
.setChunkData(writeChunkRequestProto.getChunkData());
ContainerCommandRequestProto dataContainerCommandProto =
ContainerCommandRequestProto.newBuilder(requestProto)
- .setCmdType(Type.ReadChunk)
- .setReadChunk(readChunkRequestProto)
+ .setCmdType(Type.ReadChunk).setReadChunk(readChunkRequestProto)
.build();
// read the chunk
@@ -302,25 +332,13 @@ public class ContainerStateMachine extends BaseStateMachine {
final WriteChunkRequestProto.Builder dataWriteChunkProto =
WriteChunkRequestProto.newBuilder(writeChunkRequestProto)
// adding the state machine data
- .setData(responseProto.getData())
- .setStage(Stage.WRITE_DATA);
+ .setData(responseProto.getData()).setStage(Stage.WRITE_DATA);
ContainerCommandRequestProto.Builder newStateMachineProto =
ContainerCommandRequestProto.newBuilder(requestProto)
.setWriteChunk(dataWriteChunkProto);
- return recreateLogEntryProto(entry,
- newStateMachineProto.build().toByteString());
- }
-
- private LogEntryProto recreateLogEntryProto(LogEntryProto entry,
- ByteString stateMachineData) {
- // recreate the log entry
- final SMLogEntryProto log =
- SMLogEntryProto.newBuilder(entry.getSmLogEntry())
- .setStateMachineData(stateMachineData)
- .build();
- return LogEntryProto.newBuilder(entry).setSmLogEntry(log).build();
+ return newStateMachineProto.build().toByteString();
}
/**
@@ -347,11 +365,11 @@ public class ContainerStateMachine extends BaseStateMachine {
* evicted.
*/
@Override
- public CompletableFuture<LogEntryProto> readStateMachineData(
+ public CompletableFuture<ByteString> readStateMachineData(
LogEntryProto entry) {
SMLogEntryProto smLogEntryProto = entry.getSmLogEntry();
if (!smLogEntryProto.getStateMachineData().isEmpty()) {
- return CompletableFuture.completedFuture(entry);
+ return CompletableFuture.completedFuture(ByteString.EMPTY);
}
try {
@@ -365,9 +383,7 @@ public class ContainerStateMachine extends BaseStateMachine {
readStateMachineData(entry, requestProto),
chunkExecutor);
} else if (requestProto.getCmdType() == Type.CreateContainer) {
- LogEntryProto log =
- recreateLogEntryProto(entry, requestProto.toByteString());
- return CompletableFuture.completedFuture(log);
+ return CompletableFuture.completedFuture(requestProto.toByteString());
} else {
throw new IllegalStateException("Cmd type:" + requestProto.getCmdType()
+ " cannot have state machine data");
@@ -387,13 +403,44 @@ public class ContainerStateMachine extends BaseStateMachine {
metrics.incNumApplyTransactionsOps();
ContainerCommandRequestProto requestProto =
getRequestProto(trx.getSMLogEntry().getData());
- Preconditions.checkState(!HddsUtils.isReadOnly(requestProto));
- stateMachineMap.computeIfAbsent(requestProto.getContainerID(),
- k -> new StateMachineHelper());
- long index =
- trx.getLogEntry() == null ? -1 : trx.getLogEntry().getIndex();
- return stateMachineMap.get(requestProto.getContainerID())
- .executeContainerCommand(requestProto, index);
+ Type cmdType = requestProto.getCmdType();
+ CompletableFuture<Message> future;
+ if (cmdType == Type.PutBlock) {
+ BlockData blockData;
+ ContainerProtos.BlockData blockDataProto =
+ requestProto.getPutBlock().getBlockData();
+
+ // set the blockCommitSequenceId
+ try {
+ blockData = BlockData.getFromProtoBuf(blockDataProto);
+ } catch (IOException ioe) {
+ LOG.error("unable to retrieve blockData info for Block {}",
+ blockDataProto.getBlockID());
+ return completeExceptionally(ioe);
+ }
+ blockData.setBlockCommitSequenceId(trx.getLogEntry().getIndex());
+ final ContainerProtos.PutBlockRequestProto putBlockRequestProto =
+ ContainerProtos.PutBlockRequestProto
+ .newBuilder(requestProto.getPutBlock())
+ .setBlockData(blockData.getProtoBufMessage()).build();
+ ContainerCommandRequestProto containerCommandRequestProto =
+ ContainerCommandRequestProto.newBuilder(requestProto)
+ .setPutBlock(putBlockRequestProto).build();
+ future = CompletableFuture
+ .supplyAsync(() -> runCommand(containerCommandRequestProto),
+ getCommandExecutor(requestProto));
+ } else {
+ future = CompletableFuture.supplyAsync(() -> runCommand(requestProto),
+ getCommandExecutor(requestProto));
+ }
+ // Mark the createContainerFuture complete so that writeStateMachineData
+ // for WriteChunk gets unblocked
+ if (cmdType == Type.CreateContainer) {
+ long containerID = requestProto.getContainerID();
+ future.thenApply(
+ r -> createContainerFutureMap.remove(containerID).complete(null));
+ }
+ return future;
} catch (IOException e) {
metrics.incNumApplyTransactionsFails();
return completeExceptionally(e);
@@ -419,259 +466,8 @@ public class ContainerStateMachine extends BaseStateMachine {
@Override
public void close() throws IOException {
- }
-
- /**
- * Class to manage the future tasks for writeChunks.
- */
- static class CommitChunkFutureMap {
- private final ConcurrentHashMap<Long, CompletableFuture<Message>>
- block2ChunkMap = new ConcurrentHashMap<>();
-
- synchronized int removeAndGetSize(long index) {
- block2ChunkMap.remove(index);
- return block2ChunkMap.size();
+ for (int i = 0; i < numExecutors; i++){
+ executors[i].shutdown();
}
-
- synchronized CompletableFuture<Message> add(long index,
- CompletableFuture<Message> future) {
- return block2ChunkMap.put(index, future);
- }
-
- synchronized List<CompletableFuture<Message>> getAll() {
- return new ArrayList<>(block2ChunkMap.values());
- }
- }
-
- /**
- * This class maintains maps and provide utilities to enforce synchronization
- * among createContainer, writeChunk, putBlock and closeContainer.
- */
- private class StateMachineHelper {
-
- private CompletableFuture<Message> createContainerFuture;
-
- // Map for maintaining all writeChunk futures mapped to blockId
- private final ConcurrentHashMap<Long, CommitChunkFutureMap>
- block2ChunkMap;
-
- // Map for putBlock futures
- private final ConcurrentHashMap<Long, CompletableFuture<Message>>
- blockCommitMap;
-
- StateMachineHelper() {
- createContainerFuture = null;
- block2ChunkMap = new ConcurrentHashMap<>();
- blockCommitMap = new ConcurrentHashMap<>();
- }
-
- // The following section handles writeStateMachineData transactions
- // on a container
-
- // enqueue the create container future during writeStateMachineData
- // so that the write stateMachine data phase of writeChunk wait on
- // create container to finish.
- private CompletableFuture<Message> handleCreateContainer() {
- createContainerFuture = new CompletableFuture<>();
- return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
- }
-
- // This synchronizes on create container to finish
- private CompletableFuture<Message> handleWriteChunk(
- ContainerCommandRequestProto requestProto, long entryIndex) {
- CompletableFuture<Message> containerOpFuture;
-
- if (createContainerFuture != null) {
- containerOpFuture = createContainerFuture
- .thenApplyAsync(v -> runCommand(requestProto), chunkExecutor);
- } else {
- containerOpFuture = CompletableFuture
- .supplyAsync(() -> runCommand(requestProto), chunkExecutor);
- }
- writeChunkFutureMap.put(entryIndex, containerOpFuture);
- return containerOpFuture;
- }
-
- CompletableFuture<Message> handleStateMachineData(
- final ContainerCommandRequestProto requestProto, long index) {
- Type cmdType = requestProto.getCmdType();
- if (cmdType == Type.CreateContainer) {
- return handleCreateContainer();
- } else if (cmdType == Type.WriteChunk) {
- return handleWriteChunk(requestProto, index);
- } else {
- return null;
- }
- }
-
- // The following section handles applyTransaction transactions
- // on a container
-
- private CompletableFuture<Message> handlePutBlock(
- ContainerCommandRequestProto requestProto, long index) {
- List<CompletableFuture<Message>> futureList = new ArrayList<>();
- BlockData blockData = null;
- ContainerProtos.BlockData blockDataProto =
- requestProto.getPutBlock().getBlockData();
-
- // set the blockCommitSequenceId
- try {
- blockData = BlockData.getFromProtoBuf(blockDataProto);
- } catch (IOException ioe) {
- LOG.error("unable to retrieve blockData info for Block {}",
- blockDataProto.getBlockID());
- return completeExceptionally(ioe);
- }
- blockData.setBlockCommitSequenceId(index);
- final ContainerProtos.PutBlockRequestProto putBlockRequestProto =
- ContainerProtos.PutBlockRequestProto
- .newBuilder(requestProto.getPutBlock())
- .setBlockData(blockData.getProtoBufMessage()).build();
- ContainerCommandRequestProto containerCommandRequestProto =
- ContainerCommandRequestProto.newBuilder(requestProto)
- .setPutBlock(putBlockRequestProto).build();
- long localId = blockDataProto.getBlockID().getLocalID();
- // Need not wait for create container future here as it has already
- // finished.
- if (block2ChunkMap.get(localId) != null) {
- futureList.addAll(block2ChunkMap.get(localId).getAll());
- }
- CompletableFuture<Message> effectiveFuture =
- runCommandAfterFutures(futureList, containerCommandRequestProto);
-
- CompletableFuture<Message> putBlockFuture =
- effectiveFuture.thenApply(message -> {
- blockCommitMap.remove(localId);
- return message;
- });
- blockCommitMap.put(localId, putBlockFuture);
- return putBlockFuture;
- }
-
- // Close Container should be executed only if all pending WriteType
- // container cmds get executed. Transactions which can return a future
- // are WriteChunk and PutBlock.
- private CompletableFuture<Message> handleCloseContainer(
- ContainerCommandRequestProto requestProto) {
- List<CompletableFuture<Message>> futureList = new ArrayList<>();
-
- // No need to wait for create container future here as it should have
- // already finished.
- block2ChunkMap.values().forEach(b -> futureList.addAll(b.getAll()));
- futureList.addAll(blockCommitMap.values());
-
- // There are pending write Chunk/PutBlock type requests
- // Queue this closeContainer request behind all these requests
- CompletableFuture<Message> closeContainerFuture =
- runCommandAfterFutures(futureList, requestProto);
-
- return closeContainerFuture.thenApply(message -> {
- stateMachineMap.remove(requestProto.getContainerID());
- return message;
- });
- }
-
- private CompletableFuture<Message> handleChunkCommit(
- ContainerCommandRequestProto requestProto, long index) {
- WriteChunkRequestProto write = requestProto.getWriteChunk();
- // the data field has already been removed in start Transaction
- Preconditions.checkArgument(!write.hasData());
- CompletableFuture<Message> stateMachineFuture =
- writeChunkFutureMap.remove(index);
- CompletableFuture<Message> commitChunkFuture = stateMachineFuture
- .thenComposeAsync(v -> CompletableFuture
- .completedFuture(runCommand(requestProto)));
-
- long localId = requestProto.getWriteChunk().getBlockID().getLocalID();
- // Put the applyTransaction Future again to the Map.
- // closeContainer should synchronize with this.
- block2ChunkMap
- .computeIfAbsent(localId, id -> new CommitChunkFutureMap())
- .add(index, commitChunkFuture);
- return commitChunkFuture.thenApply(message -> {
- block2ChunkMap.computeIfPresent(localId, (containerId, chunks)
- -> chunks.removeAndGetSize(index) == 0? null: chunks);
- return message;
- });
- }
-
- private CompletableFuture<Message> runCommandAfterFutures(
- List<CompletableFuture<Message>> futureList,
- ContainerCommandRequestProto requestProto) {
- CompletableFuture<Message> effectiveFuture;
- if (futureList.isEmpty()) {
- effectiveFuture = CompletableFuture
- .supplyAsync(() -> runCommand(requestProto));
-
- } else {
- CompletableFuture<Void> allFuture = CompletableFuture.allOf(
- futureList.toArray(new CompletableFuture[futureList.size()]));
- effectiveFuture = allFuture
- .thenApplyAsync(v -> runCommand(requestProto));
- }
- return effectiveFuture;
- }
-
- CompletableFuture<Message> handleCreateContainer(
- ContainerCommandRequestProto requestProto) {
- CompletableFuture<Message> future =
- CompletableFuture.completedFuture(runCommand(requestProto));
- future.thenAccept(m -> {
- createContainerFuture.complete(m);
- createContainerFuture = null;
- });
- return future;
- }
-
- CompletableFuture<Message> handleOtherCommands(
- ContainerCommandRequestProto requestProto) {
- return CompletableFuture.completedFuture(runCommand(requestProto));
- }
-
- CompletableFuture<Message> executeContainerCommand(
- ContainerCommandRequestProto requestProto, long index) {
- Type cmdType = requestProto.getCmdType();
- switch (cmdType) {
- case WriteChunk:
- return handleChunkCommit(requestProto, index);
- case CloseContainer:
- return handleCloseContainer(requestProto);
- case PutBlock:
- return handlePutBlock(requestProto, index);
- case CreateContainer:
- return handleCreateContainer(requestProto);
- default:
- return handleOtherCommands(requestProto);
- }
- }
- }
-
- @VisibleForTesting
- public ConcurrentHashMap<Long, StateMachineHelper> getStateMachineMap() {
- return stateMachineMap;
- }
-
- @VisibleForTesting
- public CompletableFuture<Message> getCreateContainerFuture(long containerId) {
- StateMachineHelper helper = stateMachineMap.get(containerId);
- return helper == null ? null : helper.createContainerFuture;
- }
-
- @VisibleForTesting
- public List<CompletableFuture<Message>> getCommitChunkFutureMap(
- long containerId) {
- StateMachineHelper helper = stateMachineMap.get(containerId);
- if (helper != null) {
- List<CompletableFuture<Message>> futureList = new ArrayList<>();
- stateMachineMap.get(containerId).block2ChunkMap.values()
- .forEach(b -> futureList.addAll(b.getAll()));
- return futureList;
- }
- return null;
- }
-
- @VisibleForTesting
- public Collection<CompletableFuture<Message>> getWriteChunkFutureMap() {
- return writeChunkFutureMap.values();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index c2ef504..9094217 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -56,9 +56,9 @@ import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.shaded.proto.RaftProtos;
-import org.apache.ratis.shaded.proto.RaftProtos.RoleInfoProto;
-import org.apache.ratis.shaded.proto.RaftProtos.ReplicationLevel;
+import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
+import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
@@ -97,6 +97,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
private final StateContext context;
private final ReplicationLevel replicationLevel;
private long nodeFailureTimeoutMs;
+ private ContainerStateMachine stateMachine;
private XceiverServerRatis(DatanodeDetails dd, int port,
ContainerDispatcher dispatcher, Configuration conf, StateContext context)
@@ -112,12 +113,15 @@ public final class XceiverServerRatis implements XceiverServerSpi {
100, TimeUnit.SECONDS,
new ArrayBlockingQueue<>(1024),
new ThreadPoolExecutor.CallerRunsPolicy());
+ final int numContainerOpExecutors = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT);
this.context = context;
this.replicationLevel =
conf.getEnum(OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT);
- ContainerStateMachine stateMachine =
- new ContainerStateMachine(dispatcher, chunkExecutor, this);
+ stateMachine = new ContainerStateMachine(dispatcher, chunkExecutor, this,
+ numContainerOpExecutors);
this.server = RaftServer.newBuilder()
.setServerId(RatisHelper.toRaftPeerId(dd))
.setProperties(serverProperties)
@@ -292,6 +296,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
public void stop() {
try {
chunkExecutor.shutdown();
+ stateMachine.close();
server.close();
} catch (IOException e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 62e328e..8bdae0f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
index 3495363..dee0c11 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
@@ -21,7 +21,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index d96fbfa..61a303f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoo
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.ratis.shaded.com.google.protobuf
+import org.apache.ratis.thirdparty.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 3aafb0c..c8a40b2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
-import org.apache.ratis.shaded.io.grpc.ManagedChannel;
-import org.apache.ratis.shaded.io.grpc.netty.NettyChannelBuilder;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
+import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
index d8f696f..30a251d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index fc622b2..fc84ae7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 6580c2c9..de666ce 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -712,6 +712,11 @@ public class ChunkGroupOutputStream extends OutputStream {
if (this.outputStream instanceof ChunkOutputStream) {
ChunkOutputStream out = (ChunkOutputStream) this.outputStream;
return out.getBlockCommitSequenceId();
+ } else if (outputStream == null) {
+ // For a pre allocated block for which no write has been initiated,
+ // the OutputStream will be null here.
+ // In such cases, the default blockCommitSequenceId will be 0
+ return 0;
}
throw new IOException("Invalid Output Stream for Key: " + key);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index f606263..da8d334 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -341,7 +341,6 @@ public class TestCloseContainerHandlingByClient {
Assert.assertTrue(key.getOutputStream() instanceof ChunkGroupOutputStream);
// With the initial size provided, it should have pre allocated 4 blocks
Assert.assertEquals(2, groupOutputStream.getStreamEntries().size());
- Assert.assertEquals(2, groupOutputStream.getLocationInfoList().size());
String dataString = fixedLengthString(keyString, (1 * blockSize));
byte[] data = dataString.getBytes();
key.write(data);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index f278479..324187c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -439,6 +439,7 @@ public final class ContainerTestHelper {
List<ContainerProtos.ChunkInfo> newList = new LinkedList<>();
newList.add(writeRequest.getChunkData());
blockData.setChunks(newList);
+ blockData.setBlockCommitSequenceId(0);
putRequest.setBlockData(blockData.getProtoBufMessage());
ContainerCommandRequestProto.Builder request =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
index 78bf008..92bad27 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index de55d9e..e6ebbf1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
-import org.apache.ratis.shaded.io.netty.channel.embedded.EmbeddedChannel;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
deleted file mode 100644
index c875a7e..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.server;
-
-
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerType;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.container.ContainerTestHelper;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis
- .ContainerStateMachine;
-import org.apache.ratis.RatisHelper;
-import org.apache.ratis.protocol.ClientId;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientRequest;
-import org.apache.ratis.shaded.proto.RaftProtos;
-import org.apache.ratis.statemachine.TransactionContext;
-import org.apache.ratis.util.ProtoUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Random;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class tests ContainerStateMachine.
- */
-public class TestContainerStateMachine {
-
- private static final AtomicLong CALL_ID_COUNTER = new AtomicLong();
-
- private static long nextCallId() {
- return CALL_ID_COUNTER.getAndIncrement() & Long.MAX_VALUE;
- }
-
- private ThreadPoolExecutor executor =
- new ThreadPoolExecutor(4, 4, 100, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(1024),
- new ThreadPoolExecutor.CallerRunsPolicy());
- private ContainerStateMachine stateMachine =
- new ContainerStateMachine(new TestContainerDispatcher(), executor, null);
-
-
- @Test
- public void testCloseContainerSynchronization() throws Exception {
- Pipeline pipeline = ContainerTestHelper.createPipeline(3);
- long containerId = new Random().nextLong();
-
- //create container request
- RaftClientRequest createContainer = getRaftClientRequest(
- ContainerTestHelper.getCreateContainerRequest(containerId, pipeline));
-
- ContainerCommandRequestProto writeChunkProto = ContainerTestHelper
- .getWriteChunkRequest(pipeline, new BlockID(containerId, nextCallId()),
- 1024);
-
- RaftClientRequest writeChunkRequest = getRaftClientRequest(writeChunkProto);
-
- // add putKey request
- ContainerCommandRequestProto putKeyProto = ContainerTestHelper
- .getPutBlockRequest(pipeline, writeChunkProto.getWriteChunk());
- RaftClientRequest putKeyRequest = getRaftClientRequest(putKeyProto);
-
- TransactionContext createContainerCtxt =
- startAndWriteStateMachineData(createContainer);
- // Start and Write into the StateMachine
- TransactionContext writeChunkcontext =
- startAndWriteStateMachineData(writeChunkRequest);
-
- TransactionContext putKeyContext =
- stateMachine.startTransaction(putKeyRequest);
- Assert.assertEquals(1, stateMachine.getStateMachineMap().size());
- Assert.assertNotNull(stateMachine.getCreateContainerFuture(containerId));
- Assert.assertEquals(1,
- stateMachine.getWriteChunkFutureMap().size());
- Assert.assertTrue(
- stateMachine.getCommitChunkFutureMap(containerId).isEmpty());
-
- //Add a closeContainerRequest
- RaftClientRequest closeRequest = getRaftClientRequest(
- ContainerTestHelper.getCloseContainer(pipeline, containerId));
-
- TransactionContext closeCtx = stateMachine.startTransaction(closeRequest);
-
- // Now apply all the transaction for the CreateContainer Command.
- // This will unblock writeChunks as well
-
- stateMachine.applyTransaction(createContainerCtxt);
- stateMachine.applyTransaction(writeChunkcontext);
- CompletableFuture<Message> putKeyFuture =
- stateMachine.applyTransaction(putKeyContext);
- waitForTransactionCompletion(putKeyFuture);
- // Make sure the putKey transaction complete
- Assert.assertTrue(putKeyFuture.isDone());
-
- // Execute the closeContainer. This should ensure all prior Write Type
- // container requests finish execution
-
- CompletableFuture<Message> closeFuture =
- stateMachine.applyTransaction(closeCtx);
- waitForTransactionCompletion(closeFuture);
- // Make sure the closeContainer transaction complete
- Assert.assertTrue(closeFuture.isDone());
- Assert.assertNull(stateMachine.getCreateContainerFuture(containerId));
- Assert.assertNull(stateMachine.getCommitChunkFutureMap(containerId));
-
- }
-
- private RaftClientRequest getRaftClientRequest(
- ContainerCommandRequestProto req) throws IOException {
- ClientId clientId = ClientId.randomId();
- return new RaftClientRequest(clientId,
- RatisHelper.toRaftPeerId(ContainerTestHelper.createDatanodeDetails()),
- RatisHelper.emptyRaftGroup().getGroupId(), nextCallId(), 0,
- Message.valueOf(req.toByteString()), RaftClientRequest
- .writeRequestType(RaftProtos.ReplicationLevel.MAJORITY));
- }
-
- private void waitForTransactionCompletion(
- CompletableFuture<Message> future) throws Exception {
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- executorService
- .invokeAll(Collections.singleton(future::get), 10,
- TimeUnit.SECONDS); // Timeout of 10 minutes.
- executorService.shutdown();
- }
-
- private TransactionContext startAndWriteStateMachineData(
- RaftClientRequest request) throws IOException {
- TransactionContext ctx = stateMachine.startTransaction(request);
- RaftProtos.LogEntryProto e = ProtoUtils
- .toLogEntryProto(ctx.getSMLogEntry(), request.getSeqNum(),
- request.getCallId(), ClientId.randomId(), request.getCallId());
- ctx.setLogEntry(e);
- stateMachine.writeStateMachineData(e);
- return ctx;
- }
-
- // ContainerDispatcher for test only purpose.
- private static class TestContainerDispatcher implements ContainerDispatcher {
- /**
- * Dispatches commands to container layer.
- *
- * @param msg - Command Request
- * @return Command Response
- */
- @Override
- public ContainerCommandResponseProto dispatch(
- ContainerCommandRequestProto msg) {
- return ContainerTestHelper.getCreateContainerResponse(msg);
- }
-
- @Override
- public void init() {
- }
-
- @Override
- public void shutdown() {
- }
-
- @Override
- public void setScmId(String scmId) {
- }
-
- @Override
- public Handler getHandler(ContainerType containerType) {
- return null;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 8811d91..362e53f 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine.DatanodeStates;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bc6d048/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 5f88371..7472a54 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -103,7 +103,7 @@
<ldap-api.version>1.0.0-M33</ldap-api.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-eca3531-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-9b84d79-SNAPSHOT</ratis.version>
<jcache.version>1.0-alpha-1</jcache.version>
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org