You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/04/14 00:09:29 UTC
[01/44] hadoop git commit: YARN-7984. Improved YARN service
stop/destroy and clean up. Contributed by Billie Rinaldi
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 66610b5fd -> 72a3743cc
YARN-7984. Improved YARN service stop/destroy and clean up.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5537990
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5537990
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5537990
Branch: refs/heads/HDFS-7240
Commit: d553799030a5a64df328319aceb35734d0b2de20
Parents: 8ab776d
Author: Eric Yang <ey...@apache.org>
Authored: Tue Apr 10 17:40:49 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Apr 10 17:40:49 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/service/webapp/ApiServer.java | 47 +++++++---
.../hadoop/yarn/service/ServiceClientTest.java | 6 ++
.../hadoop/yarn/service/TestApiServer.java | 26 ++++++
.../yarn/service/client/ServiceClient.java | 93 +++++++++++++++-----
.../hadoop/yarn/service/ServiceTestUtils.java | 15 +++-
.../yarn/service/TestYarnNativeServices.java | 42 ++++++++-
6 files changed, 191 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5537990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 59ee05d..14c77f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -231,30 +231,40 @@ public class ApiServer {
e.getCause().getMessage());
} catch (YarnException | FileNotFoundException e) {
return formatResponse(Status.NOT_FOUND, e.getMessage());
- } catch (IOException | InterruptedException e) {
+ } catch (Exception e) {
LOG.error("Fail to stop service: {}", e);
return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
}
}
private Response stopService(String appName, boolean destroy,
- final UserGroupInformation ugi) throws IOException,
- InterruptedException, YarnException, FileNotFoundException {
+ final UserGroupInformation ugi) throws Exception {
int result = ugi.doAs(new PrivilegedExceptionAction<Integer>() {
@Override
- public Integer run() throws IOException, YarnException,
- FileNotFoundException {
+ public Integer run() throws Exception {
int result = 0;
ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG);
sc.start();
- result = sc.actionStop(appName, destroy);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully stopped service {}", appName);
+ Exception stopException = null;
+ try {
+ result = sc.actionStop(appName, destroy);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ } catch (Exception e) {
+ LOG.info("Got exception stopping service", e);
+ stopException = e;
}
if (destroy) {
result = sc.actionDestroy(appName);
- LOG.info("Successfully deleted service {}", appName);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully deleted service {}", appName);
+ }
+ } else {
+ if (stopException != null) {
+ throw stopException;
+ }
}
sc.close();
return result;
@@ -262,8 +272,21 @@ public class ApiServer {
});
ServiceStatus serviceStatus = new ServiceStatus();
if (destroy) {
- serviceStatus.setDiagnostics("Successfully destroyed service " +
- appName);
+ if (result == EXIT_SUCCESS) {
+ serviceStatus.setDiagnostics("Successfully destroyed service " +
+ appName);
+ } else {
+ if (result == EXIT_NOT_FOUND) {
+ serviceStatus
+ .setDiagnostics("Service " + appName + " doesn't exist");
+ return formatResponse(Status.BAD_REQUEST, serviceStatus);
+ } else {
+ serviceStatus
+ .setDiagnostics("Service " + appName + " error cleaning up " +
+ "registry");
+ return formatResponse(Status.INTERNAL_SERVER_ERROR, serviceStatus);
+ }
+ }
} else {
if (result == EXIT_COMMAND_ARGUMENT_ERROR) {
serviceStatus
@@ -394,7 +417,7 @@ public class ApiServer {
String message = "Service is not found in hdfs: " + appName;
LOG.error(message, e);
return formatResponse(Status.NOT_FOUND, e.getMessage());
- } catch (IOException | InterruptedException e) {
+ } catch (Exception e) {
String message = "Error while performing operation for app: " + appName;
LOG.error(message, e);
return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5537990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
index 5d959da..543c583 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -103,6 +103,12 @@ public class ServiceClientTest extends ServiceClient {
}
if (serviceName.equals("jenkins")) {
return EXIT_SUCCESS;
+ } else if (serviceName.equals("jenkins-already-stopped")) {
+ return EXIT_SUCCESS;
+ } else if (serviceName.equals("jenkins-doesn't-exist")) {
+ return EXIT_NOT_FOUND;
+ } else if (serviceName.equals("jenkins-error-cleaning-registry")) {
+ return EXIT_OTHER_FAILURE;
} else {
throw new IllegalArgumentException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5537990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
index 7db6be2..9e28c96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -155,6 +155,24 @@ public class TestApiServer {
}
@Test
+ public void testBadDeleteService3() {
+ final Response actual = apiServer.deleteService(request,
+ "jenkins-doesn't-exist");
+ assertEquals("Delete service is ",
+ Response.status(Status.BAD_REQUEST).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
+ public void testBadDeleteService4() {
+ final Response actual = apiServer.deleteService(request,
+ "jenkins-error-cleaning-registry");
+ assertEquals("Delete service is ",
+ Response.status(Status.INTERNAL_SERVER_ERROR).build().getStatus(),
+ actual.getStatus());
+ }
+
+ @Test
public void testGoodDeleteService() {
final Response actual = apiServer.deleteService(request, "jenkins");
assertEquals("Delete service is ",
@@ -162,6 +180,14 @@ public class TestApiServer {
}
@Test
+ public void testDeleteStoppedService() {
+ final Response actual = apiServer.deleteService(request,
+ "jenkins-already-stopped");
+ assertEquals("Delete service is ",
+ Response.status(Status.OK).build().getStatus(), actual.getStatus());
+ }
+
+ @Test
public void testDecreaseContainerAndStop() {
Service service = new Service();
service.setState(ServiceState.STOPPED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5537990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 04ca943..4c7b72d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -375,8 +375,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
.save(fs.getFileSystem(), ServiceApiUtil.getServiceJsonPath(fs, serviceName),
persistedService, true);
+ ApplicationId appId = getAppId(serviceName);
+ if (appId == null) {
+ String message = "Application ID doesn't exist for " + serviceName;
+ LOG.error(message);
+ throw new YarnException(message);
+ }
ApplicationReport appReport =
- yarnClient.getApplicationReport(getAppId(serviceName));
+ yarnClient.getApplicationReport(appId);
if (appReport.getYarnApplicationState() != RUNNING) {
String message =
serviceName + " is at " + appReport.getYarnApplicationState()
@@ -408,10 +414,16 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
throws YarnException, IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
ApplicationId currentAppId = getAppId(serviceName);
+ if (currentAppId == null) {
+ LOG.info("Application ID doesn't exist for service {}", serviceName);
+ cleanUpRegistry(serviceName);
+ return EXIT_COMMAND_ARGUMENT_ERROR;
+ }
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (terminatedStates.contains(report.getYarnApplicationState())) {
LOG.info("Service {} is already in a terminated state {}", serviceName,
report.getYarnApplicationState());
+ cleanUpRegistry(serviceName);
return EXIT_COMMAND_ARGUMENT_ERROR;
}
if (preRunningStates.contains(report.getYarnApplicationState())) {
@@ -419,6 +431,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
+ ", forcefully killed by user!";
yarnClient.killApplication(currentAppId, msg);
LOG.info(msg);
+ cleanUpRegistry(serviceName);
return EXIT_SUCCESS;
}
if (StringUtils.isEmpty(report.getHost())) {
@@ -438,10 +451,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
yarnClient.killApplication(currentAppId,
serviceName + " is forcefully killed by user!");
LOG.info("Forcefully kill the service: " + serviceName);
+ cleanUpRegistry(serviceName);
return EXIT_SUCCESS;
}
if (!waitForAppStopped) {
+ cleanUpRegistry(serviceName);
return EXIT_SUCCESS;
}
// Wait until the app is killed.
@@ -471,6 +486,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
+ e.getMessage() + ", forcefully kill the app.");
yarnClient.killApplication(currentAppId, "Forcefully kill the app");
}
+ cleanUpRegistry(serviceName);
return EXIT_SUCCESS;
}
@@ -484,7 +500,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
FileSystem fileSystem = fs.getFileSystem();
// remove from the appId cache
cachedAppInfo.remove(serviceName);
- boolean destroySucceed = true;
+ int ret = EXIT_SUCCESS;
if (fileSystem.exists(appDir)) {
if (fileSystem.delete(appDir, true)) {
LOG.info("Successfully deleted service dir for " + serviceName + ": "
@@ -498,13 +514,34 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
} else {
LOG.info("Service '" + serviceName + "' doesn't exist at hdfs path: "
+ appDir);
- destroySucceed = false;
+ ret = EXIT_NOT_FOUND;
}
try {
deleteZKNode(serviceName);
+ // don't set destroySucceed to false if no ZK node exists because not
+ // all services use a ZK node
} catch (Exception e) {
throw new IOException("Could not delete zk node for " + serviceName, e);
}
+ if (!cleanUpRegistry(serviceName)) {
+ if (ret == EXIT_SUCCESS) {
+ ret = EXIT_OTHER_FAILURE;
+ }
+ }
+ if (ret == EXIT_SUCCESS) {
+ LOG.info("Successfully destroyed service {}", serviceName);
+ return ret;
+ } else if (ret == EXIT_NOT_FOUND) {
+ LOG.error("Error on destroy '" + serviceName + "': not found.");
+ return ret;
+ } else {
+ LOG.error("Error on destroy '" + serviceName + "': error cleaning up " +
+ "registry.");
+ return ret;
+ }
+ }
+
+ private boolean cleanUpRegistry(String serviceName) throws SliderException {
String registryPath =
ServiceRegistryUtils.registryPathForInstance(serviceName);
try {
@@ -514,18 +551,13 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
LOG.info(
"Service '" + serviceName + "' doesn't exist at ZK registry path: "
+ registryPath);
- destroySucceed = false;
+ // not counted as a failure if the registry entries don't exist
}
} catch (IOException e) {
LOG.warn("Error deleting registry entry {}", registryPath, e);
+ return false;
}
- if (destroySucceed) {
- LOG.info("Successfully destroyed service {}", serviceName);
- return EXIT_SUCCESS;
- } else {
- LOG.error("Error on destroy '" + serviceName + "': not found.");
- return -1;
- }
+ return true;
}
private synchronized RegistryOperations getRegistryClient()
@@ -540,17 +572,25 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return registryClient;
}
- private boolean deleteZKNode(String clusterName) throws Exception {
+ /**
+ * Delete service's ZK node. This is a different node from the service's
+ * registry entry and is set aside for the service to use for its own ZK data.
+ *
+ * @param serviceName service name
+ * @return true if the node was deleted, false if the node doesn't exist
+ * @throws Exception if the node couldn't be deleted
+ */
+ private boolean deleteZKNode(String serviceName) throws Exception {
CuratorFramework curatorFramework = getCuratorClient();
String user = RegistryUtils.currentUser();
- String zkPath = ServiceRegistryUtils.mkServiceHomePath(user, clusterName);
+ String zkPath = ServiceRegistryUtils.mkServiceHomePath(user, serviceName);
if (curatorFramework.checkExists().forPath(zkPath) != null) {
curatorFramework.delete().deletingChildrenIfNeeded().forPath(zkPath);
LOG.info("Deleted zookeeper path: " + zkPath);
return true;
} else {
LOG.info(
- "Service '" + clusterName + "' doesn't exist at ZK path: " + zkPath);
+ "Service '" + serviceName + "' doesn't exist at ZK path: " + zkPath);
return false;
}
}
@@ -963,6 +1003,9 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
public String updateLifetime(String serviceName, long lifetime)
throws YarnException, IOException {
ApplicationId currentAppId = getAppId(serviceName);
+ if (currentAppId == null) {
+ throw new YarnException("Application ID not found for " + serviceName);
+ }
ApplicationReport report = yarnClient.getApplicationReport(currentAppId);
if (report == null) {
throw new YarnException("Service not found for " + serviceName);
@@ -1008,8 +1051,8 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
throws IOException, YarnException {
try {
// try parsing appIdOrName, if it succeeds, it means it's appId
- ApplicationId.fromString(appIdOrName);
- return getStatusByAppId(appIdOrName);
+ ApplicationId appId = ApplicationId.fromString(appIdOrName);
+ return getStatusByAppId(appId);
} catch (IllegalArgumentException e) {
// not appId format, it could be appName.
Service status = getStatus(appIdOrName);
@@ -1017,10 +1060,10 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
}
}
- private String getStatusByAppId(String appId)
+ private String getStatusByAppId(ApplicationId appId)
throws IOException, YarnException {
ApplicationReport appReport =
- yarnClient.getApplicationReport(ApplicationId.fromString(appId));
+ yarnClient.getApplicationReport(appId);
if (appReport.getYarnApplicationState() != RUNNING) {
return "";
@@ -1037,10 +1080,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
public Service getStatus(String serviceName)
throws IOException, YarnException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
- ApplicationId currentAppId = getAppId(serviceName);
- ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
Service appSpec = new Service();
appSpec.setName(serviceName);
+ ApplicationId currentAppId = getAppId(serviceName);
+ if (currentAppId == null) {
+ LOG.info("Service {} does not have an application ID", serviceName);
+ return appSpec;
+ }
+ ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId);
appSpec.setState(convertState(appReport.getYarnApplicationState()));
ApplicationTimeout lifetime =
appReport.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
@@ -1167,7 +1214,11 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
throw new YarnException("Service " + serviceName
+ " doesn't exist on hdfs. Please check if the app exists in RM");
}
- ApplicationId currentAppId = ApplicationId.fromString(persistedService.getId());
+ if (persistedService.getId() == null) {
+ return null;
+ }
+ ApplicationId currentAppId = ApplicationId.fromString(persistedService
+ .getId());
cachedAppInfo.put(serviceName, new AppInfo(currentAppId, persistedService
.getKerberosPrincipal().getPrincipalName()));
return currentAppId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5537990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 8347eb3..d84f05f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.registry.client.impl.zk.CuratorService;
+import org.apache.hadoop.service.ServiceOperations;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -74,7 +76,8 @@ public class ServiceTestUtils {
private MiniYARNCluster yarnCluster = null;
private MiniDFSCluster hdfsCluster = null;
- TestingCluster zkCluster;
+ private TestingCluster zkCluster;
+ private CuratorService curatorService;
private FileSystem fs = null;
private Configuration conf = null;
public static final int NUM_NMS = 1;
@@ -186,6 +189,10 @@ public class ServiceTestUtils {
conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
LOG.info("ZK cluster: " + zkCluster.getConnectString());
+ curatorService = new CuratorService("testCuratorService");
+ curatorService.init(conf);
+ curatorService.start();
+
fs = FileSystem.get(conf);
basedir = new File("target", "apps");
if (basedir.exists()) {
@@ -253,6 +260,9 @@ public class ServiceTestUtils {
hdfsCluster = null;
}
}
+ if (curatorService != null) {
+ ServiceOperations.stop(curatorService);
+ }
if (zkCluster != null) {
zkCluster.stop();
}
@@ -295,6 +305,9 @@ public class ServiceTestUtils {
return client;
}
+ protected CuratorService getCuratorService() throws IOException {
+ return curatorService;
+ }
/**
* Watcher to initialize yarn service base path under target and deletes the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5537990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index e4d1914..2b44701 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -22,6 +22,8 @@ import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
@@ -40,6 +42,7 @@ import org.apache.hadoop.yarn.service.api.records.PlacementType;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.hamcrest.CoreMatchers;
@@ -59,6 +62,8 @@ import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_NOT_FOUND;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
@@ -131,6 +136,10 @@ public class TestYarnNativeServices extends ServiceTestUtils {
Assert.assertEquals(FINISHED, report.getYarnApplicationState());
Assert.assertEquals(FinalApplicationStatus.ENDED,
report.getFinalApplicationStatus());
+ String serviceZKPath = RegistryUtils.servicePath(RegistryUtils
+ .currentUser(), YarnServiceConstants.APP_TYPE, exampleApp.getName());
+ Assert.assertFalse("Registry ZK service path still exists after stop",
+ getCuratorService().zkPathExists(serviceZKPath));
LOG.info("Destroy the service");
// destroy the service and check the app dir is deleted from fs.
@@ -139,7 +148,20 @@ public class TestYarnNativeServices extends ServiceTestUtils {
Assert.assertFalse(getFS().exists(appDir));
// check that destroying again does not succeed
- Assert.assertEquals(-1, client.actionDestroy(exampleApp.getName()));
+ Assert.assertEquals(EXIT_NOT_FOUND, client.actionDestroy(exampleApp.getName()));
+ }
+
+ // Save a service without starting it and ensure that stop does not NPE and
+ // that service can be successfully destroyed
+ @Test (timeout = 200000)
+ public void testStopDestroySavedService() throws Exception {
+ setupInternal(NUM_NMS);
+ ServiceClient client = createClient(getConf());
+ Service exampleApp = createExampleApplication();
+ client.actionBuild(exampleApp);
+ Assert.assertEquals(EXIT_COMMAND_ARGUMENT_ERROR, client.actionStop(
+ exampleApp.getName()));
+ Assert.assertEquals(0, client.actionDestroy(exampleApp.getName()));
}
// Create compa with 2 containers
@@ -507,19 +529,31 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4
// When flex down to 3 instances, it should be compA-1 , compA-2, compA-3.
private void checkCompInstancesInOrder(ServiceClient client,
- Service exampleApp) throws IOException, YarnException {
+ Service exampleApp) throws IOException, YarnException,
+ TimeoutException, InterruptedException {
Service service = client.getStatus(exampleApp.getName());
for (Component comp : service.getComponents()) {
- checkEachCompInstancesInOrder(comp);
+ checkEachCompInstancesInOrder(comp, exampleApp.getName());
}
}
- private void checkEachCompInstancesInOrder(Component component) {
+ private void checkEachCompInstancesInOrder(Component component, String
+ serviceName) throws TimeoutException, InterruptedException {
long expectedNumInstances = component.getNumberOfContainers();
Assert.assertEquals(expectedNumInstances, component.getContainers().size());
TreeSet<String> instances = new TreeSet<>();
for (Container container : component.getContainers()) {
instances.add(container.getComponentInstanceName());
+ String componentZKPath = RegistryUtils.componentPath(RegistryUtils
+ .currentUser(), YarnServiceConstants.APP_TYPE, serviceName,
+ RegistryPathUtils.encodeYarnID(container.getId()));
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return getCuratorService().zkPathExists(componentZKPath);
+ } catch (IOException e) {
+ return false;
+ }
+ }, 1000, 60000);
}
int i = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/44] hadoop git commit: HDFS-13056. Expose file-level composite
CRCs in HDFS which are comparable across different instances/layouts.
Contributed by Dennis Huo.
Posted by xy...@apache.org.
HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c9cdad6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c9cdad6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c9cdad6
Branch: refs/heads/HDFS-7240
Commit: 7c9cdad6d04c98db5a83e2108219bf6e6c903daf
Parents: 6cc59a0
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Apr 10 20:56:07 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Apr 10 21:31:48 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/CompositeCrcFileChecksum.java | 82 +++++
.../main/java/org/apache/hadoop/fs/Options.java | 11 +
.../org/apache/hadoop/util/CrcComposer.java | 187 ++++++++++
.../java/org/apache/hadoop/util/CrcUtil.java | 220 +++++++++++
.../org/apache/hadoop/util/DataChecksum.java | 18 +
.../org/apache/hadoop/util/TestCrcComposer.java | 242 ++++++++++++
.../org/apache/hadoop/util/TestCrcUtil.java | 232 ++++++++++++
.../main/java/org/apache/hadoop/fs/Hdfs.java | 4 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 56 ++-
.../hadoop/hdfs/DistributedFileSystem.java | 5 +-
.../apache/hadoop/hdfs/FileChecksumHelper.java | 365 +++++++++++++------
.../hdfs/client/HdfsClientConfigKeys.java | 2 +
.../hadoop/hdfs/client/impl/DfsClientConf.java | 27 ++
.../hdfs/protocol/BlockChecksumOptions.java | 54 +++
.../hadoop/hdfs/protocol/BlockChecksumType.java | 30 ++
.../datatransfer/DataTransferProtocol.java | 12 +-
.../hdfs/protocol/datatransfer/Sender.java | 11 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 44 +++
.../src/main/proto/datatransfer.proto | 7 +-
.../src/main/proto/hdfs.proto | 21 ++
.../hdfs/protocol/datatransfer/Receiver.java | 8 +-
.../server/datanode/BlockChecksumHelper.java | 289 +++++++++++++--
.../hdfs/server/datanode/DataXceiver.java | 26 +-
...dBlockChecksumCompositeCrcReconstructor.java | 80 ++++
...StripedBlockChecksumMd5CrcReconstructor.java | 74 ++++
.../StripedBlockChecksumReconstructor.java | 66 ++--
.../erasurecode/StripedBlockReconstructor.java | 1 +
.../src/main/resources/hdfs-default.xml | 11 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 31 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 101 ++++-
.../hdfs/TestFileChecksumCompositeCrc.java | 47 +++
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 14 +
.../hadoop/tools/mapred/TestCopyMapper.java | 173 +++++++--
.../mapred/TestCopyMapperCompositeCrc.java | 50 +++
34 files changed, 2359 insertions(+), 242 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
new file mode 100644
index 0000000..e1ed5cb
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.util.CrcUtil;
+import org.apache.hadoop.util.DataChecksum;
+
+/** Composite CRC. */
+@InterfaceAudience.LimitedPrivate({"HDFS"})
+@InterfaceStability.Unstable
+public class CompositeCrcFileChecksum extends FileChecksum {
+ public static final int LENGTH = Integer.SIZE / Byte.SIZE;
+
+ private int crc;
+ private DataChecksum.Type crcType;
+ private int bytesPerCrc;
+
+ /** Create a CompositeCrcFileChecksum. */
+ public CompositeCrcFileChecksum(
+ int crc, DataChecksum.Type crcType, int bytesPerCrc) {
+ this.crc = crc;
+ this.crcType = crcType;
+ this.bytesPerCrc = bytesPerCrc;
+ }
+
+ @Override
+ public String getAlgorithmName() {
+ return "COMPOSITE-" + crcType.name();
+ }
+
+ @Override
+ public int getLength() {
+ return LENGTH;
+ }
+
+ @Override
+ public byte[] getBytes() {
+ return CrcUtil.intToBytes(crc);
+ }
+
+ @Override
+ public ChecksumOpt getChecksumOpt() {
+ return new ChecksumOpt(crcType, bytesPerCrc);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ crc = in.readInt();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(crc);
+ }
+
+ @Override
+ public String toString() {
+ return getAlgorithmName() + ":" + String.format("0x%08x", crc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index e455abf..126e754 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -504,4 +504,15 @@ public final class Options {
}
+ /**
+ * Enum for indicating what mode to use when combining chunk and block
+ * checksums to define an aggregate FileChecksum. This should be considered
+ * a client-side runtime option rather than a persistent property of any
+ * stored metadata, which is why this is not part of ChecksumOpt, which
+ * deals with properties of files at rest.
+ */
+ public enum ChecksumCombineMode {
+ MD5MD5CRC, // MD5 of block checksums, which are MD5 over chunk CRCs
+ COMPOSITE_CRC // Block/chunk-independent composite CRC
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
new file mode 100644
index 0000000..4023995
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+/**
+ * Encapsulates logic for composing multiple CRCs into one or more combined CRCs
+ * corresponding to concatenated underlying data ranges. Optimized for composing
+ * a large number of CRCs that correspond to underlying chunks of data all of
+ * same size.
+ */
+@InterfaceAudience.LimitedPrivate({"Common", "HDFS", "MapReduce", "Yarn"})
+@InterfaceStability.Unstable
+public class CrcComposer {
+ private static final int CRC_SIZE_BYTES = 4;
+ private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class);
+
+ private final int crcPolynomial;
+ private final int precomputedMonomialForHint;
+ private final long bytesPerCrcHint;
+ private final long stripeLength;
+
+ private int curCompositeCrc = 0;
+ private long curPositionInStripe = 0;
+ private ByteArrayOutputStream digestOut = new ByteArrayOutputStream();
+
+ /**
+ * Returns a CrcComposer which will collapse all ingested CRCs into a single
+ * value.
+ */
+ public static CrcComposer newCrcComposer(
+ DataChecksum.Type type, long bytesPerCrcHint)
+ throws IOException {
+ return newStripedCrcComposer(type, bytesPerCrcHint, Long.MAX_VALUE);
+ }
+
+ /**
+ * Returns a CrcComposer which will collapse CRCs for every combined
+ * underlying data size which aligns with the specified stripe boundary. For
+ * example, if "update" is called with 20 CRCs and bytesPerCrc == 5, and
+ * stripeLength == 10, then every two (10 / 5) consecutive CRCs will be
+ * combined with each other, yielding a list of 10 CRC "stripes" in the
+ * final digest, each corresponding to 10 underlying data bytes. Using
+ * a stripeLength greater than the total underlying data size is equivalent
+ * to using a non-striped CrcComposer.
+ */
+ public static CrcComposer newStripedCrcComposer(
+ DataChecksum.Type type, long bytesPerCrcHint, long stripeLength)
+ throws IOException {
+ int polynomial = DataChecksum.getCrcPolynomialForType(type);
+ return new CrcComposer(
+ polynomial,
+ CrcUtil.getMonomial(bytesPerCrcHint, polynomial),
+ bytesPerCrcHint,
+ stripeLength);
+ }
+
+ CrcComposer(
+ int crcPolynomial,
+ int precomputedMonomialForHint,
+ long bytesPerCrcHint,
+ long stripeLength) {
+ LOG.debug(
+ "crcPolynomial=0x{}, precomputedMonomialForHint=0x{}, "
+ + "bytesPerCrcHint={}, stripeLength={}",
+ Integer.toString(crcPolynomial, 16),
+ Integer.toString(precomputedMonomialForHint, 16),
+ bytesPerCrcHint,
+ stripeLength);
+ this.crcPolynomial = crcPolynomial;
+ this.precomputedMonomialForHint = precomputedMonomialForHint;
+ this.bytesPerCrcHint = bytesPerCrcHint;
+ this.stripeLength = stripeLength;
+ }
+
+ /**
+ * Composes length / CRC_SIZE_IN_BYTES more CRCs from crcBuffer, with
+ * each CRC expected to correspond to exactly {@code bytesPerCrc} underlying
+ * data bytes.
+ *
+ * @param length must be a multiple of the expected byte-size of a CRC.
+ */
+ public void update(
+ byte[] crcBuffer, int offset, int length, long bytesPerCrc)
+ throws IOException {
+ if (length % CRC_SIZE_BYTES != 0) {
+ throw new IOException(String.format(
+ "Trying to update CRC from byte array with length '%d' at offset "
+ + "'%d' which is not a multiple of %d!",
+ length, offset, CRC_SIZE_BYTES));
+ }
+ int limit = offset + length;
+ while (offset < limit) {
+ int crcB = CrcUtil.readInt(crcBuffer, offset);
+ update(crcB, bytesPerCrc);
+ offset += CRC_SIZE_BYTES;
+ }
+ }
+
+ /**
+ * Composes {@code numChecksumsToRead} additional CRCs into the current digest
+ * out of {@code checksumIn}, with each CRC expected to correspond to exactly
+ * {@code bytesPerCrc} underlying data bytes.
+ */
+ public void update(
+ DataInputStream checksumIn, long numChecksumsToRead, long bytesPerCrc)
+ throws IOException {
+ for (long i = 0; i < numChecksumsToRead; ++i) {
+ int crcB = checksumIn.readInt();
+ update(crcB, bytesPerCrc);
+ }
+ }
+
+ /**
+ * Updates with a single additional CRC which corresponds to an underlying
+ * data size of {@code bytesPerCrc}.
+ */
+ public void update(int crcB, long bytesPerCrc) throws IOException {
+ if (curCompositeCrc == 0) {
+ curCompositeCrc = crcB;
+ } else if (bytesPerCrc == bytesPerCrcHint) {
+ curCompositeCrc = CrcUtil.composeWithMonomial(
+ curCompositeCrc, crcB, precomputedMonomialForHint, crcPolynomial);
+ } else {
+ curCompositeCrc = CrcUtil.compose(
+ curCompositeCrc, crcB, bytesPerCrc, crcPolynomial);
+ }
+
+ curPositionInStripe += bytesPerCrc;
+
+ if (curPositionInStripe > stripeLength) {
+ throw new IOException(String.format(
+ "Current position in stripe '%d' after advancing by bytesPerCrc '%d' "
+ + "exceeds stripeLength '%d' without stripe alignment.",
+ curPositionInStripe, bytesPerCrc, stripeLength));
+ } else if (curPositionInStripe == stripeLength) {
+ // Hit a stripe boundary; flush the curCompositeCrc and reset for next
+ // stripe.
+ digestOut.write(CrcUtil.intToBytes(curCompositeCrc), 0, CRC_SIZE_BYTES);
+ curCompositeCrc = 0;
+ curPositionInStripe = 0;
+ }
+ }
+
+ /**
+ * Returns byte representation of composed CRCs; if no stripeLength was
+ * specified, the digest should be of length equal to exactly one CRC.
+ * Otherwise, the number of CRCs in the returned array is equal to the
+ * total sum bytesPerCrc divided by stripeLength. If the sum of bytesPerCrc
+ * is not a multiple of stripeLength, then the last CRC in the array
+ * corresponds to totalLength % stripeLength underlying data bytes.
+ */
+ public byte[] digest() {
+ if (curPositionInStripe > 0) {
+ digestOut.write(CrcUtil.intToBytes(curCompositeCrc), 0, CRC_SIZE_BYTES);
+ curCompositeCrc = 0;
+ curPositionInStripe = 0;
+ }
+ byte[] digestValue = digestOut.toByteArray();
+ digestOut.reset();
+ return digestValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
new file mode 100644
index 0000000..42eaf14
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * This class provides utilities for working with CRCs.
+ */
+@InterfaceAudience.LimitedPrivate({"Common", "HDFS", "MapReduce", "Yarn"})
+@InterfaceStability.Unstable
+public final class CrcUtil {
+ public static final int MULTIPLICATIVE_IDENTITY = 0x80000000;
+ public static final int GZIP_POLYNOMIAL = 0xEDB88320;
+ public static final int CASTAGNOLI_POLYNOMIAL = 0x82F63B78;
+
+ /**
+ * Hide default constructor for a static utils class.
+ */
+ private CrcUtil() {
+ }
+
+ /**
+ * Compute x^({@code lengthBytes} * 8) mod {@code mod}, where {@code mod} is
+ * in "reversed" (little-endian) format such that {@code mod & 1} represents
+ * x^31 and has an implicit term x^32.
+ */
+ public static int getMonomial(long lengthBytes, int mod) {
+ if (lengthBytes == 0) {
+ return MULTIPLICATIVE_IDENTITY;
+ } else if (lengthBytes < 0) {
+ throw new IllegalArgumentException(
+ "lengthBytes must be positive, got " + lengthBytes);
+ }
+
+ // Decompose into
+ // x^degree == x ^ SUM(bit[i] * 2^i) == PRODUCT(x ^ (bit[i] * 2^i))
+ // Generate each x^(2^i) by squaring.
+ // Since 'degree' is in 'bits', but we only need to support byte
+ // granularity we can begin with x^8.
+ int multiplier = MULTIPLICATIVE_IDENTITY >>> 8;
+ int product = MULTIPLICATIVE_IDENTITY;
+ long degree = lengthBytes;
+ while (degree > 0) {
+ if ((degree & 1) != 0) {
+ product = (product == MULTIPLICATIVE_IDENTITY) ? multiplier :
+ galoisFieldMultiply(product, multiplier, mod);
+ }
+ multiplier = galoisFieldMultiply(multiplier, multiplier, mod);
+ degree >>= 1;
+ }
+ return product;
+ }
+
+ /**
+ * @param monomial Precomputed x^(lengthBInBytes * 8) mod {@code mod}
+ */
+ public static int composeWithMonomial(
+ int crcA, int crcB, int monomial, int mod) {
+ return galoisFieldMultiply(crcA, monomial, mod) ^ crcB;
+ }
+
+ /**
+ * @param lengthB length of content corresponding to {@code crcB}, in bytes.
+ */
+ public static int compose(int crcA, int crcB, long lengthB, int mod) {
+ int monomial = getMonomial(lengthB, mod);
+ return composeWithMonomial(crcA, crcB, monomial, mod);
+ }
+
+ /**
+ * @return 4-byte array holding the big-endian representation of
+ * {@code value}.
+ */
+ public static byte[] intToBytes(int value) {
+ byte[] buf = new byte[4];
+ try {
+ writeInt(buf, 0, value);
+ } catch (IOException ioe) {
+ // Since this should only be able to occur from code bugs within this
+ // class rather than user input, we throw as a RuntimeException
+ // rather than requiring this method to declare throwing IOException
+ // for something the caller can't control.
+ throw new RuntimeException(ioe);
+ }
+ return buf;
+ }
+
+ /**
+ * Writes big-endian representation of {@code value} into {@code buf}
+ * starting at {@code offset}. buf.length must be greater than or
+ * equal to offset + 4.
+ */
+ public static void writeInt(byte[] buf, int offset, int value)
+ throws IOException {
+ if (offset + 4 > buf.length) {
+ throw new IOException(String.format(
+ "writeInt out of bounds: buf.length=%d, offset=%d",
+ buf.length, offset));
+ }
+ buf[offset + 0] = (byte)((value >>> 24) & 0xff);
+ buf[offset + 1] = (byte)((value >>> 16) & 0xff);
+ buf[offset + 2] = (byte)((value >>> 8) & 0xff);
+ buf[offset + 3] = (byte)(value & 0xff);
+ }
+
+ /**
+ * Reads 4-byte big-endian int value from {@code buf} starting at
+ * {@code offset}. buf.length must be greater than or equal to offset + 4.
+ */
+ public static int readInt(byte[] buf, int offset)
+ throws IOException {
+ if (offset + 4 > buf.length) {
+ throw new IOException(String.format(
+ "readInt out of bounds: buf.length=%d, offset=%d",
+ buf.length, offset));
+ }
+ int value = ((buf[offset + 0] & 0xff) << 24) |
+ ((buf[offset + 1] & 0xff) << 16) |
+ ((buf[offset + 2] & 0xff) << 8) |
+ ((buf[offset + 3] & 0xff));
+ return value;
+ }
+
+ /**
+ * For use with debug statements; verifies bytes.length on creation,
+ * expecting it to represent exactly one CRC, and returns a hex
+ * formatted value.
+ */
+ public static String toSingleCrcString(final byte[] bytes)
+ throws IOException {
+ if (bytes.length != 4) {
+ throw new IOException((String.format(
+ "Unexpected byte[] length '%d' for single CRC. Contents: %s",
+ bytes.length, Arrays.toString(bytes))));
+ }
+ return String.format("0x%08x", readInt(bytes, 0));
+ }
+
+ /**
+ * For use with debug statements; verifies bytes.length on creation,
+ * expecting it to be divisible by CRC byte size, and returns a list of
+ * hex formatted values.
+ */
+ public static String toMultiCrcString(final byte[] bytes)
+ throws IOException {
+ if (bytes.length % 4 != 0) {
+ throw new IOException((String.format(
+ "Unexpected byte[] length '%d' not divisible by 4. Contents: %s",
+ bytes.length, Arrays.toString(bytes))));
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append('[');
+ for (int i = 0; i < bytes.length; i += 4) {
+ sb.append(String.format("0x%08x", readInt(bytes, i)));
+ if (i != bytes.length - 4) {
+ sb.append(", ");
+ }
+ }
+ sb.append(']');
+ return sb.toString();
+ }
+
+ /**
+ * Galois field multiplication of {@code p} and {@code q} with the
+ * generator polynomial {@code m} as the modulus.
+ *
+ * @param m The little-endian polynomial to use as the modulus when
+ * multiplying p and q, with implicit "1" bit beyond the bottom bit.
+ */
+ private static int galoisFieldMultiply(int p, int q, int m) {
+ int summation = 0;
+
+ // Top bit is the x^0 place; each right-shift increments the degree of the
+ // current term.
+ int curTerm = MULTIPLICATIVE_IDENTITY;
+
+ // Iteratively multiply p by x mod m as we go to represent the q[i] term
+ // (of degree x^i) times p.
+ int px = p;
+
+ while (curTerm != 0) {
+ if ((q & curTerm) != 0) {
+ summation ^= px;
+ }
+
+ // Bottom bit represents highest degree since we're little-endian; before
+ // we multiply by "x" for the next term, check bottom bit to know whether
+ // the resulting px will thus have a term matching the implicit "1" term
+ // of "m" and thus will need to subtract "m" after mutiplying by "x".
+ boolean hasMaxDegree = ((px & 1) != 0);
+ px >>>= 1;
+ if (hasMaxDegree) {
+ px ^= m;
+ }
+ curTerm >>>= 1;
+ }
+ return summation;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index 43e377f..06ef8ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -104,6 +104,24 @@ public class DataChecksum implements Checksum {
}
}
+ /**
+ * @return the int representation of the polynomial associated with the
+ * CRC {@code type}, suitable for use with further CRC arithmetic.
+ * @throws IOException if there is no CRC polynomial applicable
+ * to the given {@code type}.
+ */
+ public static int getCrcPolynomialForType(Type type) throws IOException {
+ switch (type) {
+ case CRC32:
+ return CrcUtil.GZIP_POLYNOMIAL;
+ case CRC32C:
+ return CrcUtil.CASTAGNOLI_POLYNOMIAL;
+ default:
+ throw new IOException(
+ "No CRC polynomial could be associated with type: " + type);
+ }
+ }
+
public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
if ( bytesPerChecksum <= 0 ) {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcComposer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcComposer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcComposer.java
new file mode 100644
index 0000000..f08702e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcComposer.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unittests for CrcComposer.
+ */
+public class TestCrcComposer {
+ @Rule
+ public Timeout globalTimeout = new Timeout(10000);
+
+ private Random rand = new Random(1234);
+
+ private DataChecksum.Type type = DataChecksum.Type.CRC32C;
+ private DataChecksum checksum = DataChecksum.newDataChecksum(
+ type, Integer.MAX_VALUE);
+ private int dataSize = 75;
+ private byte[] data = new byte[dataSize];
+ private int chunkSize = 10;
+ private int cellSize = 20;
+
+ private int fullCrc;
+ private int[] crcsByChunk;
+ private int[] crcsByCell;
+
+ private byte[] crcBytesByChunk;
+ private byte[] crcBytesByCell;
+
+ @Before
+ public void setup() throws IOException {
+ rand.nextBytes(data);
+ fullCrc = getRangeChecksum(data, 0, dataSize);
+
+ // 7 chunks of size chunkSize, 1 chunk of size (dataSize % chunkSize).
+ crcsByChunk = new int[8];
+ for (int i = 0; i < 7; ++i) {
+ crcsByChunk[i] = getRangeChecksum(data, i * chunkSize, chunkSize);
+ }
+ crcsByChunk[7] = getRangeChecksum(
+ data, (crcsByChunk.length - 1) * chunkSize, dataSize % chunkSize);
+
+ // 3 cells of size cellSize, 1 cell of size (dataSize % cellSize).
+ crcsByCell = new int[4];
+ for (int i = 0; i < 3; ++i) {
+ crcsByCell[i] = getRangeChecksum(data, i * cellSize, cellSize);
+ }
+ crcsByCell[3] = getRangeChecksum(
+ data, (crcsByCell.length - 1) * cellSize, dataSize % cellSize);
+
+ crcBytesByChunk = intArrayToByteArray(crcsByChunk);
+ crcBytesByCell = intArrayToByteArray(crcsByCell);
+ }
+
+ private int getRangeChecksum(byte[] buf, int offset, int length) {
+ checksum.reset();
+ checksum.update(buf, offset, length);
+ return (int) checksum.getValue();
+ }
+
+ private byte[] intArrayToByteArray(int[] values) throws IOException {
+ byte[] bytes = new byte[values.length * 4];
+ for (int i = 0; i < values.length; ++i) {
+ CrcUtil.writeInt(bytes, i * 4, values[i]);
+ }
+ return bytes;
+ }
+
+ @Test
+ public void testUnstripedIncorrectChunkSize() throws IOException {
+ CrcComposer digester = CrcComposer.newCrcComposer(type, chunkSize);
+
+ // If we incorrectly specify that all CRCs ingested correspond to chunkSize
+ // when the last CRC in the array actually corresponds to
+ // dataSize % chunkSize then we expect the resulting CRC to not be equal to
+ // the fullCrc.
+ digester.update(crcBytesByChunk, 0, crcBytesByChunk.length, chunkSize);
+ byte[] digest = digester.digest();
+ assertEquals(4, digest.length);
+ int calculatedCrc = CrcUtil.readInt(digest, 0);
+ assertNotEquals(fullCrc, calculatedCrc);
+ }
+
+ @Test
+ public void testUnstripedByteArray() throws IOException {
+ CrcComposer digester = CrcComposer.newCrcComposer(type, chunkSize);
+ digester.update(crcBytesByChunk, 0, crcBytesByChunk.length - 4, chunkSize);
+ digester.update(
+ crcBytesByChunk, crcBytesByChunk.length - 4, 4, dataSize % chunkSize);
+
+ byte[] digest = digester.digest();
+ assertEquals(4, digest.length);
+ int calculatedCrc = CrcUtil.readInt(digest, 0);
+ assertEquals(fullCrc, calculatedCrc);
+ }
+
+ @Test
+ public void testUnstripedDataInputStream() throws IOException {
+ CrcComposer digester = CrcComposer.newCrcComposer(type, chunkSize);
+ DataInputStream input =
+ new DataInputStream(new ByteArrayInputStream(crcBytesByChunk));
+ digester.update(input, crcsByChunk.length - 1, chunkSize);
+ digester.update(input, 1, dataSize % chunkSize);
+
+ byte[] digest = digester.digest();
+ assertEquals(4, digest.length);
+ int calculatedCrc = CrcUtil.readInt(digest, 0);
+ assertEquals(fullCrc, calculatedCrc);
+ }
+
+ @Test
+ public void testUnstripedSingleCrcs() throws IOException {
+ CrcComposer digester = CrcComposer.newCrcComposer(type, chunkSize);
+ for (int i = 0; i < crcsByChunk.length - 1; ++i) {
+ digester.update(crcsByChunk[i], chunkSize);
+ }
+ digester.update(crcsByChunk[crcsByChunk.length - 1], dataSize % chunkSize);
+
+ byte[] digest = digester.digest();
+ assertEquals(4, digest.length);
+ int calculatedCrc = CrcUtil.readInt(digest, 0);
+ assertEquals(fullCrc, calculatedCrc);
+ }
+
+ @Test
+ public void testStripedByteArray() throws IOException {
+ CrcComposer digester =
+ CrcComposer.newStripedCrcComposer(type, chunkSize, cellSize);
+ digester.update(crcBytesByChunk, 0, crcBytesByChunk.length - 4, chunkSize);
+ digester.update(
+ crcBytesByChunk, crcBytesByChunk.length - 4, 4, dataSize % chunkSize);
+
+ byte[] digest = digester.digest();
+ assertArrayEquals(crcBytesByCell, digest);
+ }
+
+ @Test
+ public void testStripedDataInputStream() throws IOException {
+ CrcComposer digester =
+ CrcComposer.newStripedCrcComposer(type, chunkSize, cellSize);
+ DataInputStream input =
+ new DataInputStream(new ByteArrayInputStream(crcBytesByChunk));
+ digester.update(input, crcsByChunk.length - 1, chunkSize);
+ digester.update(input, 1, dataSize % chunkSize);
+
+ byte[] digest = digester.digest();
+ assertArrayEquals(crcBytesByCell, digest);
+ }
+
+ @Test
+ public void testStripedSingleCrcs() throws IOException {
+ CrcComposer digester =
+ CrcComposer.newStripedCrcComposer(type, chunkSize, cellSize);
+ for (int i = 0; i < crcsByChunk.length - 1; ++i) {
+ digester.update(crcsByChunk[i], chunkSize);
+ }
+ digester.update(crcsByChunk[crcsByChunk.length - 1], dataSize % chunkSize);
+
+ byte[] digest = digester.digest();
+ assertArrayEquals(crcBytesByCell, digest);
+ }
+
+ @Test
+ public void testMultiStageMixed() throws IOException {
+ CrcComposer digester =
+ CrcComposer.newStripedCrcComposer(type, chunkSize, cellSize);
+
+ // First combine chunks into cells.
+ DataInputStream input =
+ new DataInputStream(new ByteArrayInputStream(crcBytesByChunk));
+ digester.update(input, crcsByChunk.length - 1, chunkSize);
+ digester.update(input, 1, dataSize % chunkSize);
+ byte[] digest = digester.digest();
+
+ // Second, individually combine cells into full crc.
+ digester =
+ CrcComposer.newCrcComposer(type, cellSize);
+ for (int i = 0; i < digest.length - 4; i += 4) {
+ int cellCrc = CrcUtil.readInt(digest, i);
+ digester.update(cellCrc, cellSize);
+ }
+ digester.update(digest, digest.length - 4, 4, dataSize % cellSize);
+ digest = digester.digest();
+ assertEquals(4, digest.length);
+ int calculatedCrc = CrcUtil.readInt(digest, 0);
+ assertEquals(fullCrc, calculatedCrc);
+ }
+
+ @Test
+ public void testUpdateMismatchesStripe() throws Exception {
+ CrcComposer digester =
+ CrcComposer.newStripedCrcComposer(type, chunkSize, cellSize);
+
+ digester.update(crcsByChunk[0], chunkSize);
+
+ // Going from chunkSize to chunkSize + cellSize will cross a cellSize
+ // boundary in a single CRC, which is not allowed, since we'd lack a
+ // CRC corresponding to the actual cellSize boundary.
+ LambdaTestUtils.intercept(
+ IOException.class,
+ "stripe",
+ () -> digester.update(crcsByChunk[1], cellSize));
+ }
+
+ @Test
+ public void testUpdateByteArrayLengthUnalignedWithCrcSize()
+ throws Exception {
+ CrcComposer digester = CrcComposer.newCrcComposer(type, chunkSize);
+
+ LambdaTestUtils.intercept(
+ IOException.class,
+ "length",
+ () -> digester.update(crcBytesByChunk, 0, 6, chunkSize));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java
new file mode 100644
index 0000000..a98cb8a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unittests for CrcUtil.
+ */
+public class TestCrcUtil {
+ @Rule
+ public Timeout globalTimeout = new Timeout(10000);
+
+ private Random rand = new Random(1234);
+
+ @Test
+ public void testComposeCrc32() throws IOException {
+ byte[] data = new byte[64 * 1024];
+ rand.nextBytes(data);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 512, false);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 511, false);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 32 * 1024, false);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 32 * 1024 - 1, false);
+ }
+
+ @Test
+ public void testComposeCrc32c() throws IOException {
+ byte[] data = new byte[64 * 1024];
+ rand.nextBytes(data);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 512, false);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 511, false);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 32 * 1024, false);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 32 * 1024 - 1, false);
+ }
+
+ @Test
+ public void testComposeCrc32WithMonomial() throws IOException {
+ byte[] data = new byte[64 * 1024];
+ rand.nextBytes(data);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 512, true);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 511, true);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 32 * 1024, true);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32, 32 * 1024 - 1, true);
+ }
+
+ @Test
+ public void testComposeCrc32cWithMonomial() throws IOException {
+ byte[] data = new byte[64 * 1024];
+ rand.nextBytes(data);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 512, true);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 511, true);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 32 * 1024, true);
+ doTestComposeCrc(data, DataChecksum.Type.CRC32C, 32 * 1024 - 1, true);
+ }
+
+ @Test
+ public void testComposeCrc32ZeroLength() throws IOException {
+ doTestComposeCrcZerolength(DataChecksum.Type.CRC32);
+ }
+
+ @Test
+ public void testComposeCrc32CZeroLength() throws IOException {
+ doTestComposeCrcZerolength(DataChecksum.Type.CRC32C);
+ }
+
+ /**
+ * Helper method to compare a DataChecksum-computed end-to-end CRC against
+ * a piecewise-computed CRC that uses CrcUtil.compose on "chunk CRCs"
+ * corresponding to ever {@code chunkSize} bytes.
+ */
+ private static void doTestComposeCrc(
+ byte[] data, DataChecksum.Type type, int chunkSize, boolean useMonomial)
+ throws IOException {
+ int crcPolynomial = DataChecksum.getCrcPolynomialForType(type);
+
+ // Get full end-to-end CRC in a single shot first.
+ DataChecksum checksum = DataChecksum.newDataChecksum(
+ type, Integer.MAX_VALUE);
+ checksum.update(data, 0, data.length);
+ int fullCrc = (int) checksum.getValue();
+
+ // Now compute CRCs of each chunk individually first, and compose them in a
+ // second pass to compare to the end-to-end CRC.
+ int compositeCrc = 0;
+ int crcMonomial =
+ useMonomial ? CrcUtil.getMonomial(chunkSize, crcPolynomial) : 0;
+ for (int offset = 0;
+ offset + chunkSize <= data.length;
+ offset += chunkSize) {
+ checksum.reset();
+ checksum.update(data, offset, chunkSize);
+ int partialCrc = (int) checksum.getValue();
+ if (useMonomial) {
+ compositeCrc = CrcUtil.composeWithMonomial(
+ compositeCrc, partialCrc, crcMonomial, crcPolynomial);
+ } else {
+ compositeCrc = CrcUtil.compose(
+ compositeCrc, partialCrc, chunkSize, crcPolynomial);
+ }
+ }
+
+ // There may be a final partial chunk smaller than chunkSize.
+ int partialChunkSize = data.length % chunkSize;
+ if (partialChunkSize > 0) {
+ checksum.reset();
+ checksum.update(data, data.length - partialChunkSize, partialChunkSize);
+ int partialCrc = (int) checksum.getValue();
+ compositeCrc = CrcUtil.compose(
+ compositeCrc, partialCrc, partialChunkSize, crcPolynomial);
+ }
+ assertEquals(
+ String.format(
+ "Using CRC type '%s' with crcPolynomial '0x%08x' and chunkSize '%d'"
+ + ", expected '0x%08x', got '0x%08x'",
+ type, crcPolynomial, chunkSize, fullCrc, compositeCrc),
+ fullCrc,
+ compositeCrc);
+ }
+
+ /**
+ * Helper method for testing the behavior of composing a CRC with a
+ * zero-length second CRC.
+ */
+ private static void doTestComposeCrcZerolength(DataChecksum.Type type)
+ throws IOException {
+ // Without loss of generality, we can pick any integer as our fake crcA
+ // even if we don't happen to know the preimage.
+ int crcA = 0xCAFEBEEF;
+ int crcPolynomial = DataChecksum.getCrcPolynomialForType(type);
+ DataChecksum checksum = DataChecksum.newDataChecksum(
+ type, Integer.MAX_VALUE);
+ int crcB = (int) checksum.getValue();
+ assertEquals(crcA, CrcUtil.compose(crcA, crcB, 0, crcPolynomial));
+
+ int monomial = CrcUtil.getMonomial(0, crcPolynomial);
+ assertEquals(
+ crcA, CrcUtil.composeWithMonomial(crcA, crcB, monomial, crcPolynomial));
+ }
+
+ @Test
+ public void testIntSerialization() throws IOException {
+ byte[] bytes = CrcUtil.intToBytes(0xCAFEBEEF);
+ assertEquals(0xCAFEBEEF, CrcUtil.readInt(bytes, 0));
+
+ bytes = new byte[8];
+ CrcUtil.writeInt(bytes, 0, 0xCAFEBEEF);
+ assertEquals(0xCAFEBEEF, CrcUtil.readInt(bytes, 0));
+ CrcUtil.writeInt(bytes, 4, 0xABCDABCD);
+ assertEquals(0xABCDABCD, CrcUtil.readInt(bytes, 4));
+
+ // Assert big-endian format for general Java consistency.
+ assertEquals(0xBEEFABCD, CrcUtil.readInt(bytes, 2));
+ }
+
+ @Test
+ public void testToSingleCrcStringBadLength()
+ throws Exception {
+ LambdaTestUtils.intercept(
+ IOException.class,
+ "length",
+ () -> CrcUtil.toSingleCrcString(new byte[8]));
+ }
+
+ @Test
+ public void testToSingleCrcString() throws IOException {
+ byte[] buf = CrcUtil.intToBytes(0xcafebeef);
+ assertEquals(
+ "0xcafebeef", CrcUtil.toSingleCrcString(buf));
+ }
+
+ @Test
+ public void testToMultiCrcStringBadLength()
+ throws Exception {
+ LambdaTestUtils.intercept(
+ IOException.class,
+ "length",
+ () -> CrcUtil.toMultiCrcString(new byte[6]));
+ }
+
+ @Test
+ public void testToMultiCrcStringMultipleElements()
+ throws IOException {
+ byte[] buf = new byte[12];
+ CrcUtil.writeInt(buf, 0, 0xcafebeef);
+ CrcUtil.writeInt(buf, 4, 0xababcccc);
+ CrcUtil.writeInt(buf, 8, 0xddddefef);
+ assertEquals(
+ "[0xcafebeef, 0xababcccc, 0xddddefef]",
+ CrcUtil.toMultiCrcString(buf));
+ }
+
+ @Test
+ public void testToMultiCrcStringSingleElement()
+ throws IOException {
+ byte[] buf = new byte[4];
+ CrcUtil.writeInt(buf, 0, 0xcafebeef);
+ assertEquals(
+ "[0xcafebeef]",
+ CrcUtil.toMultiCrcString(buf));
+ }
+
+ @Test
+ public void testToMultiCrcStringNoElements()
+ throws IOException {
+ assertEquals(
+ "[]",
+ CrcUtil.toMultiCrcString(new byte[0]));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 0138195..d07d5a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -130,9 +130,9 @@ public class Hdfs extends AbstractFileSystem {
}
@Override
- public FileChecksum getFileChecksum(Path f)
+ public FileChecksum getFileChecksum(Path f)
throws IOException, UnresolvedLinkException {
- return dfs.getFileChecksum(getUriPath(f), Long.MAX_VALUE);
+ return dfs.getFileChecksumWithCombineMode(getUriPath(f), Long.MAX_VALUE);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 0875328..09154d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -76,6 +77,7 @@ import org.apache.hadoop.fs.HdfsBlockLocation;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Options.ChecksumCombineMode;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
@@ -1753,18 +1755,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return encryptionKey;
}
- /**
- * Get the checksum of the whole file or a range of the file. Note that the
- * range always starts from the beginning of the file. The file can be
- * in replicated form, or striped mode. It can be used to checksum and compare
- * two replicated files, or two striped files, but not applicable for two
- * files of different block layout forms.
- * @param src The file path
- * @param length the length of the range, i.e., the range is [0, length]
- * @return The checksum
- * @see DistributedFileSystem#getFileChecksum(Path)
- */
- public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length)
+ private FileChecksum getFileChecksumInternal(
+ String src, long length, ChecksumCombineMode combineMode)
throws IOException {
checkOpen();
Preconditions.checkArgument(length >= 0);
@@ -1779,15 +1771,51 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
maker = ecPolicy != null ?
new FileChecksumHelper.StripedFileNonStripedChecksumComputer(src,
- length, blockLocations, namenode, this, ecPolicy) :
+ length, blockLocations, namenode, this, ecPolicy, combineMode) :
new FileChecksumHelper.ReplicatedFileChecksumComputer(src, length,
- blockLocations, namenode, this);
+ blockLocations, namenode, this, combineMode);
maker.compute();
return maker.getFileChecksum();
}
+ /**
+ * Get the checksum of the whole file or a range of the file. Note that the
+ * range always starts from the beginning of the file. The file can be
+ * in replicated form, or striped mode. Depending on the
+ * dfs.checksum.combine.mode, checksums may or may not be comparable between
+ * different block layout forms.
+ *
+ * @param src The file path
+ * @param length the length of the range, i.e., the range is [0, length]
+ * @return The checksum
+ * @see DistributedFileSystem#getFileChecksum(Path)
+ */
+ public FileChecksum getFileChecksumWithCombineMode(String src, long length)
+ throws IOException {
+ ChecksumCombineMode combineMode = getConf().getChecksumCombineMode();
+ return getFileChecksumInternal(src, length, combineMode);
+ }
+
+ /**
+ * Get the checksum of the whole file or a range of the file. Note that the
+ * range always starts from the beginning of the file. The file can be
+ * in replicated form, or striped mode. It can be used to checksum and compare
+ * two replicated files, or two striped files, but not applicable for two
+ * files of different block layout forms.
+ *
+ * @param src The file path
+ * @param length the length of the range, i.e., the range is [0, length]
+ * @return The checksum
+ * @see DistributedFileSystem#getFileChecksum(Path)
+ */
+ public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length)
+ throws IOException {
+ return (MD5MD5CRC32FileChecksum) getFileChecksumInternal(
+ src, length, ChecksumCombineMode.MD5MD5CRC);
+ }
+
protected LocatedBlocks getBlockLocations(String src,
long length) throws IOException {
//get block locations for the file range
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 03cb317..1e9ed09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1681,7 +1681,8 @@ public class DistributedFileSystem extends FileSystem
return new FileSystemLinkResolver<FileChecksum>() {
@Override
public FileChecksum doCall(final Path p) throws IOException {
- return dfs.getFileChecksum(getPathName(p), Long.MAX_VALUE);
+ return dfs.getFileChecksumWithCombineMode(
+ getPathName(p), Long.MAX_VALUE);
}
@Override
@@ -1701,7 +1702,7 @@ public class DistributedFileSystem extends FileSystem
return new FileSystemLinkResolver<FileChecksum>() {
@Override
public FileChecksum doCall(final Path p) throws IOException {
- return dfs.getFileChecksum(getPathName(p), length);
+ return dfs.getFileChecksumWithCombineMode(getPathName(p), length);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
index 72cf147..8f807d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
@@ -17,9 +17,14 @@
*/
package org.apache.hadoop.hdfs;
+import org.apache.hadoop.fs.CompositeCrcFileChecksum;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
-import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
+import org.apache.hadoop.fs.Options.ChecksumCombineMode;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumOptions;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumType;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -41,6 +46,8 @@ import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.CrcComposer;
+import org.apache.hadoop.util.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,9 +74,11 @@ final class FileChecksumHelper {
private final long length;
private final DFSClient client;
private final ClientProtocol namenode;
- private final DataOutputBuffer md5out = new DataOutputBuffer();
+ private final ChecksumCombineMode combineMode;
+ private final BlockChecksumType blockChecksumType;
+ private final DataOutputBuffer blockChecksumBuf = new DataOutputBuffer();
- private MD5MD5CRC32FileChecksum fileChecksum;
+ private FileChecksum fileChecksum;
private LocatedBlocks blockLocations;
private int timeout;
@@ -88,12 +97,24 @@ final class FileChecksumHelper {
FileChecksumComputer(String src, long length,
LocatedBlocks blockLocations,
ClientProtocol namenode,
- DFSClient client) throws IOException {
+ DFSClient client,
+ ChecksumCombineMode combineMode) throws IOException {
this.src = src;
this.length = length;
this.blockLocations = blockLocations;
this.namenode = namenode;
this.client = client;
+ this.combineMode = combineMode;
+ switch (combineMode) {
+ case MD5MD5CRC:
+ this.blockChecksumType = BlockChecksumType.MD5CRC;
+ break;
+ case COMPOSITE_CRC:
+ this.blockChecksumType = BlockChecksumType.COMPOSITE_CRC;
+ break;
+ default:
+ throw new IOException("Unknown ChecksumCombineMode: " + combineMode);
+ }
this.remaining = length;
@@ -121,11 +142,19 @@ final class FileChecksumHelper {
return namenode;
}
- DataOutputBuffer getMd5out() {
- return md5out;
+ ChecksumCombineMode getCombineMode() {
+ return combineMode;
+ }
+
+ BlockChecksumType getBlockChecksumType() {
+ return blockChecksumType;
+ }
+
+ DataOutputBuffer getBlockChecksumBuf() {
+ return blockChecksumBuf;
}
- MD5MD5CRC32FileChecksum getFileChecksum() {
+ FileChecksum getFileChecksum() {
return fileChecksum;
}
@@ -226,17 +255,31 @@ final class FileChecksumHelper {
}
/**
- * Compute and aggregate block checksums block by block.
+ * Compute block checksums block by block and append the raw bytes of the
+ * block checksums into getBlockChecksumBuf().
+ *
* @throws IOException
*/
abstract void checksumBlocks() throws IOException;
/**
- * Make final file checksum result given the computing process done.
+ * Make final file checksum result given the per-block or per-block-group
+ * checksums collected into getBlockChecksumBuf().
*/
- MD5MD5CRC32FileChecksum makeFinalResult() {
+ FileChecksum makeFinalResult() throws IOException {
+ switch (combineMode) {
+ case MD5MD5CRC:
+ return makeMd5CrcResult();
+ case COMPOSITE_CRC:
+ return makeCompositeCrcResult();
+ default:
+ throw new IOException("Unknown ChecksumCombineMode: " + combineMode);
+ }
+ }
+
+ FileChecksum makeMd5CrcResult() {
//compute file MD5
- final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData());
+ final MD5Hash fileMD5 = MD5Hash.digest(blockChecksumBuf.getData());
switch (crcType) {
case CRC32:
return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC,
@@ -250,6 +293,58 @@ final class FileChecksumHelper {
}
}
+ FileChecksum makeCompositeCrcResult() throws IOException {
+ long blockSizeHint = 0;
+ if (locatedBlocks.size() > 0) {
+ blockSizeHint = locatedBlocks.get(0).getBlockSize();
+ }
+ CrcComposer crcComposer =
+ CrcComposer.newCrcComposer(getCrcType(), blockSizeHint);
+ byte[] blockChecksumBytes = blockChecksumBuf.getData();
+
+ long sumBlockLengths = 0;
+ for (int i = 0; i < locatedBlocks.size() - 1; ++i) {
+ LocatedBlock block = locatedBlocks.get(i);
+ // For everything except the last LocatedBlock, we expect getBlockSize()
+ // to accurately reflect the number of file bytes digested in the block
+ // checksum.
+ sumBlockLengths += block.getBlockSize();
+ int blockCrc = CrcUtil.readInt(blockChecksumBytes, i * 4);
+
+ crcComposer.update(blockCrc, block.getBlockSize());
+ LOG.debug(
+ "Added blockCrc 0x{} for block index {} of size {}",
+ Integer.toString(blockCrc, 16), i, block.getBlockSize());
+ }
+
+ // NB: In some cases the located blocks have their block size adjusted
+ // explicitly based on the requested length, but not all cases;
+ // these numbers may or may not reflect actual sizes on disk.
+ long reportedLastBlockSize =
+ blockLocations.getLastLocatedBlock().getBlockSize();
+ long consumedLastBlockLength = reportedLastBlockSize;
+ if (length - sumBlockLengths < reportedLastBlockSize) {
+ LOG.warn(
+ "Last block length {} is less than reportedLastBlockSize {}",
+ length - sumBlockLengths, reportedLastBlockSize);
+ consumedLastBlockLength = length - sumBlockLengths;
+ }
+ // NB: blockChecksumBytes.length may be much longer than actual bytes
+ // written into the DataOutput.
+ int lastBlockCrc = CrcUtil.readInt(
+ blockChecksumBytes, 4 * (locatedBlocks.size() - 1));
+ crcComposer.update(lastBlockCrc, consumedLastBlockLength);
+ LOG.debug(
+ "Added lastBlockCrc 0x{} for block index {} of size {}",
+ Integer.toString(lastBlockCrc, 16),
+ locatedBlocks.size() - 1,
+ consumedLastBlockLength);
+
+ int compositeCrc = CrcUtil.readInt(crcComposer.digest(), 0);
+ return new CompositeCrcFileChecksum(
+ compositeCrc, getCrcType(), bytesPerCRC);
+ }
+
/**
* Create and return a sender given an IO stream pair.
*/
@@ -267,6 +362,117 @@ final class FileChecksumHelper {
IOUtils.closeStream(pair.out);
}
}
+
+ /**
+ * Parses out various checksum properties like bytesPerCrc, crcPerBlock,
+ * and crcType from {@code checksumData} and either stores them as the
+ * authoritative value or compares them to a previously extracted value
+ * to check comppatibility.
+ *
+ * @param checksumData response from the datanode
+ * @param locatedBlock the block corresponding to the response
+ * @param datanode the datanode which produced the response
+ * @param blockIdx the block or block-group index of the response
+ */
+ void extractChecksumProperties(
+ OpBlockChecksumResponseProto checksumData,
+ LocatedBlock locatedBlock,
+ DatanodeInfo datanode,
+ int blockIdx)
+ throws IOException {
+ //read byte-per-checksum
+ final int bpc = checksumData.getBytesPerCrc();
+ if (blockIdx == 0) { //first block
+ setBytesPerCRC(bpc);
+ } else if (bpc != getBytesPerCRC()) {
+ if (getBlockChecksumType() == BlockChecksumType.COMPOSITE_CRC) {
+ LOG.warn(
+ "Current bytesPerCRC={} doesn't match next bpc={}, but "
+ + "continuing anyway because we're using COMPOSITE_CRC. "
+ + "If trying to preserve CHECKSUMTYPE, only the current "
+ + "bytesPerCRC will be preserved.", getBytesPerCRC(), bpc);
+ } else {
+ throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
+ + " but bytesPerCRC=" + getBytesPerCRC());
+ }
+ }
+
+ //read crc-per-block
+ final long cpb = checksumData.getCrcPerBlock();
+ if (getLocatedBlocks().size() > 1 && blockIdx == 0) {
+ setCrcPerBlock(cpb);
+ }
+
+ // read crc-type
+ final DataChecksum.Type ct;
+ if (checksumData.hasCrcType()) {
+ ct = PBHelperClient.convert(checksumData.getCrcType());
+ } else {
+ LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
+ "inferring checksum by reading first byte");
+ ct = getClient().inferChecksumTypeByReading(locatedBlock, datanode);
+ }
+
+ if (blockIdx == 0) {
+ setCrcType(ct);
+ } else if (getCrcType() != DataChecksum.Type.MIXED &&
+ getCrcType() != ct) {
+ if (getBlockChecksumType() == BlockChecksumType.COMPOSITE_CRC) {
+ throw new IOException(
+ "DataChecksum.Type.MIXED is not supported for COMPOSITE_CRC");
+ } else {
+ // if crc types are mixed in a file
+ setCrcType(DataChecksum.Type.MIXED);
+ }
+ }
+
+ if (blockIdx == 0) {
+ LOG.debug("set bytesPerCRC={}, crcPerBlock={}",
+ getBytesPerCRC(), getCrcPerBlock());
+ }
+ }
+
+ /**
+ * Parses out the raw blockChecksum bytes from {@code checksumData}
+ * according to the blockChecksumType and populates the cumulative
+ * blockChecksumBuf with it.
+ *
+ * @return a debug-string representation of the parsed checksum if
+ * debug is enabled, otherwise null.
+ */
+ String populateBlockChecksumBuf(OpBlockChecksumResponseProto checksumData)
+ throws IOException {
+ String blockChecksumForDebug = null;
+ switch (getBlockChecksumType()) {
+ case MD5CRC:
+ //read md5
+ final MD5Hash md5 = new MD5Hash(
+ checksumData.getBlockChecksum().toByteArray());
+ md5.write(getBlockChecksumBuf());
+ if (LOG.isDebugEnabled()) {
+ blockChecksumForDebug = md5.toString();
+ }
+ break;
+ case COMPOSITE_CRC:
+ BlockChecksumType returnedType = PBHelperClient.convert(
+ checksumData.getBlockChecksumOptions().getBlockChecksumType());
+ if (returnedType != BlockChecksumType.COMPOSITE_CRC) {
+ throw new IOException(String.format(
+ "Unexpected blockChecksumType '%s', expecting COMPOSITE_CRC",
+ returnedType));
+ }
+ byte[] crcBytes = checksumData.getBlockChecksum().toByteArray();
+ if (LOG.isDebugEnabled()) {
+ blockChecksumForDebug = CrcUtil.toSingleCrcString(crcBytes);
+ }
+ getBlockChecksumBuf().write(crcBytes);
+ break;
+ default:
+ throw new IOException(
+ "Unknown BlockChecksumType: " + getBlockChecksumType());
+ }
+ return blockChecksumForDebug;
+ }
}
/**
@@ -278,8 +484,10 @@ final class FileChecksumHelper {
ReplicatedFileChecksumComputer(String src, long length,
LocatedBlocks blockLocations,
ClientProtocol namenode,
- DFSClient client) throws IOException {
- super(src, length, blockLocations, namenode, client);
+ DFSClient client,
+ ChecksumCombineMode combineMode)
+ throws IOException {
+ super(src, length, blockLocations, namenode, client, combineMode);
}
@Override
@@ -295,7 +503,8 @@ final class FileChecksumHelper {
LocatedBlock locatedBlock = getLocatedBlocks().get(blockIdx);
if (!checksumBlock(locatedBlock)) {
- throw new IOException("Fail to get block MD5 for " + locatedBlock);
+ throw new PathIOException(
+ getSrc(), "Fail to get block MD5 for " + locatedBlock);
}
}
}
@@ -368,9 +577,11 @@ final class FileChecksumHelper {
LOG.debug("write to {}: {}, block={}", datanode,
Op.BLOCK_CHECKSUM, block);
- // get block MD5
- createSender(pair).blockChecksum(block,
- locatedBlock.getBlockToken());
+ // get block checksum
+ createSender(pair).blockChecksum(
+ block,
+ locatedBlock.getBlockToken(),
+ new BlockChecksumOptions(getBlockChecksumType()));
final BlockOpResponseProto reply = BlockOpResponseProto.parseFrom(
PBHelperClient.vintPrefixed(pair.in));
@@ -381,51 +592,11 @@ final class FileChecksumHelper {
OpBlockChecksumResponseProto checksumData =
reply.getChecksumResponse();
-
- //read byte-per-checksum
- final int bpc = checksumData.getBytesPerCrc();
- if (blockIdx == 0) { //first block
- setBytesPerCRC(bpc);
- } else if (bpc != getBytesPerCRC()) {
- throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
- + " but bytesPerCRC=" + getBytesPerCRC());
- }
-
- //read crc-per-block
- final long cpb = checksumData.getCrcPerBlock();
- if (getLocatedBlocks().size() > 1 && blockIdx == 0) {
- setCrcPerBlock(cpb);
- }
-
- //read md5
- final MD5Hash md5 = new MD5Hash(checksumData.getMd5().toByteArray());
- md5.write(getMd5out());
-
- // read crc-type
- final DataChecksum.Type ct;
- if (checksumData.hasCrcType()) {
- ct = PBHelperClient.convert(checksumData.getCrcType());
- } else {
- LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
- "inferring checksum by reading first byte");
- ct = getClient().inferChecksumTypeByReading(locatedBlock, datanode);
- }
-
- if (blockIdx == 0) { // first block
- setCrcType(ct);
- } else if (getCrcType() != DataChecksum.Type.MIXED
- && getCrcType() != ct) {
- // if crc types are mixed in a file
- setCrcType(DataChecksum.Type.MIXED);
- }
-
- if (LOG.isDebugEnabled()) {
- if (blockIdx == 0) {
- LOG.debug("set bytesPerCRC=" + getBytesPerCRC()
- + ", crcPerBlock=" + getCrcPerBlock());
- }
- LOG.debug("got reply from " + datanode + ": md5=" + md5);
- }
+ extractChecksumProperties(
+ checksumData, locatedBlock, datanode, blockIdx);
+ String blockChecksumForDebug = populateBlockChecksumBuf(checksumData);
+ LOG.debug("got reply from {}: blockChecksum={}, blockChecksumType={}",
+ datanode, blockChecksumForDebug, getBlockChecksumType());
}
}
}
@@ -442,9 +613,10 @@ final class FileChecksumHelper {
LocatedBlocks blockLocations,
ClientProtocol namenode,
DFSClient client,
- ErasureCodingPolicy ecPolicy)
+ ErasureCodingPolicy ecPolicy,
+ ChecksumCombineMode combineMode)
throws IOException {
- super(src, length, blockLocations, namenode, client);
+ super(src, length, blockLocations, namenode, client, combineMode);
this.ecPolicy = ecPolicy;
}
@@ -464,7 +636,8 @@ final class FileChecksumHelper {
LocatedStripedBlock blockGroup = (LocatedStripedBlock) locatedBlock;
if (!checksumBlockGroup(blockGroup)) {
- throw new IOException("Fail to get block MD5 for " + locatedBlock);
+ throw new PathIOException(
+ getSrc(), "Fail to get block checksum for " + locatedBlock);
}
}
}
@@ -519,16 +692,18 @@ final class FileChecksumHelper {
StripedBlockInfo stripedBlockInfo,
DatanodeInfo datanode,
long requestedNumBytes) throws IOException {
-
try (IOStreamPair pair = getClient().connectToDN(datanode,
getTimeout(), blockGroup.getBlockToken())) {
LOG.debug("write to {}: {}, blockGroup={}",
datanode, Op.BLOCK_GROUP_CHECKSUM, blockGroup);
- // get block MD5
- createSender(pair).blockGroupChecksum(stripedBlockInfo,
- blockGroup.getBlockToken(), requestedNumBytes);
+ // get block group checksum
+ createSender(pair).blockGroupChecksum(
+ stripedBlockInfo,
+ blockGroup.getBlockToken(),
+ requestedNumBytes,
+ new BlockChecksumOptions(getBlockChecksumType()));
BlockOpResponseProto reply = BlockOpResponseProto.parseFrom(
PBHelperClient.vintPrefixed(pair.in));
@@ -538,54 +713,10 @@ final class FileChecksumHelper {
DataTransferProtoUtil.checkBlockOpStatus(reply, logInfo);
OpBlockChecksumResponseProto checksumData = reply.getChecksumResponse();
-
- //read byte-per-checksum
- final int bpc = checksumData.getBytesPerCrc();
- if (bgIdx == 0) { //first block
- setBytesPerCRC(bpc);
- } else {
- if (bpc != getBytesPerCRC()) {
- throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
- + " but bytesPerCRC=" + getBytesPerCRC());
- }
- }
-
- //read crc-per-block
- final long cpb = checksumData.getCrcPerBlock();
- if (getLocatedBlocks().size() > 1 && bgIdx == 0) { // first block
- setCrcPerBlock(cpb);
- }
-
- //read md5
- final MD5Hash md5 = new MD5Hash(
- checksumData.getMd5().toByteArray());
- md5.write(getMd5out());
-
- // read crc-type
- final DataChecksum.Type ct;
- if (checksumData.hasCrcType()) {
- ct = PBHelperClient.convert(checksumData.getCrcType());
- } else {
- LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
- "inferring checksum by reading first byte");
- ct = getClient().inferChecksumTypeByReading(blockGroup, datanode);
- }
-
- if (bgIdx == 0) {
- setCrcType(ct);
- } else if (getCrcType() != DataChecksum.Type.MIXED &&
- getCrcType() != ct) {
- // if crc types are mixed in a file
- setCrcType(DataChecksum.Type.MIXED);
- }
-
- if (LOG.isDebugEnabled()) {
- if (bgIdx == 0) {
- LOG.debug("set bytesPerCRC=" + getBytesPerCRC()
- + ", crcPerBlock=" + getCrcPerBlock());
- }
- LOG.debug("got reply from " + datanode + ": md5=" + md5);
- }
+ extractChecksumProperties(checksumData, blockGroup, datanode, bgIdx);
+ String blockChecksumForDebug = populateBlockChecksumBuf(checksumData);
+ LOG.debug("got reply from {}: blockChecksum={}, blockChecksumType={}",
+ datanode, blockChecksumForDebug, getBlockChecksumType());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 52a7cd0..f2cec31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -120,6 +120,8 @@ public interface HdfsClientConfigKeys {
String DFS_CHECKSUM_TYPE_DEFAULT = "CRC32C";
String DFS_BYTES_PER_CHECKSUM_KEY = "dfs.bytes-per-checksum";
int DFS_BYTES_PER_CHECKSUM_DEFAULT = 512;
+ String DFS_CHECKSUM_COMBINE_MODE_KEY = "dfs.checksum.combine.mode";
+ String DFS_CHECKSUM_COMBINE_MODE_DEFAULT = "MD5MD5CRC";
String DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY =
"dfs.datanode.socket.write.timeout";
String DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index 2703617..e63e3f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Options.ChecksumCombineMode;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.ReplicaAccessorBuilder;
@@ -38,6 +39,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BLOCK_SIZE_
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BLOCK_SIZE_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CHECKSUM_COMBINE_MODE_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CHECKSUM_COMBINE_MODE_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CHECKSUM_TYPE_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT;
@@ -106,6 +109,7 @@ public class DfsClientConf {
private final int datanodeSocketWriteTimeout;
private final int ioBufferSize;
private final ChecksumOpt defaultChecksumOpt;
+ private final ChecksumCombineMode checksumCombineMode;
private final int writePacketSize;
private final int writeMaxPackets;
private final ByteArrayManager.Conf writeByteArrayManagerConf;
@@ -177,6 +181,7 @@ public class DfsClientConf {
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
defaultChecksumOpt = getChecksumOptFromConf(conf);
+ checksumCombineMode = getChecksumCombineModeFromConf(conf);
dataTransferTcpNoDelay = conf.getBoolean(
DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY,
DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT);
@@ -300,6 +305,21 @@ public class DfsClientConf {
}
}
+ private static ChecksumCombineMode getChecksumCombineModeFromConf(
+ Configuration conf) {
+ final String mode = conf.get(
+ DFS_CHECKSUM_COMBINE_MODE_KEY,
+ DFS_CHECKSUM_COMBINE_MODE_DEFAULT);
+ try {
+ return ChecksumCombineMode.valueOf(mode);
+ } catch(IllegalArgumentException iae) {
+ LOG.warn("Bad checksum combine mode: {}. Using default {}", mode,
+ DFS_CHECKSUM_COMBINE_MODE_DEFAULT);
+ return ChecksumCombineMode.valueOf(
+ DFS_CHECKSUM_COMBINE_MODE_DEFAULT);
+ }
+ }
+
// Construct a checksum option from conf
public static ChecksumOpt getChecksumOptFromConf(Configuration conf) {
DataChecksum.Type type = getChecksumType(conf);
@@ -393,6 +413,13 @@ public class DfsClientConf {
}
/**
+ * @return the checksumCombineMode
+ */
+ public ChecksumCombineMode getChecksumCombineMode() {
+ return checksumCombineMode;
+ }
+
+ /**
* @return the writePacketSize
*/
public int getWritePacketSize() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumOptions.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumOptions.java
new file mode 100644
index 0000000..82e07d4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumOptions.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Encapsulates various options related to how fine-grained data checksums are
+ * combined into block-level checksums.
+ */
+@InterfaceAudience.Private
+public class BlockChecksumOptions {
+ private final BlockChecksumType blockChecksumType;
+ private final long stripeLength;
+
+ public BlockChecksumOptions(
+ BlockChecksumType blockChecksumType, long stripeLength) {
+ this.blockChecksumType = blockChecksumType;
+ this.stripeLength = stripeLength;
+ }
+
+ public BlockChecksumOptions(BlockChecksumType blockChecksumType) {
+ this(blockChecksumType, 0);
+ }
+
+ public BlockChecksumType getBlockChecksumType() {
+ return blockChecksumType;
+ }
+
+ public long getStripeLength() {
+ return stripeLength;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("blockChecksumType=%s, stripedLength=%d",
+ blockChecksumType, stripeLength);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumType.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumType.java
new file mode 100644
index 0000000..cc33660
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockChecksumType.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Algorithms/types denoting how block-level checksums are computed using
+ * lower-level chunk checksums/CRCs.
+ */
+@InterfaceAudience.Private
+public enum BlockChecksumType {
+ MD5CRC, // BlockChecksum obtained by taking the MD5 digest of chunk CRCs
+ COMPOSITE_CRC // Chunk-independent CRC, optionally striped
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
index fe20c37..384f1dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumOptions;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.StripedBlockInfo;
@@ -214,11 +215,13 @@ public interface DataTransferProtocol {
*
* @param blk a block.
* @param blockToken security token for accessing the block.
+ * @param blockChecksumOptions determines how the block-level checksum is
+ * computed from underlying block metadata.
* @throws IOException
*/
void blockChecksum(ExtendedBlock blk,
- Token<BlockTokenIdentifier> blockToken) throws IOException;
-
+ Token<BlockTokenIdentifier> blockToken,
+ BlockChecksumOptions blockChecksumOptions) throws IOException;
/**
* Get striped block group checksum (MD5 of CRC32).
@@ -227,9 +230,12 @@ public interface DataTransferProtocol {
* @param blockToken security token for accessing the block.
* @param requestedNumBytes requested number of bytes in the block group
* to compute the checksum.
+ * @param blockChecksumOptions determines how the block-level checksum is
+ * computed from underlying block metadata.
* @throws IOException
*/
void blockGroupChecksum(StripedBlockInfo stripedBlockInfo,
Token<BlockTokenIdentifier> blockToken,
- long requestedNumBytes) throws IOException;
+ long requestedNumBytes,
+ BlockChecksumOptions blockChecksumOptions) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
index 8a8d20d..7526f96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumOptions;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.StripedBlockInfo;
@@ -267,9 +268,11 @@ public class Sender implements DataTransferProtocol {
@Override
public void blockChecksum(final ExtendedBlock blk,
- final Token<BlockTokenIdentifier> blockToken) throws IOException {
+ final Token<BlockTokenIdentifier> blockToken,
+ BlockChecksumOptions blockChecksumOptions) throws IOException {
OpBlockChecksumProto proto = OpBlockChecksumProto.newBuilder()
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
+ .setBlockChecksumOptions(PBHelperClient.convert(blockChecksumOptions))
.build();
send(out, Op.BLOCK_CHECKSUM, proto);
@@ -277,8 +280,9 @@ public class Sender implements DataTransferProtocol {
@Override
public void blockGroupChecksum(StripedBlockInfo stripedBlockInfo,
- Token<BlockTokenIdentifier> blockToken, long requestedNumBytes)
- throws IOException {
+ Token<BlockTokenIdentifier> blockToken,
+ long requestedNumBytes,
+ BlockChecksumOptions blockChecksumOptions) throws IOException {
OpBlockGroupChecksumProto proto = OpBlockGroupChecksumProto.newBuilder()
.setHeader(DataTransferProtoUtil.buildBaseHeader(
stripedBlockInfo.getBlock(), blockToken))
@@ -291,6 +295,7 @@ public class Sender implements DataTransferProtocol {
.setEcPolicy(PBHelperClient.convertErasureCodingPolicy(
stripedBlockInfo.getErasureCodingPolicy()))
.setRequestedNumBytes(requestedNumBytes)
+ .setBlockChecksumOptions(PBHelperClient.convert(blockChecksumOptions))
.build();
send(out, Op.BLOCK_GROUP_CHECKSUM, proto);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index d9e7aa0..ff9733c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -61,6 +61,8 @@ import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumOptions;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumType;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.BlockType;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
@@ -247,6 +249,48 @@ public class PBHelperClient {
return HdfsProtos.ChecksumTypeProto.valueOf(type.id);
}
+ public static HdfsProtos.BlockChecksumTypeProto convert(
+ BlockChecksumType type) {
+ switch(type) {
+ case MD5CRC:
+ return HdfsProtos.BlockChecksumTypeProto.MD5CRC;
+ case COMPOSITE_CRC:
+ return HdfsProtos.BlockChecksumTypeProto.COMPOSITE_CRC;
+ default:
+ throw new IllegalStateException(
+ "BUG: BlockChecksumType not found, type=" + type);
+ }
+ }
+
+ public static BlockChecksumType convert(
+ HdfsProtos.BlockChecksumTypeProto blockChecksumTypeProto) {
+ switch(blockChecksumTypeProto) {
+ case MD5CRC:
+ return BlockChecksumType.MD5CRC;
+ case COMPOSITE_CRC:
+ return BlockChecksumType.COMPOSITE_CRC;
+ default:
+ throw new IllegalStateException(
+ "BUG: BlockChecksumTypeProto not found, type="
+ + blockChecksumTypeProto);
+ }
+ }
+
+ public static HdfsProtos.BlockChecksumOptionsProto convert(
+ BlockChecksumOptions options) {
+ return HdfsProtos.BlockChecksumOptionsProto.newBuilder()
+ .setBlockChecksumType(convert(options.getBlockChecksumType()))
+ .setStripeLength(options.getStripeLength())
+ .build();
+ }
+
+ public static BlockChecksumOptions convert(
+ HdfsProtos.BlockChecksumOptionsProto options) {
+ return new BlockChecksumOptions(
+ convert(options.getBlockChecksumType()),
+ options.getStripeLength());
+ }
+
public static ExtendedBlockProto convert(final ExtendedBlock b) {
if (b == null) return null;
return ExtendedBlockProto.newBuilder().
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/44] hadoop git commit: HADOOP-15379. Make IrqHandler.bind()
public. Contributed by Ajay Kumar
Posted by xy...@apache.org.
HADOOP-15379. Make IrqHandler.bind() public. Contributed by Ajay Kumar
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec1e8c1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec1e8c1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec1e8c1a
Branch: refs/heads/HDFS-7240
Commit: ec1e8c1a8ca3d01b82ca82d73ba1132b6625d659
Parents: 9b0b9f2
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Thu Apr 12 21:51:20 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Thu Apr 12 21:51:20 2018 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/service/launcher/IrqHandler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec1e8c1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java
index 30bb91c..17aa963 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java
@@ -86,7 +86,7 @@ public final class IrqHandler implements SignalHandler {
* Bind to the interrupt handler.
* @throws IllegalArgumentException if the exception could not be set
*/
- void bind() {
+ public void bind() {
Preconditions.checkState(signal == null, "Handler already bound");
try {
signal = new Signal(name);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/44] hadoop git commit: HDFS-13386. RBF: Wrong date information in
list file(-ls) result. Contributed by Dibyendu Karmakar.
Posted by xy...@apache.org.
HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/044341b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/044341b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/044341b4
Branch: refs/heads/HDFS-7240
Commit: 044341b4e4459b23159e94c7ff0601058398fd70
Parents: b5353c7
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Apr 12 09:30:11 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Apr 12 09:30:11 2018 -0700
----------------------------------------------------------------------
.../federation/router/RouterRpcServer.java | 19 +++++-
.../federation/router/TestRouterMountTable.java | 62 +++++++++++++++++++-
2 files changed, 79 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/044341b4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index e6d2f5e..0dc2a69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -2328,7 +2328,24 @@ public class RouterRpcServer extends AbstractService
*/
private Map<String, Long> getMountPointDates(String path) {
Map<String, Long> ret = new TreeMap<>();
- // TODO add when we have a Mount Table
+ if (subclusterResolver instanceof MountTableResolver) {
+ MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
+ String srcPath;
+ try {
+ final List<String> children = subclusterResolver.getMountPoints(path);
+ for (String child : children) {
+ if (path.equals(Path.SEPARATOR)) {
+ srcPath = Path.SEPARATOR + child;
+ } else {
+ srcPath = path + Path.SEPARATOR + child;
+ }
+ MountTable entry = mountTable.getMountPoint(srcPath);
+ ret.put(child, entry.getDateModified());
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get mount point: {}", e.getMessage());
+ }
+ }
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/044341b4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
index c9e28b1..b33b998 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -17,25 +17,33 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,6 +57,7 @@ public class TestRouterMountTable {
private static NamenodeContext nnContext;
private static RouterContext routerContext;
private static MountTableResolver mountTable;
+ private static ClientProtocol routerProtocol;
@BeforeClass
public static void globalSetUp() throws Exception {
@@ -69,6 +78,7 @@ public class TestRouterMountTable {
nnContext = cluster.getRandomNamenode();
routerContext = cluster.getRandomRouter();
Router router = routerContext.getRouter();
+ routerProtocol = routerContext.getClient().getNamenode();
mountTable = (MountTableResolver) router.getSubclusterResolver();
}
@@ -140,4 +150,54 @@ public class TestRouterMountTable {
return addResponse.getStatus();
}
+
+ /**
+ * Verify that the file/dir listing contains correct date/time information.
+ */
+ @Test
+ public void testListFilesTime() throws Exception {
+ Long beforeCreatingTime = Time.now();
+ // Add mount table entry
+ MountTable addEntry = MountTable.newInstance(
+ "/testdir", Collections.singletonMap("ns0", "/testdir"));
+ assertTrue(addMountTable(addEntry));
+ addEntry = MountTable.newInstance(
+ "/testdir2", Collections.singletonMap("ns0", "/testdir2"));
+ assertTrue(addMountTable(addEntry));
+ addEntry = MountTable.newInstance(
+ "/testdir/subdir", Collections.singletonMap("ns0", "/testdir/subdir"));
+ assertTrue(addMountTable(addEntry));
+
+ // Create test dir in NN
+ final FileSystem nnFs = nnContext.getFileSystem();
+ assertTrue(nnFs.mkdirs(new Path("/newdir")));
+
+ Map<String, Long> pathModTime = new TreeMap<>();
+ for (String mount : mountTable.getMountPoints("/")) {
+ pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
+ .getDateModified());
+ }
+ FileStatus[] iterator = nnFs.listStatus(new Path("/"));
+ for (FileStatus file : iterator) {
+ pathModTime.put(file.getPath().getName(), file.getModificationTime());
+ }
+ // Fetch listing
+ DirectoryListing listing =
+ routerProtocol.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> pathModTimeIterator = pathModTime.keySet().iterator();
+
+ // Match date/time for each path returned
+ for(HdfsFileStatus f : listing.getPartialListing()) {
+ String fileName = pathModTimeIterator.next();
+ String currentFile = f.getFullPath(new Path("/")).getName();
+ Long currentTime = f.getModificationTime();
+ Long expectedTime = pathModTime.get(currentFile);
+
+ assertEquals(currentFile, fileName);
+ assertTrue(currentTime > beforeCreatingTime);
+ assertEquals(currentTime, expectedTime);
+ }
+ // Verify the total number of results found/matched
+ assertEquals(pathModTime.size(), listing.getPartialListing().length);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/44] hadoop git commit: HDFS-13403: libhdfs++ Use hdfs::IoService
object rather than asio::io_service. Contributed by James Clampffer.
Posted by xy...@apache.org.
HDFS-13403: libhdfs++ Use hdfs::IoService object rather than asio::io_service. Contributed by James Clampffer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eefe2a14
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eefe2a14
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eefe2a14
Branch: refs/heads/HDFS-7240
Commit: eefe2a147c83dbb62c4021b67d59d3b9f065f890
Parents: 7eb783e
Author: James Clampffer <jh...@apache.org>
Authored: Wed Apr 11 10:27:23 2018 -0400
Committer: James Clampffer <jh...@apache.org>
Committed: Wed Apr 11 10:27:23 2018 -0400
----------------------------------------------------------------------
.../native/libhdfspp/include/hdfspp/hdfspp.h | 53 +------
.../native/libhdfspp/include/hdfspp/ioservice.h | 140 ++++++++++++++++
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 7 +-
.../native/libhdfspp/lib/common/CMakeLists.txt | 2 +-
.../native/libhdfspp/lib/common/async_stream.h | 13 +-
.../libhdfspp/lib/common/continuation/asio.h | 5 -
.../libhdfspp/lib/common/hdfs_ioservice.cc | 146 -----------------
.../libhdfspp/lib/common/hdfs_ioservice.h | 79 ---------
.../libhdfspp/lib/common/ioservice_impl.cc | 159 +++++++++++++++++++
.../libhdfspp/lib/common/ioservice_impl.h | 76 +++++++++
.../main/native/libhdfspp/lib/common/logging.h | 3 -
.../libhdfspp/lib/common/namenode_info.cc | 15 +-
.../native/libhdfspp/lib/common/namenode_info.h | 8 +-
.../main/native/libhdfspp/lib/common/util.cc | 14 +-
.../src/main/native/libhdfspp/lib/common/util.h | 25 ++-
.../lib/connection/datanodeconnection.cc | 27 +++-
.../lib/connection/datanodeconnection.h | 26 +--
.../main/native/libhdfspp/lib/fs/filehandle.cc | 18 +--
.../main/native/libhdfspp/lib/fs/filehandle.h | 12 +-
.../main/native/libhdfspp/lib/fs/filesystem.cc | 67 ++++++--
.../main/native/libhdfspp/lib/fs/filesystem.h | 66 ++------
.../libhdfspp/lib/fs/namenode_operations.h | 4 +-
.../native/libhdfspp/lib/reader/block_reader.cc | 18 +--
.../native/libhdfspp/lib/reader/block_reader.h | 10 +-
.../native/libhdfspp/lib/reader/datatransfer.h | 4 +-
.../libhdfspp/lib/rpc/namenode_tracker.cc | 2 +-
.../native/libhdfspp/lib/rpc/namenode_tracker.h | 4 +-
.../main/native/libhdfspp/lib/rpc/request.cc | 5 +-
.../native/libhdfspp/lib/rpc/rpc_connection.h | 2 +-
.../libhdfspp/lib/rpc/rpc_connection_impl.cc | 32 ++--
.../libhdfspp/lib/rpc/rpc_connection_impl.h | 9 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 14 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 9 +-
.../native/libhdfspp/tests/bad_datanode_test.cc | 31 ++--
.../libhdfspp/tests/hdfs_ioservice_test.cc | 10 +-
.../native/libhdfspp/tests/mock_connection.h | 4 +-
.../libhdfspp/tests/remote_block_reader_test.cc | 4 +-
.../native/libhdfspp/tests/rpc_engine_test.cc | 112 ++++++-------
38 files changed, 681 insertions(+), 554 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
index 2fdeec9..e68a612 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
@@ -19,6 +19,7 @@
#define LIBHDFSPP_HDFSPP_H_
#include "hdfspp/options.h"
+#include "hdfspp/ioservice.h"
#include "hdfspp/status.h"
#include "hdfspp/events.h"
#include "hdfspp/block_location.h"
@@ -31,62 +32,10 @@
#include <functional>
#include <memory>
-#include <set>
-#include <iostream>
namespace hdfs {
/**
- * An IoService manages a queue of asynchronous tasks. All libhdfs++
- * operations are filed against a particular IoService.
- *
- * When an operation is queued into an IoService, the IoService will
- * run the callback handler associated with the operation. Note that
- * the IoService must be stopped before destructing the objects that
- * post the operations.
- *
- * From an implementation point of view the hdfs::IoService provides
- * a thin wrapper over an asio::io_service object so that additional
- * instrumentation and functionality can be added.
- **/
-
-class IoService : public std::enable_shared_from_this<IoService>
-{
- public:
- static IoService *New();
- static std::shared_ptr<IoService> MakeShared();
- virtual ~IoService();
-
- /**
- * Start up as many threads as there are logical processors.
- * Return number of threads created.
- **/
- virtual unsigned int InitDefaultWorkers() = 0;
-
- /**
- * Initialize with thread_count handler threads.
- * If thread count is less than one print a log message and default to one thread.
- * Return number of threads created.
- **/
- virtual unsigned int InitWorkers(unsigned int thread_count) = 0;
-
- /**
- * Place an item on the execution queue. Will be invoked from outside of the calling context.
- **/
- virtual void PostTask(std::function<void(void)>& asyncTask) = 0;
-
- /**
- * Run the asynchronous tasks associated with this IoService.
- **/
- virtual void Run() = 0;
- /**
- * Stop running asynchronous tasks associated with this IoService.
- * All worker threads will return as soon as they finish executing their current task.
- **/
- virtual void Stop() = 0;
-};
-
-/**
* A node exclusion rule provides a simple way of testing if the
* client should attempt to connect to a node based on the node's
* UUID. The FileSystem and FileHandle use the BadDataNodeTracker
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h
new file mode 100644
index 0000000..9805bad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * An asio::io_service maintains a queue of asynchronous tasks and invokes them
+ * when they are ready to run. Async network IO handlers become runnable when
+ * the associated IO operation has completed. The hdfs::IoService is a thin
+ * wrapper over that object to make it easier to add logging and instrumentation
+ * to tasks that have been queued.
+ *
+ * Lifecycle management:
+ * -The IoService *shall* outlive any tasks it owns. Deleting a task
+ * before it has been run **will** result in dangling reference issues.
+ * -Dependencies (including transitive dependencies) of pending tasks
+ * *shall* outlive the task. Failure to ensure this **will** result in
+ * danging reference issues.
+ * -libhdfs++ uses shared_ptr/weak_ptr heavily as a mechanism to ensure
+ * liveness of dependencies.
+ * -refcounted pointers in lambda capture lists have a poor track record
+ * for ensuring liveness in this library; it's easy to omit them because
+ * the capture list isn't context aware. Developers are encouraged to
+ * write callable classes that explicitly list dependencies.
+ *
+ * Constraints on tasks:
+ * -Tasks and async callbacks *shall* never do blocking IO or sleep().
+ * At best this hurts performance by preventing worker threads from doing
+ * useful work. It may also cause situations that look like deadlocks
+ * if the worker thread is stalled for long enough.
+ * -Tasks and async callbacks *shall* not acquire locks that guard resources
+ * that might be unavailable for an unknown amount of time. Lock acquisition
+ * when accessing shared data structures is acceptable and is often required.
+ * -Tasks and async callbacks *should* not allow exceptions to escape their
+ * scope since tasks will be executed on a different stack then where they
+ * were created. The exception will be caught by the IoService rather than
+ * being forwarded to the next task.
+ * -Tasks and async callbacks *should* not rely on thread local storage for
+ * ancillary context. The IoService does not support any sort of thread
+ * affinity that would guarantee tasks Post()ed from one thread will always
+ * be executed on the same thread. Applications that only use a single
+ * worker thread may use TLS but developers should be mindful that throughput
+ * can no longer be scaled by adding threads.
+ **/
+#ifndef INCLUDE_HDFSPP_IOSERVICE_H_
+#define INCLUDE_HDFSPP_IOSERVICE_H_
+
+#include <memory>
+
+// forward decl
+namespace asio {
+ class io_service;
+}
+
+namespace hdfs {
+
+// (Un)comment this to determine if issues are due to concurrency or logic faults
+// If tests still fail with concurrency disabled it's most likely a logic bug
+#define DISABLE_CONCURRENT_WORKERS
+
+class IoService : public std::enable_shared_from_this<IoService>
+{
+ public:
+ static IoService *New();
+ static std::shared_ptr<IoService> MakeShared();
+ virtual ~IoService();
+
+ /**
+ * Start up as many threads as there are logical processors.
+ * Return number of threads created.
+ **/
+ virtual unsigned int InitDefaultWorkers() = 0;
+
+ /**
+ * Initialize with thread_count handler threads.
+ * If thread count is less than one print a log message and default to one thread.
+ * Return number of threads created.
+ **/
+ virtual unsigned int InitWorkers(unsigned int thread_count) = 0;
+
+ /**
+ * Add a worker thread to existing pool.
+ * Return true on success, false otherwise.
+ **/
+ virtual bool AddWorkerThread() = 0;
+
+ /**
+ * Return the number of worker threads in use.
+ **/
+ virtual unsigned int GetWorkerThreadCount() = 0;
+
+ /**
+ * Enqueue an item for deferred execution. Non-blocking.
+ * Task will be invoked from outside of the calling context.
+ **/
+ virtual void PostTask(std::function<void(void)> asyncTask) = 0;
+
+ /**
+ * Provide type erasure for lambdas defined inside the argument list.
+ **/
+ template <typename LambdaInstance>
+ inline void PostLambda(LambdaInstance&& func)
+ {
+ std::function<void(void)> typeEraser = func;
+ this->PostTask(func);
+ }
+
+ /**
+ * Run the asynchronous tasks associated with this IoService.
+ **/
+ virtual void Run() = 0;
+ /**
+ * Stop running asynchronous tasks associated with this IoService.
+ * All worker threads will return as soon as they finish executing their current task.
+ **/
+ virtual void Stop() = 0;
+
+ /**
+ * Access underlying io_service object. Only to be used in asio library calls.
+ * After HDFS-11884 is complete only tests should need direct access to the asio::io_service.
+ **/
+ virtual asio::io_service& GetRaw() = 0;
+};
+
+
+} // namespace hdfs
+#endif // include guard
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 9a7c8b4..6b2468f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -17,18 +17,17 @@
*/
#include "hdfspp/hdfspp.h"
+#include "hdfspp/hdfs_ext.h"
-#include "fs/filesystem.h"
#include "common/hdfs_configuration.h"
#include "common/configuration_loader.h"
#include "common/logging.h"
+#include "fs/filesystem.h"
+#include "fs/filehandle.h"
-#include <hdfs/hdfs.h>
-#include <hdfspp/hdfs_ext.h>
#include <libgen.h>
#include "limits.h"
-
#include <string>
#include <cstring>
#include <iostream>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index 5d9e52c..1ab04d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -19,6 +19,6 @@ if(NEED_LINK_DL)
set(LIB_DL dl)
endif()
-add_library(common_obj OBJECT status.cc sasl_digest_md5.cc hdfs_ioservice.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc config_parser.cc)
+add_library(common_obj OBJECT status.cc sasl_digest_md5.cc ioservice_impl.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc config_parser.cc)
add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
target_link_libraries(common ${LIB_DL})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/async_stream.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/async_stream.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/async_stream.h
index 575904c..efe2e1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/async_stream.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/async_stream.h
@@ -19,12 +19,15 @@
#ifndef LIB_COMMON_ASYNC_STREAM_H_
#define LIB_COMMON_ASYNC_STREAM_H_
-#include <asio.hpp>
+#include <asio/buffer.hpp>
+#include <asio/error_code.hpp>
+#include <functional>
namespace hdfs {
-typedef asio::mutable_buffers_1 MutableBuffers;
-typedef asio::const_buffers_1 ConstBuffers;
+// Contiguous buffer types
+typedef asio::mutable_buffers_1 MutableBuffer;
+typedef asio::const_buffers_1 ConstBuffer;
/*
* asio-compatible stream implementation.
@@ -35,11 +38,11 @@ typedef asio::const_buffers_1 ConstBuffers;
*/
class AsyncStream {
public:
- virtual void async_read_some(const MutableBuffers &buf,
+ virtual void async_read_some(const MutableBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) = 0;
- virtual void async_write_some(const ConstBuffers &buf,
+ virtual void async_write_some(const ConstBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) = 0;
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
index 193358f..0215176 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h
@@ -20,13 +20,8 @@
#include "continuation.h"
#include "common/util.h"
-
#include "hdfspp/status.h"
-
-#include <asio/connect.hpp>
-#include <asio/read.hpp>
#include <asio/write.hpp>
-#include <asio/ip/tcp.hpp>
#include <memory>
namespace hdfs {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
deleted file mode 100644
index 578b782..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.cc
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "hdfs_ioservice.h"
-
-#include <thread>
-#include <mutex>
-#include <vector>
-
-#include "common/logging.h"
-
-namespace hdfs {
-
-IoService::~IoService() {}
-
-IoService *IoService::New() {
- return new IoServiceImpl();
-}
-
-std::shared_ptr<IoService> IoService::MakeShared() {
- return std::make_shared<IoServiceImpl>();
-}
-
-
-unsigned int IoServiceImpl::InitDefaultWorkers() {
- LOG_TRACE(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers@" << this << " called.");
- unsigned int logical_thread_count = std::thread::hardware_concurrency();
-#ifndef DISABLE_CONCURRENT_WORKERS
- if(logical_thread_count < 1) {
- LOG_WARN(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers did not detect any logical processors. Defaulting to 1 worker thread.");
- } else {
- LOG_DEBUG(kRPC, << "IoServiceImpl::InitDefaultWorkers detected " << logical_thread_count << " logical threads and will spawn a worker for each.");
- }
-#else
- if(logical_thread_count > 0) {
- LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers: " << logical_thread_count << " threads available. Concurrent workers are disabled so 1 worker thread will be used");
- }
- logical_thread_count = 1;
-#endif
- return InitWorkers(logical_thread_count);
-}
-
-unsigned int IoServiceImpl::InitWorkers(unsigned int thread_count) {
-#ifdef DISABLED_CONCURRENT_WORKERS
- LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl::InitWorkers: " << thread_count << " threads specified but concurrent workers are disabled so 1 will be used");
- thread_count = 1;
-#endif
- unsigned int created_threads = 0;
- for(unsigned int i=0; i<thread_count; i++) {
- bool created = AddWorkerThread();
- if(created) {
- created_threads++;
- } else {
- LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl@" << this << " ::InitWorkers failed to create a worker thread");
- }
- }
- if(created_threads != thread_count) {
- LOG_WARN(kAsyncRuntime, << "IoServiceImpl@" << this << " ::InitWorkers attempted to create "
- << thread_count << " but only created " << created_threads
- << " worker threads. Make sure this process has adequate resources.");
- }
- return created_threads;
-}
-
-bool IoServiceImpl::AddWorkerThread() {
- mutex_guard state_lock(state_lock_);
- auto async_worker = [this]() {
- this->ThreadStartHook();
- this->Run();
- this->ThreadExitHook();
- };
- worker_threads_.push_back(WorkerPtr( new std::thread(async_worker)) );
- return true;
-}
-
-
-void IoServiceImpl::ThreadStartHook() {
- mutex_guard state_lock(state_lock_);
- LOG_DEBUG(kAsyncRuntime, << "Worker thread #" << std::this_thread::get_id() << " for IoServiceImpl@" << this << " starting");
-}
-
-void IoServiceImpl::ThreadExitHook() {
- mutex_guard state_lock(state_lock_);
- LOG_DEBUG(kAsyncRuntime, << "Worker thread #" << std::this_thread::get_id() << " for IoServiceImpl@" << this << " exiting");
-}
-
-void IoServiceImpl::PostTask(std::function<void(void)>& asyncTask) {
- io_service_.post(asyncTask);
-}
-
-void IoServiceImpl::WorkerDeleter::operator()(std::thread *t) {
- // It is far too easy to destroy the filesystem (and thus the threadpool)
- // from within one of the worker threads, leading to a deadlock. Let's
- // provide some explicit protection.
- if(t->get_id() == std::this_thread::get_id()) {
- LOG_ERROR(kAsyncRuntime, << "FileSystemImpl::WorkerDeleter::operator(treadptr="
- << t << ") : FATAL: Attempted to destroy a thread pool"
- "from within a callback of the thread pool!");
- }
- t->join();
- delete t;
-}
-
-// As long as this just forwards to an asio::io_service method it doesn't need a lock
-void IoServiceImpl::Run() {
- // The IoService executes callbacks provided by library users in the context of worker threads,
- // there is no way of preventing those callbacks from throwing but we can at least prevent them
- // from escaping this library and crashing the process.
-
- // As recommended in http://www.boost.org/doc/libs/1_39_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.effect_of_exceptions_thrown_from_handlers
- asio::io_service::work work(io_service_);
- while(true)
- {
- try
- {
- io_service_.run();
- break;
- } catch (const std::exception & e) {
- LOG_WARN(kFileSystem, << "Unexpected exception in libhdfspp worker thread: " << e.what());
- } catch (...) {
- LOG_WARN(kFileSystem, << "Unexpected value not derived from std::exception in libhdfspp worker thread");
- }
- }
-}
-
-unsigned int IoServiceImpl::get_worker_thread_count() {
- mutex_guard state_lock(state_lock_);
- return worker_threads_.size();
-}
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
deleted file mode 100644
index 294252b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/hdfs_ioservice.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef COMMON_HDFS_IOSERVICE_H_
-#define COMMON_HDFS_IOSERVICE_H_
-
-#include "hdfspp/hdfspp.h"
-
-#include <asio/io_service.hpp>
-#include "common/util.h"
-
-#include <mutex>
-#include <thread>
-
-namespace hdfs {
-
-// Uncomment this to determine if issues are due to concurrency or logic faults
-// If tests still fail with concurrency disabled it's most likely a logic bug
-#define DISABLE_CONCURRENT_WORKERS
-
-/*
- * A thin wrapper over the asio::io_service with a few extras
- * -manages it's own worker threads
- * -some helpers for sharing with multiple modules that need to do async work
- */
-
-class IoServiceImpl : public IoService {
- public:
- IoServiceImpl() {}
-
- virtual unsigned int InitDefaultWorkers() override;
- virtual unsigned int InitWorkers(unsigned int thread_count) override;
- virtual void PostTask(std::function<void(void)>& asyncTask) override;
- virtual void Run() override;
- virtual void Stop() override { io_service_.stop(); }
-
- // Add a single worker thread, in the common case try to avoid this in favor
- // of Init[Default]Workers. Public for use by tests and rare cases where a
- // client wants very explicit control of threading for performance reasons
- // e.g. pinning threads to NUMA nodes.
- bool AddWorkerThread();
-
- // Be very careful about using this: HDFS-10241
- ::asio::io_service &io_service() { return io_service_; }
- unsigned int get_worker_thread_count();
- private:
- std::mutex state_lock_;
- ::asio::io_service io_service_;
-
- // For doing logging + resource manager updates on thread start/exit
- void ThreadStartHook();
- void ThreadExitHook();
-
- // Support for async worker threads
- struct WorkerDeleter {
- void operator()(std::thread *t);
- };
- typedef std::unique_ptr<std::thread, WorkerDeleter> WorkerPtr;
- std::vector<WorkerPtr> worker_threads_;
-};
-
-}
-
-#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.cc
new file mode 100644
index 0000000..de081ed
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.cc
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ioservice_impl.h"
+
+#include <thread>
+#include <mutex>
+#include <vector>
+
+#include "common/util.h"
+#include "common/logging.h"
+
+
+namespace hdfs {
+
+IoService::~IoService() {}
+
+IoService *IoService::New() {
+ return new IoServiceImpl();
+}
+
+std::shared_ptr<IoService> IoService::MakeShared() {
+ return std::make_shared<IoServiceImpl>();
+}
+
+
+unsigned int IoServiceImpl::InitDefaultWorkers() {
+ LOG_TRACE(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers@" << this << " called.");
+ unsigned int logical_thread_count = std::thread::hardware_concurrency();
+#ifndef DISABLE_CONCURRENT_WORKERS
+ if(logical_thread_count < 1) {
+ LOG_WARN(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers did not detect any logical processors. Defaulting to 1 worker thread.");
+ } else {
+ LOG_DEBUG(kRPC, << "IoServiceImpl::InitDefaultWorkers detected " << logical_thread_count << " logical threads and will spawn a worker for each.");
+ }
+#else
+ if(logical_thread_count > 0) {
+ LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl::InitDefaultWorkers: " << logical_thread_count << " threads available. Concurrent workers are disabled so 1 worker thread will be used");
+ }
+ logical_thread_count = 1;
+#endif
+ return InitWorkers(logical_thread_count);
+}
+
+unsigned int IoServiceImpl::InitWorkers(unsigned int thread_count) {
+#ifdef DISABLED_CONCURRENT_WORKERS
+ LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl::InitWorkers: " << thread_count << " threads specified but concurrent workers are disabled so 1 will be used");
+ thread_count = 1;
+#endif
+ unsigned int created_threads = 0;
+ for(unsigned int i=0; i<thread_count; i++) {
+ bool created = AddWorkerThread();
+ if(created) {
+ created_threads++;
+ } else {
+ LOG_DEBUG(kAsyncRuntime, << "IoServiceImpl@" << this << " ::InitWorkers failed to create a worker thread");
+ }
+ }
+ if(created_threads != thread_count) {
+ LOG_WARN(kAsyncRuntime, << "IoServiceImpl@" << this << " ::InitWorkers attempted to create "
+ << thread_count << " but only created " << created_threads
+ << " worker threads. Make sure this process has adequate resources.");
+ }
+ return created_threads;
+}
+
+bool IoServiceImpl::AddWorkerThread() {
+ mutex_guard state_lock(state_lock_);
+ auto async_worker = [this]() {
+ this->ThreadStartHook();
+ this->Run();
+ this->ThreadExitHook();
+ };
+ worker_threads_.push_back(WorkerPtr( new std::thread(async_worker)) );
+ return true;
+}
+
+
+void IoServiceImpl::ThreadStartHook() {
+ mutex_guard state_lock(state_lock_);
+ LOG_DEBUG(kAsyncRuntime, << "Worker thread #" << std::this_thread::get_id() << " for IoServiceImpl@" << this << " starting");
+}
+
+void IoServiceImpl::ThreadExitHook() {
+ mutex_guard state_lock(state_lock_);
+ LOG_DEBUG(kAsyncRuntime, << "Worker thread #" << std::this_thread::get_id() << " for IoServiceImpl@" << this << " exiting");
+}
+
+void IoServiceImpl::PostTask(std::function<void(void)> asyncTask) {
+ io_service_.post(asyncTask);
+}
+
+void IoServiceImpl::WorkerDeleter::operator()(std::thread *t) {
+ // It is far too easy to destroy the filesystem (and thus the threadpool)
+ // from within one of the worker threads, leading to a deadlock. Let's
+ // provide some explicit protection.
+ if(t->get_id() == std::this_thread::get_id()) {
+ LOG_ERROR(kAsyncRuntime, << "FileSystemImpl::WorkerDeleter::operator(treadptr="
+ << t << ") : FATAL: Attempted to destroy a thread pool"
+ "from within a callback of the thread pool!");
+ }
+ t->join();
+ delete t;
+}
+
+// As long as this just forwards to an asio::io_service method it doesn't need a lock
+void IoServiceImpl::Run() {
+ // The IoService executes callbacks provided by library users in the context of worker threads,
+ // there is no way of preventing those callbacks from throwing but we can at least prevent them
+ // from escaping this library and crashing the process.
+
+ // As recommended in http://www.boost.org/doc/libs/1_39_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.effect_of_exceptions_thrown_from_handlers
+ asio::io_service::work work(io_service_);
+ while(true)
+ {
+ try
+ {
+ io_service_.run();
+ break;
+ } catch (const std::exception & e) {
+ LOG_WARN(kFileSystem, << "Unexpected exception in libhdfspp worker thread: " << e.what());
+ } catch (...) {
+ LOG_WARN(kFileSystem, << "Caught unexpected value not derived from std::exception in libhdfspp worker thread");
+ }
+ }
+}
+
+void IoServiceImpl::Stop() {
+ // Note: This doesn't wait for running operations to stop.
+ io_service_.stop();
+}
+
+asio::io_service& IoServiceImpl::GetRaw() {
+ return io_service_;
+}
+
+unsigned int IoServiceImpl::GetWorkerThreadCount() {
+ mutex_guard state_lock(state_lock_);
+ return worker_threads_.size();
+
+}
+
+
+} // namespace hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.h
new file mode 100644
index 0000000..a29985c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.h
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_HDFS_IOSERVICE_H_
+#define COMMON_HDFS_IOSERVICE_H_
+
+#include "hdfspp/ioservice.h"
+
+#include <asio/io_service.hpp>
+#include "common/new_delete.h"
+
+#include <mutex>
+#include <thread>
+
+namespace hdfs {
+
+/*
+ * A thin wrapper over the asio::io_service with a few extras
+ * -manages it's own worker threads
+ * -some helpers for sharing with multiple modules that need to do async work
+ */
+
+class IoServiceImpl : public IoService {
+ public:
+ MEMCHECKED_CLASS(IoServiceImpl)
+ IoServiceImpl() {}
+
+ unsigned int InitDefaultWorkers() override;
+ unsigned int InitWorkers(unsigned int thread_count) override;
+ void PostTask(std::function<void(void)> asyncTask) override;
+ void Run() override;
+ void Stop() override;
+ asio::io_service& GetRaw() override;
+
+ // Add a single worker thread, in the common case try to avoid this in favor
+ // of Init[Default]Workers. Public for use by tests and rare cases where a
+ // client wants very explicit control of threading for performance reasons
+ // e.g. pinning threads to NUMA nodes.
+ bool AddWorkerThread() override;
+
+ unsigned int GetWorkerThreadCount() override;
+
+ private:
+ std::mutex state_lock_;
+ ::asio::io_service io_service_;
+
+ // For doing logging + resource manager updates on thread start/exit
+ void ThreadStartHook();
+ void ThreadExitHook();
+
+ // Support for async worker threads
+ struct WorkerDeleter {
+ void operator()(std::thread *t);
+ };
+ typedef std::unique_ptr<std::thread, WorkerDeleter> WorkerPtr;
+ std::vector<WorkerPtr> worker_threads_;
+};
+
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
index 69f9c6e..4e66a93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
@@ -23,14 +23,11 @@
#include "hdfspp/log.h"
-#include <iostream>
#include <sstream>
#include <mutex>
#include <memory>
#include <thread>
-#include <asio/ip/tcp.hpp>
-
namespace hdfs {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
index d29f1e9..a04daf1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
@@ -20,11 +20,12 @@
#include "common/util.h"
#include "common/logging.h"
+#include "hdfspp/ioservice.h"
#include <sstream>
#include <utility>
#include <future>
-#include <memory>
+
namespace hdfs {
@@ -35,8 +36,6 @@ ResolvedNamenodeInfo& ResolvedNamenodeInfo::operator=(const NamenodeInfo &info)
return *this;
}
-
-
std::string ResolvedNamenodeInfo::str() const {
std::stringstream ss;
ss << "ResolvedNamenodeInfo {nameservice: " << nameservice << ", name: " << name << ", uri: " << uri.str();
@@ -58,7 +57,7 @@ std::string ResolvedNamenodeInfo::str() const {
}
-bool ResolveInPlace(::asio::io_service *ioservice, ResolvedNamenodeInfo &info) {
+bool ResolveInPlace(std::shared_ptr<IoService> ioservice, ResolvedNamenodeInfo &info) {
// this isn't very memory friendly, but if it needs to be called often there are bigger issues at hand
info.endpoints.clear();
std::vector<ResolvedNamenodeInfo> resolved = BulkResolve(ioservice, {info});
@@ -76,7 +75,7 @@ typedef std::vector<asio::ip::tcp::endpoint> endpoint_vector;
// RAII wrapper
class ScopedResolver {
private:
- ::asio::io_service *io_service_;
+ std::shared_ptr<IoService> io_service_;
std::string host_;
std::string port_;
::asio::ip::tcp::resolver::query query_;
@@ -86,8 +85,8 @@ class ScopedResolver {
// Caller blocks on access if resolution isn't finished
std::shared_ptr<std::promise<Status>> result_status_;
public:
- ScopedResolver(::asio::io_service *service, const std::string &host, const std::string &port) :
- io_service_(service), host_(host), port_(port), query_(host, port), resolver_(*io_service_)
+ ScopedResolver(std::shared_ptr<IoService> service, const std::string &host, const std::string &port) :
+ io_service_(service), host_(host), port_(port), query_(host, port), resolver_(io_service_->GetRaw())
{
if(!io_service_)
LOG_ERROR(kAsyncRuntime, << "ScopedResolver@" << this << " passed nullptr to io_service");
@@ -140,7 +139,7 @@ class ScopedResolver {
}
};
-std::vector<ResolvedNamenodeInfo> BulkResolve(::asio::io_service *ioservice, const std::vector<NamenodeInfo> &nodes) {
+std::vector<ResolvedNamenodeInfo> BulkResolve(std::shared_ptr<IoService> ioservice, const std::vector<NamenodeInfo> &nodes) {
std::vector< std::unique_ptr<ScopedResolver> > resolvers;
resolvers.reserve(nodes.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h
index fdee8d7..f43690d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h
@@ -20,6 +20,7 @@
#define COMMON_HDFS_NAMENODE_INFO_H_
#include <asio.hpp>
+
#include <hdfspp/options.h>
#include <string>
@@ -27,6 +28,9 @@
namespace hdfs {
+// Forward decl
+class IoService;
+
// Internal representation of namenode info that keeps track
// of its endpoints.
struct ResolvedNamenodeInfo : public NamenodeInfo {
@@ -38,11 +42,11 @@ struct ResolvedNamenodeInfo : public NamenodeInfo {
// Clear endpoints if set and resolve all of them in parallel.
// Only successful lookups will be placed in the result set.
-std::vector<ResolvedNamenodeInfo> BulkResolve(::asio::io_service *ioservice, const std::vector<NamenodeInfo> &nodes);
+std::vector<ResolvedNamenodeInfo> BulkResolve(std::shared_ptr<IoService> ioservice, const std::vector<NamenodeInfo> &nodes);
// Clear endpoints, if any, and resolve them again
// Return true if endpoints were resolved
-bool ResolveInPlace(::asio::io_service *ioservice, ResolvedNamenodeInfo &info);
+bool ResolveInPlace(std::shared_ptr<IoService> ioservice, ResolvedNamenodeInfo &info);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
index 375f951..6a07987 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
@@ -19,17 +19,25 @@
#include "common/util.h"
#include "common/util_c.h"
+#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <exception>
#include <sstream>
-#include <iostream>
#include <iomanip>
#include <thread>
namespace hdfs {
+Status ToStatus(const ::asio::error_code &ec) {
+ if (ec) {
+ return Status(ec.value(), ec.message().c_str());
+ } else {
+ return Status::OK();
+ }
+}
+
bool ReadDelimitedPBMessage(::google::protobuf::io::CodedInputStream *in,
::google::protobuf::MessageLite *msg) {
uint32_t size = 0;
@@ -60,6 +68,10 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL
return buf;
}
+int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) {
+ size_t size = msg->ByteSize();
+ return ::google::protobuf::io::CodedOutputStream::VarintSize32(size) + size;
+}
std::string GetRandomClientName() {
std::vector<unsigned char>buf(8);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
index be902bd..590ba54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
@@ -21,16 +21,19 @@
#include "hdfspp/status.h"
#include "common/logging.h"
-#include <sstream>
#include <mutex>
#include <string>
#include <asio/error_code.hpp>
#include <openssl/rand.h>
-
-#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/coded_stream.h>
-#include <asio.hpp>
+
+
+namespace google {
+ namespace protobuf {
+ class MessageLite;
+ }
+}
namespace hdfs {
@@ -38,20 +41,11 @@ namespace hdfs {
typedef std::lock_guard<std::mutex> mutex_guard;
-static inline Status ToStatus(const ::asio::error_code &ec) {
- if (ec) {
- return Status(ec.value(), ec.message().c_str());
- } else {
- return Status::OK();
- }
-}
+Status ToStatus(const ::asio::error_code &ec);
// Determine size of buffer that needs to be allocated in order to serialize msg
// in delimited format
-static inline int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) {
- size_t size = msg->ByteSize();
- return ::google::protobuf::io::CodedOutputStream::VarintSize32(size) + size;
-}
+int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg);
// Construct msg from the input held in the CodedInputStream
// return false on failure, otherwise return true
@@ -84,7 +78,6 @@ bool lock_held(T & mutex) {
std::string SafeDisconnect(asio::ip::tcp::socket *sock);
-
// The following helper function is used for classes that look like the following:
//
// template <typename socket_like_object>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.cc
index 27cd666..4142482 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.cc
@@ -24,14 +24,14 @@ namespace hdfs {
DataNodeConnection::~DataNodeConnection(){}
DataNodeConnectionImpl::~DataNodeConnectionImpl(){}
-DataNodeConnectionImpl::DataNodeConnectionImpl(asio::io_service * io_service,
- const ::hadoop::hdfs::DatanodeInfoProto &dn_proto,
- const hadoop::common::TokenProto *token,
- LibhdfsEvents *event_handlers) : event_handlers_(event_handlers)
+DataNodeConnectionImpl::DataNodeConnectionImpl(std::shared_ptr<IoService> io_service,
+ const ::hadoop::hdfs::DatanodeInfoProto &dn_proto,
+ const hadoop::common::TokenProto *token,
+ LibhdfsEvents *event_handlers) : event_handlers_(event_handlers)
{
using namespace ::asio::ip;
- conn_.reset(new tcp::socket(*io_service));
+ conn_.reset(new tcp::socket(io_service->GetRaw()));
auto datanode_addr = dn_proto.id();
endpoints_[0] = tcp::endpoint(address::from_string(datanode_addr.ipaddr()),
datanode_addr.xferport());
@@ -68,5 +68,22 @@ void DataNodeConnectionImpl::Cancel() {
}
}
+void DataNodeConnectionImpl::async_read_some(const MutableBuffer &buf,
+ std::function<void (const asio::error_code & error, std::size_t bytes_transferred) > handler)
+{
+ event_handlers_->call("DN_read_req", "", "", buf.end() - buf.begin());
+
+ mutex_guard state_lock(state_lock_);
+ conn_->async_read_some(buf, handler);
+}
+
+void DataNodeConnectionImpl::async_write_some(const ConstBuffer &buf,
+ std::function<void (const asio::error_code & error, std::size_t bytes_transferred) > handler)
+{
+ event_handlers_->call("DN_write_req", "", "", buf.end() - buf.begin());
+
+ mutex_guard state_lock(state_lock_);
+ conn_->async_write_some(buf, handler);
+}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
index 21193b3..a54338f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
@@ -18,7 +18,7 @@
#ifndef LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
#define LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
-#include "common/hdfs_ioservice.h"
+#include "hdfspp/ioservice.h"
#include "common/async_stream.h"
#include "ClientNamenodeProtocol.pb.h"
#include "common/libhdfs_events_impl.h"
@@ -58,13 +58,14 @@ private:
// held (briefly) while posting async ops to the asio task queue
std::mutex state_lock_;
public:
+ MEMCHECKED_CLASS(DataNodeConnectionImpl)
std::unique_ptr<asio::ip::tcp::socket, SocketDeleter> conn_;
std::array<asio::ip::tcp::endpoint, 1> endpoints_;
std::string uuid_;
LibhdfsEvents *event_handlers_;
virtual ~DataNodeConnectionImpl();
- DataNodeConnectionImpl(asio::io_service * io_service, const ::hadoop::hdfs::DatanodeInfoProto &dn_proto,
+ DataNodeConnectionImpl(std::shared_ptr<IoService> io_service, const ::hadoop::hdfs::DatanodeInfoProto &dn_proto,
const hadoop::common::TokenProto *token,
LibhdfsEvents *event_handlers);
@@ -72,24 +73,11 @@ public:
void Cancel() override;
- void async_read_some(const MutableBuffers &buf,
- std::function<void (const asio::error_code & error, std::size_t bytes_transferred) > handler)
- override {
- event_handlers_->call("DN_read_req", "", "", buf.end() - buf.begin());
+ void async_read_some(const MutableBuffer &buf,
+ std::function<void (const asio::error_code & error, std::size_t bytes_transferred) > handler) override;
-
- mutex_guard state_lock(state_lock_);
- conn_->async_read_some(buf, handler);
- };
-
- void async_write_some(const ConstBuffers &buf,
- std::function<void (const asio::error_code & error, std::size_t bytes_transferred) > handler)
- override {
- event_handlers_->call("DN_write_req", "", "", buf.end() - buf.begin());
-
- mutex_guard state_lock(state_lock_);
- conn_->async_write_some(buf, handler);
- }
+ void async_write_some(const ConstBuffer &buf,
+ std::function<void (const asio::error_code & error, std::size_t bytes_transferred) > handler) override;
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index ba702b0..02630fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -36,10 +36,10 @@ FileHandle::~FileHandle() {}
FileHandleImpl::FileHandleImpl(const std::string & cluster_name,
const std::string & path,
- ::asio::io_service *io_service, const std::string &client_name,
- const std::shared_ptr<const struct FileInfo> file_info,
- std::shared_ptr<BadDataNodeTracker> bad_data_nodes,
- std::shared_ptr<LibhdfsEvents> event_handlers)
+ std::shared_ptr<IoService> io_service, const std::string &client_name,
+ const std::shared_ptr<const struct FileInfo> file_info,
+ std::shared_ptr<BadDataNodeTracker> bad_data_nodes,
+ std::shared_ptr<LibhdfsEvents> event_handlers)
: cluster_name_(cluster_name), path_(path), io_service_(io_service), client_name_(client_name), file_info_(file_info),
bad_node_tracker_(bad_data_nodes), offset_(0), cancel_state_(CancelTracker::New()), event_handlers_(event_handlers), bytes_read_(0) {
LOG_TRACE(kFileHandle, << "FileHandleImpl::FileHandleImpl("
@@ -167,7 +167,7 @@ bool FileHandleImpl::CheckSeekBounds(ssize_t desired_position) {
* on the FileHandle
*/
void FileHandleImpl::AsyncPreadSome(
- size_t offset, const MutableBuffers &buffers,
+ size_t offset, const MutableBuffer &buffer,
std::shared_ptr<NodeExclusionRule> excluded_nodes,
const std::function<void(const Status &, const std::string &, size_t)> handler) {
using ::hadoop::hdfs::DatanodeInfoProto;
@@ -233,7 +233,7 @@ void FileHandleImpl::AsyncPreadSome(
uint64_t offset_within_block = offset - block->offset();
uint64_t size_within_block = std::min<uint64_t>(
- block->b().numbytes() - offset_within_block, asio::buffer_size(buffers));
+ block->b().numbytes() - offset_within_block, asio::buffer_size(buffer));
LOG_DEBUG(kFileHandle, << "FileHandleImpl::AsyncPreadSome("
<< FMT_THIS_ADDR << "), ...) Datanode hostname=" << dnHostName << ", IP Address=" << dnIpAddr
@@ -268,7 +268,7 @@ void FileHandleImpl::AsyncPreadSome(
handler(status, dn_id, transferred);
};
- auto connect_handler = [handler,event_handlers,cluster_name,path,read_handler,block,offset_within_block,size_within_block, buffers, reader, dn_id, client_name]
+ auto connect_handler = [handler,event_handlers,cluster_name,path,read_handler,block,offset_within_block,size_within_block, buffer, reader, dn_id, client_name]
(Status status, std::shared_ptr<DataNodeConnection> dn) {
(void)dn;
event_response event_resp = event_handlers->call(FILE_DN_CONNECT_EVENT, cluster_name.c_str(), path.c_str(), 0);
@@ -281,7 +281,7 @@ void FileHandleImpl::AsyncPreadSome(
if (status.ok()) {
reader->AsyncReadBlock(
client_name, *block, offset_within_block,
- asio::buffer(buffers, size_within_block), read_handler);
+ asio::buffer(buffer, size_within_block), read_handler);
} else {
handler(status, dn_id, 0);
}
@@ -307,7 +307,7 @@ std::shared_ptr<BlockReader> FileHandleImpl::CreateBlockReader(const BlockReader
}
std::shared_ptr<DataNodeConnection> FileHandleImpl::CreateDataNodeConnection(
- ::asio::io_service * io_service,
+ std::shared_ptr<IoService> io_service,
const ::hadoop::hdfs::DatanodeInfoProto & dn,
const hadoop::common::TokenProto * token) {
LOG_TRACE(kFileHandle, << "FileHandleImpl::CreateDataNodeConnection("
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
index 4135156..57da237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
@@ -18,7 +18,7 @@
#ifndef LIBHDFSPP_LIB_FS_FILEHANDLE_H_
#define LIBHDFSPP_LIB_FS_FILEHANDLE_H_
-#include "common/hdfs_ioservice.h"
+#include "hdfspp/ioservice.h"
#include "common/async_stream.h"
#include "common/cancel_tracker.h"
#include "common/libhdfs_events_impl.h"
@@ -26,12 +26,10 @@
#include "reader/fileinfo.h"
#include "reader/readergroup.h"
-#include "asio.hpp"
#include "bad_datanode_tracker.h"
#include "ClientNamenodeProtocol.pb.h"
#include <mutex>
-#include <iostream>
namespace hdfs {
@@ -53,7 +51,7 @@ public:
MEMCHECKED_CLASS(FileHandleImpl)
FileHandleImpl(const std::string & cluster_name,
const std::string & path,
- ::asio::io_service *io_service, const std::string &client_name,
+ std::shared_ptr<IoService> io_service, const std::string &client_name,
const std::shared_ptr<const struct FileInfo> file_info,
std::shared_ptr<BadDataNodeTracker> bad_data_nodes,
std::shared_ptr<LibhdfsEvents> event_handlers);
@@ -93,7 +91,7 @@ public:
* If trying to begin a read past the EOF, status will be Status::InvalidOffset.
*
*/
- void AsyncPreadSome(size_t offset, const MutableBuffers &buffers,
+ void AsyncPreadSome(size_t offset, const MutableBuffer &buffer,
std::shared_ptr<NodeExclusionRule> excluded_nodes,
const std::function<void(const Status &status,
const std::string &dn_id, size_t bytes_read)> handler);
@@ -124,13 +122,13 @@ protected:
std::shared_ptr<DataNodeConnection> dn,
std::shared_ptr<hdfs::LibhdfsEvents> event_handlers);
virtual std::shared_ptr<DataNodeConnection> CreateDataNodeConnection(
- ::asio::io_service *io_service,
+ std::shared_ptr<IoService> io_service,
const ::hadoop::hdfs::DatanodeInfoProto & dn,
const hadoop::common::TokenProto * token);
private:
const std::string cluster_name_;
const std::string path_;
- ::asio::io_service * const io_service_;
+ std::shared_ptr<IoService> io_service_;
const std::string client_name_;
const std::shared_ptr<const struct FileInfo> file_info_;
std::shared_ptr<BadDataNodeTracker> bad_node_tracker_;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index 56d02d8..41cc645 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -18,6 +18,7 @@
#include "filesystem.h"
+#include "filehandle.h"
#include "common/namenode_info.h"
#include <functional>
@@ -104,6 +105,54 @@ FileSystem *FileSystem::New() {
* FILESYSTEM IMPLEMENTATION
****************************************************************************/
+struct FileSystemImpl::FindSharedState {
+ //Name pattern (can have wild-cards) to find
+ const std::string name;
+ //Maximum depth to recurse after the end of path is reached.
+ //Can be set to 0 for pure path globbing and ignoring name pattern entirely.
+ const uint32_t maxdepth;
+ //Vector of all sub-directories from the path argument (each can have wild-cards)
+ std::vector<std::string> dirs;
+ //Callback from Find
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> handler;
+ //outstanding_requests is incremented once for every GetListing call.
+ std::atomic<uint64_t> outstanding_requests;
+ //Boolean needed to abort all recursion on error or on user command
+ std::atomic<bool> aborted;
+ //Shared variables will need protection with a lock
+ std::mutex lock;
+ FindSharedState(const std::string path_, const std::string name_, const uint32_t maxdepth_,
+ const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> handler_,
+ uint64_t outstanding_recuests_, bool aborted_)
+ : name(name_),
+ maxdepth(maxdepth_),
+ handler(handler_),
+ outstanding_requests(outstanding_recuests_),
+ aborted(aborted_),
+ lock() {
+ //Constructing the list of sub-directories
+ std::stringstream ss(path_);
+ if(path_.back() != '/'){
+ ss << "/";
+ }
+ for (std::string token; std::getline(ss, token, '/'); ) {
+ dirs.push_back(token);
+ }
+ }
+};
+
+struct FileSystemImpl::FindOperationalState {
+ const std::string path;
+ const uint32_t depth;
+ const bool search_path;
+ FindOperationalState(const std::string path_, const uint32_t depth_, const bool search_path_)
+ : path(path_),
+ depth(depth_),
+ search_path(search_path_) {
+ }
+};
+
+
const std::string get_effective_user_name(const std::string &user_name) {
if (!user_name.empty())
return user_name;
@@ -134,10 +183,10 @@ const std::string get_effective_user_name(const std::string &user_name) {
}
FileSystemImpl::FileSystemImpl(IoService *&io_service, const std::string &user_name, const Options &options) :
- io_service_(static_cast<IoServiceImpl *>(io_service)), options_(options),
+ io_service_(io_service), options_(options),
client_name_(GetRandomClientName()),
nn_(
- &io_service_->io_service(), options, client_name_,
+ io_service_, options, client_name_,
get_effective_user_name(user_name), kNamenodeProtocol,
kNamenodeProtocolVersion
),
@@ -166,10 +215,10 @@ FileSystemImpl::FileSystemImpl(IoService *&io_service, const std::string &user_n
}
FileSystemImpl::FileSystemImpl(std::shared_ptr<IoService> io_service, const std::string& user_name, const Options &options) :
- io_service_(std::static_pointer_cast<IoServiceImpl>(io_service)), options_(options),
+ io_service_(io_service), options_(options),
client_name_(GetRandomClientName()),
nn_(
- &io_service_->io_service(), options, client_name_,
+ io_service_, options, client_name_,
get_effective_user_name(user_name), kNamenodeProtocol,
kNamenodeProtocolVersion
),
@@ -178,7 +227,7 @@ FileSystemImpl::FileSystemImpl(std::shared_ptr<IoService> io_service, const std:
{
LOG_DEBUG(kFileSystem, << "FileSystemImpl::FileSystemImpl("
<< FMT_THIS_ADDR << ", shared IoService@" << io_service_.get() << ") called");
- int worker_thread_count = io_service_->get_worker_thread_count();
+ int worker_thread_count = io_service_->GetWorkerThreadCount();
if(worker_thread_count < 1) {
LOG_WARN(kFileSystem, << "FileSystemImpl::FileSystemImpl IoService provided doesn't have any worker threads. "
<< "It needs at least 1 worker to connect to an HDFS cluster.")
@@ -217,7 +266,7 @@ void FileSystemImpl::Connect(const std::string &server,
auto name_service = options_.services.find(server);
if(name_service != options_.services.end()) {
cluster_name_ = name_service->first;
- resolved_namenodes = BulkResolve(&io_service_->io_service(), name_service->second);
+ resolved_namenodes = BulkResolve(io_service_, name_service->second);
} else {
cluster_name_ = server + ":" + service;
@@ -230,7 +279,7 @@ void FileSystemImpl::Connect(const std::string &server,
handler(Status::Error(("Invalid namenode " + cluster_name_ + " in config").c_str()), this);
}
- resolved_namenodes = BulkResolve(&io_service_->io_service(), {tmp_info});
+ resolved_namenodes = BulkResolve(io_service_, {tmp_info});
}
for(unsigned int i=0;i<resolved_namenodes.size();i++) {
@@ -282,7 +331,7 @@ int FileSystemImpl::WorkerThreadCount() {
if(!io_service_) {
return -1;
} else {
- return io_service_->get_worker_thread_count();
+ return io_service_->GetWorkerThreadCount();
}
}
@@ -339,7 +388,7 @@ void FileSystemImpl::Open(
LOG_DEBUG(kFileSystem, << "Operation not allowed on standby datanode");
}
}
- handler(stat, stat.ok() ? new FileHandleImpl(cluster_name_, path, &io_service_->io_service(), client_name_, file_info, bad_node_tracker_, event_handlers_)
+ handler(stat, stat.ok() ? new FileHandleImpl(cluster_name_, path, io_service_, client_name_, file_info, bad_node_tracker_, event_handlers_)
: nullptr);
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
index f2e9abd..935e7c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h
@@ -18,19 +18,18 @@
#ifndef LIBHDFSPP_LIB_FS_FILESYSTEM_H_
#define LIBHDFSPP_LIB_FS_FILESYSTEM_H_
-#include "filehandle.h"
-#include "hdfspp/hdfspp.h"
+#include "namenode_operations.h"
#include "fs/bad_datanode_tracker.h"
-#include "reader/block_reader.h"
+#include "hdfspp/hdfspp.h"
#include "reader/fileinfo.h"
-#include "asio.hpp"
-
#include <thread>
-#include "namenode_operations.h"
namespace hdfs {
+class FileHandle;
+
+
/*
* FileSystem: The consumer's main point of interaction with the cluster as
* a whole.
@@ -48,6 +47,7 @@ public:
MEMCHECKED_CLASS(FileSystemImpl)
typedef std::function<void(const Status &, FileSystem *)> ConnectCallback;
+ // Note: Longer term it'd be cleaner to take a rvalue reference to a shared_ptr to get ownership
explicit FileSystemImpl(IoService *&io_service, const std::string& user_name, const Options &options);
explicit FileSystemImpl(std::shared_ptr<IoService>, const std::string& user_name, const Options &options);
~FileSystemImpl() override;
@@ -215,7 +215,7 @@ private:
* A side effect of this is that requests may outlive the RpcEngine they
* reference.
**/
- std::shared_ptr<IoServiceImpl> io_service_;
+ std::shared_ptr<IoService> io_service_;
const Options options_;
const std::string client_name_;
std::string cluster_name_;
@@ -234,53 +234,11 @@ private:
void GetListingShim(const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more,
std::string path, const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler);
-
- struct FindSharedState {
- //Name pattern (can have wild-cards) to find
- const std::string name;
- //Maximum depth to recurse after the end of path is reached.
- //Can be set to 0 for pure path globbing and ignoring name pattern entirely.
- const uint32_t maxdepth;
- //Vector of all sub-directories from the path argument (each can have wild-cards)
- std::vector<std::string> dirs;
- //Callback from Find
- const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> handler;
- //outstanding_requests is incremented once for every GetListing call.
- std::atomic<uint64_t> outstanding_requests;
- //Boolean needed to abort all recursion on error or on user command
- std::atomic<bool> aborted;
- //Shared variables will need protection with a lock
- std::mutex lock;
- FindSharedState(const std::string path_, const std::string name_, const uint32_t maxdepth_,
- const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> handler_,
- uint64_t outstanding_recuests_, bool aborted_)
- : name(name_),
- maxdepth(maxdepth_),
- handler(handler_),
- outstanding_requests(outstanding_recuests_),
- aborted(aborted_),
- lock() {
- //Constructing the list of sub-directories
- std::stringstream ss(path_);
- if(path_.back() != '/'){
- ss << "/";
- }
- for (std::string token; std::getline(ss, token, '/'); ) {
- dirs.push_back(token);
- }
- }
- };
-
- struct FindOperationalState {
- const std::string path;
- const uint32_t depth;
- const bool search_path;
- FindOperationalState(const std::string path_, const uint32_t depth_, const bool search_path_)
- : path(path_),
- depth(depth_),
- search_path(search_path_) {
- }
- };
+ /**
+ * Helper struct to store state for recursive find
+ */
+ struct FindSharedState;
+ struct FindOperationalState;
void FindShim(const Status &stat, const std::vector<StatInfo> & stat_infos,
bool directory_has_more, std::shared_ptr<FindOperationalState> current_state, std::shared_ptr<FindSharedState> shared_state);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index f4caa18..3470a48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -42,7 +42,7 @@ namespace hdfs {
class NameNodeOperations {
public:
MEMCHECKED_CLASS(NameNodeOperations)
- NameNodeOperations(::asio::io_service *io_service, const Options &options,
+ NameNodeOperations(std::shared_ptr<IoService> io_service, const Options &options,
const std::string &client_name, const std::string &user_name,
const char *protocol_name, int protocol_version) :
io_service_(io_service),
@@ -119,7 +119,7 @@ private:
static void DirectoryListingProtoToStatInfo(std::shared_ptr<std::vector<StatInfo>> stat_infos, const ::hadoop::hdfs::DirectoryListingProto & dl);
static void GetFsStatsResponseProtoToFsInfo(hdfs::FsInfo & fs_info, const std::shared_ptr<::hadoop::hdfs::GetFsStatsResponseProto> & fs);
- ::asio::io_service * io_service_;
+ std::shared_ptr<IoService> io_service_;
// This is the only permanent owner of the RpcEngine, however the RPC layer
// needs to reference count it prevent races during FileSystem destruction.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
index ca7715d..90c02f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
@@ -431,7 +431,7 @@ private:
std::shared_ptr<DataNodeConnection> shared_conn_;
};
-void BlockReaderImpl::AsyncReadPacket(const MutableBuffers &buffers,
+void BlockReaderImpl::AsyncReadPacket(const MutableBuffer &buffer,
const std::function<void(const Status &, size_t bytes_transferred)> &handler)
{
assert(state_ != kOpen && "Not connected");
@@ -450,7 +450,7 @@ void BlockReaderImpl::AsyncReadPacket(const MutableBuffers &buffers,
.Push(new ReadChecksum(this))
.Push(new ReadPadding(this))
.Push(new ReadData(
- this, m->state().bytes_transferred, buffers))
+ this, m->state().bytes_transferred, buffer))
.Push(new AckRead(this));
auto self = this->shared_from_this();
@@ -460,14 +460,14 @@ void BlockReaderImpl::AsyncReadPacket(const MutableBuffers &buffers,
}
-size_t BlockReaderImpl::ReadPacket(const MutableBuffers &buffers, Status *status)
+size_t BlockReaderImpl::ReadPacket(const MutableBuffer &buffer, Status *status)
{
LOG_TRACE(kBlockReader, << "BlockReaderImpl::ReadPacket called");
size_t transferred = 0;
auto done = std::make_shared<std::promise<void>>();
auto future = done->get_future();
- AsyncReadPacket(buffers,
+ AsyncReadPacket(buffer,
[status, &transferred, done](const Status &stat, size_t t) {
*status = stat;
transferred = t;
@@ -504,7 +504,7 @@ private:
struct BlockReaderImpl::ReadBlockContinuation : continuation::Continuation
{
- ReadBlockContinuation(BlockReader *reader, MutableBuffers buffer, size_t *transferred)
+ ReadBlockContinuation(BlockReader *reader, MutableBuffer buffer, size_t *transferred)
: reader_(reader), buffer_(buffer), buffer_size_(asio::buffer_size(buffer)), transferred_(transferred) {}
virtual void Run(const Next &next) override {
@@ -517,7 +517,7 @@ struct BlockReaderImpl::ReadBlockContinuation : continuation::Continuation
private:
BlockReader *reader_;
- const MutableBuffers buffer_;
+ const MutableBuffer buffer_;
const size_t buffer_size_;
size_t *transferred_;
std::function<void(const Status &)> next_;
@@ -542,7 +542,7 @@ void BlockReaderImpl::AsyncReadBlock(
const std::string & client_name,
const hadoop::hdfs::LocatedBlockProto &block,
size_t offset,
- const MutableBuffers &buffers,
+ const MutableBuffer &buffer,
const std::function<void(const Status &, size_t)> handler)
{
LOG_TRACE(kBlockReader, << "BlockReaderImpl::AsyncReadBlock("
@@ -551,10 +551,10 @@ void BlockReaderImpl::AsyncReadBlock(
auto m = continuation::Pipeline<size_t>::Create(cancel_state_);
size_t * bytesTransferred = &m->state();
- size_t size = asio::buffer_size(buffers);
+ size_t size = asio::buffer_size(buffer);
m->Push(new RequestBlockContinuation(this, client_name, &block.b(), size, offset))
- .Push(new ReadBlockContinuation(this, buffers, bytesTransferred));
+ .Push(new ReadBlockContinuation(this, buffer, bytesTransferred));
m->Run([handler] (const Status &status, const size_t totalBytesTransferred) {
handler(status, totalBytesTransferred);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
index b5cbdf5..167c57d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
@@ -72,11 +72,11 @@ public:
virtual void AsyncReadBlock(
const std::string & client_name,
const hadoop::hdfs::LocatedBlockProto &block, size_t offset,
- const MutableBuffers &buffers,
+ const MutableBuffer &buffer,
const std::function<void(const Status &, size_t)> handler) = 0;
virtual void AsyncReadPacket(
- const MutableBuffers &buffers,
+ const MutableBuffer &buffer,
const std::function<void(const Status &, size_t bytes_transferred)> &handler) = 0;
virtual void AsyncRequestBlock(
@@ -98,7 +98,7 @@ public:
chunk_padding_bytes_(0), cancel_state_(cancel_state), event_handlers_(event_handlers.get()) {}
virtual void AsyncReadPacket(
- const MutableBuffers &buffers,
+ const MutableBuffer &buffer,
const std::function<void(const Status &, size_t bytes_transferred)> &handler) override;
virtual void AsyncRequestBlock(
@@ -111,12 +111,12 @@ public:
virtual void AsyncReadBlock(
const std::string & client_name,
const hadoop::hdfs::LocatedBlockProto &block, size_t offset,
- const MutableBuffers &buffers,
+ const MutableBuffer &buffer,
const std::function<void(const Status &, size_t)> handler) override;
virtual void CancelOperation() override;
- size_t ReadPacket(const MutableBuffers &buffers, Status *status);
+ size_t ReadPacket(const MutableBuffer &buffer, Status *status);
Status RequestBlock(
const std::string &client_name,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h
index 93103c5..2b36f59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h
@@ -44,13 +44,13 @@ public:
template <class Handler> void Handshake(const Handler &next);
- void async_read_some(const MutableBuffers &buf,
+ void async_read_some(const MutableBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
stream_->async_read_some(buf, handler);
}
- void async_write_some(const ConstBuffers &buf,
+ void async_write_some(const ConstBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
stream_->async_write_some(buf, handler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
index e83a28c..242c6ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.cc
@@ -35,7 +35,7 @@ static std::string format_endpoints(const std::vector<::asio::ip::tcp::endpoint>
}
HANamenodeTracker::HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
- ::asio::io_service *ioservice,
+ std::shared_ptr<IoService> ioservice,
std::shared_ptr<LibhdfsEvents> event_handlers)
: enabled_(false), resolved_(false),
ioservice_(ioservice), event_handlers_(event_handlers)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
index cc34f51..032b1d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/namenode_tracker.h
@@ -40,7 +40,7 @@ namespace hdfs {
class HANamenodeTracker {
public:
HANamenodeTracker(const std::vector<ResolvedNamenodeInfo> &servers,
- ::asio::io_service *ioservice,
+ std::shared_ptr<IoService> ioservice,
std::shared_ptr<LibhdfsEvents> event_handlers_);
virtual ~HANamenodeTracker();
@@ -66,7 +66,7 @@ class HANamenodeTracker {
bool resolved_;
// Keep service in case a second round of DNS lookup is required
- ::asio::io_service *ioservice_;
+ std::shared_ptr<IoService> ioservice_;
// Event handlers, for now this is the simplest place to catch all failover events
// and push info out to client application. Possibly move into RPCEngine.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
index 356411e..9157476 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
@@ -20,6 +20,7 @@
#include "request.h"
#include "rpc_engine.h"
#include "sasl_protocol.h"
+#include "hdfspp/ioservice.h"
#include "RpcHeader.pb.h"
#include "ProtobufRpcEngine.pb.h"
@@ -118,7 +119,7 @@ Request::Request(std::shared_ptr<LockFreeRpcEngine> engine, const std::string &m
: engine_(engine),
method_name_(method_name),
call_id_(call_id),
- timer_(engine->io_service()),
+ timer_(engine->io_service()->GetRaw()),
handler_(std::move(handler)),
retry_count_(engine->retry_policy() ? 0 : kNoRetry),
failover_count_(0)
@@ -129,7 +130,7 @@ Request::Request(std::shared_ptr<LockFreeRpcEngine> engine, const std::string &m
Request::Request(std::shared_ptr<LockFreeRpcEngine> engine, Handler &&handler)
: engine_(engine),
call_id_(-1/*Handshake ID*/),
- timer_(engine->io_service()),
+ timer_(engine->io_service()->GetRaw()),
handler_(std::move(handler)),
retry_count_(engine->retry_policy() ? 0 : kNoRetry),
failover_count_(0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index 9e54983..9f7b3bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -83,7 +83,7 @@ class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
void SetAuthInfo(const AuthInfo& auth_info);
std::weak_ptr<LockFreeRpcEngine> engine() { return engine_; }
- ::asio::io_service *GetIoService();
+ std::shared_ptr<IoService> GetIoService();
protected:
struct Response {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
index ee9b704..43111ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -70,27 +70,27 @@ RpcConnection::RpcConnection(std::shared_ptr<LockFreeRpcEngine> engine)
: engine_(engine),
connected_(kNotYetConnected) {}
-::asio::io_service *RpcConnection::GetIoService() {
+std::shared_ptr<IoService> RpcConnection::GetIoService() {
std::shared_ptr<LockFreeRpcEngine> pinnedEngine = engine_.lock();
if(!pinnedEngine) {
LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid RpcEngine");
return nullptr;
}
- return &pinnedEngine->io_service();
+ return pinnedEngine->io_service();
}
void RpcConnection::StartReading() {
auto shared_this = shared_from_this();
- ::asio::io_service *service = GetIoService();
+ std::shared_ptr<IoService> service = GetIoService();
if(!service) {
LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid IoService");
return;
}
- service->post([shared_this, this] () {
- OnRecvCompleted(::asio::error_code(), 0);
- });
+ service->PostLambda(
+ [shared_this, this] () { OnRecvCompleted(::asio::error_code(), 0); }
+ );
}
void RpcConnection::HandshakeComplete(const Status &s) {
@@ -164,13 +164,14 @@ void RpcConnection::ContextComplete(const Status &s) {
void RpcConnection::AsyncFlushPendingRequests() {
std::shared_ptr<RpcConnection> shared_this = shared_from_this();
- ::asio::io_service *service = GetIoService();
+ std::shared_ptr<IoService> service = GetIoService();
if(!service) {
LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid IoService");
return;
}
- service->post([shared_this, this]() {
+ std::function<void()> task = [shared_this, this]()
+ {
std::lock_guard<std::mutex> state_lock(connection_state_lock_);
LOG_TRACE(kRPC, << "RpcConnection::AsyncFlushPendingRequests called (connected=" << ToString(connected_) << ")");
@@ -178,7 +179,10 @@ void RpcConnection::AsyncFlushPendingRequests() {
if (!outgoing_request_) {
FlushPendingRequests();
}
- });
+ };
+
+ service->PostTask(task);
+
}
Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
@@ -228,15 +232,17 @@ Status RpcConnection::HandleRpcResponse(std::shared_ptr<Response> response) {
return status;
}
- ::asio::io_service *service = GetIoService();
+ std::shared_ptr<IoService> service = GetIoService();
if(!service) {
LOG_ERROR(kRPC, << "RpcConnection@" << this << " attempted to access invalid IoService");
return Status::Error("RpcConnection attempted to access invalid IoService");
}
- service->post([req, response, status]() {
- req->OnResponseArrived(response->in.get(), status); // Never call back while holding a lock
- });
+ service->PostLambda(
+ [req, response, status]() {
+ req->OnResponseArrived(response->in.get(), status); // Never call back while holding a lock
+ }
+ );
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/44] hadoop git commit: YARN-7941. Transitive dependencies for
component are not resolved. Contributed by Billie Rinaldi.
Posted by xy...@apache.org.
YARN-7941. Transitive dependencies for component are not resolved. Contributed by Billie Rinaldi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0487110
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0487110
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0487110
Branch: refs/heads/HDFS-7240
Commit: c0487110990958fa985d273eb178bdf76002cf3a
Parents: d919eb6
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Apr 11 09:18:50 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Apr 11 09:18:50 2018 +0530
----------------------------------------------------------------------
.../hadoop/yarn/service/component/Component.java | 1 +
.../hadoop/yarn/service/TestYarnNativeServices.java | 16 ++++++++++++----
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0487110/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index a2127c8..39897f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -179,6 +179,7 @@ public class Component implements EventHandler<ComponentEvent> {
maxContainerFailurePerComp = componentSpec.getConfiguration()
.getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10);
createNumCompInstances(component.getNumberOfContainers());
+ setDesiredContainers(component.getNumberOfContainers().intValue());
}
private void createNumCompInstances(long count) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0487110/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 2b44701..5e267bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -166,7 +166,9 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// Create compa with 2 containers
// Create compb with 2 containers which depends on compa
- // Check containers for compa started before containers for compb
+ // Create compc with 2 containers which depends on compb
+ // Check containers for compa started before containers for compb before
+ // containers for compc
@Test (timeout = 200000)
public void testComponentStartOrder() throws Exception {
setupInternal(NUM_NMS);
@@ -175,17 +177,23 @@ public class TestYarnNativeServices extends ServiceTestUtils {
exampleApp.setName("teststartorder");
exampleApp.setVersion("v1");
exampleApp.addComponent(createComponent("compa", 2, "sleep 1000"));
- Component compb = createComponent("compb", 2, "sleep 1000");
- // Let compb depedends on compa;
+ // Let compb depend on compa
+ Component compb = createComponent("compb", 2, "sleep 1000");
compb.setDependencies(Collections.singletonList("compa"));
exampleApp.addComponent(compb);
+ // Let compc depend on compb
+ Component compc = createComponent("compc", 2, "sleep 1000");
+ compc.setDependencies(Collections.singletonList("compb"));
+ exampleApp.addComponent(compc);
+
client.actionCreate(exampleApp);
waitForServiceToBeStable(client, exampleApp);
// check that containers for compa are launched before containers for compb
- checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
+ checkContainerLaunchDependencies(client, exampleApp, "compa", "compb",
+ "compc");
client.actionStop(exampleApp.getName(), true);
client.actionDestroy(exampleApp.getName());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/44] hadoop git commit: YARN-8133. Doc link broken for
yarn-service from overview page. (Rohith Sharma K S via wangda)
Posted by xy...@apache.org.
YARN-8133. Doc link broken for yarn-service from overview page. (Rohith Sharma K S via wangda)
Change-Id: Iacf9a004585dd59e1c0cd8f8c618a38f047cc0fe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d919eb6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d919eb6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d919eb6e
Branch: refs/heads/HDFS-7240
Commit: d919eb6efa1072517017c75fb323e391f4418dc8
Parents: 2bf9cc2
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Apr 10 17:33:15 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Apr 10 17:33:15 2018 -0700
----------------------------------------------------------------------
.../src/site/markdown/yarn-service/Concepts.md | 6 +++---
.../src/site/markdown/yarn-service/Overview.md | 14 +++++++-------
.../src/site/markdown/yarn-service/QuickStart.md | 12 ++++++------
.../src/site/markdown/yarn-service/RegistryDNS.md | 4 ++--
.../site/markdown/yarn-service/ServiceDiscovery.md | 4 ++--
5 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
index e567d03..5c77f17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
@@ -29,7 +29,7 @@ affinity and anti-affinity scheduling, log aggregation for services, automatical
A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users
from dealing with the low-level APIs, writing complex code to bring their services onto YARN. The REST layer acts as a unified REST based entry for
creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex,
-multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.md) for detailed API documentations.
+multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.html) for detailed API documentations.
The API-server is stateless, which means users can simply spin up multiple instances, and have a load balancer fronting them to
support HA, distribute the load etc.
@@ -37,10 +37,10 @@ support HA, distribute the load etc.
### Service Discovery
A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup.
-The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.md). And the DNS server essentially exposes the
+The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.html). And the DNS server essentially exposes the
information in YARN service registry by translating them into DNS records such as A record and SRV record.
Clients can then discover the IPs of containers via standard DNS lookup.
The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult
-to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [Service Discovery doc](ServiceDiscovery.md)
+to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [Service Discovery doc](ServiceDiscovery.html)
for more details.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 72c2d3e..8e2bf9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -46,16 +46,16 @@ The benefits of combining these workloads are two-fold:
*`This feature is in alpha state`* and so APIs, command lines are subject to change. We will continue to update the documents over time.
-[QuickStart](QuickStart.md) shows a quick tutorial that walks you through simple steps to deploy a service on YARN.
+[QuickStart](QuickStart.html) shows a quick tutorial that walks you through simple steps to deploy a service on YARN.
## How do I get my hands dirty?
-* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
-* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
-* [Service Discovery](ServiceDiscovery.md): Describes the service discovery mechanism on YARN.
-* [Registry DNS](RegistryDNS.md): Deep dives into the Registry DNS internals.
-* [Examples](Examples.md): List some example service definitions (`Yarnfile`).
-* [Configurations](Configurations.md): Describes how to configure the custom services on YARN.
+* [Concepts](Concepts.html): Describes the internals of the framework and some features in YARN core to support running services on YARN.
+* [Service REST API](YarnServiceAPI.html): The API doc for deploying/managing services on YARN.
+* [Service Discovery](ServiceDiscovery.html): Describes the service discovery mechanism on YARN.
+* [Registry DNS](RegistryDNS.html): Deep dives into the Registry DNS internals.
+* [Examples](Examples.html): List some example service definitions (`Yarnfile`).
+* [Configurations](Configurations.html): Describes how to configure the custom services on YARN.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 2981a9a..f563193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -51,11 +51,11 @@ e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
```
yarn app -launch my-sleeper sleeper
```
-For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
+For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.html).
## Manage services on YARN via CLI
Below steps walk you through deploying a services on YARN using CLI.
-Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options.
+Refer to [Yarn Commands](../YarnCommands.html) for the full list of commands and options.
### Deploy a service
```
yarn app -launch ${SERVICE_NAME} ${PATH_TO_SERVICE_DEF_FILE}
@@ -121,7 +121,7 @@ For running inside ResourceManager, add this property to `yarn-site.xml` and res
Services can be deployed on YARN through the ResourceManager web endpoint.
-Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
+Refer to [API doc](YarnServiceAPI.html) for the detailed API specificatiosn.
### Deploy a service
@@ -194,7 +194,7 @@ A new `service` tab is added in the YARN UI2 specially to show YARN Services in
The services framework posts the data into TimelineService and the `service` UI reads data from TimelineService to render its content.
### Enable Timeline Service v2
-Please refer to [TimeLineService v2 doc](../TimelineServiceV2.md) for how to enable Timeline Service v2.
+Please refer to [TimeLineService v2 doc](../TimelineServiceV2.html) for how to enable Timeline Service v2.
### Enable new YARN UI
@@ -229,7 +229,7 @@ E.g. A typical configuration looks like below:
# Run with Docker
The above example is only for a non-docker container based service. YARN Service Framework also provides first-class support for managing docker based services.
Most of the steps for managing docker based services are the same except that in docker the `Artifact` type for a component is `DOCKER` and the Artifact `id` is the name of the docker image.
-For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.md).
+For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.html).
With docker support, it also opens up a set of new possibilities to implement features such as discovering service containers on YARN with DNS.
-Check [ServiceDiscovery](ServiceDiscovery.md) for more details.
+Check [ServiceDiscovery](ServiceDiscovery.html) for more details.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
index c7f19ce..2307e5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
@@ -15,7 +15,7 @@
# Registry DNS Server
<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-The document describes the internals of Registry DNS server. It is based on the [YARN service registry](../registry/index.md) which is backed by a zookeeper cluster.
+The document describes the internals of Registry DNS server. It is based on the [YARN service registry](../registry/index.html) which is backed by a zookeeper cluster.
## Introduction
The Registry DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
@@ -153,7 +153,7 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
| Name | Description |
| ------------ | ------------- |
-|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [YARN registry](../registry/registry-configuration.md). |
+|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [YARN registry](../registry/registry-configuration.html). |
| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
index 8542899..f351e23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -18,7 +18,7 @@ This document describes the mechanism of service discovery on YARN and the
steps for enabling it.
## Overview
-A [DNS server](RegistryDNS.md) is implemented to enable discovering services on YARN via
+A [DNS server](RegistryDNS.html) is implemented to enable discovering services on YARN via
the standard mechanism: DNS lookup.
The framework ApplicationMaster posts the container information such as hostname and IP address into
@@ -71,7 +71,7 @@ assigned `0` since it is the first and only instance for the `hbasemaster` compo
## Configure Registry DNS
Below is the set of configurations in `yarn-site.xml` required for enabling Registry DNS. A full list of properties can be found in the Configuration
-section of [Registry DNS](RegistryDNS.md).
+section of [Registry DNS](RegistryDNS.html).
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/44] hadoop git commit: MAPREDUCE-7077. Pipe mapreduce job fails
with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)
Posted by xy...@apache.org.
MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)
Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4
(cherry picked from commit 035e0f97ea44b0495707949a781d8792dcf6ea6b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/995cba65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/995cba65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/995cba65
Branch: refs/heads/HDFS-7240
Commit: 995cba65fe29966583e36f9491d9a27b323918ae
Parents: 9031a76
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Apr 12 14:33:33 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Apr 13 13:52:10 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapred/pipes/Application.java | 5 ++---
.../hadoop/mapred/pipes/TestPipeApplication.java | 19 ++++++++-----------
2 files changed, 10 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
index 5c8aab9..83d2509 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskLog;
-import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
@@ -104,8 +103,8 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
// This password is used as shared secret key between this application and
// child pipes process
byte[] password = jobToken.getPassword();
- String localPasswordFile = new File(conf.get(MRConfig.LOCAL_DIR))
- + Path.SEPARATOR + "jobTokenPassword";
+ String localPasswordFile = new File(".") + Path.SEPARATOR
+ + "jobTokenPassword";
writePasswordToLocalFile(localPasswordFile, password, conf);
env.put("hadoop.pipes.shared.secret.location", localPasswordFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
index 13597e0..88d8f95 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.IFile.Writer;
-import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapred.Counters;
@@ -84,10 +83,10 @@ public class TestPipeApplication {
public void testRunner() throws Exception {
// clean old password files
- JobConf conf = new JobConf();
- File[] psw = cleanTokenPasswordFile(conf);
+ File[] psw = cleanTokenPasswordFile();
try {
RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner();
+ JobConf conf = new JobConf();
conf.set(Submitter.IS_JAVA_RR, "true");
// for stdour and stderror
@@ -163,7 +162,7 @@ public class TestPipeApplication {
TestTaskReporter reporter = new TestTaskReporter();
- File[] psw = cleanTokenPasswordFile(conf);
+ File[] psw = cleanTokenPasswordFile();
try {
conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName);
@@ -248,7 +247,7 @@ public class TestPipeApplication {
JobConf conf = new JobConf();
- File[] psw = cleanTokenPasswordFile(conf);
+ File[] psw = cleanTokenPasswordFile();
System.setProperty("test.build.data",
"target/tmp/build/TEST_SUBMITTER_MAPPER/data");
@@ -389,8 +388,8 @@ public class TestPipeApplication {
@Test
public void testPipesReduser() throws Exception {
+ File[] psw = cleanTokenPasswordFile();
JobConf conf = new JobConf();
- File[] psw = cleanTokenPasswordFile(conf);
try {
Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
@@ -507,16 +506,14 @@ public class TestPipeApplication {
}
- private File[] cleanTokenPasswordFile(JobConf conf) throws Exception {
+ private File[] cleanTokenPasswordFile() throws Exception {
File[] result = new File[2];
- result[0] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
- + "jobTokenPassword");
+ result[0] = new File("./jobTokenPassword");
if (result[0].exists()) {
FileUtil.chmod(result[0].getAbsolutePath(), "700");
assertTrue(result[0].delete());
}
- result[1] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
- + ".jobTokenPassword.crc");
+ result[1] = new File("./.jobTokenPassword.crc");
if (result[1].exists()) {
FileUtil.chmod(result[1].getAbsolutePath(), "700");
result[1].delete();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/44] hadoop git commit: HDFS-7101. Potential null dereference in
DFSck#doWork(). Contributed by skrho and Ted Yu.
Posted by xy...@apache.org.
HDFS-7101. Potential null dereference in DFSck#doWork(). Contributed by skrho and Ted Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/113af12c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/113af12c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/113af12c
Branch: refs/heads/HDFS-7240
Commit: 113af12cfb240ea9a7189bb2701693466eb8e993
Parents: 832852c
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Apr 12 17:47:37 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Apr 12 17:47:37 2018 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/113af12c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 96fca24..10b0012 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -354,7 +354,7 @@ public class DFSck extends Configured implements Tool {
BufferedReader input = new BufferedReader(new InputStreamReader(
stream, "UTF-8"));
String line = null;
- String lastLine = null;
+ String lastLine = NamenodeFsck.CORRUPT_STATUS;
int errCode = -1;
try {
while ((line = input.readLine()) != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/44] hadoop git commit: YARN-8153. Guaranteed containers always
stay in SCHEDULED on NM after restart. Contributed by Yang Wang.
Posted by xy...@apache.org.
YARN-8153. Guaranteed containers always stay in SCHEDULED on NM after restart. Contributed by Yang Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/226bedc0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/226bedc0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/226bedc0
Branch: refs/heads/HDFS-7240
Commit: 226bedc0239ba23f3ca0c40dac6aab3777d3ada6
Parents: 375654c
Author: Weiwei Yang <ww...@apache.org>
Authored: Fri Apr 13 13:17:37 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Fri Apr 13 13:17:37 2018 +0800
----------------------------------------------------------------------
.../scheduler/ContainerScheduler.java | 7 ++-
.../TestContainerManagerRecovery.java | 51 ++++++++++++++++++++
2 files changed, 56 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/226bedc0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index d9b713f..57368ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -501,8 +501,11 @@ public class ContainerScheduler extends AbstractService implements
private void startContainer(Container container) {
LOG.info("Starting container [" + container.getContainerId()+ "]");
- runningContainers.put(container.getContainerId(), container);
- this.utilizationTracker.addContainerResources(container);
+ // Skip to put into runningContainers and addUtilization when recover
+ if (!runningContainers.containsKey(container.getContainerId())) {
+ runningContainers.put(container.getContainerId(), container);
+ this.utilizationTracker.addContainerResources(container);
+ }
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
this.metrics.startOpportunisticContainer(container.getResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/226bedc0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index cad835c..bf8b500 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -91,6 +92,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
@@ -439,6 +441,54 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
assertNotNull(app);
containerStatus = getContainerStatus(context, cm, cid);
assertEquals(targetResource, containerStatus.getCapability());
+ cm.stop();
+ }
+
+ @Test
+ public void testContainerSchedulerRecovery() throws Exception {
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
+ NMStateStoreService stateStore = new NMMemoryStateStoreService();
+ stateStore.init(conf);
+ stateStore.start();
+ context = createContext(conf, stateStore);
+ ContainerManagerImpl cm = createContainerManager(context, delSrvc);
+ ((NMContext) context).setContainerManager(cm);
+ cm.init(conf);
+ cm.start();
+ // add an application by starting a container
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId attemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId cid = ContainerId.newContainerId(attemptId, 1);
+
+ commonLaunchContainer(appId, cid, cm);
+
+ Application app = context.getApplications().get(appId);
+ assertNotNull(app);
+
+ ResourceUtilization utilization =
+ ResourceUtilization.newInstance(1024, 2048, 0.25F);
+ assertEquals(cm.getContainerScheduler().getNumRunningContainers(), 1);
+ assertEquals(utilization,
+ cm.getContainerScheduler().getCurrentUtilization());
+
+ // restart and verify container scheduler has recovered correctly
+ cm.stop();
+ context = createContext(conf, stateStore);
+ cm = createContainerManager(context, delSrvc);
+ ((NMContext) context).setContainerManager(cm);
+ cm.init(conf);
+ cm.start();
+ assertEquals(1, context.getApplications().size());
+ app = context.getApplications().get(appId);
+ assertNotNull(app);
+ waitForNMContainerState(cm, cid, ContainerState.RUNNING);
+
+ assertEquals(cm.getContainerScheduler().getNumRunningContainers(), 1);
+ assertEquals(utilization,
+ cm.getContainerScheduler().getCurrentUtilization());
+ cm.stop();
}
@Test
@@ -494,6 +544,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
resourceMappings.getAssignedResources("numa").equals(numaResources));
Assert.assertTrue(
resourceMappings.getAssignedResources("fpga").equals(fpgaResources));
+ cm.stop();
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/44] hadoop git commit: YARN-7936. Add default service AM Xmx.
Contributed by Jian He
Posted by xy...@apache.org.
YARN-7936. Add default service AM Xmx. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53b3e594
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53b3e594
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53b3e594
Branch: refs/heads/HDFS-7240
Commit: 53b3e594732e7a567dda4e08b9a8af5f87a4472a
Parents: 1884459
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Apr 12 11:35:14 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Thu Apr 12 12:38:00 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/service/client/ServiceClient.java | 9 +++++++--
.../apache/hadoop/yarn/service/conf/YarnServiceConf.java | 2 ++
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53b3e594/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 4c7b72d..21fb075 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -758,8 +758,13 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
CLI.forceIPv4().headless();
- CLI.setJVMOpts(YarnServiceConf.get(YarnServiceConf.JVM_OPTS, null,
- app.getConfiguration(), conf));
+ String jvmOpts = YarnServiceConf
+ .get(YarnServiceConf.JVM_OPTS, "", app.getConfiguration(), conf);
+ if (!jvmOpts.contains("-Xmx")) {
+ jvmOpts += DEFAULT_AM_JVM_XMX;
+ }
+
+ CLI.setJVMOpts(jvmOpts);
if (hasSliderAMLog4j) {
CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53b3e594/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 14c4d15..3dd5a7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -38,6 +38,8 @@ public class YarnServiceConf {
public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
+ public static final String DEFAULT_AM_JVM_XMX = " -Xmx768m ";
+
public static final String YARN_QUEUE = "yarn.service.queue";
public static final String API_SERVER_ADDRESS = "yarn.service.api-server.address";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/44] hadoop git commit: YARN-8120. JVM can crash with SIGSEGV when
exiting due to custom leveldb logger. Contributed by Jason Lowe.
Posted by xy...@apache.org.
YARN-8120. JVM can crash with SIGSEGV when exiting due to custom leveldb logger. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bb128df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bb128df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bb128df
Branch: refs/heads/HDFS-7240
Commit: 6bb128dfb893cf0e4aa2d3ecc65440668a1fc8d7
Parents: d272056
Author: Eric E Payne <er...@oath.com>
Authored: Thu Apr 12 16:04:23 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Thu Apr 12 16:04:23 2018 +0000
----------------------------------------------------------------------
.../v2/hs/HistoryServerLeveldbStateStoreService.java | 11 -----------
.../java/org/apache/hadoop/mapred/ShuffleHandler.java | 12 ------------
.../recovery/NMLeveldbStateStoreService.java | 12 ------------
.../resourcemanager/recovery/LeveldbRMStateStore.java | 12 ------------
4 files changed, 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb128df/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
index c8741aa..6d2e407 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
@@ -74,7 +74,6 @@ public class HistoryServerLeveldbStateStoreService extends
Path storeRoot = createStorageDir(getConfig());
Options options = new Options();
options.createIfMissing(false);
- options.logger(new LeveldbLogger());
LOG.info("Using state database at " + storeRoot + " for recovery");
File dbfile = new File(storeRoot.toString());
try {
@@ -366,14 +365,4 @@ public class HistoryServerLeveldbStateStoreService extends
+ getCurrentVersion() + ", but loading version " + loadedVersion);
}
}
-
- private static class LeveldbLogger implements org.iq80.leveldb.Logger {
- private static final Logger LOG =
- LoggerFactory.getLogger(LeveldbLogger.class);
-
- @Override
- public void log(String message) {
- LOG.info(message);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb128df/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index ec992fe..aeda9cc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -93,7 +93,6 @@ import org.fusesource.leveldbjni.JniDBFactory;
import org.fusesource.leveldbjni.internal.NativeDB;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBException;
-import org.iq80.leveldb.Logger;
import org.iq80.leveldb.Options;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.buffer.ChannelBuffers;
@@ -628,7 +627,6 @@ public class ShuffleHandler extends AuxiliaryService {
private void startStore(Path recoveryRoot) throws IOException {
Options options = new Options();
options.createIfMissing(false);
- options.logger(new LevelDBLogger());
Path dbPath = new Path(recoveryRoot, STATE_DB_NAME);
LOG.info("Using state database at " + dbPath + " for recovery");
File dbfile = new File(dbPath.toString());
@@ -774,16 +772,6 @@ public class ShuffleHandler extends AuxiliaryService {
}
}
- private static class LevelDBLogger implements Logger {
- private static final org.slf4j.Logger LOG =
- LoggerFactory.getLogger(LevelDBLogger.class);
-
- @Override
- public void log(String message) {
- LOG.info(message);
- }
- }
-
static class TimeoutHandler extends IdleStateAwareChannelHandler {
private boolean enabledTimeout;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb128df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 723dd48..6f643b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -1531,7 +1531,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
Path storeRoot = createStorageDir(conf);
Options options = new Options();
options.createIfMissing(false);
- options.logger(new LeveldbLogger());
LOG.info("Using state database at " + storeRoot + " for recovery");
File dbfile = new File(storeRoot.toString());
try {
@@ -1596,17 +1595,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
}
}
- private static class LeveldbLogger implements org.iq80.leveldb.Logger {
- private static final org.slf4j.Logger LOG =
- LoggerFactory.getLogger(LeveldbLogger.class);
-
- @Override
- public void log(String message) {
- LOG.info(message);
- }
- }
-
-
Version loadVersion() throws IOException {
byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY));
// if version is not stored previously, treat it as CURRENT_VERSION_INFO.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb128df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
index a53083f..36a8dfa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
@@ -64,12 +64,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.fusesource.leveldbjni.JniDBFactory;
import org.fusesource.leveldbjni.internal.NativeDB;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBException;
-import org.iq80.leveldb.Logger;
import org.iq80.leveldb.Options;
import org.iq80.leveldb.WriteBatch;
@@ -165,7 +163,6 @@ public class LeveldbRMStateStore extends RMStateStore {
Path storeRoot = createStorageDir();
Options options = new Options();
options.createIfMissing(false);
- options.logger(new LeveldbLogger());
LOG.info("Using state database at " + storeRoot + " for recovery");
File dbfile = new File(storeRoot.toString());
try {
@@ -872,13 +869,4 @@ public class LeveldbRMStateStore extends RMStateStore {
LOG.info("Full compaction cycle completed in " + duration + " msec");
}
}
-
- private static class LeveldbLogger implements Logger {
- private static final Log LOG = LogFactory.getLog(LeveldbLogger.class);
-
- @Override
- public void log(String message) {
- LOG.info(message);
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/44] hadoop git commit: YARN-7221. Add security check for
privileged docker container. Contributed by Eric Yang
Posted by xy...@apache.org.
YARN-7221. Add security check for privileged docker container. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/933477e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/933477e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/933477e9
Branch: refs/heads/HDFS-7240
Commit: 933477e9e0526e2ed81ea454f8806de31981822a
Parents: f7d5bac
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Apr 11 08:23:20 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Wed Apr 11 11:24:23 2018 -0700
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 10 +-
.../container-executor/impl/utils/docker-util.c | 100 ++++++++++++++++++-
.../test/utils/test_docker_util.cc | 97 +++++++++---------
.../runtime/TestDockerContainerRuntime.java | 11 +-
4 files changed, 157 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 51abeb6..7106aad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -767,7 +767,11 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
throw new ContainerExecutionException(message);
}
}
- dockerRunAsUser = uid + ":" + gid;
+ if (!allowPrivilegedContainerExecution(container)) {
+ dockerRunAsUser = uid + ":" + gid;
+ } else {
+ dockerRunAsUser = ctx.getExecutionAttribute(USER);
+ }
}
//List<String> -> stored as List -> fetched/converted to List<String>
@@ -879,7 +883,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
if(enableUserReMapping) {
- runCommand.groupAdd(groups);
+ if (!allowPrivilegedContainerExecution(container)) {
+ runCommand.groupAdd(groups);
+ }
}
// use plugins to update docker run command.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 3bd94a1..fdeaeea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -16,6 +16,9 @@
* limitations under the License.
*/
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/wait.h>
#include <string.h>
#include <strings.h>
#include <stdlib.h>
@@ -25,6 +28,9 @@
#include "docker-util.h"
#include "string-utils.h"
#include "util.h"
+#include <grp.h>
+#include <pwd.h>
+#include <errno.h>
static int read_and_verify_command_file(const char *command_file, const char *docker_command,
struct configuration *command_config) {
@@ -1254,14 +1260,94 @@ static int add_rw_mounts(const struct configuration *command_config, const stru
return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
}
+static int check_privileges(const char *user) {
+ int ngroups = 0;
+ gid_t *groups = NULL;
+ struct passwd *pw;
+ struct group *gr;
+ int ret = 0;
+ int waitid = -1;
+ int statval = 0;
+
+ pw = getpwnam(user);
+ if (pw == NULL) {
+ fprintf(ERRORFILE, "User %s does not exist in host OS.\n", user);
+ exit(INITIALIZE_USER_FAILED);
+ }
+
+ int rc = getgrouplist(user, pw->pw_gid, groups, &ngroups);
+ if (rc < 0) {
+ groups = (gid_t *) alloc_and_clear_memory(ngroups, sizeof(gid_t));
+ if (groups == NULL) {
+ fprintf(ERRORFILE, "Failed to allocate buffer for group lookup for user %s.\n", user);
+ exit(OUT_OF_MEMORY);
+ }
+ if (getgrouplist(user, pw->pw_gid, groups, &ngroups) == -1) {
+ fprintf(ERRORFILE, "Fail to lookup groups for user %s.\n", user);
+ ret = 2;
+ }
+ }
+
+ if (ret != 2) {
+ for (int j = 0; j < ngroups; j++) {
+ gr = getgrgid(groups[j]);
+ if (gr != NULL) {
+ if (strcmp(gr->gr_name, "root")==0 || strcmp(gr->gr_name, "docker")==0) {
+ ret = 1;
+ break;
+ }
+ }
+ }
+ }
+
+ if (ret != 1) {
+ int child_pid = fork();
+ if (child_pid == 0) {
+ execl("/bin/sudo", "sudo", "-U", user, "-n", "-l", "docker", NULL);
+ exit(INITIALIZE_USER_FAILED);
+ } else {
+ while ((waitid = waitpid(child_pid, &statval, 0)) != child_pid) {
+ if (waitid == -1 && errno != EINTR) {
+ fprintf(ERRORFILE, "waitpid failed: %s\n", strerror(errno));
+ break;
+ }
+ }
+ if (waitid == child_pid) {
+ if (WIFEXITED(statval)) {
+ if (WEXITSTATUS(statval) == 0) {
+ ret = 1;
+ }
+ } else if (WIFSIGNALED(statval)) {
+ fprintf(ERRORFILE, "sudo terminated by signal %d\n", WTERMSIG(statval));
+ }
+ }
+ }
+ }
+ free(groups);
+ if (ret == 1) {
+ fprintf(ERRORFILE, "check privileges passed for user: %s\n", user);
+ } else {
+ fprintf(ERRORFILE, "check privileges failed for user: %s, error code: %d\n", user, ret);
+ ret = 0;
+ }
+ return ret;
+}
+
static int set_privileged(const struct configuration *command_config, const struct configuration *conf, char *out,
const size_t outlen) {
size_t tmp_buffer_size = 1024;
+ char *user = NULL;
char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
char *value = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, command_config);
char *privileged_container_enabled
= get_configuration_value("docker.privileged-containers.enabled", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
int ret = 0;
+ int allowed = 0;
+
+ user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (user == NULL) {
+ return INVALID_DOCKER_USER_NAME;
+ }
if (value != NULL && strcasecmp(value, "true") == 0 ) {
if (privileged_container_enabled != NULL) {
@@ -1273,9 +1359,16 @@ static int set_privileged(const struct configuration *command_config, const stru
ret = PRIVILEGED_CONTAINERS_DISABLED;
goto free_and_exit;
}
- ret = add_to_buffer(out, outlen, "--privileged ");
- if (ret != 0) {
- ret = BUFFER_TOO_SMALL;
+ allowed = check_privileges(user);
+ if (allowed) {
+ ret = add_to_buffer(out, outlen, "--privileged ");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled for user: %s\n", user);
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
}
} else {
fprintf(ERRORFILE, "Privileged containers are disabled\n");
@@ -1293,6 +1386,7 @@ static int set_privileged(const struct configuration *command_config, const stru
free(tmp_buffer);
free(value);
free(privileged_container_enabled);
+ free(user);
if (ret != 0) {
memset(out, 0, outlen);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index 5d9779c..35b7873 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -646,10 +646,10 @@ namespace ContainerExecutor {
FAIL();
}
ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
- ASSERT_EQ(0, ret);
- ASSERT_STREQ(itr->second.c_str(), buff);
+ ASSERT_EQ(6, ret);
+ ASSERT_EQ(0, strlen(buff));
}
- write_command_file("[docker-command-execution]\n docker-command=run\n privileged=true\n image=nothadoop/image");
+ write_command_file("[docker-command-execution]\n docker-command=run\n user=nobody\n privileged=true\n image=nothadoop/image");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
@@ -669,9 +669,7 @@ namespace ContainerExecutor {
}
file_cmd_vec.clear();
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n privileged=false", ""));
- file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run", ""));
+ "[docker-command-execution]\n docker-command=run\n user=root\n privileged=false", ""));
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
memset(buff, 0, buff_len);
write_command_file(itr->first);
@@ -683,7 +681,7 @@ namespace ContainerExecutor {
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), buff);
}
- write_command_file("[docker-command-execution]\n docker-command=run\n privileged=true");
+ write_command_file("[docker-command-execution]\n docker-command=run\n user=root\n privileged=true");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
@@ -1114,64 +1112,64 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test",
- "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' "));
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody",
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' --cap-drop='ALL' 'hadoop/docker-image' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test",
- "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'nothadoop/docker-image' "));
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody",
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' --cap-drop='ALL' 'nothadoop/docker-image' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n"
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' --cap-drop='ALL' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
// Test non-privileged conatiner with launch command
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
" -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
" --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' "
"'test_script.sh' 'arg1' 'arg2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm"
" --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
// Test non-privileged container and drop all privileges
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
" -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
"--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'"
" 'test_script.sh' 'arg1' 'arg2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge'"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm --net='bridge'"
" --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
// Test privileged container
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=root\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
@@ -1181,10 +1179,9 @@ namespace ContainerExecutor {
"--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' "
"'bash' 'test_script.sh' 'arg1' 'arg2' "));
-
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=root\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
@@ -1196,28 +1193,28 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=nobody\n hostname=host-id\n"
" network=bridge\n net=bridge\n"
" detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' --cap-drop='ALL' "
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm --net='bridge' --cap-drop='ALL' "
"--hostname='host-id' --group-add '1000' --group-add '1001' "
"'docker-image' "));
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n user=test",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n user=nobody",
static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=test\n",
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=nobody\n",
static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n",
static_cast<int>(INVALID_DOCKER_USER_NAME)));
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
@@ -1227,7 +1224,7 @@ namespace ContainerExecutor {
// invalid rw mount
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/var/log:/var/log\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
@@ -1237,7 +1234,7 @@ namespace ContainerExecutor {
// invalid ro mount
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/bin:/bin,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
@@ -1247,7 +1244,7 @@ namespace ContainerExecutor {
// invalid capability
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
@@ -1257,17 +1254,17 @@ namespace ContainerExecutor {
// invalid device
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- static_cast<int>(INVALID_DOCKER_DEVICE)));
+ static_cast<int>(PRIVILEGED_CONTAINERS_DISABLED)));
// invalid network
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
@@ -1304,59 +1301,59 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test",
- "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=nobody",
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' --cap-drop='ALL' 'docker-image' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n"
- " user=test\n launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ " user=nobody\n launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' --cap-drop='ALL' 'docker-image' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
" -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
" --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' "
"'test_script.sh' 'arg1' 'arg2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm"
" --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
" -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
"--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'"
" 'test_script.sh' 'arg1' 'arg2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
- "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge'"
+ "run --name='container_e1_12312_11111_02_000001' --user='nobody' -d --rm --net='bridge'"
" --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
- " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
" ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
@@ -1387,8 +1384,8 @@ namespace ContainerExecutor {
"--config='/my-config' stop container_e1_12312_11111_02_000001"));
input_output_map.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001\n"
- " image=docker-image\n user=test",
- "--config='/my-config' run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ " image=docker-image\n user=nobody",
+ "--config='/my-config' run --name='container_e1_12312_11111_02_000001' --user='nobody' --cap-drop='ALL' 'docker-image' "));
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
char buffer[4096];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index acb3e42..8fbfbe2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -214,7 +214,7 @@ public class TestDockerContainerRuntime {
// Prevent gid threshold failures for these tests
conf.setInt(YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD, 0);
- user = "user";
+ user = submittingUser;
appId = "app_id";
containerIdStr = containerId;
containerWorkDir = new Path("/test_container_work_dir");
@@ -957,7 +957,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -967,8 +967,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
- Assert.assertEquals(" group-add=" + String.join(",", groups),
- dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
Assert.assertEquals(
@@ -984,7 +982,8 @@ public class TestDockerContainerRuntime {
" rw-mounts=/test_container_log_dir:/test_container_log_dir,"
+ "/test_application_local_dir:/test_application_local_dir",
dockerCommands.get(counter++));
- Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + submittingUser,
+ dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
}
@@ -1303,7 +1302,7 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(op.getOperationType(),
PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
Assert.assertEquals(runAsUser, op.getArguments().get(0));
- Assert.assertEquals("user", op.getArguments().get(1));
+ Assert.assertEquals(submittingUser, op.getArguments().get(1));
Assert.assertEquals("2", op.getArguments().get(2));
Assert.assertEquals("1234", op.getArguments().get(3));
Assert.assertEquals("0", op.getArguments().get(4));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/44] hadoop git commit: HADOOP-15346 S3ARetryPolicy for
400/BadArgument to be "fail". Contributed by Steve Loughran.
Posted by xy...@apache.org.
HADOOP-15346 S3ARetryPolicy for 400/BadArgument to be "fail". Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0aff8a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0aff8a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0aff8a9
Branch: refs/heads/HDFS-7240
Commit: b0aff8a96221fbf40d1eafe51b4f530b73146a20
Parents: 7c9cdad
Author: Aaron Fabbri <fa...@apache.org>
Authored: Tue Apr 10 23:55:38 2018 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Tue Apr 10 23:55:38 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 6 +++---
.../java/org/apache/hadoop/fs/s3a/TestInvoker.java | 14 ++++----------
2 files changed, 7 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aff8a9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
index d857330..2b361fd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -175,9 +175,9 @@ public class S3ARetryPolicy implements RetryPolicy {
// which isn't going to be recovered from
policyMap.put(EOFException.class, retryIdempotentCalls);
- // policy on a 400/bad request still ambiguous. Given it
- // comes and goes on test runs: try again
- policyMap.put(AWSBadRequestException.class, connectivityFailure);
+ // policy on a 400/bad request still ambiguous.
+ // Treated as an immediate failure
+ policyMap.put(AWSBadRequestException.class, fail);
// Status 500 error code is also treated as a connectivity problem
policyMap.put(AWSStatus500Exception.class, connectivityFailure);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aff8a9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
index d29e2df..5da665c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
@@ -283,18 +283,12 @@ public class TestInvoker extends Assert {
/**
* Repeatedly retry until eventually a bad request succeeds.
*/
- @Test
- public void testRetryBadRequestIdempotent() throws Throwable {
- final AtomicInteger counter = new AtomicInteger(0);
- final int attemptsBeforeSuccess = ACTIVE_RETRY_LIMIT;
- invoker.retry("test", null, true,
+ @Test(expected = AWSBadRequestException.class)
+ public void testRetryBadRequestNotIdempotent() throws Throwable {
+ invoker.retry("test", null, false,
() -> {
- if (counter.incrementAndGet() < attemptsBeforeSuccess) {
- throw BAD_REQUEST;
- }
+ throw BAD_REQUEST;
});
- assertEquals(attemptsBeforeSuccess, counter.get());
- assertEquals("retry count ", attemptsBeforeSuccess - 1, retryCount);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/44] hadoop git commit: HADOOP-15350. [JDK10] Update maven plugin
tools to fix compile error in hadoop-maven-plugins module
Posted by xy...@apache.org.
HADOOP-15350. [JDK10] Update maven plugin tools to fix compile error in hadoop-maven-plugins module
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/832852ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/832852ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/832852ce
Branch: refs/heads/HDFS-7240
Commit: 832852ce4ff00d4aa698e89e1df39e5bf0df78b9
Parents: c7cd362
Author: Takanobu Asanuma <ta...@yahoo-corp.jp>
Authored: Thu Apr 12 17:19:35 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Apr 12 17:19:35 2018 +0900
----------------------------------------------------------------------
hadoop-maven-plugins/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/832852ce/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index bd347d6..b31d158 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -27,7 +27,7 @@
<name>Apache Hadoop Maven Plugins</name>
<properties>
<maven.dependency.version>3.0</maven.dependency.version>
- <maven.plugin-tools.version>3.4</maven.plugin-tools.version>
+ <maven.plugin-tools.version>3.5.1</maven.plugin-tools.version>
</properties>
<dependencies>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/44] hadoop git commit: HDFS-13386. RBF: Wrong date information in
list file(-ls) result. Contributed by Dibyendu Karmakar.
Posted by xy...@apache.org.
HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18de6f20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18de6f20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18de6f20
Branch: refs/heads/HDFS-7240
Commit: 18de6f2042b70f9f0d7a2620c60de022768a7b13
Parents: 933477e
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Apr 11 12:03:14 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Apr 11 12:03:14 2018 -0700
----------------------------------------------------------------------
.../federation/router/RouterRpcServer.java | 19 +++++-
.../federation/router/TestRouterMountTable.java | 62 +++++++++++++++++++-
2 files changed, 79 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18de6f20/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index e6d2f5e..0dc2a69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -2328,7 +2328,24 @@ public class RouterRpcServer extends AbstractService
*/
private Map<String, Long> getMountPointDates(String path) {
Map<String, Long> ret = new TreeMap<>();
- // TODO add when we have a Mount Table
+ if (subclusterResolver instanceof MountTableResolver) {
+ MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
+ String srcPath;
+ try {
+ final List<String> children = subclusterResolver.getMountPoints(path);
+ for (String child : children) {
+ if (path.equals(Path.SEPARATOR)) {
+ srcPath = Path.SEPARATOR + child;
+ } else {
+ srcPath = path + Path.SEPARATOR + child;
+ }
+ MountTable entry = mountTable.getMountPoint(srcPath);
+ ret.put(child, entry.getDateModified());
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get mount point: {}", e.getMessage());
+ }
+ }
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18de6f20/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
index c9e28b1..b33b998 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -17,25 +17,33 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,6 +57,7 @@ public class TestRouterMountTable {
private static NamenodeContext nnContext;
private static RouterContext routerContext;
private static MountTableResolver mountTable;
+ private static ClientProtocol routerProtocol;
@BeforeClass
public static void globalSetUp() throws Exception {
@@ -69,6 +78,7 @@ public class TestRouterMountTable {
nnContext = cluster.getRandomNamenode();
routerContext = cluster.getRandomRouter();
Router router = routerContext.getRouter();
+ routerProtocol = routerContext.getClient().getNamenode();
mountTable = (MountTableResolver) router.getSubclusterResolver();
}
@@ -140,4 +150,54 @@ public class TestRouterMountTable {
return addResponse.getStatus();
}
+
+ /**
+ * Verify that the file/dir listing contains correct date/time information.
+ */
+ @Test
+ public void testListFilesTime() throws Exception {
+ Long beforeCreatingTime = Time.now();
+ // Add mount table entry
+ MountTable addEntry = MountTable.newInstance(
+ "/testdir", Collections.singletonMap("ns0", "/testdir"));
+ assertTrue(addMountTable(addEntry));
+ addEntry = MountTable.newInstance(
+ "/testdir2", Collections.singletonMap("ns0", "/testdir2"));
+ assertTrue(addMountTable(addEntry));
+ addEntry = MountTable.newInstance(
+ "/testdir/subdir", Collections.singletonMap("ns0", "/testdir/subdir"));
+ assertTrue(addMountTable(addEntry));
+
+ // Create test dir in NN
+ final FileSystem nnFs = nnContext.getFileSystem();
+ assertTrue(nnFs.mkdirs(new Path("/newdir")));
+
+ Map<String, Long> pathModTime = new TreeMap<>();
+ for (String mount : mountTable.getMountPoints("/")) {
+ pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
+ .getDateModified());
+ }
+ FileStatus[] iterator = nnFs.listStatus(new Path("/"));
+ for (FileStatus file : iterator) {
+ pathModTime.put(file.getPath().getName(), file.getModificationTime());
+ }
+ // Fetch listing
+ DirectoryListing listing =
+ routerProtocol.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> pathModTimeIterator = pathModTime.keySet().iterator();
+
+ // Match date/time for each path returned
+ for(HdfsFileStatus f : listing.getPartialListing()) {
+ String fileName = pathModTimeIterator.next();
+ String currentFile = f.getFullPath(new Path("/")).getName();
+ Long currentTime = f.getModificationTime();
+ Long expectedTime = pathModTime.get(currentFile);
+
+ assertEquals(currentFile, fileName);
+ assertTrue(currentTime > beforeCreatingTime);
+ assertEquals(currentTime, expectedTime);
+ }
+ // Verify the total number of results found/matched
+ assertEquals(pathModTime.size(), listing.getPartialListing().length);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/44] hadoop git commit: HADOOP-14445. Delegation tokens are not
shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Posted by xy...@apache.org.
HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/583fa6ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/583fa6ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/583fa6ed
Branch: refs/heads/HDFS-7240
Commit: 583fa6ed48ad3df40bcaa9c591d5ccd07ce3ea81
Parents: e813975
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Apr 10 15:26:33 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Apr 10 15:38:25 2018 -0700
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 212 ++++----
.../crypto/key/kms/KMSDelegationToken.java | 22 +-
.../crypto/key/kms/KMSLegacyTokenRenewer.java | 56 ++
.../hadoop/crypto/key/kms/KMSTokenRenewer.java | 103 ++++
.../hadoop/crypto/key/kms/package-info.java | 18 +
.../fs/CommonConfigurationKeysPublic.java | 10 +
.../web/DelegationTokenAuthenticatedURL.java | 21 +-
.../DelegationTokenAuthenticationHandler.java | 8 +-
.../web/DelegationTokenAuthenticator.java | 2 +-
.../java/org/apache/hadoop/util/KMSUtil.java | 45 +-
.../hadoop/util/KMSUtilFaultInjector.java | 49 ++
...apache.hadoop.security.token.TokenIdentifier | 1 +
...rg.apache.hadoop.security.token.TokenRenewer | 3 +-
.../src/main/resources/core-default.xml | 20 +
.../crypto/key/kms/TestKMSClientProvider.java | 162 ++++++
.../kms/TestLoadBalancingKMSClientProvider.java | 67 ++-
.../org/apache/hadoop/util/TestKMSUtil.java | 65 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 519 ++++++++++++++++---
18 files changed, 1180 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 2eb2e21..f97fde7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -36,8 +36,9 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.security.token.TokenSelector;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.hadoop.util.KMSUtil;
@@ -82,6 +83,8 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT;
import static org.apache.hadoop.util.KMSUtil.checkNotEmpty;
import static org.apache.hadoop.util.KMSUtil.checkNotNull;
import static org.apache.hadoop.util.KMSUtil.parseJSONEncKeyVersion;
@@ -96,16 +99,13 @@ import static org.apache.hadoop.util.KMSUtil.parseJSONMetadata;
public class KMSClientProvider extends KeyProvider implements CryptoExtension,
KeyProviderDelegationTokenExtension.DelegationTokenExtension {
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(KMSClientProvider.class);
private static final String INVALID_SIGNATURE = "Invalid signature";
private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous requests are disallowed";
- public static final String TOKEN_KIND_STR = KMSDelegationToken.TOKEN_KIND_STR;
- public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND;
-
public static final String SCHEME_NAME = "kms";
private static final String UTF8 = "UTF-8";
@@ -133,12 +133,17 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private static final ObjectWriter WRITER =
new ObjectMapper().writerWithDefaultPrettyPrinter();
+ /* dtService defines the token service value for the kms token.
+ * The value can be legacy format which is ip:port format or it can be uri.
+ * If it's uri format, then the value is read from
+ * CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH at key
+ * provider creation time, and set to token's Service field.
+ * When a token is renewed / canceled, its Service field will be used to
+ * instantiate a KeyProvider, eliminating the need to read configs
+ * at that time.
+ */
private final Text dtService;
-
- // Allow fallback to default kms server port 9600 for certain tests that do
- // not specify the port explicitly in the kms provider url.
- @VisibleForTesting
- public static volatile boolean fallbackDefaultPortForTesting = false;
+ private final boolean copyLegacyToken;
private class EncryptedQueueRefiller implements
ValueQueue.QueueRefiller<EncryptedKeyVersion> {
@@ -162,66 +167,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
}
- /**
- * The KMS implementation of {@link TokenRenewer}.
- */
- public static class KMSTokenRenewer extends TokenRenewer {
- private static final Logger LOG =
- LoggerFactory.getLogger(KMSTokenRenewer.class);
-
- @Override
- public boolean handleKind(Text kind) {
- return kind.equals(TOKEN_KIND);
- }
-
- @Override
- public boolean isManaged(Token<?> token) throws IOException {
- return true;
- }
-
- @Override
- public long renew(Token<?> token, Configuration conf) throws IOException {
- LOG.debug("Renewing delegation token {}", token);
- KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
- KeyProviderFactory.KEY_PROVIDER_PATH);
- try {
- if (!(keyProvider instanceof
- KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
- LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
- "null" : keyProvider.getClass());
- return 0;
- }
- return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
- keyProvider).renewDelegationToken(token);
- } finally {
- if (keyProvider != null) {
- keyProvider.close();
- }
- }
- }
-
- @Override
- public void cancel(Token<?> token, Configuration conf) throws IOException {
- LOG.debug("Canceling delegation token {}", token);
- KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
- KeyProviderFactory.KEY_PROVIDER_PATH);
- try {
- if (!(keyProvider instanceof
- KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
- LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
- "null" : keyProvider.getClass());
- return;
- }
- ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
- keyProvider).cancelDelegationToken(token);
- } finally {
- if (keyProvider != null) {
- keyProvider.close();
- }
- }
- }
- }
-
public static class KMSEncryptedKeyVersion extends EncryptedKeyVersion {
public KMSEncryptedKeyVersion(String keyName, String keyVersionName,
byte[] iv, String encryptedVersionName, byte[] keyMaterial) {
@@ -281,13 +226,13 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
hostsPart = t[0];
}
- return createProvider(conf, origUrl, port, hostsPart);
+ return createProvider(conf, origUrl, port, hostsPart, providerUri);
}
return null;
}
- private KeyProvider createProvider(Configuration conf,
- URL origUrl, int port, String hostsPart) throws IOException {
+ private KeyProvider createProvider(Configuration conf, URL origUrl,
+ int port, String hostsPart, URI providerUri) throws IOException {
String[] hosts = hostsPart.split(";");
KMSClientProvider[] providers = new KMSClientProvider[hosts.length];
for (int i = 0; i < hosts.length; i++) {
@@ -295,7 +240,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
providers[i] =
new KMSClientProvider(
new URI("kms", origUrl.getProtocol(), hosts[i], port,
- origUrl.getPath(), null, null), conf);
+ origUrl.getPath(), null, null), conf, providerUri);
} catch (URISyntaxException e) {
throw new IOException("Could not instantiate KMSProvider.", e);
}
@@ -353,17 +298,10 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
}
- public KMSClientProvider(URI uri, Configuration conf) throws IOException {
+ public KMSClientProvider(URI uri, Configuration conf, URI providerUri) throws
+ IOException {
super(conf);
kmsUrl = createServiceURL(extractKMSPath(uri));
- int kmsPort = kmsUrl.getPort();
- if ((kmsPort == -1) && fallbackDefaultPortForTesting) {
- kmsPort = 9600;
- }
-
- InetSocketAddress addr = new InetSocketAddress(kmsUrl.getHost(), kmsPort);
- dtService = SecurityUtil.buildTokenService(addr);
-
if ("https".equalsIgnoreCase(kmsUrl.getProtocol())) {
sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
try {
@@ -376,6 +314,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_SECONDS,
CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_DEFAULT);
authRetry = conf.getInt(AUTH_RETRY, DEFAULT_AUTH_RETRY);
+ copyLegacyToken = conf.getBoolean(KMS_CLIENT_COPY_LEGACY_TOKEN_KEY,
+ KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT);
+
configurator = new TimeoutConnConfigurator(timeout, sslFactory);
encKeyVersionQueue =
new ValueQueue<KeyProviderCryptoExtension.EncryptedKeyVersion>(
@@ -400,6 +341,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
new EncryptedQueueRefiller());
authToken = new DelegationTokenAuthenticatedURL.Token();
+ dtService = new Text(providerUri.toString());
LOG.info("KMSClientProvider for KMS url: {} delegation token service: {}" +
" created.", kmsUrl, dtService);
}
@@ -473,7 +415,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
@Override
public HttpURLConnection run() throws Exception {
DelegationTokenAuthenticatedURL authUrl =
- new DelegationTokenAuthenticatedURL(configurator);
+ createKMSAuthenticatedURL();
return authUrl.openConnection(url, authToken, doAsUser);
}
});
@@ -924,7 +866,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
LOG.debug("Renewing delegation token {} with url:{}, as:{}",
token, url, doAsUser);
final DelegationTokenAuthenticatedURL authUrl =
- new DelegationTokenAuthenticatedURL(configurator);
+ createKMSAuthenticatedURL();
return getActualUgi().doAs(
new PrivilegedExceptionAction<Long>() {
@Override
@@ -956,7 +898,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
LOG.debug("Cancelling delegation token {} with url:{}, as:{}",
dToken, url, doAsUser);
final DelegationTokenAuthenticatedURL authUrl =
- new DelegationTokenAuthenticatedURL(configurator);
+ createKMSAuthenticatedURL();
authUrl.cancelDelegationToken(url, token, doAsUser);
return null;
}
@@ -1008,6 +950,17 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return token;
}
+ @VisibleForTesting
+ DelegationTokenAuthenticatedURL createKMSAuthenticatedURL() {
+ return new DelegationTokenAuthenticatedURL(configurator) {
+ @Override
+ public org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
+ getDelegationToken(URL url, Credentials creds) {
+ return selectKMSDelegationToken(creds);
+ }
+ };
+ }
+
@Override
public Token<?>[] addDelegationTokens(final String renewer,
Credentials credentials) throws IOException {
@@ -1016,7 +969,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
if (token == null) {
final URL url = createURL(null, null, null, null);
final DelegationTokenAuthenticatedURL authUrl =
- new DelegationTokenAuthenticatedURL(configurator);
+ createKMSAuthenticatedURL();
try {
final String doAsUser = getDoAsUser();
token = getActualUgi().doAs(new PrivilegedExceptionAction<Token<?>>() {
@@ -1030,9 +983,16 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
});
if (token != null) {
- LOG.debug("New token received: ({})", token);
+ if (KMSDelegationToken.TOKEN_KIND.equals(token.getKind())) {
+ // do not set service for legacy kind, for compatibility.
+ token.setService(dtService);
+ }
+ LOG.info("New token created: ({})", token);
credentials.addToken(token.getService(), token);
- tokens = new Token<?>[] { token };
+ Token<?> legacyToken = createAndAddLegacyToken(credentials, token);
+ tokens = legacyToken == null ?
+ new Token<?>[] {token} :
+ new Token<?>[] {token, legacyToken};
} else {
throw new IOException("Got NULL as delegation token");
}
@@ -1049,13 +1009,75 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return tokens;
}
- private boolean containsKmsDt(UserGroupInformation ugi) throws IOException {
- // Add existing credentials from the UGI, since provider is cached.
- Credentials creds = ugi.getCredentials();
+ /**
+ * If {@link CommonConfigurationKeysPublic#KMS_CLIENT_COPY_LEGACY_TOKEN_KEY}
+ * is true when creating the provider, then copy the passed-in token of
+ * {@link KMSDelegationToken#TOKEN_KIND} and create a new token of
+ * {@link KMSDelegationToken#TOKEN_LEGACY_KIND}, and add it to credentials.
+ *
+ * @return The legacy token, or null if one should not be created.
+ */
+ private Token<?> createAndAddLegacyToken(Credentials credentials,
+ Token<?> token) {
+ if (!copyLegacyToken || !KMSDelegationToken.TOKEN_KIND
+ .equals(token.getKind())) {
+ LOG.debug("Not creating legacy token because copyLegacyToken={}, "
+ + "token={}", copyLegacyToken, token);
+ return null;
+ }
+ // copy a KMS_DELEGATION_TOKEN and create a new kms-dt with the same
+ // underlying token for backwards-compatibility. Old clients/renewers
+ // does not parse the new token and can only work with kms-dt.
+ final Token<?> legacyToken = token.copyToken();
+ legacyToken.setKind(KMSDelegationToken.TOKEN_LEGACY_KIND);
+ final InetSocketAddress addr =
+ new InetSocketAddress(kmsUrl.getHost(), kmsUrl.getPort());
+ final Text fallBackServiceText = SecurityUtil.buildTokenService(addr);
+ legacyToken.setService(fallBackServiceText);
+ LOG.info("Copied token to legacy kind: {}", legacyToken);
+ credentials.addToken(legacyToken.getService(), legacyToken);
+ return legacyToken;
+ }
+
+ @VisibleForTesting
+ public Text getDelegationTokenService() {
+ return dtService;
+ }
+
+ /**
+ * Given a list of tokens, return the token that should be used for KMS
+ * authentication.
+ */
+ @VisibleForTesting
+ Token selectKMSDelegationToken(Credentials creds) {
+ // always look for TOKEN_KIND first
+ final TokenSelector<AbstractDelegationTokenIdentifier> tokenSelector =
+ new AbstractDelegationTokenSelector<AbstractDelegationTokenIdentifier>(
+ KMSDelegationToken.TOKEN_KIND) {
+ };
+ Token token = tokenSelector.selectToken(dtService, creds.getAllTokens());
+ LOG.debug("Searching service {} found token {}", dtService, token);
+ if (token != null) {
+ return token;
+ }
+
+ // fall back to look for token by service, regardless of kind.
+ // this is old behavior, keeping for compatibility reasons (for example,
+ // even if KMS server is new, if the job is submitted with an old kms
+ // client, job runners on new version should be able to find the token).
+ final InetSocketAddress addr =
+ new InetSocketAddress(kmsUrl.getHost(), kmsUrl.getPort());
+ final Text fallBackServiceText = SecurityUtil.buildTokenService(addr);
+ token = creds.getToken(fallBackServiceText);
+ LOG.debug("Selected delegation token {} using service:{}", token,
+ fallBackServiceText);
+ return token;
+ }
+
+ private boolean containsKmsDt(UserGroupInformation ugi) {
+ final Credentials creds = ugi.getCredentials();
if (!creds.getAllTokens().isEmpty()) {
- LOG.debug("Searching for token that matches service: {}", dtService);
- org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
- dToken = creds.getToken(dtService);
+ final Token dToken = selectKMSDelegationToken(creds);
if (dToken != null) {
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
index adeebf2..2642e79 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
@@ -27,7 +27,10 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier
@InterfaceAudience.Private
public final class KMSDelegationToken {
- public static final String TOKEN_KIND_STR = "kms-dt";
+ public static final String TOKEN_LEGACY_KIND_STR = "kms-dt";
+ public static final Text TOKEN_LEGACY_KIND = new Text(TOKEN_LEGACY_KIND_STR);
+
+ public static final String TOKEN_KIND_STR = "KMS_DELEGATION_TOKEN";
public static final Text TOKEN_KIND = new Text(TOKEN_KIND_STR);
// Utility class is not supposed to be instantiated.
@@ -49,4 +52,21 @@ public final class KMSDelegationToken {
return TOKEN_KIND;
}
}
+
+ /**
+ * DelegationTokenIdentifier used for the KMS for legacy tokens.
+ */
+ @Deprecated
+ public static class KMSLegacyDelegationTokenIdentifier
+ extends DelegationTokenIdentifier {
+
+ public KMSLegacyDelegationTokenIdentifier() {
+ super(TOKEN_LEGACY_KIND);
+ }
+
+ @Override
+ public Text getKind() {
+ return TOKEN_LEGACY_KIND;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java
new file mode 100644
index 0000000..fd27073
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.KMSUtil;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
+
+/**
+ * The {@link KMSTokenRenewer} that supports legacy tokens.
+ */
+@InterfaceAudience.Private
+@Deprecated
+public class KMSLegacyTokenRenewer extends KMSTokenRenewer {
+
+ @Override
+ public boolean handleKind(Text kind) {
+ return kind.equals(TOKEN_LEGACY_KIND);
+ }
+
+ /**
+ * Create a key provider for token renewal / cancellation.
+ * Caller is responsible for closing the key provider.
+ */
+ @Override
+ protected KeyProvider createKeyProvider(Token<?> token,
+ Configuration conf) throws IOException {
+ assert token.getKind().equals(TOKEN_LEGACY_KIND);
+ // Legacy tokens get service from configuration.
+ return KMSUtil.createKeyProvider(conf,
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java
new file mode 100644
index 0000000..908ad39
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.util.KMSUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
+
+/**
+ * The KMS implementation of {@link TokenRenewer}.
+ */
+@InterfaceAudience.Private
+public class KMSTokenRenewer extends TokenRenewer {
+
+ public static final Logger LOG = LoggerFactory
+ .getLogger(org.apache.hadoop.crypto.key.kms.KMSTokenRenewer.class);
+
+ @Override
+ public boolean handleKind(Text kind) {
+ return kind.equals(TOKEN_KIND);
+ }
+
+ @Override
+ public boolean isManaged(Token<?> token) throws IOException {
+ return true;
+ }
+
+ @Override
+ public long renew(Token<?> token, Configuration conf) throws IOException {
+ LOG.debug("Renewing delegation token {}", token);
+ final KeyProvider keyProvider = createKeyProvider(token, conf);
+ try {
+ if (!(keyProvider instanceof
+ KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+ LOG.warn("keyProvider {} cannot renew token {}.",
+ keyProvider == null ? "null" : keyProvider.getClass(), token);
+ return 0;
+ }
+ return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+ keyProvider).renewDelegationToken(token);
+ } finally {
+ if (keyProvider != null) {
+ keyProvider.close();
+ }
+ }
+ }
+
+ @Override
+ public void cancel(Token<?> token, Configuration conf) throws IOException {
+ LOG.debug("Canceling delegation token {}", token);
+ final KeyProvider keyProvider = createKeyProvider(token, conf);
+ try {
+ if (!(keyProvider instanceof
+ KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+ LOG.warn("keyProvider {} cannot cancel token {}.",
+ keyProvider == null ? "null" : keyProvider.getClass(), token);
+ return;
+ }
+ ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+ keyProvider).cancelDelegationToken(token);
+ } finally {
+ if (keyProvider != null) {
+ keyProvider.close();
+ }
+ }
+ }
+
+ /**
+ * Create a key provider for token renewal / cancellation.
+ * Caller is responsible for closing the key provider.
+ */
+ protected KeyProvider createKeyProvider(Token<?> token,
+ Configuration conf) throws IOException {
+ return KMSUtil
+ .createKeyProviderFromTokenService(conf, token.getService().toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java
new file mode 100644
index 0000000..eea93c2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 8cd753a..be1c7bc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -775,6 +775,16 @@ public class CommonConfigurationKeysPublic {
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
* core-default.xml</a>
*/
+ public static final String KMS_CLIENT_COPY_LEGACY_TOKEN_KEY =
+ "hadoop.security.kms.client.copy.legacy.token";
+ /** Default value is true. */
+ public static final boolean KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT = true;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY =
"hadoop.security.kms.client.failover.sleep.max.millis";
/** Default value is 2 secs. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index 0b1fdf8..0ddc4fc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -300,11 +300,7 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
creds.getAllTokens());
}
if (!creds.getAllTokens().isEmpty()) {
- InetSocketAddress serviceAddr = new InetSocketAddress(url.getHost(),
- url.getPort());
- Text service = SecurityUtil.buildTokenService(serviceAddr);
- dToken = creds.getToken(service);
- LOG.debug("Using delegation token {} from service:{}", dToken, service);
+ dToken = getDelegationToken(url, creds);
if (dToken != null) {
if (useQueryStringForDelegationToken()) {
// delegation token will go in the query string, injecting it
@@ -341,6 +337,21 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
}
/**
+ * Select a delegation token from all tokens in credentials, based on url.
+ */
+ @InterfaceAudience.Private
+ public org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
+ getDelegationToken(URL url, Credentials creds) {
+ final InetSocketAddress serviceAddr =
+ new InetSocketAddress(url.getHost(), url.getPort());
+ final Text service = SecurityUtil.buildTokenService(serviceAddr);
+ org.apache.hadoop.security.token.Token<? extends TokenIdentifier> dToken =
+ creds.getToken(service);
+ LOG.debug("Selected delegation token {} using service:{}", dToken, service);
+ return dToken;
+ }
+
+ /**
* Requests a delegation token using the configured <code>Authenticator</code>
* for authentication.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index 6ee59f1..0ef102e7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -81,7 +81,7 @@ import com.google.common.annotations.VisibleForTesting;
@InterfaceStability.Evolving
public abstract class DelegationTokenAuthenticationHandler
implements AuthenticationHandler {
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(DelegationTokenAuthenticationHandler.class);
protected static final String TYPE_POSTFIX = "-dt";
@@ -224,7 +224,8 @@ public abstract class DelegationTokenAuthenticationHandler
HttpServletRequest request, HttpServletResponse response)
throws IOException, AuthenticationException {
boolean requestContinues = true;
- LOG.trace("Processing operation for req=({}), token: {}", request, token);
+ LOG.trace("Processing operation for req=({}), token: {}",
+ request.getRequestURL(), token);
String op = ServletUtils.getParameter(request,
KerberosDelegationTokenAuthenticator.OP_PARAM);
op = (op != null) ? StringUtils.toUpperCase(op) : null;
@@ -407,7 +408,8 @@ public abstract class DelegationTokenAuthenticationHandler
HttpServletResponse.SC_FORBIDDEN, new AuthenticationException(ex));
}
} else {
- LOG.debug("Falling back to {} (req={})", authHandler.getClass(), request);
+ LOG.debug("Falling back to {} (req={})", authHandler.getClass(),
+ request.getRequestURL());
token = authHandler.authenticate(request, response);
}
return token;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 617773b..7e83781 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -50,7 +50,7 @@ import java.util.Map;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public abstract class DelegationTokenAuthenticator implements Authenticator {
- private static Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(DelegationTokenAuthenticator.class);
private static final String CONTENT_TYPE = "Content-Type";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
index c96c6fb..8077076 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -41,8 +42,7 @@ import java.util.Map;
*/
@InterfaceAudience.Private
public final class KMSUtil {
- public static final Logger LOG =
- LoggerFactory.getLogger(KMSUtil.class);
+ public static final Logger LOG = LoggerFactory.getLogger(KMSUtil.class);
private KMSUtil() { /* Hidden constructor */ }
@@ -64,6 +64,13 @@ public final class KMSUtil {
if (providerUriStr == null || providerUriStr.isEmpty()) {
return null;
}
+ KeyProvider kp = KMSUtilFaultInjector.get().createKeyProviderForTests(
+ providerUriStr, conf);
+ if (kp != null) {
+ LOG.info("KeyProvider is created with uri: {}. This should happen only " +
+ "in tests.", providerUriStr);
+ return kp;
+ }
return createKeyProviderFromUri(conf, URI.create(providerUriStr));
}
@@ -205,4 +212,38 @@ public final class KMSUtil {
}
return metadata;
}
+
+ /**
+ * Creates a key provider from token service field, which must be URI format.
+ *
+ * @param conf
+ * @param tokenServiceValue
+ * @return new KeyProvider or null
+ * @throws IOException
+ */
+ public static KeyProvider createKeyProviderFromTokenService(
+ final Configuration conf, final String tokenServiceValue)
+ throws IOException {
+ LOG.debug("Creating key provider from token service value {}. ",
+ tokenServiceValue);
+ final KeyProvider kp = KMSUtilFaultInjector.get()
+ .createKeyProviderForTests(tokenServiceValue, conf);
+ if (kp != null) {
+ LOG.info("KeyProvider is created with uri: {}. This should happen only "
+ + "in tests.", tokenServiceValue);
+ return kp;
+ }
+ if (!tokenServiceValue.contains("://")) {
+ throw new IllegalArgumentException(
+ "Invalid token service " + tokenServiceValue);
+ }
+ final URI tokenServiceUri;
+ try {
+ tokenServiceUri = new URI(tokenServiceValue);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(
+ "Invalid token service " + tokenServiceValue, e);
+ }
+ return createKeyProviderFromUri(conf, tokenServiceUri);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java
new file mode 100644
index 0000000..46d5069
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+
+import java.io.IOException;
+
+/**
+ * Used for returning custom KeyProvider from test methods.
+ */
+@VisibleForTesting
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class KMSUtilFaultInjector {
+ private static KMSUtilFaultInjector instance = new KMSUtilFaultInjector();
+
+ public static KMSUtilFaultInjector get() {
+ return instance;
+ }
+
+ public static void set(KMSUtilFaultInjector injector) {
+ instance = injector;
+ }
+
+ public KeyProvider createKeyProviderForTests(String value, Configuration conf)
+ throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index b65f151..43d06e2 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -12,3 +12,4 @@
# limitations under the License.
#
org.apache.hadoop.crypto.key.kms.KMSDelegationToken$KMSDelegationTokenIdentifier
+org.apache.hadoop.crypto.key.kms.KMSDelegationToken$KMSLegacyDelegationTokenIdentifier
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
index 56320fb..5b6082c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -11,4 +11,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-org.apache.hadoop.crypto.key.kms.KMSClientProvider$KMSTokenRenewer
\ No newline at end of file
+org.apache.hadoop.crypto.key.kms.KMSTokenRenewer
+org.apache.hadoop.crypto.key.kms.KMSLegacyTokenRenewer
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index ad24f56..f32268b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2603,6 +2603,26 @@
</property>
<property>
+ <name>hadoop.security.kms.client.copy.legacy.token</name>
+ <value>true</value>
+ <description>
+ Expert only. Whether the KMS client provider should copy a token to legacy
+ kind. This is for KMS_DELEGATION_TOKEN to be backwards compatible. With the
+ default value set to true, the client will locally duplicate the
+ KMS_DELEGATION_TOKEN token and create a kms-dt token, with the service field
+ conforming to kms-dt. All other parts of the token remain the same.
+ Then the new clients will use KMS_DELEGATION_TOKEN and old clients will
+ use kms-dt to authenticate. Default value is true.
+ You should only change this to false if you know all the KMS servers
+ , clients (including both job submitters and job runners) and the
+ token renewers (usually Yarn RM) are on a version that supports
+ KMS_DELEGATION_TOKEN.
+ Turning this off prematurely may result in old clients failing to
+ authenticate with new servers.
+ </description>
+</property>
+
+<property>
<name>hadoop.security.kms.client.failover.sleep.max.millis</name>
<value>2000</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java
new file mode 100644
index 0000000..56aace5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.event.Level;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.net.URL;
+
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit test for {@link KMSClientProvider} class.
+ */
+public class TestKMSClientProvider {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestKMSClientProvider.class);
+
+ private final Token token = new Token();
+ private final Token legacyToken = new Token();
+ private final String uriString = "kms://https@host:16000/kms";
+ private final String legacyTokenService = "host:16000";
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(30000);
+
+ {
+ GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE);
+ }
+
+ @Before
+ public void setup() {
+ SecurityUtil.setTokenServiceUseIp(false);
+ token.setKind(TOKEN_KIND);
+ token.setService(new Text(uriString));
+ legacyToken.setKind(TOKEN_LEGACY_KIND);
+ legacyToken.setService(new Text(legacyTokenService));
+ }
+
+ @Test
+ public void testNotCopyFromLegacyToken() throws Exception {
+ final DelegationTokenAuthenticatedURL url =
+ mock(DelegationTokenAuthenticatedURL.class);
+ final Configuration conf = new Configuration();
+ final URI uri = new URI(uriString);
+ final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
+ try {
+ final KMSClientProvider spyKp = spy(kp);
+ when(spyKp.createKMSAuthenticatedURL()).thenReturn(url);
+ when(url.getDelegationToken(any(URL.class),
+ any(DelegationTokenAuthenticatedURL.Token.class), any(String.class),
+ any(String.class))).thenReturn(legacyToken);
+
+ final Credentials creds = new Credentials();
+ final Token<?>[] tokens = spyKp.addDelegationTokens("yarn", creds);
+ LOG.info("Got tokens: {}", tokens);
+ assertEquals(1, tokens.length);
+ LOG.info("uri:" + uriString);
+ // if KMS server returned a legacy token, new client should leave the
+ // service being legacy and not set uri string
+ assertEquals(legacyTokenService, tokens[0].getService().toString());
+ } finally {
+ kp.close();
+ }
+ }
+
+ @Test
+ public void testCopyFromToken() throws Exception {
+ final DelegationTokenAuthenticatedURL url =
+ mock(DelegationTokenAuthenticatedURL.class);
+ final Configuration conf = new Configuration();
+ final URI uri = new URI(uriString);
+ final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
+ try {
+ final KMSClientProvider spyKp = spy(kp);
+ when(spyKp.createKMSAuthenticatedURL()).thenReturn(url);
+ when(url.getDelegationToken(any(URL.class),
+ any(DelegationTokenAuthenticatedURL.Token.class), any(String.class),
+ any(String.class))).thenReturn(token);
+
+ final Credentials creds = new Credentials();
+ final Token<?>[] tokens = spyKp.addDelegationTokens("yarn", creds);
+ LOG.info("Got tokens: {}", tokens);
+ assertEquals(2, tokens.length);
+ assertTrue(creds.getAllTokens().contains(token));
+ assertNotNull(creds.getToken(legacyToken.getService()));
+ } finally {
+ kp.close();
+ }
+ }
+
+ @Test
+ public void testSelectTokenWhenBothExist() throws Exception {
+ final Credentials creds = new Credentials();
+ final Configuration conf = new Configuration();
+ final URI uri = new URI(uriString);
+ final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
+ try {
+ creds.addToken(token.getService(), token);
+ creds.addToken(legacyToken.getService(), legacyToken);
+ Token t = kp.selectKMSDelegationToken(creds);
+ assertEquals(token, t);
+ } finally {
+ kp.close();
+ }
+ }
+
+ @Test
+ public void testSelectTokenLegacyService() throws Exception {
+ final Configuration conf = new Configuration();
+ final URI uri = new URI(uriString);
+ final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
+ try {
+ Text legacyService = new Text(legacyTokenService);
+ token.setService(legacyService);
+ final Credentials creds = new Credentials();
+ creds.addToken(legacyService, token);
+ Token t = kp.selectKMSDelegationToken(creds);
+ assertEquals(token, t);
+ } finally {
+ kp.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
index bd68dca..e6a9fe0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
@@ -42,7 +42,8 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authorize.AuthorizationException;
-import org.junit.After;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.KMSUtilFaultInjector;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
@@ -56,33 +57,68 @@ public class TestLoadBalancingKMSClientProvider {
SecurityUtil.setTokenServiceUseIp(false);
}
- @After
- public void teardown() throws IOException {
- KMSClientProvider.fallbackDefaultPortForTesting = false;
+ private void setKMSUtilFaultInjector() {
+ KMSUtilFaultInjector injector = new KMSUtilFaultInjector() {
+ @Override
+ public KeyProvider createKeyProviderForTests(
+ String value, Configuration conf) throws IOException {
+ return TestLoadBalancingKMSClientProvider
+ .createKeyProviderForTests(value, conf);
+ }
+ };
+ KMSUtilFaultInjector.set(injector);
+ }
+
+ public static KeyProvider createKeyProviderForTests(
+ String value, Configuration conf) throws IOException {
+ // The syntax for kms servers will be
+ // kms://http@localhost:port1/kms,kms://http@localhost:port2/kms
+ if (!value.contains(",")) {
+ return null;
+ }
+ String[] keyProviderUrisStr = value.split(",");
+ KMSClientProvider[] keyProviderArr =
+ new KMSClientProvider[keyProviderUrisStr.length];
+
+ int i = 0;
+ for (String keyProviderUri: keyProviderUrisStr) {
+ KMSClientProvider kmcp =
+ new KMSClientProvider(URI.create(keyProviderUri), conf, URI
+ .create(value));
+ keyProviderArr[i] = kmcp;
+ i++;
+ }
+ LoadBalancingKMSClientProvider lbkcp =
+ new LoadBalancingKMSClientProvider(keyProviderArr, conf);
+ return lbkcp;
}
@Test
public void testCreation() throws Exception {
Configuration conf = new Configuration();
- KMSClientProvider.fallbackDefaultPortForTesting = true;
KeyProvider kp = new KMSClientProvider.Factory().createProvider(new URI(
- "kms://http@host1/kms/foo"), conf);
+ "kms://http@host1:9600/kms/foo"), conf);
assertTrue(kp instanceof LoadBalancingKMSClientProvider);
KMSClientProvider[] providers =
((LoadBalancingKMSClientProvider) kp).getProviders();
assertEquals(1, providers.length);
- assertEquals(Sets.newHashSet("http://host1/kms/foo/v1/"),
+ assertEquals(Sets.newHashSet("http://host1:9600/kms/foo/v1/"),
Sets.newHashSet(providers[0].getKMSUrl()));
-
- kp = new KMSClientProvider.Factory().createProvider(new URI(
- "kms://http@host1;host2;host3/kms/foo"), conf);
+ setKMSUtilFaultInjector();
+ String uriStr = "kms://http@host1:9600/kms/foo," +
+ "kms://http@host2:9600/kms/foo," +
+ "kms://http@host3:9600/kms/foo";
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ uriStr);
+ kp = KMSUtil.createKeyProvider(conf, CommonConfigurationKeysPublic
+ .HADOOP_SECURITY_KEY_PROVIDER_PATH);
assertTrue(kp instanceof LoadBalancingKMSClientProvider);
providers =
((LoadBalancingKMSClientProvider) kp).getProviders();
assertEquals(3, providers.length);
- assertEquals(Sets.newHashSet("http://host1/kms/foo/v1/",
- "http://host2/kms/foo/v1/",
- "http://host3/kms/foo/v1/"),
+ assertEquals(Sets.newHashSet("http://host1:9600/kms/foo/v1/",
+ "http://host2:9600/kms/foo/v1/",
+ "http://host3:9600/kms/foo/v1/"),
Sets.newHashSet(providers[0].getKMSUrl(),
providers[1].getKMSUrl(),
providers[2].getKMSUrl()));
@@ -208,7 +244,7 @@ public class TestLoadBalancingKMSClientProvider {
private class MyKMSClientProvider extends KMSClientProvider {
public MyKMSClientProvider(URI uri, Configuration conf) throws IOException {
- super(uri, conf);
+ super(uri, conf, uri);
}
@Override
@@ -245,9 +281,8 @@ public class TestLoadBalancingKMSClientProvider {
@Test
public void testClassCastException() throws Exception {
Configuration conf = new Configuration();
- KMSClientProvider.fallbackDefaultPortForTesting = true;
KMSClientProvider p1 = new MyKMSClientProvider(
- new URI("kms://http@host1/kms/foo"), conf);
+ new URI("kms://http@host1:9600/kms/foo"), conf);
LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
new KMSClientProvider[] {p1}, 0, conf);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java
new file mode 100644
index 0000000..77f52ee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test {@link KMSUtil}.
+ */
+public class TestKMSUtil {
+
+ public static final Logger LOG = LoggerFactory.getLogger(TestKMSUtil.class);
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(90000);
+
+ @Test
+ public void testCreateKeyProviderFromTokenService() throws Exception {
+ final Configuration conf = new Configuration();
+ KeyProvider kp = KMSUtil.createKeyProviderFromTokenService(conf,
+ "kms://https@localhost:9600/kms");
+ assertNotNull(kp);
+ kp.close();
+
+ kp = KMSUtil.createKeyProviderFromTokenService(conf,
+ "kms://https@localhost:9600/kms,kms://localhost1:9600/kms");
+ assertNotNull(kp);
+ kp.close();
+
+ String invalidService = "whatever:9600";
+ try {
+ KMSUtil.createKeyProviderFromTokenService(conf, invalidService);
+ } catch (Exception ex) {
+ LOG.info("Expected exception:", ex);
+ assertTrue(ex instanceof IllegalArgumentException);
+ GenericTestUtils.assertExceptionContains(
+ "Invalid token service " + invalidService, ex);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/44] hadoop git commit: HDFS-13426. Fix javadoc in
FsDatasetAsyncDiskService#removeVolume. Contributed by Shashikant Banerjee.
Posted by xy...@apache.org.
HDFS-13426. Fix javadoc in FsDatasetAsyncDiskService#removeVolume. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ed8511a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ed8511a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ed8511a
Branch: refs/heads/HDFS-7240
Commit: 7ed8511ad8daff19f765e78e4dca07cdebc2c2b2
Parents: b859785
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Thu Apr 12 20:12:31 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Thu Apr 12 20:12:31 2018 +0530
----------------------------------------------------------------------
.../server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ed8511a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 9174cb0..4929b5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
@@ -137,7 +138,7 @@ class FsDatasetAsyncDiskService {
/**
* Stops AsyncDiskService for a volume.
- * @param volume the root of the volume.
+ * @param storageId id of {@link StorageDirectory}.
*/
synchronized void removeVolume(String storageId) {
if (executors == null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/44] hadoop git commit: YARN-8147.
TestClientRMService#testGetApplications sporadically fails. Contributed by
Jason Lowe
Posted by xy...@apache.org.
YARN-8147. TestClientRMService#testGetApplications sporadically fails. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18844599
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18844599
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18844599
Branch: refs/heads/HDFS-7240
Commit: 18844599aef42f79d2af4500aa2eee472dda95cb
Parents: 044341b
Author: Eric E Payne <er...@oath.com>
Authored: Thu Apr 12 17:53:57 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Thu Apr 12 17:53:57 2018 +0000
----------------------------------------------------------------------
.../server/resourcemanager/TestClientRMService.java | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18844599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 1c50dd3..d66a866 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1064,7 +1064,7 @@ public class TestClientRMService {
}
@Test
- public void testGetApplications() throws IOException, YarnException {
+ public void testGetApplications() throws Exception {
/**
* 1. Submit 3 applications alternately in two queues
* 2. Test each of the filters
@@ -1113,8 +1113,12 @@ public class TestClientRMService {
SubmitApplicationRequest submitRequest = mockSubmitAppRequest(
appId, appNames[i], queues[i % queues.length],
new HashSet<String>(tags.subList(0, i + 1)));
+ // make sure each app is submitted at a different time
+ Thread.sleep(1);
rmService.submitApplication(submitRequest);
- submitTimeMillis[i] = System.currentTimeMillis();
+ submitTimeMillis[i] = rmService.getApplicationReport(
+ GetApplicationReportRequest.newInstance(appId))
+ .getApplicationReport().getStartTime();
}
// Test different cases of ClientRMService#getApplications()
@@ -1129,19 +1133,19 @@ public class TestClientRMService {
// Check start range
request = GetApplicationsRequest.newInstance();
- request.setStartRange(submitTimeMillis[0], System.currentTimeMillis());
+ request.setStartRange(submitTimeMillis[0] + 1, System.currentTimeMillis());
// 2 applications are submitted after first timeMills
assertEquals("Incorrect number of matching start range",
2, rmService.getApplications(request).getApplicationList().size());
// 1 application is submitted after the second timeMills
- request.setStartRange(submitTimeMillis[1], System.currentTimeMillis());
+ request.setStartRange(submitTimeMillis[1] + 1, System.currentTimeMillis());
assertEquals("Incorrect number of matching start range",
1, rmService.getApplications(request).getApplicationList().size());
// no application is submitted after the third timeMills
- request.setStartRange(submitTimeMillis[2], System.currentTimeMillis());
+ request.setStartRange(submitTimeMillis[2] + 1, System.currentTimeMillis());
assertEquals("Incorrect number of matching start range",
0, rmService.getApplications(request).getApplicationList().size());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/44] hadoop git commit: YARN-7931. [atsv2 read acls] Include
domain table creation as part of schema creator. (Vrushali C via Haibo Chen)
Posted by xy...@apache.org.
YARN-7931. [atsv2 read acls] Include domain table creation as part of schema creator. (Vrushali C via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8597858
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8597858
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8597858
Branch: refs/heads/HDFS-7240
Commit: b8597858b17e40a99611e3a384cdd241293af83f
Parents: 113af12
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Apr 12 06:38:30 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Apr 12 06:38:30 2018 -0700
----------------------------------------------------------------------
.../storage/TimelineSchemaCreator.java | 10 ++
.../storage/domain/DomainTableRW.java | 92 ++++++++++
.../storage/domain/package-info.java | 28 +++
.../storage/domain/DomainColumn.java | 111 ++++++++++++
.../storage/domain/DomainColumnFamily.java | 52 ++++++
.../storage/domain/DomainRowKey.java | 179 +++++++++++++++++++
.../storage/domain/DomainTable.java | 45 +++++
.../storage/domain/package-info.java | 28 +++
.../storage/common/TestRowKeys.java | 32 ++++
9 files changed, 577 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index e9e4770..37ed50c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableR
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTableRW;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTableRW;
import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainTableRW;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
@@ -357,6 +358,15 @@ public final class TimelineSchemaCreator {
throw e;
}
}
+ try {
+ new DomainTableRW().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
} finally {
if (conn != null) {
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java
new file mode 100644
index 0000000..1d58e40
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.domain;
+
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Create, read and write to the domain Table.
+ */
+public class DomainTableRW extends BaseTableRW<DomainTable> {
+ /** domain prefix. */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "domain";
+
+ /** config param name that specifies the domain table name. */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for domain table name. */
+ private static final String DEFAULT_TABLE_NAME = "timelineservice.domain";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DomainTableRW.class);
+
+ public DomainTableRW() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTableRW#
+ * createTable(org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor domainTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor mappCF =
+ new HColumnDescriptor(DomainColumnFamily.INFO.getBytes());
+ mappCF.setBloomFilterType(BloomType.ROWCOL);
+ domainTableDescp.addFamily(mappCF);
+
+ domainTableDescp
+ .setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ domainTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(domainTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java
new file mode 100644
index 0000000..49882e6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.domain contains
+ * a set of classes used for the domain table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.domain;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumn.java
new file mode 100644
index 0000000..73233af
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumn.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.domain;
+
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies fully qualified columns for the {@link DomainTable}.
+ */
+public enum DomainColumn implements Column<DomainTable> {
+
+ /**
+ * The created time.
+ */
+ CREATED_TIME(DomainColumnFamily.INFO, "created_time"),
+
+ /**
+ * The description of the domain.
+ */
+ DESCRIPTION(DomainColumnFamily.INFO, "description"),
+
+ /**
+ * The modification time.
+ */
+ MODIFICATION_TIME(DomainColumnFamily.INFO, "modification_time"),
+
+ /**
+ * The owner.
+ */
+ OWNER(DomainColumnFamily.INFO, "owner"),
+
+ /**
+ * The readers.
+ */
+ READERS(DomainColumnFamily.INFO, "readers"),
+
+ /**
+ * The Writers.
+ */
+ WRITERS(DomainColumnFamily.INFO, "writers");
+
+
+ private final ColumnFamily<DomainTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+ private final ValueConverter valueConverter;
+
+ DomainColumn(ColumnFamily<DomainTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.valueConverter = GenericConverter.getInstance();
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimestamp() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumnFamily.java
new file mode 100644
index 0000000..276e1ad
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainColumnFamily.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.domain;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the domain table column families.
+ */
+public enum DomainColumnFamily implements ColumnFamily<DomainTable> {
+ /**
+ * Info column family houses known columns such as created_time, owners,
+ * readers.
+ */
+ INFO("i");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ DomainColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainRowKey.java
new file mode 100644
index 0000000..6ffaef7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainRowKey.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.domain;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+
+/**
+ * Represents a row key for the domain table, which is the
+ * cluster ! domain id.
+ */
+public class DomainRowKey {
+ private final String clusterId;
+ private final String domainId;
+ private final DomainRowKeyConverter domainIdKeyConverter =
+ new DomainRowKeyConverter();
+
+ public DomainRowKey(String clusterId, String domainId) {
+ this.clusterId = clusterId;
+ this.domainId = domainId;
+ }
+
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getDomainId() {
+ return domainId;
+ }
+
+ /**
+ * Constructs a row key prefix for the domain table.
+ *
+ * @return byte array with the row key
+ */
+ public byte[] getRowKey() {
+
+ return domainIdKeyConverter.encode(this);
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey a rowkey represented as a byte array.
+ * @return an <cite>DomainRowKey</cite> object.
+ */
+ public static DomainRowKey parseRowKey(byte[] rowKey) {
+ return new DomainRowKeyConverter().decode(rowKey);
+ }
+
+ /**
+ * Constructs a row key for the domain table as follows:
+ * <p>
+ * {@code clusterId!domainId}.
+ * </p>
+ * @return String representation of row key.
+ */
+ public String getRowKeyAsString() {
+ return domainIdKeyConverter.encodeAsString(this);
+ }
+
+ /**
+ * Given the encoded row key as string, returns the row key as an object.
+ * @param encodedRowKey String representation of row key.
+ * @return A <cite>DomainRowKey</cite> object.
+ */
+ public static DomainRowKey parseRowKeyFromString(String encodedRowKey) {
+ return new DomainRowKeyConverter().decodeFromString(encodedRowKey);
+ }
+
+ /**
+ * Encodes and decodes row key for the domain table.
+ * The row key is of the
+ * form : domainId
+ * <p>
+ */
+ final private static class DomainRowKeyConverter
+ implements KeyConverter<DomainRowKey>,
+ KeyConverterToString<DomainRowKey> {
+
+ private DomainRowKeyConverter() {
+ }
+
+ /**
+ * The domain row key is of the form
+ * clusterId!domainId with each segment separated by !.
+ * The sizes below indicate sizes of each one of
+ * these segements in sequence.
+ * clusterId and domainId are strings.
+ * Strings are variable in size
+ * (i.e. they end whenever separator is encountered).
+ * This is used while
+ * decoding and helps in determining where to split.
+ */
+ private static final int[] SEGMENT_SIZES = {
+ Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE};
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes DomainRowKey object into a byte array
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(DomainRowKey rowKey) {
+ if (rowKey == null) {
+ return Separator.EMPTY_BYTES;
+ }
+
+ byte[] cluster =
+ Separator.encode(rowKey.getClusterId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+ byte[] domainIdBytes =
+ Separator.encode(rowKey.getDomainId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+
+ return Separator.QUALIFIERS.join(cluster, domainIdBytes);
+ }
+
+ @Override
+ public DomainRowKey decode(byte[] rowKey) {
+ byte[][] rowKeyComponents =
+ Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
+ if (rowKeyComponents.length != 2) {
+ throw new IllegalArgumentException("the row key is not valid for "
+ + "a domain id");
+ }
+ String clusterId =
+ Separator.decode(Bytes.toString(rowKeyComponents[0]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+
+ String domainId =
+ Separator.decode(Bytes.toString(rowKeyComponents[1]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+
+ return new DomainRowKey(clusterId, domainId);
+ }
+
+ @Override
+ public String encodeAsString(DomainRowKey key) {
+ return TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] {key.clusterId, key.domainId});
+ }
+
+ @Override
+ public DomainRowKey decodeFromString(String encodedRowKey) {
+ List<String> split = TimelineReaderUtils.split(encodedRowKey);
+ if (split == null || split.size() != 2) {
+ throw new IllegalArgumentException(
+ "Invalid row key for domain id.");
+ }
+ return new DomainRowKey(split.get(0), split.get(1));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTable.java
new file mode 100644
index 0000000..c02fed3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTable.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.domain;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The domain table has column family info. Info stores
+ * information about a timeline domain object
+ *
+ * Example domain table record:
+ *
+ * <pre>
+ * |-------------------------------------------|
+ * | Row | Column Family |
+ * | key | info |
+ * |-------------------------------------------|
+ * | clusterId! | created_time:1521676928000 |
+ * | domainI | description: "domain |
+ * | | information for XYZ job" |
+ * | | owners: "user1, yarn" |
+ * | | readers: |
+ * | | "user2,user33 yarn,group2" |
+ * | | |
+ * |-------------------------------------------|
+ * </pre>
+ */
+public final class DomainTable extends BaseTable<DomainTable> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java
new file mode 100644
index 0000000..39e53fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.domain
+ * contains classes related to implementation for domain table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.domain;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
index d05cbad..5b7c7dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRo
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainRowKey;
import org.junit.Test;
@@ -273,4 +274,35 @@ public class TestRowKeys {
assertEquals(USER, rowKey.getUserId());
}
+ @Test
+ public void testDomainRowKey() {
+ String clusterId = "cluster1@dc1";
+ String domainId = "helloworld";
+ byte[] byteRowKey =
+ new DomainRowKey(clusterId, domainId).getRowKey();
+ DomainRowKey rowKey = DomainRowKey.parseRowKey(byteRowKey);
+ assertEquals(clusterId, rowKey.getClusterId());
+ assertEquals(domainId, rowKey.getDomainId());
+
+ String rowKeyStr = rowKey.getRowKeyAsString();
+ DomainRowKey drk = DomainRowKey.parseRowKeyFromString(rowKeyStr);
+ assertEquals(drk.getClusterId(), rowKey.getClusterId());
+ assertEquals(drk.getDomainId(), rowKey.getDomainId());
+ }
+
+ @Test
+ public void testDomainRowKeySpecialChars() {
+ String clusterId = "cluster1!temp!dc1";
+ String domainId = "hello=world";
+ byte[] byteRowKey =
+ new DomainRowKey(clusterId, domainId).getRowKey();
+ DomainRowKey rowKey = DomainRowKey.parseRowKey(byteRowKey);
+ assertEquals(clusterId, rowKey.getClusterId());
+ assertEquals(domainId, rowKey.getDomainId());
+
+ String rowKeyStr = rowKey.getRowKeyAsString();
+ DomainRowKey drk = DomainRowKey.parseRowKeyFromString(rowKeyStr);
+ assertEquals(drk.getClusterId(), rowKey.getClusterId());
+ assertEquals(drk.getDomainId(), rowKey.getDomainId());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/44] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by xy...@apache.org.
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72a3743c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72a3743c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72a3743c
Branch: refs/heads/HDFS-7240
Commit: 72a3743cc49d9c7b8d2eaec8064a25b8d890c267
Parents: 66610b5 995cba6
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Apr 13 17:00:19 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Fri Apr 13 17:00:19 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 11 +-
.../crypto/key/kms/KMSClientProvider.java | 212 ++++----
.../crypto/key/kms/KMSDelegationToken.java | 22 +-
.../crypto/key/kms/KMSLegacyTokenRenewer.java | 56 ++
.../hadoop/crypto/key/kms/KMSTokenRenewer.java | 103 ++++
.../hadoop/crypto/key/kms/package-info.java | 18 +
.../apache/hadoop/fs/ChecksumFileSystem.java | 9 +-
.../fs/CommonConfigurationKeysPublic.java | 10 +
.../hadoop/fs/CompositeCrcFileChecksum.java | 82 +++
.../java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../main/java/org/apache/hadoop/fs/Options.java | 11 +
.../org/apache/hadoop/fs/shell/Command.java | 69 ++-
.../apache/hadoop/fs/shell/CopyCommands.java | 6 +
.../java/org/apache/hadoop/fs/shell/Ls.java | 26 +-
.../org/apache/hadoop/fs/shell/PathData.java | 27 +
.../web/DelegationTokenAuthenticatedURL.java | 21 +-
.../DelegationTokenAuthenticationHandler.java | 8 +-
.../web/DelegationTokenAuthenticator.java | 2 +-
.../hadoop/service/launcher/IrqHandler.java | 2 +-
.../org/apache/hadoop/util/CrcComposer.java | 187 +++++++
.../java/org/apache/hadoop/util/CrcUtil.java | 220 ++++++++
.../org/apache/hadoop/util/DataChecksum.java | 18 +
.../java/org/apache/hadoop/util/KMSUtil.java | 45 +-
.../hadoop/util/KMSUtilFaultInjector.java | 49 ++
...apache.hadoop.security.token.TokenIdentifier | 1 +
...rg.apache.hadoop.security.token.TokenRenewer | 3 +-
.../src/main/resources/core-default.xml | 20 +
.../apache/hadoop/conf/TestConfiguration.java | 26 +-
.../crypto/key/kms/TestKMSClientProvider.java | 162 ++++++
.../kms/TestLoadBalancingKMSClientProvider.java | 67 ++-
.../apache/hadoop/fs/shell/find/TestFind.java | 34 +-
.../org/apache/hadoop/util/TestCrcComposer.java | 242 +++++++++
.../org/apache/hadoop/util/TestCrcUtil.java | 232 +++++++++
.../org/apache/hadoop/util/TestKMSUtil.java | 65 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 519 ++++++++++++++++---
.../main/java/org/apache/hadoop/fs/Hdfs.java | 4 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 56 +-
.../hadoop/hdfs/DistributedFileSystem.java | 5 +-
.../apache/hadoop/hdfs/FileChecksumHelper.java | 365 ++++++++-----
.../hdfs/client/HdfsClientConfigKeys.java | 2 +
.../hadoop/hdfs/client/impl/DfsClientConf.java | 27 +
.../hdfs/protocol/BlockChecksumOptions.java | 54 ++
.../hadoop/hdfs/protocol/BlockChecksumType.java | 30 ++
.../datatransfer/DataTransferProtocol.java | 12 +-
.../hdfs/protocol/datatransfer/Sender.java | 11 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 44 ++
.../ha/RequestHedgingProxyProvider.java | 3 -
.../hdfs/shortcircuit/ShortCircuitCache.java | 11 +-
.../src/main/proto/datatransfer.proto | 7 +-
.../src/main/proto/hdfs.proto | 21 +
.../ha/TestRequestHedgingProxyProvider.java | 34 --
.../native/libhdfspp/include/hdfspp/hdfspp.h | 53 +-
.../native/libhdfspp/include/hdfspp/ioservice.h | 140 +++++
.../native/libhdfspp/lib/bindings/c/hdfs.cc | 7 +-
.../native/libhdfspp/lib/common/CMakeLists.txt | 2 +-
.../native/libhdfspp/lib/common/async_stream.h | 13 +-
.../libhdfspp/lib/common/continuation/asio.h | 5 -
.../libhdfspp/lib/common/hdfs_ioservice.cc | 146 ------
.../libhdfspp/lib/common/hdfs_ioservice.h | 79 ---
.../libhdfspp/lib/common/ioservice_impl.cc | 159 ++++++
.../libhdfspp/lib/common/ioservice_impl.h | 76 +++
.../main/native/libhdfspp/lib/common/logging.h | 3 -
.../libhdfspp/lib/common/namenode_info.cc | 15 +-
.../native/libhdfspp/lib/common/namenode_info.h | 8 +-
.../main/native/libhdfspp/lib/common/util.cc | 14 +-
.../src/main/native/libhdfspp/lib/common/util.h | 25 +-
.../lib/connection/datanodeconnection.cc | 27 +-
.../lib/connection/datanodeconnection.h | 26 +-
.../main/native/libhdfspp/lib/fs/filehandle.cc | 18 +-
.../main/native/libhdfspp/lib/fs/filehandle.h | 12 +-
.../main/native/libhdfspp/lib/fs/filesystem.cc | 67 ++-
.../main/native/libhdfspp/lib/fs/filesystem.h | 66 +--
.../libhdfspp/lib/fs/namenode_operations.h | 4 +-
.../native/libhdfspp/lib/reader/block_reader.cc | 18 +-
.../native/libhdfspp/lib/reader/block_reader.h | 10 +-
.../native/libhdfspp/lib/reader/datatransfer.h | 4 +-
.../libhdfspp/lib/rpc/namenode_tracker.cc | 2 +-
.../native/libhdfspp/lib/rpc/namenode_tracker.h | 4 +-
.../main/native/libhdfspp/lib/rpc/request.cc | 5 +-
.../native/libhdfspp/lib/rpc/rpc_connection.h | 2 +-
.../libhdfspp/lib/rpc/rpc_connection_impl.cc | 32 +-
.../libhdfspp/lib/rpc/rpc_connection_impl.h | 9 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 14 +-
.../main/native/libhdfspp/lib/rpc/rpc_engine.h | 9 +-
.../native/libhdfspp/tests/bad_datanode_test.cc | 31 +-
.../libhdfspp/tests/hdfs_ioservice_test.cc | 10 +-
.../native/libhdfspp/tests/mock_connection.h | 4 +-
.../libhdfspp/tests/remote_block_reader_test.cc | 4 +-
.../native/libhdfspp/tests/rpc_engine_test.cc | 112 ++--
.../resolver/FederationNamespaceInfo.java | 5 +
.../federation/resolver/MountTableResolver.java | 4 +-
.../federation/resolver/RemoteLocation.java | 35 +-
.../router/RemoteLocationContext.java | 7 +
.../federation/router/RouterRpcClient.java | 67 ++-
.../federation/router/RouterRpcServer.java | 19 +-
.../store/driver/impl/StateStoreFileImpl.java | 11 +-
.../federation/store/records/MountTable.java | 2 +-
.../store/records/impl/pb/MountTablePBImpl.java | 2 +-
.../hdfs/server/federation/MockResolver.java | 12 +-
.../federation/router/TestRouterAdmin.java | 4 +-
.../federation/router/TestRouterMountTable.java | 62 ++-
.../server/federation/router/TestRouterRpc.java | 48 ++
.../store/records/TestMountTable.java | 4 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +
.../hadoop/hdfs/net/DFSNetworkTopology.java | 10 +-
.../hdfs/protocol/datatransfer/Receiver.java | 8 +-
.../protocol/datatransfer/package-info.java | 8 +-
.../common/blockaliasmap/package-info.java | 9 +-
.../server/datanode/BlockChecksumHelper.java | 289 +++++++++--
.../hdfs/server/datanode/DataXceiver.java | 26 +-
...dBlockChecksumCompositeCrcReconstructor.java | 80 +++
...StripedBlockChecksumMd5CrcReconstructor.java | 74 +++
.../StripedBlockChecksumReconstructor.java | 66 ++-
.../erasurecode/StripedBlockReconstructor.java | 1 +
.../impl/FsDatasetAsyncDiskService.java | 3 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 2 +-
.../diskbalancer/connectors/package-info.java | 3 +-
.../diskbalancer/datamodel/package-info.java | 4 +-
.../hdfs/server/diskbalancer/package-info.java | 2 +-
.../diskbalancer/planner/package-info.java | 2 +-
.../org/apache/hadoop/hdfs/tools/DFSck.java | 2 +-
.../src/main/resources/hdfs-default.xml | 25 +
.../src/site/markdown/TransparentEncryption.md | 57 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 31 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 3 +
.../apache/hadoop/hdfs/TestFileChecksum.java | 101 +++-
.../hdfs/TestFileChecksumCompositeCrc.java | 47 ++
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 14 +
.../blockmanagement/TestDatanodeManager.java | 52 ++
.../shortcircuit/TestShortCircuitCache.java | 26 +
.../apache/hadoop/mapred/MapReduceChildJVM.java | 73 +--
.../v2/app/job/impl/TaskAttemptImpl.java | 8 +-
.../v2/app/job/impl/TestMapReduceChildJVM.java | 24 +-
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 10 +
.../java/org/apache/hadoop/mapred/JobConf.java | 18 +
.../apache/hadoop/mapred/pipes/Application.java | 5 +-
.../src/main/resources/mapred-default.xml | 67 ++-
.../src/site/markdown/MapReduceTutorial.md | 6 +
.../HistoryServerLeveldbStateStoreService.java | 11 -
.../org/apache/hadoop/mapred/YARNRunner.java | 11 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 26 +-
.../mapred/pipes/TestPipeApplication.java | 19 +-
.../mapreduce/MiniHadoopClusterManager.java | 1 +
.../apache/hadoop/mapred/ShuffleHandler.java | 12 -
hadoop-maven-plugins/pom.xml | 2 +-
.../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 6 +-
.../org/apache/hadoop/fs/s3a/TestInvoker.java | 14 +-
.../hadoop/tools/mapred/TestCopyMapper.java | 173 +++++--
.../mapred/TestCopyMapperCompositeCrc.java | 50 ++
.../hadoop/yarn/api/ApplicationConstants.java | 10 +
.../hadoop/yarn/service/webapp/ApiServer.java | 47 +-
.../hadoop/yarn/service/ServiceClientTest.java | 6 +
.../hadoop/yarn/service/TestApiServer.java | 26 +
.../hadoop/yarn/service/ClientAMService.java | 1 +
.../hadoop/yarn/service/ServiceScheduler.java | 41 +-
.../yarn/service/client/ServiceClient.java | 102 +++-
.../yarn/service/component/Component.java | 1 +
.../yarn/service/conf/YarnServiceConf.java | 2 +
.../hadoop/yarn/service/ServiceTestUtils.java | 26 +-
.../yarn/service/TestYarnNativeServices.java | 129 ++++-
.../java/org/apache/hadoop/yarn/util/Apps.java | 115 +++-
.../org/apache/hadoop/yarn/util/TestApps.java | 136 +++++
.../server/nodemanager/ContainerExecutor.java | 11 +
.../nodemanager/DefaultContainerExecutor.java | 6 +
.../nodemanager/LinuxContainerExecutor.java | 124 +++--
.../container/ContainerImpl.java | 3 +-
.../launcher/ContainerLaunch.java | 20 +-
.../launcher/ContainerRelaunch.java | 2 +-
.../runtime/DefaultLinuxContainerRuntime.java | 6 +
.../DelegatingLinuxContainerRuntime.java | 9 +
.../runtime/DockerLinuxContainerRuntime.java | 50 +-
.../JavaSandboxLinuxContainerRuntime.java | 10 +
.../runtime/docker/DockerCommandExecutor.java | 14 +
.../runtime/docker/DockerStartCommand.java | 29 ++
.../runtime/ContainerRuntime.java | 10 +
.../scheduler/ContainerScheduler.java | 7 +-
.../recovery/NMLeveldbStateStoreService.java | 16 +-
.../impl/container-executor.c | 2 +-
.../container-executor/impl/utils/docker-util.c | 140 ++++-
.../container-executor/impl/utils/docker-util.h | 11 +
.../test/utils/test_docker_util.cc | 117 +++--
.../nodemanager/TestLinuxContainerExecutor.java | 12 +
.../TestContainerManagerRecovery.java | 51 ++
.../launcher/TestContainerRelaunch.java | 2 +-
.../runtime/TestDockerContainerRuntime.java | 101 +++-
.../runtime/docker/TestDockerStartCommand.java | 53 ++
.../TestContainersMonitorResourceChange.java | 5 +
.../TestNMLeveldbStateStoreService.java | 16 +
.../recovery/LeveldbRMStateStore.java | 12 -
.../scheduler/common/fica/FiCaSchedulerApp.java | 10 +
.../resourcemanager/TestClientRMService.java | 14 +-
.../TestCapacitySchedulerAsyncScheduling.java | 91 ++++
.../storage/TimelineSchemaCreator.java | 10 +
.../storage/domain/DomainTableRW.java | 92 ++++
.../storage/domain/package-info.java | 28 +
.../storage/domain/DomainColumn.java | 111 ++++
.../storage/domain/DomainColumnFamily.java | 52 ++
.../storage/domain/DomainRowKey.java | 179 +++++++
.../storage/domain/DomainTable.java | 45 ++
.../storage/domain/package-info.java | 28 +
.../storage/common/TestRowKeys.java | 32 ++
.../site/markdown/PlacementConstraints.md.vm | 15 +-
.../src/site/markdown/yarn-service/Concepts.md | 6 +-
.../src/site/markdown/yarn-service/Overview.md | 14 +-
.../site/markdown/yarn-service/QuickStart.md | 12 +-
.../site/markdown/yarn-service/RegistryDNS.md | 4 +-
.../markdown/yarn-service/ServiceDiscovery.md | 4 +-
207 files changed, 6996 insertions(+), 1510 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72a3743c/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/44] hadoop git commit: HDFS-13045. RBF: Improve error message
returned from subcluster. Contributed by Inigo Goiri.
Posted by xy...@apache.org.
HDFS-13045. RBF: Improve error message returned from subcluster. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c93d43f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c93d43f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c93d43f
Branch: refs/heads/HDFS-7240
Commit: 0c93d43f3d624a4fd17b3b050443d9e7e20d4f0a
Parents: eefe2a1
Author: Wei Yan <we...@apache.org>
Authored: Wed Apr 11 08:37:43 2018 -0700
Committer: Wei Yan <we...@apache.org>
Committed: Wed Apr 11 08:37:43 2018 -0700
----------------------------------------------------------------------
.../resolver/FederationNamespaceInfo.java | 5 ++
.../federation/resolver/MountTableResolver.java | 4 +-
.../federation/resolver/RemoteLocation.java | 35 ++++++----
.../router/RemoteLocationContext.java | 7 ++
.../federation/router/RouterRpcClient.java | 67 +++++++++++++++++++-
.../federation/store/records/MountTable.java | 2 +-
.../store/records/impl/pb/MountTablePBImpl.java | 2 +-
.../hdfs/server/federation/MockResolver.java | 12 +++-
.../federation/router/TestRouterAdmin.java | 4 +-
.../server/federation/router/TestRouterRpc.java | 48 ++++++++++++++
.../store/records/TestMountTable.java | 4 +-
11 files changed, 165 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
index edcd308..33edd30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
@@ -48,6 +48,11 @@ public class FederationNamespaceInfo extends RemoteLocationContext {
return this.nameserviceId;
}
+ @Override
+ public String getSrc() {
+ return null;
+ }
+
/**
* The HDFS cluster id for this namespace.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index 9713138..3f6efd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -388,7 +388,7 @@ public class MountTableResolver
} else {
// Not found, use default location
RemoteLocation remoteLocation =
- new RemoteLocation(defaultNameService, path);
+ new RemoteLocation(defaultNameService, path, path);
List<RemoteLocation> locations =
Collections.singletonList(remoteLocation);
ret = new PathLocation(null, locations);
@@ -519,7 +519,7 @@ public class MountTableResolver
newPath += Path.SEPARATOR;
}
newPath += remainingPath;
- RemoteLocation remoteLocation = new RemoteLocation(nsId, newPath);
+ RemoteLocation remoteLocation = new RemoteLocation(nsId, newPath, path);
locations.add(remoteLocation);
}
DestinationOrder order = entry.getDestOrder();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
index 6aa12ce..77d0500 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.federation.resolver;
import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
/**
- * A single in a remote namespace consisting of a nameservice ID
- * and a HDFS path.
+ * A location in a remote namespace consisting of a nameservice ID and a HDFS
+ * path (destination). It also contains the federated location (source).
*/
public class RemoteLocation extends RemoteLocationContext {
@@ -30,16 +30,19 @@ public class RemoteLocation extends RemoteLocationContext {
/** Identifier of the namenode in the namespace for this location. */
private final String namenodeId;
/** Path in the remote location. */
- private final String path;
+ private final String dstPath;
+ /** Original path in federation. */
+ private final String srcPath;
/**
* Create a new remote location.
*
- * @param nsId
- * @param pPath
+ * @param nsId Destination namespace.
+ * @param dPath Path in the destination namespace.
+ * @param sPath Path in the federated level.
*/
- public RemoteLocation(String nsId, String pPath) {
- this(nsId, null, pPath);
+ public RemoteLocation(String nsId, String dPath, String sPath) {
+ this(nsId, null, dPath, sPath);
}
/**
@@ -47,12 +50,15 @@ public class RemoteLocation extends RemoteLocationContext {
* namespace.
*
* @param nsId Destination namespace.
- * @param pPath Path in the destination namespace.
+ * @param nnId Destination namenode.
+ * @param dPath Path in the destination namespace.
+ * @param sPath Path in the federated level
*/
- public RemoteLocation(String nsId, String nnId, String pPath) {
+ public RemoteLocation(String nsId, String nnId, String dPath, String sPath) {
this.nameserviceId = nsId;
this.namenodeId = nnId;
- this.path = pPath;
+ this.dstPath = dPath;
+ this.srcPath = sPath;
}
@Override
@@ -66,11 +72,16 @@ public class RemoteLocation extends RemoteLocationContext {
@Override
public String getDest() {
- return this.path;
+ return this.dstPath;
+ }
+
+ @Override
+ public String getSrc() {
+ return this.srcPath;
}
@Override
public String toString() {
- return getNameserviceId() + "->" + this.path;
+ return getNameserviceId() + "->" + this.dstPath;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
index a90c460..0959eaa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
@@ -39,6 +39,13 @@ public abstract class RemoteLocationContext
*/
public abstract String getDest();
+ /**
+ * Original source location.
+ *
+ * @return Source path.
+ */
+ public abstract String getSrc();
+
@Override
public int hashCode() {
return new HashCodeBuilder(17, 31)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index e2c9cb4..6c657b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.federation.router;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
@@ -62,6 +63,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -611,7 +613,7 @@ public class RouterRpcClient {
UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
List<? extends FederationNamenodeContext> nns =
getNamenodesForNameservice(nsId);
- RemoteLocationContext loc = new RemoteLocation(nsId, "/");
+ RemoteLocationContext loc = new RemoteLocation(nsId, "/", "/");
Class<?> proto = method.getProtocol();
Method m = method.getMethod();
Object[] params = method.getParams(loc);
@@ -727,8 +729,12 @@ public class RouterRpcClient {
firstResult = result;
}
} catch (IOException ioe) {
+ // Localize the exception
+
+ ioe = processException(ioe, loc);
+
// Record it and move on
- lastThrownException = (IOException) ioe;
+ lastThrownException = ioe;
if (firstThrownException == null) {
firstThrownException = lastThrownException;
}
@@ -757,6 +763,63 @@ public class RouterRpcClient {
}
/**
+ * Exception messages might contain local subcluster paths. This method
+ * generates a new exception with the proper message.
+ * @param ioe Original IOException.
+ * @param loc Location we are processing.
+ * @return Exception processed for federation.
+ */
+ private IOException processException(
+ IOException ioe, RemoteLocationContext loc) {
+
+ if (ioe instanceof RemoteException) {
+ RemoteException re = (RemoteException)ioe;
+ String newMsg = processExceptionMsg(
+ re.getMessage(), loc.getDest(), loc.getSrc());
+ RemoteException newException =
+ new RemoteException(re.getClassName(), newMsg);
+ newException.setStackTrace(ioe.getStackTrace());
+ return newException;
+ }
+
+ if (ioe instanceof FileNotFoundException) {
+ String newMsg = processExceptionMsg(
+ ioe.getMessage(), loc.getDest(), loc.getSrc());
+ FileNotFoundException newException = new FileNotFoundException(newMsg);
+ newException.setStackTrace(ioe.getStackTrace());
+ return newException;
+ }
+
+ return ioe;
+ }
+
+ /**
+ * Process a subcluster message and make it federated.
+ * @param msg Original exception message.
+ * @param dst Path in federation.
+ * @param src Path in the subcluster.
+ * @return Message processed for federation.
+ */
+ @VisibleForTesting
+ static String processExceptionMsg(
+ final String msg, final String dst, final String src) {
+ if (dst.equals(src) || !dst.startsWith("/") || !src.startsWith("/")) {
+ return msg;
+ }
+
+ String newMsg = msg.replaceFirst(dst, src);
+ int minLen = Math.min(dst.length(), src.length());
+ for (int i = 0; newMsg.equals(msg) && i < minLen; i++) {
+ // Check if we can replace sub folders
+ String dst1 = dst.substring(0, dst.length() - 1 - i);
+ String src1 = src.substring(0, src.length() - 1 - i);
+ newMsg = msg.replaceFirst(dst1, src1);
+ }
+
+ return newMsg;
+ }
+
+ /**
* Checks if a result matches the required result class.
*
* @param expectedClass Required result class, null to skip the check.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
index 60496ef..005882e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -134,7 +134,7 @@ public abstract class MountTable extends BaseRecord {
for (Entry<String, String> entry : destinations.entrySet()) {
String nsId = entry.getKey();
String path = normalizeFileSystemPath(entry.getValue());
- RemoteLocation location = new RemoteLocation(nsId, path);
+ RemoteLocation location = new RemoteLocation(nsId, path, src);
locations.add(location);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
index 48f93bc..e62d0a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
@@ -102,7 +102,7 @@ public class MountTablePBImpl extends MountTable implements PBRecord {
for (RemoteLocationProto dest : destList) {
String nsId = dest.getNameserviceId();
String path = dest.getPath();
- RemoteLocation loc = new RemoteLocation(nsId, path);
+ RemoteLocation loc = new RemoteLocation(nsId, path, getSourcePath());
ret.add(loc);
}
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index 151d731..0ce0944 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -80,7 +80,8 @@ public class MockResolver
this.locations.put(mount, locationsList);
}
- final RemoteLocation remoteLocation = new RemoteLocation(nsId, location);
+ final RemoteLocation remoteLocation =
+ new RemoteLocation(nsId, location, mount);
if (!locationsList.contains(remoteLocation)) {
locationsList.add(remoteLocation);
}
@@ -270,10 +271,15 @@ public class MockResolver
for (String key : keys) {
if (path.startsWith(key)) {
for (RemoteLocation location : this.locations.get(key)) {
- String finalPath = location.getDest() + path.substring(key.length());
+ String finalPath = location.getDest();
+ String extraPath = path.substring(key.length());
+ if (finalPath.endsWith("/") && extraPath.startsWith("/")) {
+ extraPath = extraPath.substring(1);
+ }
+ finalPath += extraPath;
String nameservice = location.getNameserviceId();
RemoteLocation remoteLocation =
- new RemoteLocation(nameservice, finalPath);
+ new RemoteLocation(nameservice, finalPath, path);
remoteLocations.add(remoteLocation);
}
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
index 3c1a136..10b71d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
@@ -251,7 +251,7 @@ public class TestRouterAdmin {
// Verify starting condition
MountTable entry = getMountTableEntry("/");
assertEquals(
- Collections.singletonList(new RemoteLocation("ns0", "/")),
+ Collections.singletonList(new RemoteLocation("ns0", "/", "/")),
entry.getDestinations());
// Edit the entry for /
@@ -264,7 +264,7 @@ public class TestRouterAdmin {
// Verify edited condition
entry = getMountTableEntry("/");
assertEquals(
- Collections.singletonList(new RemoteLocation("ns1", "/")),
+ Collections.singletonList(new RemoteLocation("ns1", "/", "/")),
entry.getDestinations());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index 414e112..d5ec093 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.delet
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING;
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -31,6 +32,7 @@ import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.lang.reflect.Method;
@@ -1023,6 +1025,52 @@ public class TestRouterRpc {
}
@Test
+ public void testProxyExceptionMessages() throws IOException {
+
+ // Install a mount point to a different path to check
+ MockResolver resolver =
+ (MockResolver)router.getRouter().getSubclusterResolver();
+ String ns0 = cluster.getNameservices().get(0);
+ resolver.addLocation("/mnt", ns0, "/");
+
+ try {
+ FsPermission permission = new FsPermission("777");
+ routerProtocol.mkdirs("/mnt/folder0/folder1", permission, false);
+ fail("mkdirs for non-existing parent folder should have failed");
+ } catch (IOException ioe) {
+ assertExceptionContains("/mnt/folder0", ioe,
+ "Wrong path in exception for mkdirs");
+ }
+
+ try {
+ FsPermission permission = new FsPermission("777");
+ routerProtocol.setPermission("/mnt/testfile.txt", permission);
+ fail("setPermission for non-existing file should have failed");
+ } catch (IOException ioe) {
+ assertExceptionContains("/mnt/testfile.txt", ioe,
+ "Wrong path in exception for setPermission");
+ }
+
+ try {
+ FsPermission permission = new FsPermission("777");
+ routerProtocol.mkdirs("/mnt/folder0/folder1", permission, false);
+ routerProtocol.delete("/mnt/folder0", false);
+ fail("delete for non-existing file should have failed");
+ } catch (IOException ioe) {
+ assertExceptionContains("/mnt/folder0", ioe,
+ "Wrong path in exception for delete");
+ }
+
+ resolver.cleanRegistrations();
+
+ // Check corner cases
+ assertEquals(
+ "Parent directory doesn't exist: /ns1/a/a/b",
+ RouterRpcClient.processExceptionMsg(
+ "Parent directory doesn't exist: /a/a/b", "/a", "/ns1/a"));
+ }
+
+ @Test
public void testErasureCoding() throws IOException {
LOG.info("List the available erasurce coding policies");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
index d5fb9ba..43cf176 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
@@ -50,8 +50,8 @@ public class TestMountTable {
private static final String DST_PATH_1 = "/path/path2";
private static final List<RemoteLocation> DST = new LinkedList<>();
static {
- DST.add(new RemoteLocation(DST_NS_0, DST_PATH_0));
- DST.add(new RemoteLocation(DST_NS_1, DST_PATH_1));
+ DST.add(new RemoteLocation(DST_NS_0, DST_PATH_0, SRC));
+ DST.add(new RemoteLocation(DST_NS_1, DST_PATH_1, SRC));
}
private static final Map<String, String> DST_MAP = new LinkedHashMap<>();
static {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/44] hadoop git commit: HDFS-13403: libhdfs++ Use hdfs::IoService
object rather than asio::io_service. Contributed by James Clampffer.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
index 8e579a2..1dd43af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
@@ -26,6 +26,7 @@
#include "common/logging.h"
#include "common/util.h"
#include "common/libhdfs_events_impl.h"
+#include "hdfspp/ioservice.h"
#include <asio/connect.hpp>
#include <asio/read.hpp>
@@ -76,8 +77,8 @@ template <class Socket>
RpcConnectionImpl<Socket>::RpcConnectionImpl(std::shared_ptr<RpcEngine> engine)
: RpcConnection(engine),
options_(engine->options()),
- socket_(engine->io_service()),
- connect_timer_(engine->io_service())
+ socket_(engine->io_service()->GetRaw()),
+ connect_timer_(engine->io_service()->GetRaw())
{
LOG_TRACE(kRPC, << "RpcConnectionImpl::RpcConnectionImpl called &" << (void*)this);
}
@@ -353,7 +354,7 @@ void RpcConnectionImpl<Socket>::FlushPendingRequests() {
OnSendCompleted(ec, size);
});
} else { // Nothing to send for this request, inform the handler immediately
- ::asio::io_service *service = GetIoService();
+ std::shared_ptr<IoService> service = GetIoService();
if(!service) {
LOG_ERROR(kRPC, << "RpcConnectionImpl@" << this << " attempted to access null IoService");
// No easy way to bail out of this context, but the only way to get here is when
@@ -361,7 +362,7 @@ void RpcConnectionImpl<Socket>::FlushPendingRequests() {
return;
}
- service->post(
+ service->PostTask(
// Never hold locks when calling a callback
[req]() { req->OnResponseArrived(nullptr, Status::OK()); }
);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index 0ca7c6a..ad6c9b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -30,7 +30,7 @@ template <class T>
using optional = std::experimental::optional<T>;
-RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
+RpcEngine::RpcEngine(std::shared_ptr<IoService> io_service, const Options &options,
const std::string &client_name, const std::string &user_name,
const char *protocol_name, int protocol_version)
: io_service_(io_service),
@@ -40,7 +40,7 @@ RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options,
protocol_name_(protocol_name),
protocol_version_(protocol_version),
call_id_(0),
- retry_timer(*io_service),
+ retry_timer(io_service->GetRaw()),
event_handlers_(std::make_shared<LibhdfsEvents>()),
connect_canceled_(false)
{
@@ -86,7 +86,7 @@ bool RpcEngine::CancelPendingConnect() {
void RpcEngine::Shutdown() {
LOG_DEBUG(kRPC, << "RpcEngine::Shutdown called");
- io_service_->post([this]() {
+ io_service_->PostLambda([this]() {
std::lock_guard<std::mutex> state_lock(engine_state_lock_);
conn_.reset();
});
@@ -154,7 +154,7 @@ void RpcEngine::AsyncRpc(
// In case user-side code isn't checking the status of Connect before doing RPC
if(connect_canceled_) {
- io_service_->post(
+ io_service_->PostLambda(
[handler](){ handler(Status::Canceled()); }
);
return;
@@ -190,7 +190,7 @@ void RpcEngine::AsyncRpcCommsError(
std::vector<std::shared_ptr<Request>> pendingRequests) {
LOG_ERROR(kRPC, << "RpcEngine::AsyncRpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << std::to_string(pendingRequests.size()));
- io_service().post([this, status, failedConnection, pendingRequests]() {
+ io_service_->PostLambda([this, status, failedConnection, pendingRequests]() {
RpcCommsError(status, failedConnection, pendingRequests);
});
}
@@ -238,7 +238,7 @@ void RpcEngine::RpcCommsError(
// on. There might be a good argument for caching the first error
// rather than the last one, that gets messy
- io_service().post([req, status]() {
+ io_service()->PostLambda([req, status]() {
req->OnResponseArrived(nullptr, status); // Never call back while holding a lock
});
it = pendingRequests.erase(it);
@@ -283,7 +283,7 @@ void RpcEngine::RpcCommsError(
for(unsigned int i=0; i<pendingRequests.size(); i++) {
std::shared_ptr<Request> sharedCurrentRequest = pendingRequests[i];
- io_service().post([sharedCurrentRequest, badEndpointStatus]() {
+ io_service()->PostLambda([sharedCurrentRequest, badEndpointStatus]() {
sharedCurrentRequest->OnResponseArrived(nullptr, badEndpointStatus); // Never call back while holding a lock
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
index 9f45fcf..845eaf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.h
@@ -60,6 +60,7 @@ class RpcConnection;
class SaslProtocol;
class RpcConnection;
class Request;
+class IoService;
/*
* These methods of the RpcEngine will never acquire locks, and are safe for
@@ -83,7 +84,7 @@ public:
virtual const std::string &user_name() = 0;
virtual const std::string &protocol_name() = 0;
virtual int protocol_version() = 0;
- virtual ::asio::io_service &io_service() = 0;
+ virtual std::shared_ptr<IoService> io_service() const = 0;
virtual const Options &options() = 0;
};
@@ -107,7 +108,7 @@ class RpcEngine : public LockFreeRpcEngine, public std::enable_shared_from_this<
kCallIdSasl = -33
};
- RpcEngine(::asio::io_service *io_service, const Options &options,
+ RpcEngine(std::shared_ptr<IoService> service, const Options &options,
const std::string &client_name, const std::string &user_name,
const char *protocol_name, int protocol_version);
@@ -145,7 +146,7 @@ class RpcEngine : public LockFreeRpcEngine, public std::enable_shared_from_this<
const std::string &user_name() override { return auth_info_.getUser(); }
const std::string &protocol_name() override { return protocol_name_; }
int protocol_version() override { return protocol_version_; }
- ::asio::io_service &io_service() override { return *io_service_; }
+ std::shared_ptr<IoService> io_service() const override { return io_service_; }
const Options &options() override { return options_; }
static std::string GetRandomClientName();
@@ -162,7 +163,7 @@ protected:
std::vector<::asio::ip::tcp::endpoint> last_endpoints_;
private:
- ::asio::io_service * const io_service_;
+ mutable std::shared_ptr<IoService> io_service_;
const Options options_;
const std::string client_name_;
const std::string client_id_;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
index 00bbf3d..23de015 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
@@ -16,11 +16,12 @@
* limitations under the License.
*/
-#include "fs/filesystem.h"
-#include "fs/bad_datanode_tracker.h"
#include "common/libhdfs_events_impl.h"
-
#include "common/util.h"
+#include "fs/filesystem.h"
+#include "fs/filehandle.h"
+#include "fs/bad_datanode_tracker.h"
+#include "reader/block_reader.h"
#include <gmock/gmock.h>
@@ -54,7 +55,7 @@ public:
const std::string & client_name,
const hadoop::hdfs::LocatedBlockProto &block,
size_t offset,
- const MutableBuffers &buffers,
+ const MutableBuffer &buffer,
const std::function<void(const Status &, size_t)> handler));
virtual void CancelOperation() override {
@@ -67,14 +68,14 @@ class MockDNConnection : public DataNodeConnection, public std::enable_shared_fr
handler(Status::OK(), shared_from_this());
}
- void async_read_some(const MutableBuffers &buf,
+ void async_read_some(const MutableBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
(void)buf;
handler(asio::error::fault, 0);
}
- void async_write_some(const ConstBuffers &buf,
+ void async_write_some(const ConstBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
(void)buf;
@@ -101,7 +102,7 @@ protected:
return mock_reader_;
}
std::shared_ptr<DataNodeConnection> CreateDataNodeConnection(
- ::asio::io_service *io_service,
+ std::shared_ptr<IoService> io_service,
const ::hadoop::hdfs::DatanodeInfoProto & dn,
const hadoop::common::TokenProto * token) override {
(void) io_service; (void) dn; (void) token;
@@ -130,12 +131,12 @@ TEST(BadDataNodeTest, TestNoNodes) {
char buf[4096] = {
0,
};
- IoServiceImpl io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
auto bad_node_tracker = std::make_shared<BadDataNodeTracker>();
auto monitors = std::make_shared<LibhdfsEvents>();
bad_node_tracker->AddBadNode("foo");
- PartialMockFileHandle is("cluster", "file", &io_service.io_service(), GetRandomClientName(), file_info, bad_node_tracker, monitors);
+ PartialMockFileHandle is("cluster", "file", io_service, GetRandomClientName(), file_info, bad_node_tracker, monitors);
Status stat;
size_t read = 0;
@@ -170,7 +171,7 @@ TEST(BadDataNodeTest, NNEventCallback) {
char buf[4096] = {
0,
};
- IoServiceImpl io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
auto tracker = std::make_shared<BadDataNodeTracker>();
@@ -191,7 +192,7 @@ TEST(BadDataNodeTest, NNEventCallback) {
return event_response::make_ok();
});
- PartialMockFileHandle is("cluster", "file", &io_service.io_service(), GetRandomClientName(), file_info, tracker, monitors);
+ PartialMockFileHandle is("cluster", "file", io_service, GetRandomClientName(), file_info, tracker, monitors);
Status stat;
size_t read = 0;
@@ -234,10 +235,10 @@ TEST(BadDataNodeTest, RecoverableError) {
char buf[4096] = {
0,
};
- IoServiceImpl io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
auto tracker = std::make_shared<BadDataNodeTracker>();
auto monitors = std::make_shared<LibhdfsEvents>();
- PartialMockFileHandle is("cluster", "file", &io_service.io_service(), GetRandomClientName(), file_info, tracker, monitors);
+ PartialMockFileHandle is("cluster", "file", io_service, GetRandomClientName(), file_info, tracker, monitors);
Status stat;
size_t read = 0;
EXPECT_CALL(*is.mock_reader_, AsyncReadBlock(_,_,_,_,_))
@@ -285,10 +286,10 @@ TEST(BadDataNodeTest, InternalError) {
char buf[4096] = {
0,
};
- IoServiceImpl io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
auto tracker = std::make_shared<BadDataNodeTracker>();
auto monitors = std::make_shared<LibhdfsEvents>();
- PartialMockFileHandle is("cluster", "file", &io_service.io_service(), GetRandomClientName(), file_info, tracker, monitors);
+ PartialMockFileHandle is("cluster", "file", io_service, GetRandomClientName(), file_info, tracker, monitors);
Status stat;
size_t read = 0;
EXPECT_CALL(*is.mock_reader_, AsyncReadBlock(_,_,_,_,_))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
index 5ee9789..2fdbd80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
@@ -16,13 +16,15 @@
* limitations under the License.
*/
-#include "common/hdfs_ioservice.h"
+#include "hdfspp/ioservice.h"
#include <future>
#include <functional>
#include <thread>
#include <string>
+
+#include <google/protobuf/stubs/common.h>
#include <gmock/gmock.h>
using ::testing::_;
@@ -34,7 +36,7 @@ using namespace hdfs;
// Make sure IoService spins up specified number of threads
TEST(IoServiceTest, InitThreads) {
#ifndef DISABLE_CONCURRENT_WORKERS
- std::shared_ptr<IoServiceImpl> service = std::static_pointer_cast<IoServiceImpl>(IoService::MakeShared());
+ std::shared_ptr<IoService> service = IoService::MakeShared();
EXPECT_NE(service, nullptr);
unsigned int thread_count = 4;
@@ -50,7 +52,7 @@ TEST(IoServiceTest, InitThreads) {
// Make sure IoService defaults to logical thread count
TEST(IoServiceTest, InitDefaultThreads) {
#ifndef DISABLE_CONCURRENT_WORKERS
- std::shared_ptr<IoServiceImpl> service = std::static_pointer_cast<IoServiceImpl>(IoService::MakeShared());
+ std::shared_ptr<IoService> service = IoService::MakeShared();
EXPECT_NE(service, nullptr);
unsigned int thread_count = std::thread::hardware_concurrency();
@@ -66,7 +68,7 @@ TEST(IoServiceTest, InitDefaultThreads) {
// Check IoService::PostTask
TEST(IoServiceTest, SimplePost) {
- std::shared_ptr<IoServiceImpl> service = std::static_pointer_cast<IoServiceImpl>(IoService::MakeShared());
+ std::shared_ptr<IoService> service = IoService::MakeShared();
EXPECT_NE(service, nullptr);
unsigned int thread_count = std::thread::hardware_concurrency();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h
index cd1fc12..de234ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h
@@ -49,7 +49,7 @@ public:
virtual ~MockConnectionBase();
typedef std::pair<asio::error_code, std::string> ProducerResult;
- void async_read_some(const MutableBuffers &buf,
+ void async_read_some(const MutableBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
if (produced_.size() == 0) {
@@ -72,7 +72,7 @@ public:
io_service_->post(std::bind(handler, asio::error_code(), len));
}
- void async_write_some(const ConstBuffers &buf,
+ void async_write_some(const ConstBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
// CompletionResult res = OnWrite(buf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
index 80127f3..4b909b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
@@ -69,14 +69,14 @@ public:
/* event handler to trigger side effects */
std::function<void(void)> OnRead;
- void async_read_some(const MutableBuffers &buf,
+ void async_read_some(const MutableBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
this->OnRead();
this->MockConnectionBase::async_read_some(buf, handler);
}
- void async_write_some(const ConstBuffers &buf,
+ void async_write_some(const ConstBuffer &buf,
std::function<void (const asio::error_code & error,
std::size_t bytes_transferred) > handler) override {
this->MockConnectionBase::async_write_some(buf, handler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index f998c7f..6bbe725 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+#include "hdfspp/ioservice.h"
+
#include "mock_connection.h"
#include "test.pb.h"
#include "RpcHeader.pb.h"
@@ -23,7 +25,6 @@
#include "common/namenode_info.h"
#include <google/protobuf/io/coded_stream.h>
-
#include <gmock/gmock.h>
using ::hadoop::common::RpcResponseHeaderProto;
@@ -104,9 +105,10 @@ static inline std::pair<error_code, string> RpcResponse(
using namespace hdfs;
TEST(RpcEngineTest, TestRoundTrip) {
- ::asio::io_service io_service;
+
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
Options options;
- std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(io_service, options, "foo", "", "protocol", 1);
auto conn =
std::make_shared<RpcConnectionImpl<MockRPCConnection> >(engine);
conn->TEST_set_connected(true);
@@ -129,20 +131,20 @@ TEST(RpcEngineTest, TestRoundTrip) {
EchoRequestProto req;
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine->AsyncRpc("test", &req, resp, [resp, &complete,&io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [resp, &complete,io_service](const Status &stat) {
ASSERT_TRUE(stat.ok());
ASSERT_EQ("foo", resp->message());
complete = true;
- io_service.stop();
+ io_service->Stop();
});
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
TEST(RpcEngineTest, TestConnectionResetAndFail) {
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
Options options;
- std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(io_service, options, "foo", "", "protocol", 1);
auto conn =
std::make_shared<RpcConnectionImpl<MockRPCConnection> >(engine);
conn->TEST_set_connected(true);
@@ -164,23 +166,23 @@ TEST(RpcEngineTest, TestConnectionResetAndFail) {
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_FALSE(stat.ok());
});
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
TEST(RpcEngineTest, TestConnectionResetAndRecover) {
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 0;
std::shared_ptr<SharedConnectionEngine> engine
- = std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
+ = std::make_shared<SharedConnectionEngine>(io_service, options, "foo", "", "protocol", 1);
// Normally determined during RpcEngine::Connect, but in this case options
// provides enough info to determine policy here.
@@ -206,22 +208,22 @@ TEST(RpcEngineTest, TestConnectionResetAndRecover) {
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_TRUE(stat.ok());
});
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
TEST(RpcEngineTest, TestConnectionResetAndRecoverWithDelay) {
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
Options options;
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 1;
std::shared_ptr<SharedConnectionEngine> engine =
- std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::make_shared<SharedConnectionEngine>(io_service, options, "foo", "", "protocol", 1);
// Normally determined during RpcEngine::Connect, but in this case options
// provides enough info to determine policy here.
@@ -246,17 +248,17 @@ TEST(RpcEngineTest, TestConnectionResetAndRecoverWithDelay) {
req.set_message("foo");
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
- engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_TRUE(stat.ok());
});
- ::asio::deadline_timer timer(io_service);
+ ::asio::deadline_timer timer(io_service->GetRaw());
timer.expires_from_now(std::chrono::hours(100));
timer.async_wait([](const asio::error_code & err){(void)err; ASSERT_FALSE("Timed out"); });
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
@@ -267,7 +269,7 @@ TEST(RpcEngineTest, TestConnectionFailure)
SharedMockConnection::SetSharedConnectionData(producer);
// Error and no retry
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
bool complete = false;
@@ -275,16 +277,16 @@ TEST(RpcEngineTest, TestConnectionFailure)
options.max_rpc_retries = 0;
options.rpc_retry_delay_ms = 0;
std::shared_ptr<SharedConnectionEngine> engine
- = std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
+ = std::make_shared<SharedConnectionEngine>(io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")));
- engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_FALSE(stat.ok());
});
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
@@ -294,7 +296,7 @@ TEST(RpcEngineTest, TestConnectionFailureRetryAndFailure)
producer->checkProducerForConnect = true;
SharedMockConnection::SetSharedConnectionData(producer);
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
bool complete = false;
@@ -302,18 +304,18 @@ TEST(RpcEngineTest, TestConnectionFailureRetryAndFailure)
options.max_rpc_retries = 2;
options.rpc_retry_delay_ms = 0;
std::shared_ptr<SharedConnectionEngine> engine =
- std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::make_shared<SharedConnectionEngine>(io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")));
- engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_FALSE(stat.ok());
});
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
@@ -323,7 +325,7 @@ TEST(RpcEngineTest, TestConnectionFailureAndRecover)
producer->checkProducerForConnect = true;
SharedMockConnection::SetSharedConnectionData(producer);
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
bool complete = false;
@@ -331,29 +333,30 @@ TEST(RpcEngineTest, TestConnectionFailureAndRecover)
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 0;
std::shared_ptr<SharedConnectionEngine> engine =
- std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::make_shared<SharedConnectionEngine>(io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(::asio::error_code(), "")))
.WillOnce(Return(std::make_pair(::asio::error::would_block, "")));
- engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_TRUE(stat.ok());
});
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
TEST(RpcEngineTest, TestEventCallbacks)
{
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
+
Options options;
options.max_rpc_retries = 99;
options.rpc_retry_delay_ms = 0;
std::shared_ptr<SharedConnectionEngine> engine =
- std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::make_shared<SharedConnectionEngine>(io_service, options, "foo", "", "protocol", 1);
// Normally determined during RpcEngine::Connect, but in this case options
// provides enough info to determine policy here.
@@ -399,17 +402,18 @@ TEST(RpcEngineTest, TestEventCallbacks)
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
bool complete = false;
- engine->AsyncRpc("test", &req, resp, [&complete, &io_service](const Status &stat) {
+ engine->AsyncRpc("test", &req, resp, [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_TRUE(stat.ok());
});
// If you're adding event hooks you'll most likely need to update this.
// It's a brittle test but makes it hard to miss control flow changes in RPC retry.
- for(const auto& m : callbacks)
+ for(const auto& m : callbacks) {
std::cerr << m << std::endl;
- io_service.run();
+ }
+ io_service->Run();
ASSERT_TRUE(complete);
ASSERT_EQ(9, callbacks.size());
ASSERT_EQ(FS_NN_CONNECT_EVENT, callbacks[0]); // error
@@ -430,7 +434,7 @@ TEST(RpcEngineTest, TestConnectionFailureAndAsyncRecover)
producer->checkProducerForConnect = true;
SharedMockConnection::SetSharedConnectionData(producer);
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
bool complete = false;
@@ -438,31 +442,31 @@ TEST(RpcEngineTest, TestConnectionFailureAndAsyncRecover)
options.max_rpc_retries = 1;
options.rpc_retry_delay_ms = 1;
std::shared_ptr<SharedConnectionEngine> engine =
- std::make_shared<SharedConnectionEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::make_shared<SharedConnectionEngine>(io_service, options, "foo", "", "protocol", 1);
EXPECT_CALL(*producer, Produce())
.WillOnce(Return(std::make_pair(make_error_code(::asio::error::connection_reset), "")))
.WillOnce(Return(std::make_pair(::asio::error_code(), "")))
.WillOnce(Return(std::make_pair(::asio::error::would_block, "")));
- engine->Connect("", make_endpoint(), [&complete, &io_service](const Status &stat) {
+ engine->Connect("", make_endpoint(), [&complete, io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_TRUE(stat.ok());
});
- ::asio::deadline_timer timer(io_service);
+ ::asio::deadline_timer timer(io_service->GetRaw());
timer.expires_from_now(std::chrono::hours(100));
timer.async_wait([](const asio::error_code & err){(void)err; ASSERT_FALSE("Timed out"); });
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
TEST(RpcEngineTest, TestTimeout) {
- ::asio::io_service io_service;
+ std::shared_ptr<IoService> io_service = IoService::MakeShared();
Options options;
options.rpc_timeout = 1;
- std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(&io_service, options, "foo", "", "protocol", 1);
+ std::shared_ptr<RpcEngine> engine = std::make_shared<RpcEngine>(io_service, options, "foo", "", "protocol", 1);
auto conn =
std::make_shared<RpcConnectionImpl<MockRPCConnection> >(engine);
conn->TEST_set_connected(true);
@@ -481,15 +485,15 @@ TEST(RpcEngineTest, TestTimeout) {
std::shared_ptr<EchoResponseProto> resp(new EchoResponseProto());
engine->AsyncRpc("test", &req, resp, [resp, &complete,&io_service](const Status &stat) {
complete = true;
- io_service.stop();
+ io_service->Stop();
ASSERT_FALSE(stat.ok());
});
- ::asio::deadline_timer timer(io_service);
+ ::asio::deadline_timer timer(io_service->GetRaw());
timer.expires_from_now(std::chrono::hours(100));
timer.async_wait([](const asio::error_code & err){(void)err; ASSERT_FALSE("Timed out"); });
- io_service.run();
+ io_service->Run();
ASSERT_TRUE(complete);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/44] hadoop git commit: YARN-8154. Fix missing titles in
PlacementConstraints document. Contributed by Weiwei Yang.
Posted by xy...@apache.org.
YARN-8154. Fix missing titles in PlacementConstraints document. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/375654c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/375654c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/375654c3
Branch: refs/heads/HDFS-7240
Commit: 375654c36a8bfa4337c9011fcd86737462dfa61e
Parents: ec1e8c1
Author: Weiwei Yang <ww...@apache.org>
Authored: Fri Apr 13 13:06:47 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Fri Apr 13 13:06:47 2018 +0800
----------------------------------------------------------------------
.../src/site/markdown/PlacementConstraints.md.vm | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/375654c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
index 6af62e7..cb34c3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
@@ -12,6 +12,9 @@
limitations under the License. See accompanying LICENSE file.
-->
+#set ( $H3 = '###' )
+#set ( $H4 = '####' )
+
Placement Constraints
=====================
@@ -35,7 +38,7 @@ Quick Guide
We first describe how to enable scheduling with placement constraints and then provide examples of how to experiment with this feature using the distributed shell, an application that allows to run a given shell command on a set of containers.
-### Enabling placement constraints
+$H3 Enabling placement constraints
To enable placement constraints, the following property has to be set to `placement-processor` or `scheduler` in **conf/yarn-site.xml**:
@@ -51,7 +54,7 @@ We now give more details about each of the three placement constraint handlers:
The `placement-processor` handler supports a wider range of constraints and can allow more containers to be placed, especially when applications have demanding constraints or the cluster is highly-utilized (due to considering multiple containers at a time). However, if respecting task priority within an application is important for the user and the capacity scheduler is used, then the `scheduler` handler should be used instead.
-### Experimenting with placement constraints using distributed shell
+$H3 Experimenting with placement constraints using distributed shell
Users can experiment with placement constraints by using the distributed shell application through the following command:
@@ -89,18 +92,18 @@ The above encodes two constraints:
Defining Placement Constraints
------------------------------
-### Allocation tags
+$H3 Allocation tags
Allocation tags are string tags that an application can associate with (groups of) its containers. Tags are used to identify components of applications. For example, an HBase Master allocation can be tagged with "hbase-m", and Region Servers with "hbase-rs". Other examples are "latency-critical" to refer to the more general demands of the allocation, or "app_0041" to denote the job ID. Allocation tags play a key role in constraints, as they allow to refer to multiple allocations that share a common tag.
Note that instead of using the `ResourceRequest` object to define allocation tags, we use the new `SchedulingRequest` object. This has many similarities with the `ResourceRequest`, but better separates the sizing of the requested allocations (number and size of allocations, priority, execution type, etc.), and the constraints dictating how these allocations should be placed (resource name, relaxed locality). Applications can still use `ResourceRequest` objects, but in order to define allocation tags and constraints, they need to use the `SchedulingRequest` object. Within a single `AllocateRequest`, an application should use either the `ResourceRequest` or the `SchedulingRequest` objects, but not both of them.
-#### Differences between node labels, node attributes and allocation tags
+$H4 Differences between node labels, node attributes and allocation tags
The difference between allocation tags and node labels or node attributes (YARN-3409), is that allocation tags are attached to allocations and not to nodes. When an allocation gets allocated to a node by the scheduler, the set of tags of that allocation are automatically added to the node for the duration of the allocation. Hence, a node inherits the tags of the allocations that are currently allocated to the node. Likewise, a rack inherits the tags of its nodes. Moreover, similar to node labels and unlike node attributes, allocation tags have no value attached to them. As we show below, our constraints can refer to allocation tags, as well as node labels and node attributes.
-### Placement constraints API
+$H3 Placement constraints API
Applications can use the public API in the `PlacementConstraints` to construct placement constraint. Before describing the methods for building constraints, we describe the methods of the `PlacementTargets` class that are used to construct the target expressions that will then be used in constraints:
@@ -127,7 +130,7 @@ The methods of the `PlacementConstraints` class for building constraints are the
The `PlacementConstraints` class also includes method for building compound constraints (AND/OR expressions with multiple constraints). Adding support for compound constraints is work in progress.
-### Specifying constraints in applications
+$H3 Specifying constraints in applications
Applications have to specify the containers for which each constraint will be enabled. To this end, applications can provide a mapping from a set of allocation tags (source tags) to a placement constraint. For example, an entry of this mapping could be "hbase"->constraint1, which means that constraint1 will be applied when scheduling each allocation with tag "hbase".
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/44] hadoop git commit: HDFS-13428. RBF: Remove LinkedList From
StateStoreFileImpl.java. Contributed by BELUGA BEHR.
Posted by xy...@apache.org.
HDFS-13428. RBF: Remove LinkedList From StateStoreFileImpl.java. Contributed by BELUGA BEHR.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7d5bace
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7d5bace
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7d5bace
Branch: refs/heads/HDFS-7240
Commit: f7d5bace435a8de151b94ccc3599a6c4de8f7daf
Parents: 0c93d43
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Apr 11 10:40:30 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Apr 11 10:43:43 2018 -0700
----------------------------------------------------------------------
.../federation/store/driver/impl/StateStoreFileImpl.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5bace/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
index 7d9ddc6..6b288b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
@@ -26,9 +26,11 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
-import java.util.LinkedList;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.slf4j.Logger;
@@ -134,15 +136,16 @@ public class StateStoreFileImpl extends StateStoreFileBaseImpl {
@Override
protected List<String> getChildren(String path) {
- List<String> ret = new LinkedList<>();
File dir = new File(path);
File[] files = dir.listFiles();
- if (files != null) {
+ if (ArrayUtils.isNotEmpty(files)) {
+ List<String> ret = new ArrayList<>(files.length);
for (File file : files) {
String filename = file.getName();
ret.add(filename);
}
+ return ret;
}
- return ret;
+ return Collections.emptyList();
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/44] hadoop git commit: HDFS-13430. Fix TestEncryptionZonesWithKMS
failure due to HADOOP-14445.
Posted by xy...@apache.org.
HDFS-13430. Fix TestEncryptionZonesWithKMS failure due to HADOOP-14445.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65035937
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65035937
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65035937
Branch: refs/heads/HDFS-7240
Commit: 650359371175fba416331e73aa03d2a96ccb90e5
Parents: fa8b88a
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Apr 13 09:04:52 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Apr 13 09:05:17 2018 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/65035937/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 6f9ef29..51c6c4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -110,6 +110,7 @@ import org.junit.Test;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
@@ -192,6 +193,8 @@ public class TestEncryptionZones {
// Lower the batch size for testing
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
2);
+ // disable kms client copy legacy token logic because it's irrelevant.
+ conf.setBoolean(KMS_CLIENT_COPY_LEGACY_TOKEN_KEY, false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/44] hadoop git commit: HDFS-13056. Expose file-level composite
CRCs in HDFS which are comparable across different instances/layouts.
Contributed by Dennis Huo.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
index 2356201..384da54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
@@ -148,8 +148,9 @@ message OpCopyBlockProto {
required BaseHeaderProto header = 1;
}
-message OpBlockChecksumProto {
+message OpBlockChecksumProto {
required BaseHeaderProto header = 1;
+ optional BlockChecksumOptionsProto blockChecksumOptions = 2;
}
message OpBlockGroupChecksumProto {
@@ -160,6 +161,7 @@ message OpBlockGroupChecksumProto {
required ErasureCodingPolicyProto ecPolicy = 4;
repeated uint32 blockIndices = 5;
required uint64 requestedNumBytes = 6;
+ optional BlockChecksumOptionsProto blockChecksumOptions = 7;
}
/**
@@ -313,8 +315,9 @@ message DNTransferAckProto {
message OpBlockChecksumResponseProto {
required uint32 bytesPerCrc = 1;
required uint64 crcPerBlock = 2;
- required bytes md5 = 3;
+ required bytes blockChecksum = 3;
optional ChecksumTypeProto crcType = 4;
+ optional BlockChecksumOptionsProto blockChecksumOptions = 5;
}
message OpCustomProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 29d0b4e..441b9d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -480,6 +480,27 @@ enum ChecksumTypeProto {
CHECKSUM_CRC32C = 2;
}
+enum BlockChecksumTypeProto {
+ MD5CRC = 1; // BlockChecksum obtained by taking the MD5 digest of chunk CRCs
+ COMPOSITE_CRC = 2; // Chunk-independent CRC, optionally striped
+}
+
+/**
+ * Algorithms/types denoting how block-level checksums are computed using
+ * lower-level chunk checksums/CRCs.
+ * These options should be kept in sync with
+ * org.apache.hadoop.hdfs.protocol.BlockChecksumOptions.
+ */
+message BlockChecksumOptionsProto {
+ optional BlockChecksumTypeProto blockChecksumType = 1 [default = MD5CRC];
+
+ // Only used if blockChecksumType specifies a striped format, such as
+ // COMPOSITE_CRC. If so, then the blockChecksum in the response is expected
+ // to be the concatenation of N crcs, where
+ // N == ((requestedLength - 1) / stripedLength) + 1
+ optional uint64 stripeLength = 2;
+}
+
/**
* HDFS Server Defaults
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index bab2e8d..5d2d1f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -301,8 +301,9 @@ public abstract class Receiver implements DataTransferProtocol {
TraceScope traceScope = continueTraceSpan(proto.getHeader(),
proto.getClass().getSimpleName());
try {
- blockChecksum(PBHelperClient.convert(proto.getHeader().getBlock()),
- PBHelperClient.convert(proto.getHeader().getToken()));
+ blockChecksum(PBHelperClient.convert(proto.getHeader().getBlock()),
+ PBHelperClient.convert(proto.getHeader().getToken()),
+ PBHelperClient.convert(proto.getBlockChecksumOptions()));
} finally {
if (traceScope != null) traceScope.close();
}
@@ -325,7 +326,8 @@ public abstract class Receiver implements DataTransferProtocol {
try {
blockGroupChecksum(stripedBlockInfo,
PBHelperClient.convert(proto.getHeader().getToken()),
- proto.getRequestedNumBytes());
+ proto.getRequestedNumBytes(),
+ PBHelperClient.convert(proto.getBlockChecksumOptions()));
} finally {
if (traceScope != null) {
traceScope.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
index e99911b..3388855 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
@@ -21,6 +21,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumOptions;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumType;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -32,6 +34,8 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.erasurecode.StripedBlockChecksumCompositeCrcReconstructor;
+import org.apache.hadoop.hdfs.server.datanode.erasurecode.StripedBlockChecksumMd5CrcReconstructor;
import org.apache.hadoop.hdfs.server.datanode.erasurecode.StripedBlockChecksumReconstructor;
import org.apache.hadoop.hdfs.server.datanode.erasurecode.StripedReconstructionInfo;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
@@ -40,6 +44,8 @@ import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.CrcComposer;
+import org.apache.hadoop.util.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +76,7 @@ final class BlockChecksumHelper {
*/
static abstract class AbstractBlockChecksumComputer {
private final DataNode datanode;
+ private final BlockChecksumOptions blockChecksumOptions;
private byte[] outBytes;
private int bytesPerCRC = -1;
@@ -77,8 +84,11 @@ final class BlockChecksumHelper {
private long crcPerBlock = -1;
private int checksumSize = -1;
- AbstractBlockChecksumComputer(DataNode datanode) throws IOException {
+ AbstractBlockChecksumComputer(
+ DataNode datanode,
+ BlockChecksumOptions blockChecksumOptions) throws IOException {
this.datanode = datanode;
+ this.blockChecksumOptions = blockChecksumOptions;
}
abstract void compute() throws IOException;
@@ -92,6 +102,10 @@ final class BlockChecksumHelper {
return datanode;
}
+ BlockChecksumOptions getBlockChecksumOptions() {
+ return blockChecksumOptions;
+ }
+
InputStream getBlockInputStream(ExtendedBlock block, long seekOffset)
throws IOException {
return datanode.data.getBlockInputStream(block, seekOffset);
@@ -155,8 +169,10 @@ final class BlockChecksumHelper {
private DataChecksum checksum;
BlockChecksumComputer(DataNode datanode,
- ExtendedBlock block) throws IOException {
- super(datanode);
+ ExtendedBlock block,
+ BlockChecksumOptions blockChecksumOptions)
+ throws IOException {
+ super(datanode, blockChecksumOptions);
this.block = block;
this.requestLength = block.getNumBytes();
Preconditions.checkArgument(requestLength >= 0);
@@ -268,8 +284,10 @@ final class BlockChecksumHelper {
static class ReplicatedBlockChecksumComputer extends BlockChecksumComputer {
ReplicatedBlockChecksumComputer(DataNode datanode,
- ExtendedBlock block) throws IOException {
- super(datanode, block);
+ ExtendedBlock block,
+ BlockChecksumOptions blockChecksumOptions)
+ throws IOException {
+ super(datanode, block, blockChecksumOptions);
}
@Override
@@ -277,22 +295,38 @@ final class BlockChecksumHelper {
try {
readHeader();
- MD5Hash md5out;
- if (isPartialBlk() && getCrcPerBlock() > 0) {
- md5out = checksumPartialBlock();
- } else {
- md5out = checksumWholeBlock();
+ BlockChecksumType type =
+ getBlockChecksumOptions().getBlockChecksumType();
+ switch (type) {
+ case MD5CRC:
+ computeMd5Crc();
+ break;
+ case COMPOSITE_CRC:
+ computeCompositeCrc(getBlockChecksumOptions().getStripeLength());
+ break;
+ default:
+ throw new IOException(String.format(
+ "Unrecognized BlockChecksumType: %s", type));
}
- setOutBytes(md5out.getDigest());
-
- LOG.debug("block={}, bytesPerCRC={}, crcPerBlock={}, md5out={}",
- getBlock(), getBytesPerCRC(), getCrcPerBlock(), md5out);
} finally {
IOUtils.closeStream(getChecksumIn());
IOUtils.closeStream(getMetadataIn());
}
}
+ private void computeMd5Crc() throws IOException {
+ MD5Hash md5out;
+ if (isPartialBlk() && getCrcPerBlock() > 0) {
+ md5out = checksumPartialBlock();
+ } else {
+ md5out = checksumWholeBlock();
+ }
+ setOutBytes(md5out.getDigest());
+
+ LOG.debug("block={}, bytesPerCRC={}, crcPerBlock={}, md5out={}",
+ getBlock(), getBytesPerCRC(), getCrcPerBlock(), md5out);
+ }
+
private MD5Hash checksumWholeBlock() throws IOException {
MD5Hash md5out = MD5Hash.digest(getChecksumIn());
return md5out;
@@ -320,6 +354,56 @@ final class BlockChecksumHelper {
return new MD5Hash(digester.digest());
}
+
+ private void computeCompositeCrc(long stripeLength) throws IOException {
+ long checksumDataLength =
+ Math.min(getVisibleLength(), getRequestLength());
+ if (stripeLength <= 0 || stripeLength > checksumDataLength) {
+ stripeLength = checksumDataLength;
+ }
+
+ CrcComposer crcComposer = CrcComposer.newStripedCrcComposer(
+ getCrcType(), getBytesPerCRC(), stripeLength);
+ DataInputStream checksumIn = getChecksumIn();
+
+ // Whether getting the checksum for the entire block (which itself may
+ // not be a full block size and may have a final chunk smaller than
+ // getBytesPerCRC()), we begin with a number of full chunks, all of size
+ // getBytesPerCRC().
+ long numFullChunks = checksumDataLength / getBytesPerCRC();
+ crcComposer.update(checksumIn, numFullChunks, getBytesPerCRC());
+
+ // There may be a final partial chunk that is not full-sized. Unlike the
+ // MD5 case, we still consider this a "partial chunk" even if
+ // getRequestLength() == getVisibleLength(), since the CRC composition
+ // depends on the byte size of that final chunk, even if it already has a
+ // precomputed CRC stored in metadata. So there are two cases:
+ // 1. Reading only part of a block via getRequestLength(); we get the
+ // crcPartialBlock() explicitly.
+ // 2. Reading full visible length; the partial chunk already has a CRC
+ // stored in block metadata, so we just continue reading checksumIn.
+ long partialChunkSize = checksumDataLength % getBytesPerCRC();
+ if (partialChunkSize > 0) {
+ if (isPartialBlk()) {
+ byte[] partialChunkCrcBytes = crcPartialBlock();
+ crcComposer.update(
+ partialChunkCrcBytes, 0, partialChunkCrcBytes.length,
+ partialChunkSize);
+ } else {
+ int partialChunkCrc = checksumIn.readInt();
+ crcComposer.update(partialChunkCrc, partialChunkSize);
+ }
+ }
+
+ byte[] composedCrcs = crcComposer.digest();
+ setOutBytes(composedCrcs);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "block={}, getBytesPerCRC={}, crcPerBlock={}, compositeCrc={}",
+ getBlock(), getBytesPerCRC(), getCrcPerBlock(),
+ CrcUtil.toMultiCrcString(composedCrcs));
+ }
+ }
}
/**
@@ -335,19 +419,29 @@ final class BlockChecksumHelper {
private final byte[] blockIndices;
private final long requestedNumBytes;
- private final DataOutputBuffer md5writer = new DataOutputBuffer();
+ private final DataOutputBuffer blockChecksumBuf = new DataOutputBuffer();
+
+ // Keeps track of the positions within blockChecksumBuf where each data
+ // block's checksum begins; for fixed-size block checksums this is easily
+ // calculated as a multiple of the checksum size, but for striped block
+ // CRCs, it's less error-prone to simply keep track of exact byte offsets
+ // before each block checksum is populated into the buffer.
+ private final int[] blockChecksumPositions;
- BlockGroupNonStripedChecksumComputer(DataNode datanode,
- StripedBlockInfo stripedBlockInfo,
- long requestedNumBytes)
+ BlockGroupNonStripedChecksumComputer(
+ DataNode datanode,
+ StripedBlockInfo stripedBlockInfo,
+ long requestedNumBytes,
+ BlockChecksumOptions blockChecksumOptions)
throws IOException {
- super(datanode);
+ super(datanode, blockChecksumOptions);
this.blockGroup = stripedBlockInfo.getBlock();
this.ecPolicy = stripedBlockInfo.getErasureCodingPolicy();
this.datanodes = stripedBlockInfo.getDatanodes();
this.blockTokens = stripedBlockInfo.getBlockTokens();
this.blockIndices = stripedBlockInfo.getBlockIndices();
this.requestedNumBytes = requestedNumBytes;
+ this.blockChecksumPositions = new int[this.ecPolicy.getNumDataUnits()];
}
private static class LiveBlockInfo {
@@ -383,6 +477,9 @@ final class BlockChecksumHelper {
}
long checksumLen = 0;
for (int idx = 0; idx < numDataUnits && idx < blkIndxLen; idx++) {
+ // Before populating the blockChecksum at this index, record the byte
+ // offset where it will begin.
+ blockChecksumPositions[idx] = blockChecksumBuf.getLength();
try {
ExtendedBlock block = getInternalBlock(numDataUnits, idx);
@@ -409,8 +506,75 @@ final class BlockChecksumHelper {
}
}
- MD5Hash md5out = MD5Hash.digest(md5writer.getData());
- setOutBytes(md5out.getDigest());
+ BlockChecksumType type = getBlockChecksumOptions().getBlockChecksumType();
+ switch (type) {
+ case MD5CRC:
+ MD5Hash md5out = MD5Hash.digest(blockChecksumBuf.getData());
+ setOutBytes(md5out.getDigest());
+ break;
+ case COMPOSITE_CRC:
+ byte[] digest = reassembleNonStripedCompositeCrc(checksumLen);
+ setOutBytes(digest);
+ break;
+ default:
+ throw new IOException(String.format(
+ "Unrecognized BlockChecksumType: %s", type));
+ }
+ }
+
+ /**
+ * @param checksumLen The sum of bytes associated with the block checksum
+ * data being digested into a block-group level checksum.
+ */
+ private byte[] reassembleNonStripedCompositeCrc(long checksumLen)
+ throws IOException {
+ int numDataUnits = ecPolicy.getNumDataUnits();
+ CrcComposer crcComposer = CrcComposer.newCrcComposer(
+ getCrcType(), ecPolicy.getCellSize());
+
+ // This should hold all the cell-granularity checksums of blk0
+ // followed by all cell checksums of blk1, etc. We must unstripe the
+ // cell checksums in order of logical file bytes. Also, note that the
+ // length of this array may not equal the the number of actually valid
+ // bytes in the buffer (blockChecksumBuf.getLength()).
+ byte[] flatBlockChecksumData = blockChecksumBuf.getData();
+
+ // Initialize byte-level cursors to where each block's checksum begins
+ // inside the combined flattened buffer.
+ int[] blockChecksumCursors = new int[numDataUnits];
+ for (int idx = 0; idx < numDataUnits; ++idx) {
+ blockChecksumCursors[idx] = blockChecksumPositions[idx];
+ }
+
+ // Reassemble cell-level CRCs in the right order.
+ long numFullCells = checksumLen / ecPolicy.getCellSize();
+ for (long cellIndex = 0; cellIndex < numFullCells; ++cellIndex) {
+ int blockIndex = (int) (cellIndex % numDataUnits);
+ int checksumCursor = blockChecksumCursors[blockIndex];
+ int cellCrc = CrcUtil.readInt(
+ flatBlockChecksumData, checksumCursor);
+ blockChecksumCursors[blockIndex] += 4;
+ crcComposer.update(cellCrc, ecPolicy.getCellSize());
+ }
+ if (checksumLen % ecPolicy.getCellSize() != 0) {
+ // Final partial cell.
+ int blockIndex = (int) (numFullCells % numDataUnits);
+ int checksumCursor = blockChecksumCursors[blockIndex];
+ int cellCrc = CrcUtil.readInt(
+ flatBlockChecksumData, checksumCursor);
+ blockChecksumCursors[blockIndex] += 4;
+ crcComposer.update(cellCrc, checksumLen % ecPolicy.getCellSize());
+ }
+ byte[] digest = crcComposer.digest();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("flatBlockChecksumData.length={}, numDataUnits={}, "
+ + "checksumLen={}, digest={}",
+ flatBlockChecksumData.length,
+ numDataUnits,
+ checksumLen,
+ CrcUtil.toSingleCrcString(digest));
+ }
+ return digest;
}
private ExtendedBlock getInternalBlock(int numDataUnits, int idx) {
@@ -437,8 +601,26 @@ final class BlockChecksumHelper {
LOG.debug("write to {}: {}, block={}",
getDatanode(), Op.BLOCK_CHECKSUM, block);
- // get block MD5
- createSender(pair).blockChecksum(block, blockToken);
+ // get block checksum
+ // A BlockGroupCheckum of type COMPOSITE_CRC uses underlying
+ // BlockChecksums also of type COMPOSITE_CRC but with
+ // stripeLength == ecPolicy.getCellSize().
+ BlockChecksumOptions childOptions;
+ BlockChecksumType groupChecksumType =
+ getBlockChecksumOptions().getBlockChecksumType();
+ switch (groupChecksumType) {
+ case MD5CRC:
+ childOptions = getBlockChecksumOptions();
+ break;
+ case COMPOSITE_CRC:
+ childOptions = new BlockChecksumOptions(
+ BlockChecksumType.COMPOSITE_CRC, ecPolicy.getCellSize());
+ break;
+ default:
+ throw new IOException(
+ "Unknown BlockChecksumType: " + groupChecksumType);
+ }
+ createSender(pair).blockChecksum(block, blockToken, childOptions);
final DataTransferProtos.BlockOpResponseProto reply =
DataTransferProtos.BlockOpResponseProto.parseFrom(
@@ -463,10 +645,37 @@ final class BlockChecksumHelper {
setOrVerifyChecksumProperties(blockIdx, checksumData.getBytesPerCrc(),
checksumData.getCrcPerBlock(), ct);
- //read md5
- final MD5Hash md5 = new MD5Hash(checksumData.getMd5().toByteArray());
- md5.write(md5writer);
- LOG.debug("got reply from datanode:{}, md5={}", targetDatanode, md5);
+
+ switch (groupChecksumType) {
+ case MD5CRC:
+ //read md5
+ final MD5Hash md5 =
+ new MD5Hash(checksumData.getBlockChecksum().toByteArray());
+ md5.write(blockChecksumBuf);
+ LOG.debug("got reply from datanode:{}, md5={}",
+ targetDatanode, md5);
+ break;
+ case COMPOSITE_CRC:
+ BlockChecksumType returnedType = PBHelperClient.convert(
+ checksumData.getBlockChecksumOptions().getBlockChecksumType());
+ if (returnedType != BlockChecksumType.COMPOSITE_CRC) {
+ throw new IOException(String.format(
+ "Unexpected blockChecksumType '%s', expecting COMPOSITE_CRC",
+ returnedType));
+ }
+ byte[] checksumBytes =
+ checksumData.getBlockChecksum().toByteArray();
+ blockChecksumBuf.write(checksumBytes, 0, checksumBytes.length);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("got reply from datanode:{} for blockIdx:{}, checksum:{}",
+ targetDatanode, blockIdx,
+ CrcUtil.toMultiCrcString(checksumBytes));
+ }
+ break;
+ default:
+ throw new IOException(
+ "Unknown BlockChecksumType: " + groupChecksumType);
+ }
}
}
@@ -489,10 +698,16 @@ final class BlockChecksumHelper {
StripedReconstructionInfo stripedReconInfo =
new StripedReconstructionInfo(
blockGroup, ecPolicy, blockIndices, datanodes, errIndices);
+ BlockChecksumType groupChecksumType =
+ getBlockChecksumOptions().getBlockChecksumType();
final StripedBlockChecksumReconstructor checksumRecon =
- new StripedBlockChecksumReconstructor(
+ groupChecksumType == BlockChecksumType.COMPOSITE_CRC ?
+ new StripedBlockChecksumCompositeCrcReconstructor(
getDatanode().getErasureCodingWorker(), stripedReconInfo,
- md5writer, blockLength);
+ blockChecksumBuf, blockLength) :
+ new StripedBlockChecksumMd5CrcReconstructor(
+ getDatanode().getErasureCodingWorker(), stripedReconInfo,
+ blockChecksumBuf, blockLength);
checksumRecon.reconstruct();
DataChecksum checksum = checksumRecon.getChecksum();
@@ -501,8 +716,8 @@ final class BlockChecksumHelper {
setOrVerifyChecksumProperties(errBlkIndex,
checksum.getBytesPerChecksum(), crcPerBlock,
checksum.getChecksumType());
- LOG.debug("Recalculated checksum for the block index:{}, md5={}",
- errBlkIndex, checksumRecon.getMD5());
+ LOG.debug("Recalculated checksum for the block index:{}, checksum={}",
+ errBlkIndex, checksumRecon.getDigestObject());
}
private void setOrVerifyChecksumProperties(int blockIdx, int bpc,
@@ -524,8 +739,16 @@ final class BlockChecksumHelper {
setCrcType(ct);
} else if (getCrcType() != DataChecksum.Type.MIXED &&
getCrcType() != ct) {
- // if crc types are mixed in a file
- setCrcType(DataChecksum.Type.MIXED);
+ BlockChecksumType groupChecksumType =
+ getBlockChecksumOptions().getBlockChecksumType();
+ if (groupChecksumType == BlockChecksumType.COMPOSITE_CRC) {
+ throw new IOException(String.format(
+ "BlockChecksumType COMPOSITE_CRC doesn't support MIXED "
+ + "underlying types; previous block was %s, next block is %s",
+ getCrcType(), ct));
+ } else {
+ setCrcType(DataChecksum.Type.MIXED);
+ }
}
if (blockIdx == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index d0ded89..0bb1987 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.ExtendedBlockId;
import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumOptions;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -968,15 +969,16 @@ class DataXceiver extends Receiver implements Runnable {
@Override
public void blockChecksum(ExtendedBlock block,
- Token<BlockTokenIdentifier> blockToken)
+ Token<BlockTokenIdentifier> blockToken,
+ BlockChecksumOptions blockChecksumOptions)
throws IOException {
updateCurrentThreadName("Getting checksum for block " + block);
final DataOutputStream out = new DataOutputStream(
getOutputStream());
checkAccess(out, true, block, blockToken, Op.BLOCK_CHECKSUM,
BlockTokenIdentifier.AccessMode.READ);
- BlockChecksumComputer maker =
- new ReplicatedBlockChecksumComputer(datanode, block);
+ BlockChecksumComputer maker = new ReplicatedBlockChecksumComputer(
+ datanode, block, blockChecksumOptions);
try {
maker.compute();
@@ -987,8 +989,10 @@ class DataXceiver extends Receiver implements Runnable {
.setChecksumResponse(OpBlockChecksumResponseProto.newBuilder()
.setBytesPerCrc(maker.getBytesPerCRC())
.setCrcPerBlock(maker.getCrcPerBlock())
- .setMd5(ByteString.copyFrom(maker.getOutBytes()))
- .setCrcType(PBHelperClient.convert(maker.getCrcType())))
+ .setBlockChecksum(ByteString.copyFrom(maker.getOutBytes()))
+ .setCrcType(PBHelperClient.convert(maker.getCrcType()))
+ .setBlockChecksumOptions(
+ PBHelperClient.convert(blockChecksumOptions)))
.build()
.writeDelimitedTo(out);
out.flush();
@@ -1007,7 +1011,9 @@ class DataXceiver extends Receiver implements Runnable {
@Override
public void blockGroupChecksum(final StripedBlockInfo stripedBlockInfo,
- final Token<BlockTokenIdentifier> blockToken, long requestedNumBytes)
+ final Token<BlockTokenIdentifier> blockToken,
+ long requestedNumBytes,
+ BlockChecksumOptions blockChecksumOptions)
throws IOException {
final ExtendedBlock block = stripedBlockInfo.getBlock();
updateCurrentThreadName("Getting checksum for block group" +
@@ -1018,7 +1024,7 @@ class DataXceiver extends Receiver implements Runnable {
AbstractBlockChecksumComputer maker =
new BlockGroupNonStripedChecksumComputer(datanode, stripedBlockInfo,
- requestedNumBytes);
+ requestedNumBytes, blockChecksumOptions);
try {
maker.compute();
@@ -1029,8 +1035,10 @@ class DataXceiver extends Receiver implements Runnable {
.setChecksumResponse(OpBlockChecksumResponseProto.newBuilder()
.setBytesPerCrc(maker.getBytesPerCRC())
.setCrcPerBlock(maker.getCrcPerBlock())
- .setMd5(ByteString.copyFrom(maker.getOutBytes()))
- .setCrcType(PBHelperClient.convert(maker.getCrcType())))
+ .setBlockChecksum(ByteString.copyFrom(maker.getOutBytes()))
+ .setCrcType(PBHelperClient.convert(maker.getCrcType()))
+ .setBlockChecksumOptions(
+ PBHelperClient.convert(blockChecksumOptions)))
.build()
.writeDelimitedTo(out);
out.flush();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumCompositeCrcReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumCompositeCrcReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumCompositeCrcReconstructor.java
new file mode 100644
index 0000000..afcc8cb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumCompositeCrcReconstructor.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.erasurecode;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.util.CrcComposer;
+
+/**
+ * Computes striped composite CRCs over reconstructed chunk CRCs.
+ */
+@InterfaceAudience.Private
+public class StripedBlockChecksumCompositeCrcReconstructor
+ extends StripedBlockChecksumReconstructor {
+ private final int ecPolicyCellSize;
+
+ private byte[] digestValue;
+ private CrcComposer digester;
+
+ public StripedBlockChecksumCompositeCrcReconstructor(
+ ErasureCodingWorker worker,
+ StripedReconstructionInfo stripedReconInfo,
+ DataOutputBuffer checksumWriter,
+ long requestedBlockLength) throws IOException {
+ super(worker, stripedReconInfo, checksumWriter, requestedBlockLength);
+ this.ecPolicyCellSize = stripedReconInfo.getEcPolicy().getCellSize();
+ }
+
+ @Override
+ public Object getDigestObject() {
+ return digestValue;
+ }
+
+ @Override
+ void prepareDigester() throws IOException {
+ digester = CrcComposer.newStripedCrcComposer(
+ getChecksum().getChecksumType(),
+ getChecksum().getBytesPerChecksum(),
+ ecPolicyCellSize);
+ }
+
+ @Override
+ void updateDigester(byte[] checksumBytes, int dataBytesPerChecksum)
+ throws IOException {
+ if (digester == null) {
+ throw new IOException(String.format(
+ "Called updatedDigester with checksumBytes.length=%d, "
+ + "dataBytesPerChecksum=%d but digester is null",
+ checksumBytes.length, dataBytesPerChecksum));
+ }
+ digester.update(
+ checksumBytes, 0, checksumBytes.length, dataBytesPerChecksum);
+ }
+
+ @Override
+ void commitDigest() throws IOException {
+ if (digester == null) {
+ throw new IOException("Called commitDigest() but digester is null");
+ }
+ digestValue = digester.digest();
+ getChecksumWriter().write(digestValue, 0, digestValue.length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumMd5CrcReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumMd5CrcReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumMd5CrcReconstructor.java
new file mode 100644
index 0000000..2f809b8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumMd5CrcReconstructor.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.erasurecode;
+
+import java.io.IOException;
+import java.security.MessageDigest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.MD5Hash;
+
+/**
+ * Computes running MD5-of-CRC over reconstructed chunk CRCs.
+ */
+@InterfaceAudience.Private
+public class StripedBlockChecksumMd5CrcReconstructor
+ extends StripedBlockChecksumReconstructor {
+ private MD5Hash md5;
+ private MessageDigest digester;
+
+ public StripedBlockChecksumMd5CrcReconstructor(ErasureCodingWorker worker,
+ StripedReconstructionInfo stripedReconInfo,
+ DataOutputBuffer checksumWriter,
+ long requestedBlockLength) throws IOException {
+ super(worker, stripedReconInfo, checksumWriter, requestedBlockLength);
+ }
+
+ @Override
+ public Object getDigestObject() {
+ return md5;
+ }
+
+ @Override
+ void prepareDigester() throws IOException {
+ digester = MD5Hash.getDigester();
+ }
+
+ @Override
+ void updateDigester(byte[] checksumBytes, int dataBytesPerChecksum)
+ throws IOException {
+ if (digester == null) {
+ throw new IOException(String.format(
+ "Called updatedDigester with checksumBytes.length=%d, "
+ + "dataBytesPerChecksum=%d but digester is null",
+ checksumBytes.length, dataBytesPerChecksum));
+ }
+ digester.update(checksumBytes, 0, checksumBytes.length);
+ }
+
+ @Override
+ void commitDigest() throws IOException {
+ if (digester == null) {
+ throw new IOException("Called commitDigest() but digester is null");
+ }
+ byte[] digest = digester.digest();
+ md5 = new MD5Hash(digest);
+ md5.write(getChecksumWriter());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
index d530a8e..b2e6496 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
@@ -19,12 +19,10 @@ package org.apache.hadoop.hdfs.server.datanode.erasurecode;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.security.MessageDigest;
import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.MD5Hash;
/**
* StripedBlockChecksumReconstructor reconstruct one or more missed striped
@@ -33,18 +31,17 @@ import org.apache.hadoop.io.MD5Hash;
* using the newly reconstructed block.
*/
@InterfaceAudience.Private
-public class StripedBlockChecksumReconstructor extends StripedReconstructor {
-
+public abstract class StripedBlockChecksumReconstructor
+ extends StripedReconstructor {
private ByteBuffer targetBuffer;
private final byte[] targetIndices;
private byte[] checksumBuf;
private DataOutputBuffer checksumWriter;
- private MD5Hash md5;
private long checksumDataLen;
private long requestedLen;
- public StripedBlockChecksumReconstructor(ErasureCodingWorker worker,
+ protected StripedBlockChecksumReconstructor(ErasureCodingWorker worker,
StripedReconstructionInfo stripedReconInfo,
DataOutputBuffer checksumWriter,
long requestedBlockLength) throws IOException {
@@ -72,8 +69,9 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
checksumBuf = new byte[tmpLen];
}
+ @Override
public void reconstruct() throws IOException {
- MessageDigest digester = MD5Hash.getDigester();
+ prepareDigester();
long maxTargetLength = getMaxTargetLength();
try {
while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) {
@@ -88,24 +86,54 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
reconstructTargets(toReconstructLen);
// step3: calculate checksum
- checksumDataLen += checksumWithTargetOutput(targetBuffer.array(),
- toReconstructLen, digester);
+ checksumDataLen += checksumWithTargetOutput(
+ targetBuffer.array(), toReconstructLen);
updatePositionInBlock(toReconstructLen);
requestedLen -= toReconstructLen;
clearBuffers();
}
- byte[] digest = digester.digest();
- md5 = new MD5Hash(digest);
- md5.write(checksumWriter);
+ commitDigest();
} finally {
cleanup();
}
}
- private long checksumWithTargetOutput(byte[] outputData, int toReconstructLen,
- MessageDigest digester) throws IOException {
+ /**
+ * Should return a representation of a completed/reconstructed digest which
+ * is suitable for debug printing.
+ */
+ public abstract Object getDigestObject();
+
+ /**
+ * This will be called before starting reconstruction.
+ */
+ abstract void prepareDigester() throws IOException;
+
+ /**
+ * This will be called repeatedly with chunked checksums computed in-flight
+ * over reconstructed data.
+ *
+ * @param dataBytesPerChecksum the number of underlying data bytes
+ * corresponding to each checksum inside {@code checksumBytes}.
+ */
+ abstract void updateDigester(byte[] checksumBytes, int dataBytesPerChecksum)
+ throws IOException;
+
+ /**
+ * This will be called when reconstruction of entire requested length is
+ * complete and any final digests should be committed to
+ * implementation-specific output fields.
+ */
+ abstract void commitDigest() throws IOException;
+
+ protected DataOutputBuffer getChecksumWriter() {
+ return checksumWriter;
+ }
+
+ private long checksumWithTargetOutput(byte[] outputData, int toReconstructLen)
+ throws IOException {
long checksumDataLength = 0;
// Calculate partial block checksum. There are two cases.
// case-1) length of data bytes which is fraction of bytesPerCRC
@@ -128,7 +156,7 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
checksumBuf = new byte[checksumRemaining];
getChecksum().calculateChunkedSums(outputData, dataOffset,
remainingLen, checksumBuf, 0);
- digester.update(checksumBuf, 0, checksumBuf.length);
+ updateDigester(checksumBuf, getChecksum().getBytesPerChecksum());
checksumDataLength = checksumBuf.length;
dataOffset = remainingLen;
}
@@ -139,7 +167,7 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
getChecksum().reset();
getChecksum().update(outputData, dataOffset, partialLength);
getChecksum().writeValue(partialCrc, 0, true);
- digester.update(partialCrc);
+ updateDigester(partialCrc, partialLength);
checksumDataLength += partialCrc.length;
}
@@ -151,7 +179,7 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
outputData.length, checksumBuf, 0);
// updates digest using the checksum array of bytes
- digester.update(checksumBuf, 0, checksumBuf.length);
+ updateDigester(checksumBuf, getChecksum().getBytesPerChecksum());
return checksumBuf.length;
}
@@ -176,10 +204,6 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
targetBuffer.clear();
}
- public MD5Hash getMD5() {
- return md5;
- }
-
public long getChecksumDataLen() {
return checksumDataLen;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index ce1254c..29c0078 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -79,6 +79,7 @@ class StripedBlockReconstructor extends StripedReconstructor
}
}
+ @Override
void reconstruct() throws IOException {
while (getPositionInBlock() < getMaxTargetLength()) {
DataNodeFaultInjector.get().stripedBlockReconstruction();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 1e83325..f8cce60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3541,6 +3541,17 @@
</property>
<property>
+ <name>dfs.checksum.combine.mode</name>
+ <value>MD5MD5CRC</value>
+ <description>
+ Defines how lower-level chunk/block checksums are combined into file-level
+ checksums; the original MD5MD5CRC mode is not comparable between files
+ with different block layouts, while modes like COMPOSITE_CRC are
+ comparable independently of block layout.
+ </description>
+</property>
+
+<property>
<name>dfs.client.block.write.locateFollowingBlock.retries</name>
<value>5</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 4f9f260..adede5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -857,6 +857,20 @@ public class DFSTestUtil {
}
}
+ /* Write the given bytes to the given file using the specified blockSize */
+ public static void writeFile(
+ FileSystem fs, Path p, byte[] bytes, long blockSize)
+ throws IOException {
+ if (fs.exists(p)) {
+ fs.delete(p, true);
+ }
+ try (InputStream is = new ByteArrayInputStream(bytes);
+ FSDataOutputStream os = fs.create(
+ p, false, 4096, fs.getDefaultReplication(p), blockSize)) {
+ IOUtils.copyBytes(is, os, bytes.length);
+ }
+ }
+
/* Write the given string to the given file */
public static void writeFile(FileSystem fs, Path p, String s)
throws IOException {
@@ -901,14 +915,27 @@ public class DFSTestUtil {
*/
public static void appendFileNewBlock(DistributedFileSystem fs,
Path p, int length) throws IOException {
- assert fs.exists(p);
assert length >= 0;
byte[] toAppend = new byte[length];
Random random = new Random();
random.nextBytes(toAppend);
+ appendFileNewBlock(fs, p, toAppend);
+ }
+
+ /**
+ * Append specified bytes to a given file, starting with new block.
+ *
+ * @param fs The file system
+ * @param p Path of the file to append
+ * @param bytes The data to append
+ * @throws IOException
+ */
+ public static void appendFileNewBlock(DistributedFileSystem fs,
+ Path p, byte[] bytes) throws IOException {
+ assert fs.exists(p);
try (FSDataOutputStream out = fs.append(p,
EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) {
- out.write(toAppend);
+ out.write(bytes);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
index d201ce1..0ff2d4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -30,7 +31,9 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -74,6 +77,9 @@ public class TestFileChecksum {
private String stripedFile2 = ecDir + "/stripedFileChecksum2";
private String replicatedFile = "/replicatedFileChecksum";
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
@Before
public void setup() throws IOException {
int numDNs = dataBlocks + parityBlocks + 2;
@@ -83,6 +89,7 @@ public class TestFileChecksum {
false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+ customizeConf(conf);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
Path ecPath = new Path(ecDir);
cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault());
@@ -106,6 +113,39 @@ public class TestFileChecksum {
}
}
+ /**
+ * Subclasses may customize the conf to run the full set of tests under
+ * different conditions.
+ */
+ protected void customizeConf(Configuration preparedConf) {
+ }
+
+ /**
+ * Subclasses may override this method to indicate whether equivalent files
+ * in striped and replicated formats are expected to have the same
+ * overall FileChecksum.
+ */
+ protected boolean expectComparableStripedAndReplicatedFiles() {
+ return false;
+ }
+
+ /**
+ * Subclasses may override this method to indicate whether equivalent files
+ * in replicated formats with different block sizes are expected to have the
+ * same overall FileChecksum.
+ */
+ protected boolean expectComparableDifferentBlockSizeReplicatedFiles() {
+ return false;
+ }
+
+ /**
+ * Subclasses may override this method to indicate whether checksums are
+ * supported for files where different blocks have different bytesPerCRC.
+ */
+ protected boolean expectSupportForSingleFileMixedBytesPerChecksum() {
+ return false;
+ }
+
@Test(timeout = 90000)
public void testStripedFileChecksum1() throws Exception {
int length = 0;
@@ -182,7 +222,30 @@ public class TestFileChecksum {
FileChecksum replicatedFileChecksum = getFileChecksum(replicatedFile,
10, false);
- Assert.assertFalse(stripedFileChecksum1.equals(replicatedFileChecksum));
+ if (expectComparableStripedAndReplicatedFiles()) {
+ Assert.assertEquals(stripedFileChecksum1, replicatedFileChecksum);
+ } else {
+ Assert.assertNotEquals(stripedFileChecksum1, replicatedFileChecksum);
+ }
+ }
+
+ @Test(timeout = 90000)
+ public void testDifferentBlockSizeReplicatedFileChecksum() throws Exception {
+ byte[] fileData = StripedFileTestUtil.generateBytes(fileSize);
+ String replicatedFile1 = "/replicatedFile1";
+ String replicatedFile2 = "/replicatedFile2";
+ DFSTestUtil.writeFile(
+ fs, new Path(replicatedFile1), fileData, blockSize);
+ DFSTestUtil.writeFile(
+ fs, new Path(replicatedFile2), fileData, blockSize / 2);
+ FileChecksum checksum1 = getFileChecksum(replicatedFile1, -1, false);
+ FileChecksum checksum2 = getFileChecksum(replicatedFile2, -1, false);
+
+ if (expectComparableDifferentBlockSizeReplicatedFiles()) {
+ Assert.assertEquals(checksum1, checksum2);
+ } else {
+ Assert.assertNotEquals(checksum1, checksum2);
+ }
}
@Test(timeout = 90000)
@@ -471,6 +534,40 @@ public class TestFileChecksum {
bytesPerCRC - 1);
}
+ @Test(timeout = 90000)
+ public void testMixedBytesPerChecksum() throws Exception {
+ int fileLength = bytesPerCRC * 3;
+ byte[] fileData = StripedFileTestUtil.generateBytes(fileLength);
+ String replicatedFile1 = "/replicatedFile1";
+
+ // Split file into two parts.
+ byte[] fileDataPart1 = new byte[bytesPerCRC * 2];
+ System.arraycopy(fileData, 0, fileDataPart1, 0, fileDataPart1.length);
+ byte[] fileDataPart2 = new byte[fileData.length - fileDataPart1.length];
+ System.arraycopy(
+ fileData, fileDataPart1.length, fileDataPart2, 0, fileDataPart2.length);
+
+ DFSTestUtil.writeFile(fs, new Path(replicatedFile1), fileDataPart1);
+
+ // Modify bytesPerCRC for second part that we append as separate block.
+ conf.setInt(
+ HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, bytesPerCRC / 2);
+ DFSTestUtil.appendFileNewBlock(
+ ((DistributedFileSystem) FileSystem.newInstance(conf)),
+ new Path(replicatedFile1), fileDataPart2);
+
+ if (expectSupportForSingleFileMixedBytesPerChecksum()) {
+ String replicatedFile2 = "/replicatedFile2";
+ DFSTestUtil.writeFile(fs, new Path(replicatedFile2), fileData);
+ FileChecksum checksum1 = getFileChecksum(replicatedFile1, -1, false);
+ FileChecksum checksum2 = getFileChecksum(replicatedFile2, -1, false);
+ Assert.assertEquals(checksum1, checksum2);
+ } else {
+ exception.expect(IOException.class);
+ FileChecksum checksum = getFileChecksum(replicatedFile1, -1, false);
+ }
+ }
+
private FileChecksum getFileChecksum(String filePath, int range,
boolean killDn) throws Exception {
int dnIdxToDie = -1;
@@ -537,4 +634,4 @@ public class TestFileChecksum {
return -1;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksumCompositeCrc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksumCompositeCrc.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksumCompositeCrc.java
new file mode 100644
index 0000000..87fb7da
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksumCompositeCrc.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+
+/**
+ * End-to-end tests for COMPOSITE_CRC combine mode.
+ */
+public class TestFileChecksumCompositeCrc extends TestFileChecksum {
+ @Override
+ protected void customizeConf(Configuration conf) {
+ conf.set(
+ HdfsClientConfigKeys.DFS_CHECKSUM_COMBINE_MODE_KEY, "COMPOSITE_CRC");
+ }
+
+ @Override
+ protected boolean expectComparableStripedAndReplicatedFiles() {
+ return true;
+ }
+
+ @Override
+ protected boolean expectComparableDifferentBlockSizeReplicatedFiles() {
+ return true;
+ }
+
+ @Override
+ protected boolean expectSupportForSingleFileMixedBytesPerChecksum() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index da56c15..22f84c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockChecksumType;
import org.apache.hadoop.hdfs.protocol.BlockType;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -683,6 +684,19 @@ public class TestPBHelper {
}
@Test
+ public void testBlockChecksumTypeProto() {
+ assertEquals(BlockChecksumType.MD5CRC,
+ PBHelperClient.convert(HdfsProtos.BlockChecksumTypeProto.MD5CRC));
+ assertEquals(BlockChecksumType.COMPOSITE_CRC,
+ PBHelperClient.convert(
+ HdfsProtos.BlockChecksumTypeProto.COMPOSITE_CRC));
+ assertEquals(PBHelperClient.convert(BlockChecksumType.MD5CRC),
+ HdfsProtos.BlockChecksumTypeProto.MD5CRC);
+ assertEquals(PBHelperClient.convert(BlockChecksumType.COMPOSITE_CRC),
+ HdfsProtos.BlockChecksumTypeProto.COMPOSITE_CRC);
+ }
+
+ @Test
public void testAclEntryProto() {
// All fields populated.
AclEntry e1 = new AclEntry.Builder().setName("test")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
index bbe7e8d..30e4683 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
@@ -73,20 +73,42 @@ public class TestCopyMapper {
private static final String SOURCE_PATH = "/tmp/source";
private static final String TARGET_PATH = "/tmp/target";
- private static Configuration configuration;
-
@BeforeClass
public static void setup() throws Exception {
- configuration = getConfigurationForCluster();
- cluster = new MiniDFSCluster.Builder(configuration)
+ Configuration configuration = getConfigurationForCluster();
+ setCluster(new MiniDFSCluster.Builder(configuration)
.numDataNodes(1)
.format(true)
- .build();
+ .build());
+ }
+
+ /**
+ * Subclasses may override this method to indicate whether copying files with
+ * non-default block sizes without setting BLOCKSIZE as a preserved attribute
+ * is expected to succeed with CRC checks enabled.
+ */
+ protected boolean expectDifferentBlockSizesMultipleBlocksToSucceed() {
+ return false;
+ }
+
+ /**
+ * Subclasses may override this method to indicate whether copying files with
+ * non-default bytes-per-crc without setting CHECKSUMTYPE as a preserved
+ * attribute is expected to succeed with CRC checks enabled.
+ */
+ protected boolean expectDifferentBytesPerCrcToSucceed() {
+ return false;
+ }
+
+ protected static void setCluster(MiniDFSCluster c) {
+ cluster = c;
}
- private static Configuration getConfigurationForCluster() throws IOException {
+ protected static Configuration getConfigurationForCluster()
+ throws IOException {
Configuration configuration = new Configuration();
- System.setProperty("test.build.data", "target/tmp/build/TEST_COPY_MAPPER/data");
+ System.setProperty(
+ "test.build.data", "target/tmp/build/TEST_COPY_MAPPER/data");
configuration.set("hadoop.log.dir", "target/tmp");
configuration.set("dfs.namenode.fs-limits.min-block-size", "0");
LOG.debug("fs.default.name == " + configuration.get("fs.default.name"));
@@ -136,7 +158,8 @@ public class TestCopyMapper {
}
}
- private static void createSourceDataWithDifferentBlockSize() throws Exception {
+ private static void createSourceDataWithDifferentBlockSize()
+ throws Exception {
mkdirs(SOURCE_PATH + "/1");
mkdirs(SOURCE_PATH + "/2");
mkdirs(SOURCE_PATH + "/2/3/4");
@@ -163,6 +186,21 @@ public class TestCopyMapper {
512));
}
+ private static void createSourceDataWithDifferentBytesPerCrc()
+ throws Exception {
+ mkdirs(SOURCE_PATH + "/1");
+ mkdirs(SOURCE_PATH + "/2");
+ mkdirs(SOURCE_PATH + "/2/3/4");
+ mkdirs(SOURCE_PATH + "/2/3");
+ mkdirs(SOURCE_PATH + "/5");
+ touchFile(SOURCE_PATH + "/5/6", false,
+ new ChecksumOpt(DataChecksum.Type.CRC32C, 32));
+ mkdirs(SOURCE_PATH + "/7");
+ mkdirs(SOURCE_PATH + "/7/8");
+ touchFile(SOURCE_PATH + "/7/8/9", false,
+ new ChecksumOpt(DataChecksum.Type.CRC32C, 64));
+ }
+
private static void mkdirs(String path) throws Exception {
FileSystem fileSystem = cluster.getFileSystem();
final Path qualifiedPath = new Path(path).makeQualified(fileSystem.getUri(),
@@ -281,7 +319,7 @@ public class TestCopyMapper {
path)), context);
}
- verifyCopy(fs, false);
+ verifyCopy(fs, false, true);
// verify that we only copied new appended data
Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE * 2, stubContext
.getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED)
@@ -317,6 +355,11 @@ public class TestCopyMapper {
EnumSet<DistCpOptions.FileAttribute> fileAttributes
= EnumSet.of(DistCpOptions.FileAttribute.REPLICATION);
if (preserveChecksum) {
+ // We created source files with both different checksum types and
+ // non-default block sizes; here we don't explicitly add BLOCKSIZE
+ // as a preserved attribute, but the current behavior is that
+ // preserving CHECKSUMTYPE also automatically implies preserving
+ // BLOCKSIZE.
fileAttributes.add(DistCpOptions.FileAttribute.CHECKSUMTYPE);
}
configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
@@ -339,7 +382,7 @@ public class TestCopyMapper {
}
// Check that the maps worked.
- verifyCopy(fs, preserveChecksum);
+ verifyCopy(fs, preserveChecksum, true);
Assert.assertEquals(numFiles, stubContext.getReporter()
.getCounter(CopyMapper.Counter.COPY).getValue());
Assert.assertEquals(numDirs, stubContext.getReporter()
@@ -361,7 +404,8 @@ public class TestCopyMapper {
}
}
- private void verifyCopy(FileSystem fs, boolean preserveChecksum)
+ private void verifyCopy(
+ FileSystem fs, boolean preserveChecksum, boolean preserveReplication)
throws Exception {
for (Path path : pathList) {
final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH,
@@ -370,8 +414,10 @@ public class TestCopyMapper {
Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path));
FileStatus sourceStatus = fs.getFileStatus(path);
FileStatus targetStatus = fs.getFileStatus(targetPath);
- Assert.assertEquals(sourceStatus.getReplication(),
- targetStatus.getReplication());
+ if (preserveReplication) {
+ Assert.assertEquals(sourceStatus.getReplication(),
+ targetStatus.getReplication());
+ }
if (preserveChecksum) {
Assert.assertEquals(sourceStatus.getBlockSize(),
targetStatus.getBlockSize());
@@ -505,7 +551,7 @@ public class TestCopyMapper {
@Override
public FileSystem run() {
try {
- return FileSystem.get(configuration);
+ return FileSystem.get(cluster.getConfiguration(0));
} catch (IOException e) {
LOG.error("Exception encountered ", e);
Assert.fail("Test failed: " + e.getMessage());
@@ -574,7 +620,7 @@ public class TestCopyMapper {
@Override
public FileSystem run() {
try {
- return FileSystem.get(configuration);
+ return FileSystem.get(cluster.getConfiguration(0));
} catch (IOException e) {
LOG.error("Exception encountered ", e);
Assert.fail("Test failed: " + e.getMessage());
@@ -649,7 +695,7 @@ public class TestCopyMapper {
@Override
public FileSystem run() {
try {
- return FileSystem.get(configuration);
+ return FileSystem.get(cluster.getConfiguration(0));
} catch (IOException e) {
LOG.error("Exception encountered ", e);
Assert.fail("Test failed: " + e.getMessage());
@@ -730,7 +776,7 @@ public class TestCopyMapper {
@Override
public FileSystem run() {
try {
- return FileSystem.get(configuration);
+ return FileSystem.get(cluster.getConfiguration(0));
} catch (IOException e) {
LOG.error("Exception encountered ", e);
Assert.fail("Test failed: " + e.getMessage());
@@ -887,7 +933,7 @@ public class TestCopyMapper {
@Override
public FileSystem run() {
try {
- return FileSystem.get(configuration);
+ return FileSystem.get(cluster.getConfiguration(0));
} catch (IOException e) {
LOG.error("Exception encountered when get FileSystem.", e);
throw new RuntimeException(e);
@@ -938,12 +984,13 @@ public class TestCopyMapper {
}
@Test(timeout=40000)
- public void testCopyFailOnBlockSizeDifference() throws Exception {
+ public void testCopyWithDifferentBlockSizes() throws Exception {
try {
deleteState();
createSourceDataWithDifferentBlockSize();
FileSystem fs = cluster.getFileSystem();
+
CopyMapper copyMapper = new CopyMapper();
StubContext stubContext = new StubContext(getConfiguration(), null, 0);
Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
@@ -959,17 +1006,79 @@ public class TestCopyMapper {
for (Path path : pathList) {
final FileStatus fileStatus = fs.getFileStatus(path);
- copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH),
- path)), new CopyListingFileStatus(fileStatus), context);
+ copyMapper.map(
+ new Text(
+ DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
+ new CopyListingFileStatus(fileStatus), context);
}
- Assert.fail("Copy should have failed because of block-size difference.");
+ if (expectDifferentBlockSizesMultipleBlocksToSucceed()) {
+ verifyCopy(fs, false, false);
+ } else {
+ Assert.fail(
+ "Copy should have failed because of block-size difference.");
+ }
+ } catch (Exception exception) {
+ if (expectDifferentBlockSizesMultipleBlocksToSucceed()) {
+ throw exception;
+ } else {
+ // Check that the exception suggests the use of -pb/-skipcrccheck.
+ // This could be refactored to use LambdaTestUtils if we add support
+ // for listing multiple different independent substrings to expect
+ // in the exception message and add support for LambdaTestUtils to
+ // inspect the transitive cause and/or suppressed exceptions as well.
+ Throwable cause = exception.getCause().getCause();
+ GenericTestUtils.assertExceptionContains("-pb", cause);
+ GenericTestUtils.assertExceptionContains("-skipcrccheck", cause);
+ }
}
- catch (IOException exception) {
- // Check that the exception suggests the use of -pb/-skipcrccheck.
- Throwable cause = exception.getCause().getCause();
- GenericTestUtils.assertExceptionContains("-pb", cause);
- GenericTestUtils.assertExceptionContains("-skipcrccheck", cause);
+ }
+
+ @Test(timeout=40000)
+ public void testCopyWithDifferentBytesPerCrc() throws Exception {
+ try {
+ deleteState();
+ createSourceDataWithDifferentBytesPerCrc();
+
+ FileSystem fs = cluster.getFileSystem();
+
+ CopyMapper copyMapper = new CopyMapper();
+ StubContext stubContext = new StubContext(getConfiguration(), null, 0);
+ Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
+ = stubContext.getContext();
+
+ Configuration configuration = context.getConfiguration();
+ EnumSet<DistCpOptions.FileAttribute> fileAttributes
+ = EnumSet.noneOf(DistCpOptions.FileAttribute.class);
+ configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
+ DistCpUtils.packAttributes(fileAttributes));
+
+ copyMapper.setup(context);
+
+ for (Path path : pathList) {
+ final FileStatus fileStatus = fs.getFileStatus(path);
+ copyMapper.map(
+ new Text(
+ DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
+ new CopyListingFileStatus(fileStatus), context);
+ }
+
+ if (expectDifferentBytesPerCrcToSucceed()) {
+ verifyCopy(fs, false, false);
+ } else {
+ Assert.fail(
+ "Copy should have failed because of bytes-per-crc difference.");
+ }
+ } catch (Exception exception) {
+ if (expectDifferentBytesPerCrcToSucceed()) {
+ throw exception;
+ } else {
+ // This could be refactored to use LambdaTestUtils if we add support
+ // for LambdaTestUtils to inspect the transitive cause and/or
+ // suppressed exceptions as well.
+ Throwable cause = exception.getCause().getCause();
+ GenericTestUtils.assertExceptionContains("mismatch", cause);
+ }
}
}
@@ -980,6 +1089,7 @@ public class TestCopyMapper {
createSourceData();
FileSystem fs = cluster.getFileSystem();
+
CopyMapper copyMapper = new CopyMapper();
StubContext stubContext = new StubContext(getConfiguration(), null, 0);
Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
@@ -1010,6 +1120,12 @@ public class TestCopyMapper {
final FileStatus source = fs.getFileStatus(path);
final FileStatus target = fs.getFileStatus(targetPath);
if (!source.isDirectory() ) {
+ // The reason the checksum check succeeds despite block sizes not
+ // matching between the two is that when only one block is ever
+ // written (partial or complete), the crcPerBlock is not included
+ // in the FileChecksum algorithmName. If we had instead written
+ // a large enough file to exceed the blocksize, then the copy
+ // would not have succeeded.
Assert.assertTrue(preserve ||
source.getBlockSize() != target.getBlockSize());
Assert.assertTrue(preserve ||
@@ -1020,8 +1136,7 @@ public class TestCopyMapper {
source.getReplication() == target.getReplication());
}
}
- }
- catch (Exception e) {
+ } catch (Exception e) {
Assert.assertTrue("Unexpected exception: " + e.getMessage(), false);
e.printStackTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapperCompositeCrc.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapperCompositeCrc.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapperCompositeCrc.java
new file mode 100644
index 0000000..6ed86e3
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapperCompositeCrc.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+
+import org.junit.BeforeClass;
+
+/**
+ * End-to-end tests for COMPOSITE_CRC combine mode.
+ */
+public class TestCopyMapperCompositeCrc extends TestCopyMapper {
+ @BeforeClass
+ public static void setup() throws Exception {
+ Configuration configuration = TestCopyMapper.getConfigurationForCluster();
+ configuration.set(
+ HdfsClientConfigKeys.DFS_CHECKSUM_COMBINE_MODE_KEY, "COMPOSITE_CRC");
+ TestCopyMapper.setCluster(new MiniDFSCluster.Builder(configuration)
+ .numDataNodes(1)
+ .format(true)
+ .build());
+ }
+
+ @Override
+ protected boolean expectDifferentBlockSizesMultipleBlocksToSucceed() {
+ return true;
+ }
+
+ @Override
+ protected boolean expectDifferentBytesPerCrcToSucceed() {
+ return true;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/44] hadoop git commit: HADOOP-14970. MiniHadoopClusterManager
doesn't respect lack of format option. Contributed by Erik Krogen
Posted by xy...@apache.org.
HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a407bc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a407bc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a407bc9
Branch: refs/heads/HDFS-7240
Commit: 1a407bc9906306801690bc75ff0f0456f8f265fd
Parents: 226bedc
Author: Erik Krogen <ek...@linkedin.com>
Authored: Thu Apr 12 23:27:51 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Apr 12 23:27:51 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a407bc9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
index b89cdc0..d29dd34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
@@ -156,6 +156,7 @@ public class MiniHadoopClusterManager {
if (!noDFS) {
dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort)
.nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes)
+ .format(dfsOpts == StartupOption.FORMAT)
.startupOption(dfsOpts).build();
LOG.info("Started MiniDFSCluster -- namenode on port "
+ dfs.getNameNodePort());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/44] hadoop git commit: YARN-8142. Improve SIGTERM handling for
YARN Service Application Master. Contributed by Billie Rinaldi
Posted by xy...@apache.org.
YARN-8142. Improve SIGTERM handling for YARN Service Application Master.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9031a76d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9031a76d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9031a76d
Branch: refs/heads/HDFS-7240
Commit: 9031a76d447f0c5eaa392144fd17c5b9812e1b20
Parents: e66e287
Author: Eric Yang <ey...@apache.org>
Authored: Fri Apr 13 15:34:33 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Apr 13 15:34:33 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/service/ClientAMService.java | 1 +
.../hadoop/yarn/service/ServiceScheduler.java | 41 +++++++----
.../hadoop/yarn/service/ServiceTestUtils.java | 11 +++
.../yarn/service/TestYarnNativeServices.java | 71 ++++++++++++++++++++
4 files changed, 110 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index 08c36f4..3d037e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -125,6 +125,7 @@ public class ClientAMService extends AbstractService
LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser());
context.scheduler.getDiagnostics()
.append("Stopped by user " + UserGroupInformation.getCurrentUser());
+ context.scheduler.setGracefulStop();
// Stop the service in 2 seconds delay to make sure this rpc call is completed.
// shutdown hook will be executed which will stop AM gracefully.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 0fcca16..7eddef9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -156,6 +156,8 @@ public class ServiceScheduler extends CompositeService {
// requests for a single service is not recommended.
private boolean hasAtLeastOnePlacementConstraint;
+ private boolean gracefulStop = false;
+
public ServiceScheduler(ServiceContext context) {
super(context.service.getName());
this.context = context;
@@ -199,6 +201,7 @@ public class ServiceScheduler extends CompositeService {
addIfService(amRMClient);
nmClient = createNMClient();
+ nmClient.getClient().cleanupRunningContainersOnStop(false);
addIfService(nmClient);
dispatcher = new AsyncDispatcher("Component dispatcher");
@@ -252,6 +255,11 @@ public class ServiceScheduler extends CompositeService {
.createAMRMClientAsync(1000, new AMRMClientCallback());
}
+ protected void setGracefulStop() {
+ this.gracefulStop = true;
+ nmClient.getClient().cleanupRunningContainersOnStop(true);
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
try {
@@ -266,26 +274,31 @@ public class ServiceScheduler extends CompositeService {
public void serviceStop() throws Exception {
LOG.info("Stopping service scheduler");
- // Mark component-instances/containers as STOPPED
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- for (ContainerId containerId : getLiveInstances().keySet()) {
- serviceTimelinePublisher.componentInstanceFinished(containerId,
- KILLED_AFTER_APP_COMPLETION, diagnostics.toString());
- }
- }
if (executorService != null) {
executorService.shutdownNow();
}
DefaultMetricsSystem.shutdown();
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- serviceTimelinePublisher
- .serviceAttemptUnregistered(context, diagnostics.toString());
+
+ // only stop the entire service when a graceful stop has been initiated
+ // (e.g. via client RPC, not through the AM receiving a SIGTERM)
+ if (gracefulStop) {
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ // mark component-instances/containers as STOPPED
+ for (ContainerId containerId : getLiveInstances().keySet()) {
+ serviceTimelinePublisher.componentInstanceFinished(containerId,
+ KILLED_AFTER_APP_COMPLETION, diagnostics.toString());
+ }
+ // mark attempt as unregistered
+ serviceTimelinePublisher
+ .serviceAttemptUnregistered(context, diagnostics.toString());
+ }
+ // unregister AM
+ amRMClient.unregisterApplicationMaster(FinalApplicationStatus.ENDED,
+ diagnostics.toString(), "");
+ LOG.info("Service {} unregistered with RM, with attemptId = {} " +
+ ", diagnostics = {} ", app.getName(), context.attemptId, diagnostics);
}
- amRMClient.unregisterApplicationMaster(FinalApplicationStatus.ENDED,
- diagnostics.toString(), "");
- LOG.info("Service {} unregistered with RM, with attemptId = {} " +
- ", diagnostics = {} ", app.getName(), context.attemptId, diagnostics);
super.serviceStop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index d84f05f..599b8a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.registry.client.impl.zk.CuratorService;
import org.apache.hadoop.service.ServiceOperations;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -305,6 +306,16 @@ public class ServiceTestUtils {
return client;
}
+ /**
+ * Creates a YarnClient for test purposes.
+ */
+ public static YarnClient createYarnClient(Configuration conf) {
+ YarnClient client = YarnClient.createYarnClient();
+ client.init(conf);
+ client.start();
+ return client;
+ }
+
protected CuratorService getCuratorService() throws IOException {
return curatorService;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 5e267bb..443ba0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -489,6 +490,76 @@ public class TestYarnNativeServices extends ServiceTestUtils {
client.actionDestroy(exampleApp.getName());
}
+ @Test(timeout = 200000)
+ public void testAMSigtermDoesNotKillApplication() throws Exception {
+ runAMSignalTest(SignalContainerCommand.GRACEFUL_SHUTDOWN);
+ }
+
+ @Test(timeout = 200000)
+ public void testAMSigkillDoesNotKillApplication() throws Exception {
+ runAMSignalTest(SignalContainerCommand.FORCEFUL_SHUTDOWN);
+ }
+
+ public void runAMSignalTest(SignalContainerCommand signal) throws Exception {
+ setupInternal(NUM_NMS);
+ ServiceClient client = createClient(getConf());
+ Service exampleApp = createExampleApplication();
+ client.actionCreate(exampleApp);
+ waitForServiceToBeStable(client, exampleApp);
+ Service appStatus1 = client.getStatus(exampleApp.getName());
+ ApplicationId exampleAppId = ApplicationId.fromString(appStatus1.getId());
+
+ YarnClient yarnClient = createYarnClient(getConf());
+ ApplicationReport applicationReport = yarnClient.getApplicationReport(
+ exampleAppId);
+
+ ApplicationAttemptId firstAttemptId = applicationReport
+ .getCurrentApplicationAttemptId();
+ ApplicationAttemptReport attemptReport = yarnClient
+ .getApplicationAttemptReport(firstAttemptId);
+
+ // the AM should not perform a graceful shutdown since the operation was not
+ // initiated through the service client
+ yarnClient.signalToContainer(attemptReport.getAMContainerId(), signal);
+
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ApplicationReport ar = client.getYarnClient()
+ .getApplicationReport(exampleAppId);
+ YarnApplicationState state = ar.getYarnApplicationState();
+ Assert.assertTrue(state == YarnApplicationState.RUNNING ||
+ state == YarnApplicationState.ACCEPTED);
+ if (state != YarnApplicationState.RUNNING) {
+ return false;
+ }
+ if (ar.getCurrentApplicationAttemptId() == null ||
+ ar.getCurrentApplicationAttemptId().equals(firstAttemptId)) {
+ return false;
+ }
+ Service appStatus2 = client.getStatus(exampleApp.getName());
+ if (appStatus2.getState() != ServiceState.STABLE) {
+ return false;
+ }
+ Assert.assertEquals(getSortedContainerIds(appStatus1).toString(),
+ getSortedContainerIds(appStatus2).toString());
+ return true;
+ } catch (YarnException | IOException e) {
+ throw new RuntimeException("while waiting", e);
+ }
+ }, 2000, 200000);
+ }
+
+ private static List<String> getSortedContainerIds(Service s) {
+ List<String> containerIds = new ArrayList<>();
+ for (Component component : s.getComponents()) {
+ for (Container container : component.getContainers()) {
+ containerIds.add(container.getId());
+ }
+ }
+ Collections.sort(containerIds);
+ return containerIds;
+ }
+
// Check containers launched are in dependency order
// Get all containers into a list and sort based on container launch time e.g.
// compa-c1, compa-c2, compb-c1, compb-c2;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/44] hadoop git commit: YARN-7973. Added ContainerRelaunch feature
for Docker containers. Contributed by Shane Kumpf
Posted by xy...@apache.org.
YARN-7973. Added ContainerRelaunch feature for Docker containers.
Contributed by Shane Kumpf
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c467f311
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c467f311
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c467f311
Branch: refs/heads/HDFS-7240
Commit: c467f311d0c7155c09052d93fac12045af925583
Parents: 583fa6e
Author: Eric Yang <ey...@apache.org>
Authored: Tue Apr 10 19:25:00 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Apr 10 19:25:00 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/api/ApplicationConstants.java | 10 ++
.../server/nodemanager/ContainerExecutor.java | 11 ++
.../nodemanager/DefaultContainerExecutor.java | 6 +
.../nodemanager/LinuxContainerExecutor.java | 124 +++++++++++--------
.../launcher/ContainerLaunch.java | 20 ++-
.../launcher/ContainerRelaunch.java | 2 +-
.../runtime/DefaultLinuxContainerRuntime.java | 6 +
.../DelegatingLinuxContainerRuntime.java | 9 ++
.../runtime/DockerLinuxContainerRuntime.java | 40 +++++-
.../JavaSandboxLinuxContainerRuntime.java | 10 ++
.../runtime/docker/DockerCommandExecutor.java | 14 +++
.../runtime/docker/DockerStartCommand.java | 29 +++++
.../runtime/ContainerRuntime.java | 10 ++
.../impl/container-executor.c | 2 +-
.../container-executor/impl/utils/docker-util.c | 40 ++++++
.../container-executor/impl/utils/docker-util.h | 11 ++
.../test/utils/test_docker_util.cc | 20 +++
.../nodemanager/TestLinuxContainerExecutor.java | 12 ++
.../launcher/TestContainerRelaunch.java | 2 +-
.../runtime/TestDockerContainerRuntime.java | 90 ++++++++++++++
.../runtime/docker/TestDockerStartCommand.java | 53 ++++++++
.../TestContainersMonitorResourceChange.java | 5 +
22 files changed, 470 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index 64bcc44..38ad596 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -99,6 +99,16 @@ public interface ApplicationConstants {
public static final String STDOUT = "stdout";
/**
+ * The type of launch for the container.
+ */
+ @Public
+ @Unstable
+ enum ContainerLaunchType {
+ LAUNCH,
+ RELAUNCH
+ }
+
+ /**
* Environment for Applications.
*
* Some of the environment variables for applications are <em>final</em>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index f566f48..8e335350 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -182,6 +182,17 @@ public abstract class ContainerExecutor implements Configurable {
IOException, ConfigurationException;
/**
+ * Relaunch the container on the node. This is a blocking call and returns
+ * only when the container exits.
+ * @param ctx Encapsulates information necessary for relaunching containers.
+ * @return the return status of the relaunch
+ * @throws IOException if the container relaunch fails
+ * @throws ConfigurationException if config error was found
+ */
+ public abstract int relaunchContainer(ContainerStartContext ctx) throws
+ IOException, ConfigurationException;
+
+ /**
* Signal container with the specified signal.
*
* @param ctx Encapsulates information necessary for signaling containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 828b8fd..446cbe4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -339,6 +339,12 @@ public class DefaultContainerExecutor extends ContainerExecutor {
return 0;
}
+ @Override
+ public int relaunchContainer(ContainerStartContext ctx)
+ throws IOException, ConfigurationException {
+ return launchContainer(ctx);
+ }
+
/**
* Create a new {@link ShellCommandExecutor} using the parameters.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index f0b0714..dfcc6ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -474,6 +474,20 @@ public class LinuxContainerExecutor extends ContainerExecutor {
@Override
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
+ return handleLaunchForLaunchType(ctx,
+ ApplicationConstants.ContainerLaunchType.LAUNCH);
+ }
+
+ @Override
+ public int relaunchContainer(ContainerStartContext ctx)
+ throws IOException, ConfigurationException {
+ return handleLaunchForLaunchType(ctx,
+ ApplicationConstants.ContainerLaunchType.RELAUNCH);
+ }
+
+ private int handleLaunchForLaunchType(ContainerStartContext ctx,
+ ApplicationConstants.ContainerLaunchType type) throws IOException,
+ ConfigurationException {
Container container = ctx.getContainer();
String user = ctx.getUser();
@@ -542,62 +556,19 @@ public class LinuxContainerExecutor extends ContainerExecutor {
ContainerRuntimeContext runtimeContext = buildContainerRuntimeContext(
ctx, pidFilePath, resourcesOptions, tcCommandFile, numaArgs);
- linuxContainerRuntime.launchContainer(runtimeContext);
+ if (type.equals(ApplicationConstants.ContainerLaunchType.RELAUNCH)) {
+ linuxContainerRuntime.relaunchContainer(runtimeContext);
+ } else {
+ linuxContainerRuntime.launchContainer(runtimeContext);
+ }
+
} else {
LOG.info(
"Container was marked as inactive. Returning terminated error");
return ContainerExecutor.ExitCode.TERMINATED.getExitCode();
}
} catch (ContainerExecutionException e) {
- int exitCode = e.getExitCode();
- LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
- // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
- // terminated/killed forcefully. In all other cases, log the
- // output
- if (exitCode != ContainerExecutor.ExitCode.FORCE_KILLED.getExitCode()
- && exitCode != ContainerExecutor.ExitCode.TERMINATED.getExitCode()) {
- LOG.warn("Exception from container-launch with container ID: "
- + containerId + " and exit code: " + exitCode, e);
-
- StringBuilder builder = new StringBuilder();
- builder.append("Exception from container-launch.\n");
- builder.append("Container id: " + containerId + "\n");
- builder.append("Exit code: " + exitCode + "\n");
- builder.append("Exception message: " + e.getMessage() + "\n");
- if (!Optional.fromNullable(e.getErrorOutput()).or("").isEmpty()) {
- builder.append("Shell error output: " + e.getErrorOutput() + "\n");
- }
- //Skip stack trace
- String output = e.getOutput();
- if (output != null && !output.isEmpty()) {
- builder.append("Shell output: " + output + "\n");
- }
- String diagnostics = builder.toString();
- logOutput(diagnostics);
- container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
- diagnostics));
- if (exitCode ==
- ExitCode.INVALID_CONTAINER_EXEC_PERMISSIONS.getExitCode() ||
- exitCode ==
- ExitCode.INVALID_CONFIG_FILE.getExitCode() ||
- exitCode ==
- ExitCode.COULD_NOT_CREATE_SCRIPT_COPY.getExitCode() ||
- exitCode ==
- ExitCode.COULD_NOT_CREATE_CREDENTIALS_FILE.getExitCode() ||
- exitCode ==
- ExitCode.COULD_NOT_CREATE_WORK_DIRECTORIES.getExitCode() ||
- exitCode ==
- ExitCode.COULD_NOT_CREATE_APP_LOG_DIRECTORIES.getExitCode() ||
- exitCode ==
- ExitCode.COULD_NOT_CREATE_TMP_DIRECTORIES.getExitCode()) {
- throw new ConfigurationException(
- "Linux Container Executor reached unrecoverable exception", e);
- }
- } else {
- container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
- "Container killed on request. Exit code is " + exitCode));
- }
- return exitCode;
+ return handleExitCode(e, container, containerId);
} finally {
resourcesHandler.postExecute(containerId);
@@ -614,6 +585,59 @@ public class LinuxContainerExecutor extends ContainerExecutor {
return 0;
}
+ private int handleExitCode(ContainerExecutionException e, Container container,
+ ContainerId containerId) throws ConfigurationException {
+ int exitCode = e.getExitCode();
+ LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
+ // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
+ // terminated/killed forcefully. In all other cases, log the
+ // output
+ if (exitCode != ContainerExecutor.ExitCode.FORCE_KILLED.getExitCode()
+ && exitCode != ContainerExecutor.ExitCode.TERMINATED.getExitCode()) {
+ LOG.warn("Exception from container-launch with container ID: "
+ + containerId + " and exit code: " + exitCode, e);
+
+ StringBuilder builder = new StringBuilder();
+ builder.append("Exception from container-launch.\n");
+ builder.append("Container id: " + containerId + "\n");
+ builder.append("Exit code: " + exitCode + "\n");
+ builder.append("Exception message: " + e.getMessage() + "\n");
+ if (!Optional.fromNullable(e.getErrorOutput()).or("").isEmpty()) {
+ builder.append("Shell error output: " + e.getErrorOutput() + "\n");
+ }
+ //Skip stack trace
+ String output = e.getOutput();
+ if (output != null && !output.isEmpty()) {
+ builder.append("Shell output: " + output + "\n");
+ }
+ String diagnostics = builder.toString();
+ logOutput(diagnostics);
+ container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
+ diagnostics));
+ if (exitCode ==
+ ExitCode.INVALID_CONTAINER_EXEC_PERMISSIONS.getExitCode() ||
+ exitCode ==
+ ExitCode.INVALID_CONFIG_FILE.getExitCode() ||
+ exitCode ==
+ ExitCode.COULD_NOT_CREATE_SCRIPT_COPY.getExitCode() ||
+ exitCode ==
+ ExitCode.COULD_NOT_CREATE_CREDENTIALS_FILE.getExitCode() ||
+ exitCode ==
+ ExitCode.COULD_NOT_CREATE_WORK_DIRECTORIES.getExitCode() ||
+ exitCode ==
+ ExitCode.COULD_NOT_CREATE_APP_LOG_DIRECTORIES.getExitCode() ||
+ exitCode ==
+ ExitCode.COULD_NOT_CREATE_TMP_DIRECTORIES.getExitCode()) {
+ throw new ConfigurationException(
+ "Linux Container Executor reached unrecoverable exception", e);
+ }
+ } else {
+ container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
+ "Container killed on request. Exit code is " + exitCode));
+ }
+ return exitCode;
+ }
+
private ContainerRuntimeContext buildContainerRuntimeContext(
ContainerStartContext ctx, Path pidFilePath, String resourcesOptions,
String tcCommandFile, List<String> numaArgs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 3875cbc..9efe686 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -487,6 +487,24 @@ public class ContainerLaunch implements Callable<Integer> {
@SuppressWarnings("unchecked")
protected int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
+ int launchPrep = prepareForLaunch(ctx);
+ if (launchPrep == 0) {
+ return exec.launchContainer(ctx);
+ }
+ return launchPrep;
+ }
+
+ @SuppressWarnings("unchecked")
+ protected int relaunchContainer(ContainerStartContext ctx)
+ throws IOException, ConfigurationException {
+ int launchPrep = prepareForLaunch(ctx);
+ if (launchPrep == 0) {
+ return exec.relaunchContainer(ctx);
+ }
+ return launchPrep;
+ }
+
+ protected int prepareForLaunch(ContainerStartContext ctx) throws IOException {
ContainerId containerId = container.getContainerId();
if (container.isMarkedForKilling()) {
LOG.info("Container " + containerId + " not launched as it has already "
@@ -508,8 +526,8 @@ public class ContainerLaunch implements Callable<Integer> {
return ExitCode.TERMINATED.getExitCode();
} else {
exec.activateContainer(containerId, pidFilePath);
- return exec.launchContainer(ctx);
}
+ return ExitCode.SUCCESS.getExitCode();
}
protected void setContainerCompletedStatus(int exitCode) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
index b3add30..c6e3ed4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
@@ -108,7 +108,7 @@ public class ContainerRelaunch extends ContainerLaunch {
+ dirsHandler.getDisksHealthReport(false));
}
- ret = launchContainer(new ContainerStartContext.Builder()
+ ret = relaunchContainer(new ContainerStartContext.Builder()
.setContainer(container)
.setLocalizedResources(localResources)
.setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 83380ee..d8db6ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -126,6 +126,12 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public void relaunchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ launchContainer(ctx);
+ }
+
+ @Override
public void signalContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
PrivilegedOperation signalOp = new PrivilegedOperation(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 675bffb..f936169 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -142,6 +142,15 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public void relaunchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ Container container = ctx.getContainer();
+ LinuxContainerRuntime runtime = pickContainerRuntime(container);
+
+ runtime.relaunchContainer(ctx);
+ }
+
+ @Override
public void signalContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
Container container = ctx.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 132ae38..51abeb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -23,9 +23,11 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStartCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
@@ -910,6 +912,40 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ @Override
+ public void relaunchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ Container container = ctx.getContainer();
+ String containerIdStr = container.getContainerId().toString();
+ // Check to see if the container already exists for relaunch
+ DockerCommandExecutor.DockerContainerStatus containerStatus =
+ DockerCommandExecutor.getContainerStatus(containerIdStr, conf,
+ privilegedOperationExecutor);
+ if (containerStatus != null &&
+ DockerCommandExecutor.isStartable(containerStatus)) {
+ DockerStartCommand startCommand = new DockerStartCommand(containerIdStr);
+ String commandFile = dockerClient.writeCommandToTempFile(startCommand,
+ containerIdStr);
+ PrivilegedOperation launchOp = buildLaunchOp(ctx, commandFile,
+ startCommand);
+
+ try {
+ privilegedOperationExecutor.executePrivilegedOperation(null,
+ launchOp, null, null, false, false);
+ } catch (PrivilegedOperationException e) {
+ LOG.warn("Relaunch container failed. Exception: ", e);
+ LOG.info("Docker command used: " + startCommand);
+
+ throw new ContainerExecutionException("Launch container failed", e
+ .getExitCode(), e.getOutput(), e.getErrorOutput());
+ }
+ } else {
+ throw new ContainerExecutionException("Container is not in a startable "
+ + "state, unable to relaunch: " + containerIdStr);
+ }
+
+ }
+
/**
* Signal the docker container.
*
@@ -1057,7 +1093,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx,
- String commandFile, DockerRunCommand runCommand) {
+ String commandFile, DockerCommand command) {
String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
String containerIdStr = ctx.getContainer().getContainerId().toString();
@@ -1096,7 +1132,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp.appendArgs(tcCommandFile);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Launching container with cmd: " + runCommand);
+ LOG.debug("Launching container with cmd: " + command);
}
return launchOp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
index 1ab1fc5..947c4bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -268,6 +268,16 @@ public class JavaSandboxLinuxContainerRuntime
}
}
+ @Override
+ public void relaunchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ try {
+ super.relaunchContainer(ctx);
+ } finally {
+ deletePolicyFiles(ctx);
+ }
+ }
+
/**
* Determine if JVMSandboxLinuxContainerRuntime should be used. This is
* decided based on the value of
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
index fd1812b..f11f460 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
@@ -225,4 +225,18 @@ public final class DockerCommandExecutor {
&& !containerStatus.equals(DockerContainerStatus.REMOVING)
&& !containerStatus.equals(DockerContainerStatus.RUNNING);
}
+
+ /**
+ * Is the container in a startable state?
+ *
+ * @param containerStatus the container's {@link DockerContainerStatus}.
+ * @return is the container in a startable state.
+ */
+ public static boolean isStartable(DockerContainerStatus containerStatus) {
+ if (containerStatus.equals(DockerContainerStatus.EXITED)
+ || containerStatus.equals(DockerContainerStatus.STOPPED)) {
+ return true;
+ }
+ return false;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStartCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStartCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStartCommand.java
new file mode 100644
index 0000000..b77590e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStartCommand.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+/**
+ * Encapsulates the docker start command and its command line arguments.
+ */
+public class DockerStartCommand extends DockerCommand {
+ private static final String START_COMMAND = "start";
+
+ public DockerStartCommand(String containerName) {
+ super(START_COMMAND);
+ super.addCommandArguments("name", containerName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index 7caa0ed..01995d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -55,6 +55,16 @@ public interface ContainerRuntime {
throws ContainerExecutionException;
/**
+ * Relaunch a container.
+ *
+ * @param ctx the {@link ContainerRuntimeContext}
+ * @throws ContainerExecutionException if an error occurs while relaunching
+ * the container
+ */
+ void relaunchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException;
+
+ /**
* Signal a container. Signals may be a request to terminate, a status check,
* etc.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 14c0da5..af7f8ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1542,7 +1542,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
fprintf(LOGFILE, "Launching docker container...\n");
fprintf(LOGFILE, "Docker run command: %s\n", docker_command_with_binary);
FILE* start_docker = popen(docker_command_with_binary, "r");
- if (pclose (start_docker) != 0)
+ if (WEXITSTATUS(pclose (start_docker)) != 0)
{
fprintf (ERRORFILE,
"Could not invoke docker %s.\n", docker_command_with_binary);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index ccc21fa..3bd94a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -357,6 +357,8 @@ int get_docker_command(const char *command_file, const struct configuration *con
return get_docker_stop_command(command_file, conf, out, outlen);
} else if (strcmp(DOCKER_VOLUME_COMMAND, command) == 0) {
return get_docker_volume_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_START_COMMAND, command) == 0) {
+ return get_docker_start_command(command_file, conf, out, outlen);
} else {
return UNKNOWN_DOCKER_COMMAND;
}
@@ -813,6 +815,44 @@ int get_docker_kill_command(const char *command_file, const struct configuration
return BUFFER_TOO_SMALL;
}
+int get_docker_start_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_START_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_START_COMMAND);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, container_name);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+free_and_exit:
+ free(container_name);
+ return ret;
+}
+
static int detach_container(const struct configuration *command_config, char *out, const size_t outlen) {
return add_param_to_command(command_config, "detach", "-d ", 0, out, outlen);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
index 8299acd..c797ecd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -32,6 +32,7 @@
#define DOCKER_STOP_COMMAND "stop"
#define DOCKER_KILL_COMMAND "kill"
#define DOCKER_VOLUME_COMMAND "volume"
+#define DOCKER_START_COMMAND "start"
enum docker_error_codes {
@@ -162,6 +163,16 @@ int get_docker_volume_command(const char *command_file, const struct configurati
const size_t outlen);
/**
+ * Get the Docker start command line string. The function will verify that the params file is meant for the start command.
+ * @param command_file File containing the params for the Docker start command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the start command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_start_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
* Give an error message for the supplied error code
* @param error_code the error code
* @return const string containing the error message
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index 4bc3404..5d9779c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -338,6 +338,26 @@ namespace ContainerExecutor {
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_kill_command);
}
+ TEST_F(TestDockerUtil, test_docker_start) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=start\n name=container_e1_12312_11111_02_000001",
+ "start container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=start\n name=ctr-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=start\n name=", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=start", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_start_command);
+ }
+
TEST_F(TestDockerUtil, test_detach_container) {
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index 31be858..ddbf3b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -675,6 +675,18 @@ public class TestLinuxContainerExecutor {
verify(lce, times(1)).reapContainer(ctx);
}
+ @Test
+ public void testRelaunchContainer() throws Exception {
+ Container container = mock(Container.class);
+ LinuxContainerExecutor lce = mock(LinuxContainerExecutor.class);
+ ContainerStartContext.Builder builder =
+ new ContainerStartContext.Builder();
+ builder.setContainer(container).setUser("foo");
+ ContainerStartContext ctx = builder.build();
+ lce.relaunchContainer(ctx);
+ verify(lce, times(1)).relaunchContainer(ctx);
+ }
+
private static class TestResourceHandler implements LCEResourcesHandler {
static Set<ContainerId> postExecContainers = new HashSet<ContainerId>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java
index f3dd972..4374e66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerRelaunch.java
@@ -78,7 +78,7 @@ public class TestContainerRelaunch {
assertEquals("relaunch failed", 0, result);
ArgumentCaptor<ContainerStartContext> captor =
ArgumentCaptor.forClass(ContainerStartContext.class);
- verify(mockExecutor).launchContainer(captor.capture());
+ verify(mockExecutor).relaunchContainer(captor.capture());
ContainerStartContext csc = captor.getValue();
assertNotNull("app ID null", csc.getAppId());
assertNotNull("container null", csc.getContainer());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 2d80592..acb3e42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -42,10 +42,12 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerClient;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStartCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin;
@@ -99,6 +101,7 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.RESOURCES_OPTIONS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.RUN_AS_USER;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.SIGNAL;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.TC_COMMAND_FILE;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER_FILECACHE_DIRS;
import static org.mockito.Mockito.any;
@@ -1864,6 +1867,32 @@ public class TestDockerContainerRuntime {
dockerCommands.get(counter++));
}
+ @Test
+ public void testDockerContainerRelaunch()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ DockerLinuxContainerRuntime runtime = new MockRuntime(mockExecutor,
+ DockerCommandExecutor.DockerContainerStatus.EXITED, false);
+ runtime.initialize(conf, null);
+ runtime.relaunchContainer(builder.build());
+
+ PrivilegedOperation op = capturePrivilegedOperation();
+ List<String> args = op.getArguments();
+ String dockerCommandFile = args.get(11);
+
+ List<String> dockerCommands = Files.readAllLines(
+ Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+
+ int expected = 3;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=start",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter));
+ }
+
class MockRuntime extends DockerLinuxContainerRuntime {
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -1929,5 +1958,66 @@ public class TestDockerContainerRuntime {
}
}
}
+
+ @Override
+ public void relaunchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ if (DockerCommandExecutor.isRemovable(containerStatus)) {
+ String relaunchContainerIdStr =
+ ctx.getContainer().getContainerId().toString();
+ DockerStartCommand startCommand =
+ new DockerStartCommand(containerIdStr);
+ DockerClient dockerClient = new DockerClient(conf);
+ String commandFile = dockerClient.writeCommandToTempFile(startCommand,
+ relaunchContainerIdStr);
+ String relaunchRunAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
+ Path relaunchNmPrivateContainerScriptPath = ctx.getExecutionAttribute(
+ NM_PRIVATE_CONTAINER_SCRIPT_PATH);
+ Path relaunchContainerWorkDir =
+ ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
+ //we can't do better here thanks to type-erasure
+ @SuppressWarnings("unchecked")
+ List<String> relaunchLocalDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
+ @SuppressWarnings("unchecked")
+ List<String> relaunchLogDirs = ctx.getExecutionAttribute(LOG_DIRS);
+ String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
+
+ PrivilegedOperation launchOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
+
+ launchOp.appendArgs(relaunchRunAsUser, ctx.getExecutionAttribute(USER),
+ Integer.toString(PrivilegedOperation
+ .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()),
+ ctx.getExecutionAttribute(APPID),
+ relaunchContainerIdStr,
+ relaunchContainerWorkDir.toString(),
+ relaunchNmPrivateContainerScriptPath.toUri().getPath(),
+ ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(),
+ ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ relaunchLocalDirs),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ relaunchLogDirs),
+ commandFile,
+ resourcesOpts);
+
+ String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
+
+ if (tcCommandFile != null) {
+ launchOp.appendArgs(tcCommandFile);
+ }
+
+ try {
+ privilegedOperationExecutor.executePrivilegedOperation(null,
+ launchOp, null, null, false, false);
+ } catch (PrivilegedOperationException e) {
+ LOG.warn("Relaunch container failed. Exception: ", e);
+ LOG.info("Docker command used: " + startCommand);
+
+ throw new ContainerExecutionException("Launch container failed", e
+ .getExitCode(), e.getOutput(), e.getErrorOutput());
+ }
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStartCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStartCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStartCommand.java
new file mode 100644
index 0000000..d8c5772
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStartCommand.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import org.apache.hadoop.util.StringUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests the docker start command and any command line arguments.
+ */
+public class TestDockerStartCommand {
+
+ private DockerStartCommand dockerStartCommand;
+
+ private static final String CONTAINER_NAME = "foo";
+
+ @Before
+ public void setUp() {
+ dockerStartCommand = new DockerStartCommand(CONTAINER_NAME);
+ }
+
+ @Test
+ public void testGetCommandOption() {
+ assertEquals("start", dockerStartCommand.getCommandOption());
+ }
+
+ @Test
+ public void testGetCommandWithArguments() {
+ assertEquals("start", StringUtils.join(",",
+ dockerStartCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerStartCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(2, dockerStartCommand.getDockerCommandWithArguments().size());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index d7d826c..c5fdccd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -86,6 +86,11 @@ public class TestContainersMonitorResourceChange {
return 0;
}
@Override
+ public int relaunchContainer(ContainerStartContext ctx) throws
+ IOException, ConfigurationException {
+ return 0;
+ }
+ @Override
public boolean signalContainer(ContainerSignalContext ctx)
throws IOException {
return true;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/44] hadoop git commit: YARN-8116. Nodemanager fails with
NumberFormatException: For input string: . (Chandni Singh via wangda)
Posted by xy...@apache.org.
YARN-8116. Nodemanager fails with NumberFormatException: For input string: . (Chandni Singh via wangda)
Change-Id: Idd30cfca59982d3fc6e47aa1b88f844a78fae94d
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bf9cc2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bf9cc2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bf9cc2c
Branch: refs/heads/HDFS-7240
Commit: 2bf9cc2c73944c9f7cde56714b8cf6995cfa539b
Parents: c467f31
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Apr 10 17:32:38 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Apr 10 17:32:38 2018 -0700
----------------------------------------------------------------------
.../containermanager/container/ContainerImpl.java | 3 ++-
.../recovery/NMLeveldbStateStoreService.java | 4 +++-
.../recovery/TestNMLeveldbStateStoreService.java | 16 ++++++++++++++++
3 files changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 2115100..c09c7f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -2191,7 +2191,8 @@ public class ContainerImpl implements Container {
}
private void storeRetryContext() {
- if (windowRetryContext.getRestartTimes() != null) {
+ if (windowRetryContext.getRestartTimes() != null &&
+ !windowRetryContext.getRestartTimes().isEmpty()) {
try {
stateStore.storeContainerRestartTimes(containerId,
windowRetryContext.getRestartTimes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index bf4c0ad..723dd48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -347,7 +347,9 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
value.substring(1, value.length() - 1).split(", ");
List<Long> restartTimes = new ArrayList<>();
for (String restartTime : unparsedRestartTimes) {
- restartTimes.add(Long.parseLong(restartTime));
+ if (!restartTime.isEmpty()) {
+ restartTimes.add(Long.parseLong(restartTime));
+ }
}
rcs.setRestartTimes(restartTimes);
} else if (suffix.equals(CONTAINER_WORK_DIR_KEY_SUFFIX)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index c270199..265b3e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -1216,6 +1216,22 @@ public class TestNMLeveldbStateStoreService {
Assert.fail("Expected exception not thrown");
}
+ @Test
+ public void testEmptyRestartTimes() throws IOException {
+ List<Long> restartTimes = new ArrayList<>();
+ ApplicationId appId = ApplicationId.newInstance(1234, 3);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId,
+ 4);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId, 5);
+ storeMockContainer(containerId);
+ stateStore.storeContainerRestartTimes(containerId,
+ restartTimes);
+ restartStateStore();
+ RecoveredContainerState rcs = stateStore.loadContainersState().get(0);
+ List<Long> recoveredRestartTimes = rcs.getRestartTimes();
+ assertTrue(recoveredRestartTimes.isEmpty());
+ }
+
private StartContainerRequest storeMockContainer(ContainerId containerId)
throws IOException {
// create a container request
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/44] hadoop git commit: HADOOP-15357.
Configuration.getPropsWithPrefix no longer does variable substitution.
Contributed by Jim Brennan
Posted by xy...@apache.org.
HADOOP-15357. Configuration.getPropsWithPrefix no longer does variable substitution. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8139754
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8139754
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8139754
Branch: refs/heads/HDFS-7240
Commit: e81397545a273cf9a090010eb644b836e0ef8c7b
Parents: d553799
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Apr 10 16:44:03 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Apr 10 16:44:55 2018 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 11 +++------
.../apache/hadoop/conf/TestConfiguration.java | 26 +++++++++++++++-----
2 files changed, 24 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8139754/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 78a2e9f..f1e2a9d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2869,15 +2869,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
*/
public Map<String, String> getPropsWithPrefix(String confPrefix) {
Properties props = getProps();
- Enumeration e = props.propertyNames();
Map<String, String> configMap = new HashMap<>();
- String name = null;
- while (e.hasMoreElements()) {
- name = (String) e.nextElement();
+ for (String name : props.stringPropertyNames()) {
if (name.startsWith(confPrefix)) {
- String value = props.getProperty(name);
- name = name.substring(confPrefix.length());
- configMap.put(name, value);
+ String value = get(name);
+ String keyName = name.substring(confPrefix.length());
+ configMap.put(keyName, value);
}
}
return configMap;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8139754/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index b0bb0d7..33a9880 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -2320,19 +2320,33 @@ public class TestConfiguration {
FileUtil.fullyDelete(tmpDir);
}
+ @Test
public void testGettingPropertiesWithPrefix() throws Exception {
Configuration conf = new Configuration();
for (int i = 0; i < 10; i++) {
- conf.set("prefix" + ".name" + i, "value");
+ conf.set("prefix." + "name" + i, "value" + i);
}
conf.set("different.prefix" + ".name", "value");
- Map<String, String> props = conf.getPropsWithPrefix("prefix");
- assertEquals(props.size(), 10);
+ Map<String, String> prefixedProps = conf.getPropsWithPrefix("prefix.");
+ assertEquals(prefixedProps.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ assertEquals("value" + i, prefixedProps.get("name" + i));
+ }
+ // Repeat test with variable substitution
+ conf.set("foo", "bar");
+ for (int i = 0; i < 10; i++) {
+ conf.set("subprefix." + "subname" + i, "value_${foo}" + i);
+ }
+ prefixedProps = conf.getPropsWithPrefix("subprefix.");
+ assertEquals(prefixedProps.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ assertEquals("value_bar" + i, prefixedProps.get("subname" + i));
+ }
// test call with no properties for a given prefix
- props = conf.getPropsWithPrefix("none");
- assertNotNull(props.isEmpty());
- assertTrue(props.isEmpty());
+ prefixedProps = conf.getPropsWithPrefix("none");
+ assertNotNull(prefixedProps.isEmpty());
+ assertTrue(prefixedProps.isEmpty());
}
public static void main(String[] argv) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/44] hadoop git commit: HDFS-13330.
ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.
Posted by xy...@apache.org.
HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e66e287e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e66e287e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e66e287e
Branch: refs/heads/HDFS-7240
Commit: e66e287efe2b43e710137a628f03c7df3ebdf498
Parents: 6503593
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Fri Apr 13 09:17:34 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Fri Apr 13 09:17:34 2018 -0700
----------------------------------------------------------------------
.../hdfs/shortcircuit/ShortCircuitCache.java | 11 ++++++---
.../shortcircuit/TestShortCircuitCache.java | 26 ++++++++++++++++++++
2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index b26652b..c2f0350 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable {
unref(replica);
}
+ static final int FETCH_OR_CREATE_RETRY_TIMES = 3;
/**
* Fetch or create a replica.
*
@@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable {
*/
public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key,
ShortCircuitReplicaCreator creator) {
- Waitable<ShortCircuitReplicaInfo> newWaitable = null;
+ Waitable<ShortCircuitReplicaInfo> newWaitable;
lock.lock();
try {
ShortCircuitReplicaInfo info = null;
- do {
+ for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){
if (closed) {
LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.",
this, key);
@@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable {
if (waitable != null) {
try {
info = fetch(key, waitable);
+ break;
} catch (RetriableException e) {
LOG.debug("{}: retrying {}", this, e.getMessage());
}
}
- } while (false);
+ }
if (info != null) return info;
// We need to load the replica ourselves.
newWaitable = new Waitable<>(lock.newCondition());
@@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable {
*
* @throws RetriableException If the caller needs to retry.
*/
- private ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
+ @VisibleForTesting // ONLY for testing
+ protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
Waitable<ShortCircuitReplicaInfo> waitable) throws RetriableException {
// Another thread is already in the process of loading this
// ShortCircuitReplica. So we simply wait for it to complete.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 7ba0edc..5da6a25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -793,4 +794,29 @@ public class TestShortCircuitCache {
cluster.shutdown();
sockDir.close();
}
+
+ @Test
+ public void testFetchOrCreateRetries() throws Exception {
+ try(ShortCircuitCache cache = Mockito
+ .spy(new ShortCircuitCache(10, 10000000, 10, 10000000, 1, 10000, 0))) {
+ final TestFileDescriptorPair pair = new TestFileDescriptorPair();
+ ExtendedBlockId extendedBlockId = new ExtendedBlockId(123, "test_bp1");
+ SimpleReplicaCreator sRC = new SimpleReplicaCreator(123, cache, pair);
+
+ // Arrange that fetch will throw RetriableException for any call
+ Mockito.doThrow(new RetriableException("Retry")).when(cache)
+ .fetch(Mockito.eq(extendedBlockId), Mockito.any());
+
+ // Act: calling fetchOrCreate two times
+ // first call: it will create and put entry to replicaInfoMap
+ // second call: it will call fetch to get info for entry, and should
+ // retry 3 times because RetriableException thrown
+ cache.fetchOrCreate(extendedBlockId, sRC);
+ cache.fetchOrCreate(extendedBlockId, sRC);
+
+ // Assert that fetchOrCreate retried to fetch at least 3 times
+ Mockito.verify(cache, Mockito.atLeast(3))
+ .fetch(Mockito.eq(extendedBlockId), Mockito.any());
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/44] hadoop git commit: YARN-8127. Resource leak when async
scheduling is enabled. Contributed by Tao Yang.
Posted by xy...@apache.org.
YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb783e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb783e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb783e2
Branch: refs/heads/HDFS-7240
Commit: 7eb783e2634d8c11fb646f1f2fdf597336325312
Parents: b0aff8a
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Apr 11 17:15:25 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Apr 11 17:15:25 2018 +0800
----------------------------------------------------------------------
.../scheduler/common/fica/FiCaSchedulerApp.java | 10 +++
.../TestCapacitySchedulerAsyncScheduling.java | 91 ++++++++++++++++++++
2 files changed, 101 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 32b2cad..3ec8191 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -339,6 +339,16 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return false;
}
}
+ // If allocate from reserved container, make sure node is still reserved
+ if (allocation.getAllocateFromReservedContainer() != null
+ && reservedContainerOnNode == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Try to allocate from reserved container " + allocation
+ .getAllocateFromReservedContainer().getRmContainer()
+ .getContainerId() + ", but node is not reserved");
+ }
+ return false;
+ }
// Do we have enough space on this node?
Resource availableResource = Resources.clone(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 18cd942..338b9f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -594,6 +594,97 @@ public class TestCapacitySchedulerAsyncScheduling {
}
}
+ // Testcase for YARN-8127
+ @Test (timeout = 30000)
+ public void testCommitDuplicatedAllocateFromReservedProposals()
+ throws Exception {
+ // disable async-scheduling for simulating complex scene
+ Configuration disableAsyncConf = new Configuration(conf);
+ disableAsyncConf.setBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+ // init RM & NMs
+ final MockRM rm = new MockRM(disableAsyncConf);
+ rm.start();
+ final MockNM nm1 = rm.registerNode("192.168.0.1:1234", 8 * GB);
+ rm.registerNode("192.168.0.2:2234", 8 * GB);
+
+ // init scheduler & nodes
+ while (
+ ((CapacityScheduler) rm.getRMContext().getScheduler()).getNodeTracker()
+ .nodeCount() < 2) {
+ Thread.sleep(10);
+ }
+ Assert.assertEquals(2,
+ ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+ .getNodeTracker().nodeCount());
+ CapacityScheduler cs =
+ (CapacityScheduler) rm.getRMContext().getScheduler();
+ SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId());
+
+ // launch app
+ RMApp app = rm.submitApp(1 * GB, "app", "user", null, false, "default",
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+ FiCaSchedulerApp schedulerApp =
+ cs.getApplicationAttempt(am.getApplicationAttemptId());
+
+ // app asks 1 * 6G container
+ // nm1 runs 2 container(container_01/AM, container_02)
+ allocateAndLaunchContainers(am, nm1, rm, 1,
+ Resources.createResource(6 * GB), 0, 2);
+ Assert.assertEquals(2, sn1.getNumContainers());
+ Assert.assertEquals(1 * GB, sn1.getUnallocatedResource().getMemorySize());
+
+ // app asks 5 * 2G container
+ // nm1 reserves 1 * 2G containers
+ am.allocate(Arrays.asList(ResourceRequest
+ .newInstance(Priority.newInstance(0), "*",
+ Resources.createResource(2 * GB), 5)), null);
+ cs.handle(new NodeUpdateSchedulerEvent(sn1.getRMNode()));
+ Assert.assertEquals(1, schedulerApp.getReservedContainers().size());
+
+ // rm kills 1 * 6G container_02
+ for (RMContainer rmContainer : sn1.getCopiedListOfRunningContainers()) {
+ if (rmContainer.getContainerId().getContainerId() != 1) {
+ cs.completedContainer(rmContainer, ContainerStatus
+ .newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL);
+ }
+ }
+ Assert.assertEquals(7 * GB, sn1.getUnallocatedResource().getMemorySize());
+
+ final CapacityScheduler spyCs = Mockito.spy(cs);
+ // handle CapacityScheduler#tryCommit, submit duplicated proposals
+ // that do allocation for reserved container for three times,
+ // to simulate that case in YARN-8127
+ Mockito.doAnswer(new Answer<Object>() {
+ public Boolean answer(InvocationOnMock invocation) throws Exception {
+ ResourceCommitRequest request =
+ (ResourceCommitRequest) invocation.getArguments()[1];
+ if (request.getFirstAllocatedOrReservedContainer()
+ .getAllocateFromReservedContainer() != null) {
+ for (int i=0; i<3; i++) {
+ cs.tryCommit((Resource) invocation.getArguments()[0],
+ (ResourceCommitRequest) invocation.getArguments()[1],
+ (Boolean) invocation.getArguments()[2]);
+ }
+ Assert.assertEquals(2, sn1.getCopiedListOfRunningContainers().size());
+ Assert.assertEquals(5 * GB,
+ sn1.getUnallocatedResource().getMemorySize());
+ }
+ return true;
+ }
+ }).when(spyCs).tryCommit(Mockito.any(Resource.class),
+ Mockito.any(ResourceCommitRequest.class), Mockito.anyBoolean());
+
+ spyCs.handle(new NodeUpdateSchedulerEvent(sn1.getRMNode()));
+
+ rm.stop();
+ }
+
private void keepNMHeartbeat(List<MockNM> mockNMs, int interval) {
if (nmHeartbeatThread != null) {
nmHeartbeatThread.setShouldStop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/44] hadoop git commit: Revert "HDFS-13386. RBF: Wrong date
information in list file(-ls) result. Contributed by Dibyendu Karmakar."
Posted by xy...@apache.org.
Revert "HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar."
This reverts commit 18de6f2042b70f9f0d7a2620c60de022768a7b13.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d272056f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d272056f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d272056f
Branch: refs/heads/HDFS-7240
Commit: d272056fcb23f6a9252b19d349acd718d7837079
Parents: 7ed8511
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Apr 12 08:42:19 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Apr 12 08:42:19 2018 -0700
----------------------------------------------------------------------
.../federation/router/RouterRpcServer.java | 19 +-----
.../federation/router/TestRouterMountTable.java | 62 +-------------------
2 files changed, 2 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d272056f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 0dc2a69..e6d2f5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -2328,24 +2328,7 @@ public class RouterRpcServer extends AbstractService
*/
private Map<String, Long> getMountPointDates(String path) {
Map<String, Long> ret = new TreeMap<>();
- if (subclusterResolver instanceof MountTableResolver) {
- MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
- String srcPath;
- try {
- final List<String> children = subclusterResolver.getMountPoints(path);
- for (String child : children) {
- if (path.equals(Path.SEPARATOR)) {
- srcPath = Path.SEPARATOR + child;
- } else {
- srcPath = path + Path.SEPARATOR + child;
- }
- MountTable entry = mountTable.getMountPoint(srcPath);
- ret.put(child, entry.getDateModified());
- }
- } catch (IOException e) {
- LOG.error("Cannot get mount point: {}", e.getMessage());
- }
- }
+ // TODO add when we have a Mount Table
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d272056f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
index b33b998..c9e28b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -17,33 +17,25 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Collections;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -57,7 +49,6 @@ public class TestRouterMountTable {
private static NamenodeContext nnContext;
private static RouterContext routerContext;
private static MountTableResolver mountTable;
- private static ClientProtocol routerProtocol;
@BeforeClass
public static void globalSetUp() throws Exception {
@@ -78,7 +69,6 @@ public class TestRouterMountTable {
nnContext = cluster.getRandomNamenode();
routerContext = cluster.getRandomRouter();
Router router = routerContext.getRouter();
- routerProtocol = routerContext.getClient().getNamenode();
mountTable = (MountTableResolver) router.getSubclusterResolver();
}
@@ -150,54 +140,4 @@ public class TestRouterMountTable {
return addResponse.getStatus();
}
-
- /**
- * Verify that the file/dir listing contains correct date/time information.
- */
- @Test
- public void testListFilesTime() throws Exception {
- Long beforeCreatingTime = Time.now();
- // Add mount table entry
- MountTable addEntry = MountTable.newInstance(
- "/testdir", Collections.singletonMap("ns0", "/testdir"));
- assertTrue(addMountTable(addEntry));
- addEntry = MountTable.newInstance(
- "/testdir2", Collections.singletonMap("ns0", "/testdir2"));
- assertTrue(addMountTable(addEntry));
- addEntry = MountTable.newInstance(
- "/testdir/subdir", Collections.singletonMap("ns0", "/testdir/subdir"));
- assertTrue(addMountTable(addEntry));
-
- // Create test dir in NN
- final FileSystem nnFs = nnContext.getFileSystem();
- assertTrue(nnFs.mkdirs(new Path("/newdir")));
-
- Map<String, Long> pathModTime = new TreeMap<>();
- for (String mount : mountTable.getMountPoints("/")) {
- pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
- .getDateModified());
- }
- FileStatus[] iterator = nnFs.listStatus(new Path("/"));
- for (FileStatus file : iterator) {
- pathModTime.put(file.getPath().getName(), file.getModificationTime());
- }
- // Fetch listing
- DirectoryListing listing =
- routerProtocol.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
- Iterator<String> pathModTimeIterator = pathModTime.keySet().iterator();
-
- // Match date/time for each path returned
- for(HdfsFileStatus f : listing.getPartialListing()) {
- String fileName = pathModTimeIterator.next();
- String currentFile = f.getFullPath(new Path("/")).getName();
- Long currentTime = f.getModificationTime();
- Long expectedTime = pathModTime.get(currentFile);
-
- assertEquals(currentFile, fileName);
- assertTrue(currentTime > beforeCreatingTime);
- assertEquals(currentTime, expectedTime);
- }
- // Verify the total number of results found/matched
- assertEquals(pathModTime.size(), listing.getPartialListing().length);
- }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/44] hadoop git commit: HDFS-13438. Fix javadoc in
FsVolumeList#removeVolume. Contributed by Shashikant Banerjee.
Posted by xy...@apache.org.
HDFS-13438. Fix javadoc in FsVolumeList#removeVolume. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa8b88ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa8b88ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa8b88ab
Branch: refs/heads/HDFS-7240
Commit: fa8b88ab2b272b29cf116a5de038d78fc4357b9d
Parents: 0725953
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Fri Apr 13 08:56:02 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Fri Apr 13 08:56:02 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa8b88ab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 75baf84..8f52ea7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -335,7 +335,7 @@ class FsVolumeList {
/**
* Dynamically remove volume in the list.
- * @param volume the volume to be removed.
+ * @param storageLocation {@link StorageLocation} of the volume to be removed.
* @param clearFailure set true to remove failure info for this volume.
*/
void removeVolume(StorageLocation storageLocation, boolean clearFailure) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/44] hadoop git commit: HDFS-13418. NetworkTopology should be
configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
Posted by xy...@apache.org.
HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0725953e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0725953e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0725953e
Branch: refs/heads/HDFS-7240
Commit: 0725953efec89b35b7586b846abb01f7c5963b37
Parents: 1a407bc
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Apr 13 17:55:45 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Apr 13 17:55:45 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +++
.../hadoop/hdfs/net/DFSNetworkTopology.java | 10 +++-
.../src/main/resources/hdfs-default.xml | 14 ++++++
.../blockmanagement/TestDatanodeManager.java | 52 ++++++++++++++++++++
4 files changed, 81 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b4b9d97..b4dab4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
@@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.use.dfs.network.topology";
public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true;
+ public static final String DFS_NET_TOPOLOGY_IMPL_KEY =
+ "dfs.net.topology.impl";
+
+ public static final Class<DFSNetworkTopology> DFS_NET_TOPOLOGY_IMPL_DEFAULT =
+ DFSNetworkTopology.class;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index e74cdec..7889ef4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -22,11 +22,13 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.util.ReflectionUtils;
import java.util.ArrayList;
import java.util.Collection;
@@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology {
private static final Random RANDOM = new Random();
public static DFSNetworkTopology getInstance(Configuration conf) {
- DFSNetworkTopology nt = new DFSNetworkTopology();
- return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY);
+
+ DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass(
+ DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY,
+ DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT,
+ DFSNetworkTopology.class), conf);
+ return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index f8cce60..4ed4690 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4859,6 +4859,20 @@
<value>true</value>
<description>
Enables DFSNetworkTopology to choose nodes for placing replicas.
+ When enabled, NetworkTopology will be instantiated as class defined in
+ property dfs.net.topology.impl, otherwise NetworkTopology will be
+ instantiated as class defined in property net.topology.impl.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.net.topology.impl</name>
+ <value>org.apache.hadoop.hdfs.net.DFSNetworkTopology</value>
+ <description>
+ The implementation class of NetworkTopology used in HDFS. By default,
+ the class org.apache.hadoop.hdfs.net.DFSNetworkTopology is specified and
+ used in block placement.
+ This property only works when dfs.use.dfs.network.topology is true.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index 96841cf..f708740 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -41,6 +41,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -61,11 +63,13 @@ import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.net.DNSToSwitchMapping;
+import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.util.Shell;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
+
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.*;
@@ -622,4 +626,52 @@ public class TestDatanodeManager {
// Approximately load tasks if the ratio between queue length is large.
verifyPendingRecoveryTasks(400, 1, 20, 20, 1);
}
+
+ @Test
+ public void testNetworkTopologyInstantiation() throws Exception {
+ // case 1, dfs.use.dfs.network.topology=true, use the default
+ // DFSNetworkTopology impl.
+ Configuration conf1 = new HdfsConfiguration();
+ FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
+ DatanodeManager dm1 = mockDatanodeManager(fsn, conf1);
+ assertEquals(DFSNetworkTopology.class, dm1.getNetworkTopology().getClass());
+
+ // case 2, dfs.use.dfs.network.topology=false, use the default
+ // NetworkTopology impl.
+ Configuration conf2 = new HdfsConfiguration();
+ conf2.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
+ DatanodeManager dm2 = mockDatanodeManager(fsn, conf2);
+ assertEquals(NetworkTopology.class, dm2.getNetworkTopology()
+ .getClass());
+
+ // case 3, dfs.use.dfs.network.topology=false, and specify the
+ // net.topology.impl property.
+ Configuration conf3 = new HdfsConfiguration();
+ conf3.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
+ MockDfsNetworkTopology.class, NetworkTopology.class);
+ conf3.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
+ DatanodeManager dm3 = mockDatanodeManager(fsn, conf3);
+ assertEquals(MockDfsNetworkTopology.class, dm3.getNetworkTopology()
+ .getClass());
+
+ // case 4, dfs.use.dfs.network.topology=true, and specify the
+ // dfs.net.topology.impl property.
+ Configuration conf4 = new HdfsConfiguration();
+ conf4.setClass(DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY,
+ MockDfsNetworkTopology.class, NetworkTopology.class);
+ conf4.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true);
+ DatanodeManager dm4 = mockDatanodeManager(fsn, conf4);
+ assertEquals(MockDfsNetworkTopology.class, dm4.getNetworkTopology()
+ .getClass());
+ }
+
+ /**
+ * A NetworkTopology implementation for test.
+ *
+ */
+ public static class MockDfsNetworkTopology extends DFSNetworkTopology {
+ public MockDfsNetworkTopology(){
+ super();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/44] hadoop git commit: MAPREDUCE-7062. Update mapreduce.job.tags
description for making use for ATSv2 purpose. Contributed by Charan Hebri.
Posted by xy...@apache.org.
MAPREDUCE-7062. Update mapreduce.job.tags description for making use for ATSv2 purpose. Contributed by Charan Hebri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cc59a09
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cc59a09
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cc59a09
Branch: refs/heads/HDFS-7240
Commit: 6cc59a09e7330dc893b386d84c8f2aa86c02eace
Parents: c048711
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Apr 11 09:45:39 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Apr 11 09:45:39 2018 +0530
----------------------------------------------------------------------
.../src/main/resources/mapred-default.xml | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc59a09/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index cf8be33..d47c0ff 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1045,6 +1045,12 @@
<value></value>
<description> Tags for the job that will be passed to YARN at submission
time. Queries to YARN for applications can filter on these tags.
+ If these tags are intended to be used with The YARN Timeline Service v.2,
+ prefix them with the appropriate tag names for flow name, flow version and
+ flow run id. Example:
+ timeline_flow_name_tag:foo,
+ timeline_flow_version_tag:3df8b0d6100530080d2e0decf9e528e57c42a90a,
+ timeline_flow_run_id_tag:1465246348599
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/44] hadoop git commit: MAPREDUCE-7069. Add ability to specify
user environment variables individually. Contributed by Jim Brennan
Posted by xy...@apache.org.
MAPREDUCE-7069. Add ability to specify user environment variables individually. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4571351c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4571351c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4571351c
Branch: refs/heads/HDFS-7240
Commit: 4571351cccf6d4977469d3d623cf045b06a5f5f0
Parents: 6bb128d
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Apr 12 11:04:22 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Apr 12 11:12:46 2018 -0500
----------------------------------------------------------------------
.../apache/hadoop/mapred/MapReduceChildJVM.java | 73 +++++-----
.../v2/app/job/impl/TaskAttemptImpl.java | 8 +-
.../v2/app/job/impl/TestMapReduceChildJVM.java | 24 +++-
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 10 ++
.../java/org/apache/hadoop/mapred/JobConf.java | 18 +++
.../src/main/resources/mapred-default.xml | 61 +++++++--
.../src/site/markdown/MapReduceTutorial.md | 6 +
.../org/apache/hadoop/mapred/YARNRunner.java | 11 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 26 +++-
.../java/org/apache/hadoop/yarn/util/Apps.java | 115 +++++++++++++---
.../org/apache/hadoop/yarn/util/TestApps.java | 136 +++++++++++++++++++
11 files changed, 407 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
index 936dc5a..d305f9f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.mapred;
import java.net.InetSocketAddress;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
@@ -28,7 +27,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskLog.LogName;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -42,50 +40,53 @@ public class MapReduceChildJVM {
filter.toString();
}
- private static String getChildEnv(JobConf jobConf, boolean isMap) {
+ private static String getChildEnvProp(JobConf jobConf, boolean isMap) {
if (isMap) {
- return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV,
- jobConf.get(JobConf.MAPRED_TASK_ENV));
+ return JobConf.MAPRED_MAP_TASK_ENV;
}
- return jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV,
- jobConf.get(JobConf.MAPRED_TASK_ENV));
+ return JobConf.MAPRED_REDUCE_TASK_ENV;
+ }
+
+ private static String getChildEnvDefaultValue(JobConf jobConf) {
+ // There is no default value for these - use the fallback value instead.
+ return jobConf.get(JobConf.MAPRED_TASK_ENV);
}
public static void setVMEnv(Map<String, String> environment,
Task task) {
JobConf conf = task.conf;
- // Add the env variables passed by the user
- String mapredChildEnv = getChildEnv(conf, task.isMapTask());
- MRApps.setEnvFromInputString(environment, mapredChildEnv, conf);
-
- // Set logging level in the environment.
- // This is so that, if the child forks another "bin/hadoop" (common in
- // streaming) it will have the correct loglevel.
- environment.put(
- "HADOOP_ROOT_LOGGER",
- MRApps.getChildLogLevel(conf, task.isMapTask()) + ",console");
-
- // TODO: The following is useful for instance in streaming tasks. Should be
- // set in ApplicationMaster's env by the RM.
- String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS");
- if (hadoopClientOpts == null) {
- hadoopClientOpts = "";
- } else {
- hadoopClientOpts = hadoopClientOpts + " ";
+ boolean isMap = task.isMapTask();
+
+ // Remove these before adding the user variables to prevent
+ // MRApps.setEnvFromInputProperty() from appending to them.
+ String hadoopRootLoggerKey = "HADOOP_ROOT_LOGGER";
+ String hadoopClientOptsKey = "HADOOP_CLIENT_OPTS";
+ environment.remove(hadoopRootLoggerKey);
+ environment.remove(hadoopClientOptsKey);
+
+ // Add the environment variables passed by the user
+ MRApps.setEnvFromInputProperty(environment, getChildEnvProp(conf, isMap),
+ getChildEnvDefaultValue(conf), conf);
+
+ // Set HADOOP_ROOT_LOGGER and HADOOP_CLIENTS if the user did not set them.
+ if (!environment.containsKey(hadoopRootLoggerKey)) {
+ // Set the value for logging level in the environment.
+ // This is so that, if the child forks another "bin/hadoop" (common in
+ // streaming) it will have the correct loglevel.
+ environment.put(hadoopRootLoggerKey,
+ MRApps.getChildLogLevel(conf, task.isMapTask()) + ",console");
}
- environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
-
- // setEnvFromInputString above will add env variable values from
- // mapredChildEnv to existing variables. We want to overwrite
- // HADOOP_ROOT_LOGGER and HADOOP_CLIENT_OPTS if the user set it explicitly.
- Map<String, String> tmpEnv = new HashMap<String, String>();
- MRApps.setEnvFromInputString(tmpEnv, mapredChildEnv, conf);
- String[] keys = { "HADOOP_ROOT_LOGGER", "HADOOP_CLIENT_OPTS" };
- for (String key : keys) {
- if (tmpEnv.containsKey(key)) {
- environment.put(key, tmpEnv.get(key));
+ if (!environment.containsKey(hadoopClientOptsKey)) {
+ // TODO: The following is useful for instance in streaming tasks.
+ // Should be set in ApplicationMaster's env by the RM.
+ String hadoopClientOptsValue = System.getenv(hadoopClientOptsKey);
+ if (hadoopClientOptsValue == null) {
+ hadoopClientOptsValue = "";
+ } else {
+ hadoopClientOptsValue = hadoopClientOptsValue + " ";
}
+ environment.put(hadoopClientOptsKey, hadoopClientOptsValue);
}
// Add stdout/stderr env
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 6632f27..0c830f8 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -953,10 +953,10 @@ public abstract class TaskAttemptImpl implements
MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
// Add the env variables passed by the admin
- MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MAPRED_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
- conf);
+ MRApps.setEnvFromInputProperty(environment,
+ MRJobConfig.MAPRED_ADMIN_USER_ENV,
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV, conf);
+
return environment;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
index 4a5d480..f00ff28 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
@@ -289,11 +289,11 @@ public class TestMapReduceChildJVM {
app.verifyCompleted();
Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job",
- app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
+ app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
Assert.assertEquals("WARN,console",
- app.cmdEnvironment.get("HADOOP_ROOT_LOGGER"));
+ app.cmdEnvironment.get("HADOOP_ROOT_LOGGER"));
Assert.assertTrue("HADOOP_CLIENT_OPTS not set for job",
- app.cmdEnvironment.containsKey("HADOOP_CLIENT_OPTS"));
+ app.cmdEnvironment.containsKey("HADOOP_CLIENT_OPTS"));
Assert.assertEquals("test", app.cmdEnvironment.get("HADOOP_CLIENT_OPTS"));
// Try one more.
@@ -305,8 +305,22 @@ public class TestMapReduceChildJVM {
app.verifyCompleted();
Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job",
- app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
+ app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
Assert.assertEquals("trace",
- app.cmdEnvironment.get("HADOOP_ROOT_LOGGER"));
+ app.cmdEnvironment.get("HADOOP_ROOT_LOGGER"));
+
+ // Try one using the mapreduce.task.env.var=value syntax
+ app = new MyMRApp(1, 0, true, this.getClass().getName(), true);
+ conf = new Configuration();
+ conf.set(JobConf.MAPRED_MAP_TASK_ENV + ".HADOOP_ROOT_LOGGER",
+ "DEBUG,console");
+ job = app.submit(conf);
+ app.waitForState(job, JobState.SUCCEEDED);
+ app.verifyCompleted();
+
+ Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job",
+ app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
+ Assert.assertEquals("DEBUG,console",
+ app.cmdEnvironment.get("HADOOP_ROOT_LOGGER"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index d435a1f..1b087a7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -708,6 +708,16 @@ public class MRApps extends Apps {
Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
+ public static void setEnvFromInputProperty(Map<String, String> env,
+ String propName, String defaultPropValue, Configuration conf) {
+ String classPathSeparator =
+ conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
+ MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
+ ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
+ Apps.setEnvFromInputProperty(env, propName, defaultPropValue, conf,
+ classPathSeparator);
+ }
+
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index 8c57d1b..c0dd650 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -312,6 +312,15 @@ public class JobConf extends Configuration {
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
* </ul>
+ *
+ * You can also add environment variables individually by appending
+ * <code>.VARNAME</code> to this configuration key, where VARNAME is
+ * the name of the environment variable.
+ *
+ * Example:
+ * <ul>
+ * <li>mapreduce.map.env.VARNAME=value</li>
+ * </ul>
*/
public static final String MAPRED_MAP_TASK_ENV = JobContext.MAP_ENV;
@@ -326,6 +335,15 @@ public class JobConf extends Configuration {
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
* </ul>
+ *
+ * You can also add environment variables individually by appending
+ * <code>.VARNAME</code> to this configuration key, where VARNAME is
+ * the name of the environment variable.
+ *
+ * Example:
+ * <ul>
+ * <li>mapreduce.reduce.env.VARNAME=value</li>
+ * </ul>
*/
public static final String MAPRED_REDUCE_TASK_ENV = JobContext.REDUCE_ENV;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index d47c0ff..dcb312c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -350,11 +350,22 @@
<property>
<name>mapred.child.env</name>
<value></value>
- <description>User added environment variables for the task processes.
+ <description>User added environment variables for the task processes,
+ specified as a comma separated list.
Example :
1) A=foo This will set the env variable A to foo
2) B=$B:c This is inherit nodemanager's B env variable on Unix.
3) B=%B%;c This is inherit nodemanager's B env variable on Windows.
+
+ To specify a comma separated list of environment variables specifically for
+ map or reduce tasks, use the mapreduce.map.env or mapreduce.reduce.env
+ properties.
+
+ To define environment variables individually for map or reduce tasks,
+ you can specify multiple properties of the form mapreduce.map.env.VARNAME
+ or mapreduce.reduce.env.VARNAME, where VARNAME is the name of the
+ environment variable. This is the only way to add a variable when its value
+ contains commas.
</description>
</property>
@@ -362,7 +373,15 @@
<property>
<name>mapreduce.map.env</name>
<value></value>
- <description>User added environment variables for the map task processes.
+ <description>User added environment variables for the map task processes,
+ specified as a comma separated list.
+ Example:
+ VAR1=value1,VAR2=value2
+
+ To define environment variables individually, you can specify
+ multiple properties of the form mapreduce.map.env.VARNAME,
+ where VARNAME is the name of the environment variable. This is the only
+ way to add a variable when its value contains commas.
</description>
</property>
-->
@@ -371,7 +390,16 @@
<property>
<name>mapreduce.reduce.env</name>
<value></value>
- <description>User added environment variables for the reduce task processes.
+ <description>User added environment variables for the reduce task processes,
+ specified as a comma separated list.
+ Example:
+ VAR1=value1,VAR2=value2
+
+ To define environment variables individually, you can specify
+ multiple properties of the form mapreduce.reduce.env.VARNAME,
+ where VARNAME is the name of the environment variable. This is the only
+ way to add a variable when its value contains commas.
+ contains commas.
</description>
</property>
-->
@@ -385,9 +413,14 @@
You must preserve the original value if you want your map and
reduce tasks to have access to native libraries (compression, etc).
When this value is empty, the command to set execution
- envrionment will be OS dependent:
+ environment will be OS dependent:
For linux, use LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native.
For windows, use PATH = %PATH%;%HADOOP_COMMON_HOME%\\bin.
+
+ To define environment variables individually, you can specify
+ multiple properties of the form mapreduce.admin.user.env.VARNAME,
+ where VARNAME is the name of the environment variable. This is the only
+ way to add a variable when its value contains commas.
</description>
</property>
@@ -1365,20 +1398,32 @@
<name>yarn.app.mapreduce.am.env</name>
<value></value>
<description>User added environment variables for the MR App Master
- processes. Example :
+ processes, specified as a comma separated list.
+ Example :
1) A=foo This will set the env variable A to foo
2) B=$B:c This is inherit tasktracker's B env variable.
+
+ To define environment variables individually, you can specify
+ multiple properties of the form yarn.app.mapreduce.am.env.VARNAME,
+ where VARNAME is the name of the environment variable. This is the only
+ way to add a variable when its value contains commas.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.admin.user.env</name>
<value></value>
- <description> Environment variables for the MR App Master
- processes for admin purposes. These values are set first and can be
- overridden by the user env (yarn.app.mapreduce.am.env) Example :
+ <description>Environment variables for the MR App Master
+ processes for admin purposes, specified as a comma separated list
+ These values are set first and can be overridden by the user env
+ (yarn.app.mapreduce.am.env). Example :
1) A=foo This will set the env variable A to foo
2) B=$B:c This is inherit app master's B env variable.
+
+ To define environment variables individually, you can specify
+ multiple properties of the form yarn.app.mapreduce.am.admin.user.env.VARNAME,
+ where VARNAME is the name of the environment variable. This is the only
+ way to add a variable when its value contains commas.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
index 574c404..33ede3a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
@@ -206,6 +206,12 @@ For example,
Here, the files dir1/dict.txt and dir2/dict.txt can be accessed by tasks using the symbolic names dict1 and dict2 respectively. The archive mytar.tgz will be placed and unarchived into a directory by the name "tgzdir".
+Applications can specify environment variables for mapper, reducer, and application master tasks by specifying them on the command line using the options -Dmapreduce.map.env, -Dmapreduce.reduce.env, and -Dyarn.app.mapreduce.am.env, respectively.
+
+For example the following sets environment variables FOO_VAR=bar and LIST_VAR=a,b,c for the mappers and reducers,
+
+ bin/hadoop jar hadoop-mapreduce-examples-<ver>.jar wordcount -Dmapreduce.map.env.FOO_VAR=bar -Dmapreduce.map.env.LIST_VAR=a,b,c -Dmapreduce.reduce.env.FOO_VAR=bar -Dmapreduce.reduce.env.LIST_VAR=a,b,c input output
+
### Walk-through
The `WordCount` application is quite straight-forward.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 127e1dc..496ff10 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -533,12 +533,13 @@ public class YARNRunner implements ClientProtocol {
MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
// Setup the environment variables for Admin first
- MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf);
+ MRApps.setEnvFromInputProperty(environment,
+ MRJobConfig.MR_AM_ADMIN_USER_ENV,
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV,
+ conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
- MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MR_AM_ENV), conf);
+ MRApps.setEnvFromInputProperty(environment, MRJobConfig.MR_AM_ENV, null,
+ conf);
// Parse distributed cache
MRApps.setupDistributedCache(jobConf, localResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 10d4c21..932e808 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -811,15 +811,22 @@ public class TestYARNRunner {
@Test
public void testAMStandardEnvWithDefaultLibPath() throws Exception {
- testAMStandardEnv(false);
+ testAMStandardEnv(false, false);
}
@Test
public void testAMStandardEnvWithCustomLibPath() throws Exception {
- testAMStandardEnv(true);
+ testAMStandardEnv(true, false);
}
- private void testAMStandardEnv(boolean customLibPath) throws Exception {
+ @Test
+ public void testAMStandardEnvWithCustomLibPathWithSeparateEnvProps()
+ throws Exception {
+ testAMStandardEnv(true, true);
+ }
+
+ private void testAMStandardEnv(boolean customLibPath,
+ boolean useSeparateEnvProps) throws Exception {
// the Windows behavior is different and this test currently doesn't really
// apply
// MAPREDUCE-6588 should revisit this test
@@ -832,9 +839,16 @@ public class TestYARNRunner {
String pathKey = Environment.LD_LIBRARY_PATH.name();
if (customLibPath) {
- jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" +
- ADMIN_LIB_PATH);
- jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH);
+ if (useSeparateEnvProps) {
+ // Specify these as individual variables instead of k=v lists
+ jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV + "." + pathKey,
+ ADMIN_LIB_PATH);
+ jobConf.set(MRJobConfig.MR_AM_ENV + "." + pathKey, USER_LIB_PATH);
+ } else {
+ jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" +
+ ADMIN_LIB_PATH);
+ jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH);
+ }
}
jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 1c90d55..7a02874 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import java.io.File;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.regex.Matcher;
@@ -32,6 +33,7 @@ import java.util.regex.Pattern;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -80,34 +82,113 @@ public class Apps {
throw new YarnRuntimeException(join("Error parsing ", name, ": ", s));
}
+ private static void setEnvFromString(Map<String, String> env,
+ String envVar, String varString, String classPathSeparator) {
+ Matcher m = VAR_SUBBER.matcher(varString);
+ StringBuffer sb = new StringBuffer();
+ while (m.find()) {
+ String var = m.group(1);
+ // do variable substitution of $var from passed in environment or from
+ // system environment and default to empty string if undefined in both.
+ String replace = env.get(var);
+ if (replace == null) {
+ replace = System.getenv(var);
+ }
+ if (replace == null) {
+ replace = "";
+ }
+ m.appendReplacement(sb, Matcher.quoteReplacement(replace));
+ }
+ m.appendTail(sb);
+ addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
+ }
+
public static void setEnvFromInputString(Map<String, String> env,
String envString, String classPathSeparator) {
if (envString != null && envString.length() > 0) {
Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
while (varValMatcher.find()) {
String envVar = varValMatcher.group(1);
- Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2));
- StringBuffer sb = new StringBuffer();
- while (m.find()) {
- String var = m.group(1);
- // replace $env with the child's env constructed by tt's
- String replace = env.get(var);
- // if this key is not configured by the tt for the child .. get it
- // from the tt's env
- if (replace == null)
- replace = System.getenv(var);
- // the env key is note present anywhere .. simply set it
- if (replace == null)
- replace = "";
- m.appendReplacement(sb, Matcher.quoteReplacement(replace));
- }
- m.appendTail(sb);
- addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
+ String varString = varValMatcher.group(2);
+ setEnvFromString(env, envVar, varString, classPathSeparator);
+ }
+ }
+ }
+
+ /**
+ * Set environment from string without doing any variable substitution.
+ * Used internally to avoid double expansion.
+ * @param env environment to set
+ * @param envString comma-separated k=v pairs.
+ * @param classPathSeparator Separator to use when appending to an existing
+ * environment variable.
+ */
+ private static void setEnvFromInputStringNoExpand(Map<String, String> env,
+ String envString, String classPathSeparator) {
+ if (envString != null && envString.length() > 0) {
+ Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
+ while (varValMatcher.find()) {
+ String envVar = varValMatcher.group(1);
+ String varString = varValMatcher.group(2);
+ addToEnvironment(env, envVar, varString, classPathSeparator);
}
}
}
/**
+ * Set environment variables from map of input properties.
+ * @param env environment to update
+ * @param inputMap environment variable property keys and values
+ * @param classPathSeparator separator to use when appending to an existing
+ * environment variable
+ */
+ private static void setEnvFromInputStringMap(Map<String, String> env,
+ Map<String, String> inputMap, String classPathSeparator) {
+ for(Map.Entry<String, String> inputVar: inputMap.entrySet()) {
+ String envVar = inputVar.getKey();
+ String varString = inputVar.getValue();
+ setEnvFromString(env, envVar, varString, classPathSeparator);
+ }
+ }
+
+ /**
+ * Set environment variables from the given environment input property.
+ * For example, given the property mapreduce.map.env, this method
+ * will extract environment variables from:
+ * the comma-separated string value of mapreduce.map.env, and
+ * the values of any properties of the form mapreduce.map.env.VAR_NAME
+ * Variables specified via the latter syntax take precedence over those
+ * specified using the former syntax.
+ * @param env the environment to update
+ * @param propName the name of the property
+ * @param defaultPropValue the default value for propName
+ * @param conf configuration containing properties
+ * @param classPathSeparator Separator used when appending to an existing var
+ */
+ public static void setEnvFromInputProperty(Map<String, String> env,
+ String propName, String defaultPropValue, Configuration conf,
+ String classPathSeparator) {
+
+ String envString = conf.get(propName, defaultPropValue);
+
+ // Get k,v pairs from string into a tmp env. Note that we don't want
+ // to expand the env var values, because we will do that below -
+ // don't want to do it twice.
+ Map<String, String> tmpEnv = new HashMap<String, String>();
+ Apps.setEnvFromInputStringNoExpand(tmpEnv, envString, classPathSeparator);
+
+ // Get map of props with prefix propName.
+ // (e.g., map.reduce.env.ENV_VAR_NAME=value)
+ Map<String, String> inputMap = conf.getPropsWithPrefix(propName + ".");
+
+ // Entries from map should override entries from input string.
+ tmpEnv.putAll(inputMap);
+
+ // Add them to the environment
+ setEnvFromInputStringMap(env, tmpEnv, classPathSeparator);
+ }
+
+ /**
*
* @param envString String containing env variable definitions
* @param classPathSeparator String that separates the definitions
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
index 66d2d23..0d65d65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.util;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
import org.junit.Test;
@@ -25,6 +26,8 @@ import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
public class TestApps {
@Test
@@ -58,4 +61,137 @@ public class TestApps {
assertEquals("=", environment.get("e1"));
assertEquals("a1=a2", environment.get("e2"));
}
+
+ @Test
+ public void testSetEnvFromInputProperty() {
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<>();
+ String propName = "mapreduce.map.env";
+ String defaultPropName = "mapreduce.child.env";
+ // Setup environment input properties
+ conf.set(propName, "env1=env1_val,env2=env2_val,env3=env3_val");
+ conf.set(propName + ".env4", "env4_val");
+ conf.set(propName + ".env2", "new_env2_val");
+ // Setup some default values - we shouldn't see these values
+ conf.set(defaultPropName, "env1=def1_val,env2=def2_val,env3=def3_val");
+ String defaultPropValue = conf.get(defaultPropName);
+ // These should never be referenced.
+ conf.set(defaultPropName + ".env4", "def4_val");
+ conf.set(defaultPropName + ".env2", "new_def2_val");
+ Apps.setEnvFromInputProperty(env, propName, defaultPropValue, conf,
+ File.pathSeparator);
+ // Check values from string
+ assertEquals("env1_val", env.get("env1"));
+ assertEquals("env3_val", env.get("env3"));
+ // Check individual value
+ assertEquals("env4_val", env.get("env4"));
+ // Check individual value that eclipses one in string
+ assertEquals("new_env2_val", env.get("env2"));
+ }
+
+ @Test
+ public void testSetEnvFromInputPropertyDefault() {
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<>();
+ String propName = "mapreduce.map.env";
+ String defaultPropName = "mapreduce.child.env";
+ // Setup environment input properties
+ conf.set(propName, "env1=env1_val,env2=env2_val,env3=env3_val");
+ conf.set(propName + ".env4", "env4_val");
+ conf.set(propName + ".env2", "new_env2_val");
+ // Setup some default values
+ conf.set(defaultPropName, "env1=def1_val,env2=def2_val,env3=def3_val");
+ String defaultPropValue = conf.get(defaultPropName);
+ // These should never be referenced.
+ conf.set(defaultPropName + ".env4", "def4_val");
+ conf.set(defaultPropName + ".env2", "new_def2_val");
+
+ // Test using default value for the string.
+ // Individually specified env properties do not have defaults,
+ // so this should just get things from the defaultPropName string.
+ String bogusProp = propName + "bogus";
+ Apps.setEnvFromInputProperty(env, bogusProp, defaultPropValue, conf,
+ File.pathSeparator);
+ // Check values from string
+ assertEquals("def1_val", env.get("env1"));
+ assertEquals("def2_val", env.get("env2"));
+ assertEquals("def3_val", env.get("env3"));
+ // Check individual value is not set.
+ assertNull(env.get("env4"));
+ }
+
+ @Test
+ public void testSetEnvFromInputPropertyOverrideDefault() {
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<>();
+
+ // Try using default value, but specify some individual values using
+ // the main propName, but no main value, so it should get values from
+ // the default string, and then the individual values.
+ String propName = "mapreduce.reduce.env";
+ conf.set(propName + ".env2", "new2_val");
+ conf.set(propName + ".env4", "new4_val");
+ // Setup some default values - we shouldn't see these values
+ String defaultPropName = "mapreduce.child.env";
+ conf.set(defaultPropName, "env1=def1_val,env2=def2_val,env3=def3_val");
+ String defaultPropValue = conf.get(defaultPropName);
+ // These should never be referenced.
+ conf.set(defaultPropName + ".env4", "def4_val");
+ conf.set(defaultPropName + ".env2", "new_def2_val");
+ Apps.setEnvFromInputProperty(env, propName, defaultPropValue, conf,
+ File.pathSeparator);
+
+ // Check values from string
+ assertEquals("def1_val", env.get("env1"));
+ assertEquals("def3_val", env.get("env3"));
+ // Check individual value
+ assertEquals("new4_val", env.get("env4"));
+ // Check individual value that eclipses one in string
+ assertEquals("new2_val", env.get("env2"));
+ }
+
+ @Test
+ public void testSetEnvFromInputPropertyCommas() {
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<>();
+ String propName = "mapreduce.reduce.env";
+ conf.set(propName, "env1=env1_val,env2=env2_val,env3=env3_val");
+ conf.set(propName + ".env2", "new2_val1,new2_val2,new2_val3");
+ conf.set(propName + ".env4", "new4_valwith=equals");
+ // Setup some default values - we shouldn't see these values
+ String defaultPropName = "mapreduce.child.env";
+ conf.set(defaultPropName, "env1=def1_val,env2=def2_val,env3=def3_val");
+ String defaultPropValue = conf.get(defaultPropName);
+
+ Apps.setEnvFromInputProperty(env, propName, defaultPropValue, conf,
+ File.pathSeparator);
+ // Check values from string
+ assertEquals("env1_val", env.get("env1"));
+ assertEquals("env3_val", env.get("env3"));
+ // Check individual value
+ assertEquals("new4_valwith=equals", env.get("env4"));
+ // Check individual value that eclipses one in string
+ assertEquals("new2_val1,new2_val2,new2_val3", env.get("env2"));
+ }
+
+ @Test
+ public void testSetEnvFromInputPropertyNull() {
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<>();
+ String propName = "mapreduce.map.env";
+ String defaultPropName = "mapreduce.child.env";
+ // Setup environment input properties
+ conf.set(propName, "env1=env1_val,env2=env2_val,env3=env3_val");
+ conf.set(propName + ".env4", "env4_val");
+ conf.set(propName + ".env2", "new_env2_val");
+ // Setup some default values - we shouldn't see these values
+ conf.set(defaultPropName, "env1=def1_val,env2=def2_val,env3=def3_val");
+ String defaultPropValue = conf.get(defaultPropName);
+ // These should never be referenced.
+ conf.set(defaultPropName + ".env4", "def4_val");
+ conf.set(defaultPropName + ".env2", "new_def2_val");
+ // Try with null inputs
+ Apps.setEnvFromInputProperty(env, "bogus1", null, conf, File.pathSeparator);
+ assertTrue(env.isEmpty());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/44] hadoop git commit: HADOOP-14445. Delegation tokens are not
shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 1517b04..c171143 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -31,26 +32,35 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
+import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer;
import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider;
import org.apache.hadoop.crypto.key.kms.ValueQueue;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.KMSUtilFaultInjector;
import org.apache.hadoop.util.Time;
import org.apache.http.client.utils.URIBuilder;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
@@ -71,7 +81,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -96,6 +105,10 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
+import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -113,6 +126,20 @@ public class TestKMS {
private SSLFactory sslFactory;
+ private final KMSUtilFaultInjector oldInjector =
+ KMSUtilFaultInjector.get();
+
+ // Injector to create providers with different ports. Can only happen in tests
+ private final KMSUtilFaultInjector testInjector =
+ new KMSUtilFaultInjector() {
+ @Override
+ public KeyProvider createKeyProviderForTests(String value,
+ Configuration conf) throws IOException {
+ return TestLoadBalancingKMSClientProvider
+ .createKeyProviderForTests(value, conf);
+ }
+ };
+
// Keep track of all key providers created during a test case, so they can be
// closed at test tearDown.
private List<KeyProvider> providersCreated = new LinkedList<>();
@@ -122,7 +149,12 @@ public class TestKMS {
@Before
public void setUp() throws Exception {
- setUpMiniKdc();
+ GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE);
+ GenericTestUtils
+ .setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE);
+ GenericTestUtils
+ .setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE);
// resetting kerberos security
Configuration conf = new Configuration();
UserGroupInformation.setConfiguration(conf);
@@ -141,24 +173,78 @@ public class TestKMS {
}
public static abstract class KMSCallable<T> implements Callable<T> {
- private URL kmsUrl;
+ private List<URL> kmsUrl;
protected URL getKMSUrl() {
- return kmsUrl;
+ return kmsUrl.get(0);
+ }
+
+ protected URL[] getKMSHAUrl() {
+ URL[] urls = new URL[kmsUrl.size()];
+ return kmsUrl.toArray(urls);
+ }
+
+ protected void addKMSUrl(URL url) {
+ if (kmsUrl == null) {
+ kmsUrl = new ArrayList<URL>();
+ }
+ kmsUrl.add(url);
+ }
+
+ /*
+ * The format of the returned value will be
+ * kms://http:kms1.example1.com:port1,kms://http:kms2.example2.com:port2
+ */
+ protected String generateLoadBalancingKeyProviderUriString() {
+ if (kmsUrl == null || kmsUrl.size() == 0) {
+ return null;
+ }
+ StringBuffer sb = new StringBuffer();
+
+ for (int i = 0; i < kmsUrl.size(); i++) {
+ sb.append(KMSClientProvider.SCHEME_NAME + "://" +
+ kmsUrl.get(0).getProtocol() + "@");
+ URL url = kmsUrl.get(i);
+ sb.append(url.getAuthority());
+ if (url.getPath() != null) {
+ sb.append(url.getPath());
+ }
+ if (i < kmsUrl.size() - 1) {
+ sb.append(",");
+ }
+ }
+ return sb.toString();
}
}
protected KeyProvider createProvider(URI uri, Configuration conf)
throws IOException {
final KeyProvider ret = new LoadBalancingKMSClientProvider(
- new KMSClientProvider[] {new KMSClientProvider(uri, conf)}, conf);
+ new KMSClientProvider[] {new KMSClientProvider(uri, conf, uri)}, conf);
providersCreated.add(ret);
return ret;
}
+ /**
+ * create a LoadBalancingKMSClientProvider from an array of URIs.
+ * @param uris an array of KMS URIs
+ * @param conf configuration object
+ * @return a LoadBalancingKMSClientProvider object
+ * @throws IOException
+ */
+ protected LoadBalancingKMSClientProvider createHAProvider(URI[] uris,
+ Configuration conf, String originalUri) throws IOException {
+ KMSClientProvider[] providers = new KMSClientProvider[uris.length];
+ for (int i = 0; i < providers.length; i++) {
+ providers[i] =
+ new KMSClientProvider(uris[i], conf, URI.create(originalUri));
+ }
+ return new LoadBalancingKMSClientProvider(providers, conf);
+ }
+
private KMSClientProvider createKMSClientProvider(URI uri, Configuration conf)
throws IOException {
- final KMSClientProvider ret = new KMSClientProvider(uri, conf);
+ final KMSClientProvider ret = new KMSClientProvider(uri, conf, uri);
providersCreated.add(ret);
return ret;
}
@@ -170,22 +256,33 @@ public class TestKMS {
protected <T> T runServer(int port, String keystore, String password, File confDir,
KMSCallable<T> callable) throws Exception {
+ return runServer(new int[] {port}, keystore, password, confDir, callable);
+ }
+
+ protected <T> T runServer(int[] ports, String keystore, String password,
+ File confDir, KMSCallable<T> callable) throws Exception {
MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder().setKmsConfDir(confDir)
.setLog4jConfFile("log4j.properties");
if (keystore != null) {
miniKMSBuilder.setSslConf(new File(keystore), password);
}
- if (port > 0) {
- miniKMSBuilder.setPort(port);
+ final List<MiniKMS> kmsList = new ArrayList<>();
+ for (int i=0; i< ports.length; i++) {
+ if (ports[i] > 0) {
+ miniKMSBuilder.setPort(ports[i]);
+ }
+ MiniKMS miniKMS = miniKMSBuilder.build();
+ kmsList.add(miniKMS);
+ miniKMS.start();
+ LOG.info("Test KMS running at: " + miniKMS.getKMSUrl());
+ callable.addKMSUrl(miniKMS.getKMSUrl());
}
- MiniKMS miniKMS = miniKMSBuilder.build();
- miniKMS.start();
try {
- System.out.println("Test KMS running at: " + miniKMS.getKMSUrl());
- callable.kmsUrl = miniKMS.getKMSUrl();
return callable.call();
} finally {
- miniKMS.stop();
+ for (MiniKMS miniKMS: kmsList) {
+ miniKMS.stop();
+ }
}
}
@@ -240,6 +337,13 @@ public class TestKMS {
return new URI("kms://" + str);
}
+ public static URI[] createKMSHAUri(URL[] kmsUrls) throws Exception {
+ URI[] uris = new URI[kmsUrls.length];
+ for (int i = 0; i < kmsUrls.length; i++) {
+ uris[i] = createKMSUri(kmsUrls[i]);
+ }
+ return uris;
+ }
private static class KerberosConfiguration
extends javax.security.auth.login.Configuration {
@@ -315,19 +419,17 @@ public class TestKMS {
principals.toArray(new String[principals.size()]));
}
- private void setUpMiniKdc() throws Exception {
+ @BeforeClass
+ public static void setUpMiniKdc() throws Exception {
Properties kdcConf = MiniKdc.createConf();
setUpMiniKdc(kdcConf);
}
@After
public void tearDown() throws Exception {
- if (kdc != null) {
- kdc.stop();
- kdc = null;
- }
UserGroupInformation.setShouldRenewImmediatelyForTests(false);
UserGroupInformation.reset();
+ KMSUtilFaultInjector.set(oldInjector);
if (!providersCreated.isEmpty()) {
final MultipleIOException.Builder b = new MultipleIOException.Builder();
for (KeyProvider kp : providersCreated) {
@@ -345,6 +447,14 @@ public class TestKMS {
}
}
+ @AfterClass
+ public static void shutdownMiniKdc() {
+ if (kdc != null) {
+ kdc.stop();
+ kdc = null;
+ }
+ }
+
private <T> T doAs(String user, final PrivilegedExceptionAction<T> action)
throws Exception {
UserGroupInformation.loginUserFromKeytab(user, keytab.getAbsolutePath());
@@ -501,8 +611,10 @@ public class TestKMS {
Token<?>[] tokens =
((KeyProviderDelegationTokenExtension.DelegationTokenExtension)kp)
.addDelegationTokens("myuser", new Credentials());
- Assert.assertEquals(1, tokens.length);
- Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
+ assertEquals(2, tokens.length);
+ assertEquals(KMSDelegationToken.TOKEN_KIND,
+ tokens[0].getKind());
+ kp.close();
return null;
}
});
@@ -518,8 +630,9 @@ public class TestKMS {
Token<?>[] tokens =
((KeyProviderDelegationTokenExtension.DelegationTokenExtension)kp)
.addDelegationTokens("myuser", new Credentials());
- Assert.assertEquals(1, tokens.length);
- Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
+ assertEquals(2, tokens.length);
+ assertEquals(KMSDelegationToken.TOKEN_KIND, tokens[0].getKind());
+ kp.close();
}
return null;
}
@@ -2011,7 +2124,6 @@ public class TestKMS {
return null;
}
});
-
nonKerberosUgi.addCredentials(credentials);
try {
@@ -2067,6 +2179,17 @@ public class TestKMS {
testDelegationTokensOps(true, true);
}
+ private Text getTokenService(KeyProvider provider) {
+ assertTrue("KeyProvider should be an instance of KMSClientProvider",
+ (provider instanceof LoadBalancingKMSClientProvider));
+ assertEquals("Num client providers should be 1", 1,
+ ((LoadBalancingKMSClientProvider)provider).getProviders().length);
+ Text tokenService =
+ (((LoadBalancingKMSClientProvider)provider).getProviders()[0])
+ .getDelegationTokenService();
+ return tokenService;
+ }
+
private void testDelegationTokensOps(final boolean ssl, final boolean kerb)
throws Exception {
final File confDir = getTestDir();
@@ -2098,11 +2221,16 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
clientConf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
createKMSUri(getKMSUrl()).toString());
+ clientConf.setBoolean(KMS_CLIENT_COPY_LEGACY_TOKEN_KEY, false);
doAs("client", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
KeyProvider kp = createProvider(uri, clientConf);
+ // Unset the conf value for key provider path just to be sure that
+ // the key provider created for renew and cancel token is from
+ // token service field.
+ clientConf.unset(HADOOP_SECURITY_KEY_PROVIDER_PATH);
// test delegation token retrieval
KeyProviderDelegationTokenExtension kpdte =
KeyProviderDelegationTokenExtension.
@@ -2110,13 +2238,10 @@ public class TestKMS {
final Credentials credentials = new Credentials();
final Token<?>[] tokens =
kpdte.addDelegationTokens("client1", credentials);
- Assert.assertEquals(1, credentials.getAllTokens().size());
- InetSocketAddress kmsAddr =
- new InetSocketAddress(getKMSUrl().getHost(),
- getKMSUrl().getPort());
- Assert.assertEquals(KMSDelegationToken.TOKEN_KIND,
- credentials.getToken(SecurityUtil.buildTokenService(kmsAddr)).
- getKind());
+ Text tokenService = getTokenService(kp);
+ assertEquals(1, credentials.getAllTokens().size());
+ assertEquals(TOKEN_KIND,
+ credentials.getToken(tokenService).getKind());
// Test non-renewer user cannot renew.
for (Token<?> token : tokens) {
@@ -2243,12 +2368,11 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
clientConf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
createKMSUri(getKMSUrl()).toString());
+ clientConf.setBoolean(KMS_CLIENT_COPY_LEGACY_TOKEN_KEY, false);
final KeyProvider kp = createProvider(uri, clientConf);
final KeyProviderDelegationTokenExtension kpdte =
KeyProviderDelegationTokenExtension.
createKeyProviderDelegationTokenExtension(kp);
- final InetSocketAddress kmsAddr =
- new InetSocketAddress(getKMSUrl().getHost(), getKMSUrl().getPort());
// Job 1 (e.g. YARN log aggregation job), with user DT.
final Collection<Token<?>> job1Token = new HashSet<>();
@@ -2258,16 +2382,17 @@ public class TestKMS {
// Get a DT and use it.
final Credentials credentials = new Credentials();
kpdte.addDelegationTokens("client", credentials);
+ Text tokenService = getTokenService(kp);
Assert.assertEquals(1, credentials.getAllTokens().size());
- Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, credentials.
- getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
+
UserGroupInformation.getCurrentUser().addCredentials(credentials);
LOG.info("Added kms dt to credentials: {}", UserGroupInformation.
getCurrentUser().getCredentials().getAllTokens());
- Token<?> token =
+ final Token<?> token =
UserGroupInformation.getCurrentUser().getCredentials()
- .getToken(SecurityUtil.buildTokenService(kmsAddr));
- Assert.assertNotNull(token);
+ .getToken(tokenService);
+ assertNotNull(token);
+ assertEquals(TOKEN_KIND, token.getKind());
job1Token.add(token);
// Decode the token to get max time.
@@ -2302,17 +2427,16 @@ public class TestKMS {
// Get a new DT, but don't use it yet.
final Credentials newCreds = new Credentials();
kpdte.addDelegationTokens("client", newCreds);
- Assert.assertEquals(1, newCreds.getAllTokens().size());
- Assert.assertEquals(KMSDelegationToken.TOKEN_KIND,
- newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).
- getKind());
+ assertEquals(1, newCreds.getAllTokens().size());
+ final Text tokenService = getTokenService(kp);
+ assertEquals(TOKEN_KIND,
+ newCreds.getToken(tokenService).getKind());
// Using job 1's DT should fail.
final Credentials oldCreds = new Credentials();
for (Token<?> token : job1Token) {
- if (token.getKind().equals(KMSDelegationToken.TOKEN_KIND)) {
- oldCreds
- .addToken(SecurityUtil.buildTokenService(kmsAddr), token);
+ if (token.getKind().equals(TOKEN_KIND)) {
+ oldCreds.addToken(tokenService, token);
}
}
UserGroupInformation.getCurrentUser().addCredentials(oldCreds);
@@ -2326,12 +2450,11 @@ public class TestKMS {
}
// Using the new DT should succeed.
- Assert.assertEquals(1, newCreds.getAllTokens().size());
- Assert.assertEquals(KMSDelegationToken.TOKEN_KIND,
- newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).
- getKind());
+ assertEquals(1, newCreds.getAllTokens().size());
+ assertEquals(TOKEN_KIND,
+ newCreds.getToken(tokenService).getKind());
UserGroupInformation.getCurrentUser().addCredentials(newCreds);
- LOG.info("Credetials now are: {}", UserGroupInformation
+ LOG.info("Credentials now are: {}", UserGroupInformation
.getCurrentUser().getCredentials().getAllTokens());
kp.getKeys();
return null;
@@ -2357,7 +2480,13 @@ public class TestKMS {
doKMSWithZK(true, true);
}
- public void doKMSWithZK(boolean zkDTSM, boolean zkSigner) throws Exception {
+ private <T> T runServerWithZooKeeper(boolean zkDTSM, boolean zkSigner,
+ KMSCallable<T> callable) throws Exception {
+ return runServerWithZooKeeper(zkDTSM, zkSigner, callable, 1);
+ }
+
+ private <T> T runServerWithZooKeeper(boolean zkDTSM, boolean zkSigner,
+ KMSCallable<T> callable, int kmsSize) throws Exception {
TestingServer zkServer = null;
try {
zkServer = new TestingServer();
@@ -2403,43 +2532,265 @@ public class TestKMS {
writeConf(testDir, conf);
- KMSCallable<KeyProvider> c =
- new KMSCallable<KeyProvider>() {
- @Override
- public KeyProvider call() throws Exception {
- final Configuration conf = new Configuration();
- conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
- final URI uri = createKMSUri(getKMSUrl());
-
- final KeyProvider kp =
- doAs("SET_KEY_MATERIAL",
- new PrivilegedExceptionAction<KeyProvider>() {
- @Override
- public KeyProvider run() throws Exception {
- KeyProvider kp = createProvider(uri, conf);
- kp.createKey("k1", new byte[16],
- new KeyProvider.Options(conf));
- kp.createKey("k2", new byte[16],
- new KeyProvider.Options(conf));
- kp.createKey("k3", new byte[16],
- new KeyProvider.Options(conf));
- return kp;
- }
- });
- return kp;
- }
- };
-
- runServer(null, null, testDir, c);
+ int[] ports = new int[kmsSize];
+ for (int i = 0; i < ports.length; i++) {
+ ports[i] = -1;
+ }
+ return runServer(ports, null, null, testDir, callable);
} finally {
if (zkServer != null) {
zkServer.stop();
zkServer.close();
}
}
+ }
+
+ public void doKMSWithZK(boolean zkDTSM, boolean zkSigner) throws Exception {
+ KMSCallable<KeyProvider> c =
+ new KMSCallable<KeyProvider>() {
+ @Override
+ public KeyProvider call() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
+ final URI uri = createKMSUri(getKMSUrl());
+
+ final KeyProvider kp =
+ doAs("SET_KEY_MATERIAL",
+ new PrivilegedExceptionAction<KeyProvider>() {
+ @Override
+ public KeyProvider run() throws Exception {
+ KeyProvider kp = createProvider(uri, conf);
+ kp.createKey("k1", new byte[16],
+ new KeyProvider.Options(conf));
+ kp.createKey("k2", new byte[16],
+ new KeyProvider.Options(conf));
+ kp.createKey("k3", new byte[16],
+ new KeyProvider.Options(conf));
+ return kp;
+ }
+ });
+ return kp;
+ }
+ };
+
+ runServerWithZooKeeper(zkDTSM, zkSigner, c);
+ }
+
+ @Test
+ public void doKMSHAZKWithDelegationTokenAccess() throws Exception {
+ KMSCallable<Void> c = new KMSCallable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
+ final URI[] uris = createKMSHAUri(getKMSHAUrl());
+ final Credentials credentials = new Credentials();
+ final String lbUri = generateLoadBalancingKeyProviderUriString();
+ final LoadBalancingKMSClientProvider lbkp =
+ createHAProvider(uris, conf, lbUri);
+ conf.unset(HADOOP_SECURITY_KEY_PROVIDER_PATH);
+ // Login as a Kerberos user principal using keytab.
+ // Connect to KMS to create a delegation token and add it to credentials
+ final String keyName = "k0";
+ doAs("SET_KEY_MATERIAL",
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KeyProviderDelegationTokenExtension kpdte =
+ KeyProviderDelegationTokenExtension.
+ createKeyProviderDelegationTokenExtension(lbkp);
+ kpdte.addDelegationTokens("SET_KEY_MATERIAL", credentials);
+ kpdte.createKey(keyName, new KeyProvider.Options(conf));
+ return null;
+ }
+ });
+
+ assertTokenIdentifierEquals(credentials);
+
+ final LoadBalancingKMSClientProvider lbkp1 =
+ createHAProvider(uris, conf, lbUri);
+ // verify both tokens can be used to authenticate
+ for (Token t : credentials.getAllTokens()) {
+ assertTokenAccess(lbkp1, keyName, t);
+ }
+ return null;
+ }
+ };
+ runServerWithZooKeeper(true, true, c, 2);
+ }
+
+ /**
+ * Assert that the passed in credentials have 2 tokens, of kind
+ * {@link KMSDelegationToken#TOKEN_KIND} and
+ * {@link KMSDelegationToken#TOKEN_LEGACY_KIND}. Assert that the 2 tokens have
+ * the same identifier.
+ */
+ private void assertTokenIdentifierEquals(Credentials credentials)
+ throws IOException {
+ // verify the 2 tokens have the same identifier
+ assertEquals(2, credentials.getAllTokens().size());
+ Token token = null;
+ Token legacyToken = null;
+ for (Token t : credentials.getAllTokens()) {
+ if (KMSDelegationToken.TOKEN_KIND.equals(t.getKind())) {
+ token = t;
+ } else if (KMSDelegationToken.TOKEN_LEGACY_KIND.equals(t.getKind())) {
+ legacyToken = t;
+ }
+ }
+ assertNotNull(token);
+ assertNotNull(legacyToken);
+ final DelegationTokenIdentifier tokenId =
+ (DelegationTokenIdentifier) token.decodeIdentifier();
+ final DelegationTokenIdentifier legacyTokenId =
+ (DelegationTokenIdentifier) legacyToken.decodeIdentifier();
+ assertEquals("KMS DT and legacy dt should have identical identifier",
+ tokenId, legacyTokenId);
+ }
+ /**
+ * Tests token access with each providers in the
+ * {@link LoadBalancingKMSClientProvider}. This is to make sure the 2 token
+ * kinds are compatible and can both be used to authenticate.
+ */
+ private void assertTokenAccess(final LoadBalancingKMSClientProvider lbkp,
+ final String keyName, final Token token) throws Exception {
+ UserGroupInformation tokenUgi =
+ UserGroupInformation.createUserForTesting("test", new String[] {});
+ // Verify the tokens can authenticate to any KMS
+ tokenUgi.addToken(token);
+ tokenUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // Create a kms client with one provider at a time. Must use one
+ // provider so that if it fails to authenticate, it does not fall
+ // back to the next KMS instance.
+ // It should succeed because its delegation token can access any
+ // KMS instances.
+ for (KMSClientProvider provider : lbkp.getProviders()) {
+ if (token.getKind().equals(TOKEN_LEGACY_KIND) && !token.getService()
+ .equals(provider.getDelegationTokenService())) {
+ // Historically known issue: Legacy token can only work with the
+ // key provider specified in the token's Service
+ continue;
+ }
+ LOG.info("Rolling key {} via provider {} with token {}.", keyName,
+ provider, token);
+ provider.rollNewVersion(keyName);
+ }
+ return null;
+ }
+ });
}
+ @Test
+ public void testKMSHAZKDelegationTokenRenewCancel() throws Exception {
+ testKMSHAZKDelegationTokenRenewCancel(TOKEN_KIND);
+ }
+
+ @Test
+ public void testKMSHAZKDelegationTokenRenewCancelLegacy() throws Exception {
+ testKMSHAZKDelegationTokenRenewCancel(TOKEN_LEGACY_KIND);
+ }
+
+ private void testKMSHAZKDelegationTokenRenewCancel(final Text tokenKind)
+ throws Exception {
+ GenericTestUtils.setLogLevel(KMSTokenRenewer.LOG, Level.TRACE);
+ assertTrue(tokenKind == TOKEN_KIND || tokenKind == TOKEN_LEGACY_KIND);
+ KMSCallable<Void> c = new KMSCallable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final Configuration conf = new Configuration();
+ final URI[] uris = createKMSHAUri(getKMSHAUrl());
+ final Credentials credentials = new Credentials();
+ // Create a UGI without Kerberos auth. It will be authenticated with
+ // delegation token.
+ final UserGroupInformation nonKerberosUgi =
+ UserGroupInformation.getCurrentUser();
+ final String lbUri = generateLoadBalancingKeyProviderUriString();
+ final LoadBalancingKMSClientProvider lbkp =
+ createHAProvider(uris, conf, lbUri);
+ conf.unset(HADOOP_SECURITY_KEY_PROVIDER_PATH);
+ // Login as a Kerberos user principal using keytab.
+ // Connect to KMS to create a delegation token and add it to credentials
+ doAs("SET_KEY_MATERIAL",
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KeyProviderDelegationTokenExtension kpdte =
+ KeyProviderDelegationTokenExtension.
+ createKeyProviderDelegationTokenExtension(lbkp);
+ kpdte.addDelegationTokens("SET_KEY_MATERIAL", credentials);
+ return null;
+ }
+ });
+
+ // Test token renewal and cancellation
+ final Collection<Token<? extends TokenIdentifier>> tokens =
+ credentials.getAllTokens();
+ doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Assert.assertEquals(2, tokens.size());
+ boolean tokenFound = false;
+ for (Token token : tokens) {
+ if (!tokenKind.equals(token.getKind())) {
+ continue;
+ } else {
+ tokenFound = true;
+ }
+ KMSUtilFaultInjector.set(testInjector);
+ setupConfForToken(token.getKind(), conf, lbUri);
+
+ LOG.info("Testing token: {}", token);
+ long tokenLife = token.renew(conf);
+ LOG.info("Renewed token {}, new lifetime:{}", token, tokenLife);
+ Thread.sleep(10);
+ long newTokenLife = token.renew(conf);
+ LOG.info("Renewed token {}, new lifetime:{}", token,
+ newTokenLife);
+ assertTrue(newTokenLife > tokenLife);
+
+ boolean canceled = false;
+ // test delegation token cancellation
+ if (!canceled) {
+ token.cancel(conf);
+ LOG.info("Cancelled token {}", token);
+ canceled = true;
+ }
+ assertTrue("token should have been canceled", canceled);
+ try {
+ token.renew(conf);
+ fail("should not be able to renew a canceled token " + token);
+ } catch (Exception e) {
+ LOG.info("Expected exception when renewing token", e);
+ }
+ }
+ assertTrue("Should have found token kind " + tokenKind + " from "
+ + tokens, tokenFound);
+ return null;
+ }
+ });
+ return null;
+ }
+ };
+ runServerWithZooKeeper(true, true, c, 2);
+ }
+
+ /**
+ * Set or unset the key provider configuration based on token kind.
+ */
+ private void setupConfForToken(Text tokenKind, Configuration conf,
+ String lbUri) {
+ if (tokenKind.equals(TOKEN_KIND)) {
+ conf.unset(HADOOP_SECURITY_KEY_PROVIDER_PATH);
+ } else {
+ // conf is only required for legacy tokens to create provider,
+ // new tokens create provider by parsing its own Service field
+ assertEquals(TOKEN_LEGACY_KIND, tokenKind);
+ conf.set(HADOOP_SECURITY_KEY_PROVIDER_PATH, lbUri);
+ }
+ }
@Test
public void testProxyUserKerb() throws Exception {
@@ -2558,6 +2909,16 @@ public class TestKMS {
@Test
public void testTGTRenewal() throws Exception {
+ shutdownMiniKdc();
+ try {
+ testTgtRenewalInt();
+ } finally {
+ shutdownMiniKdc();
+ setUpMiniKdc();
+ }
+ }
+
+ private void testTgtRenewalInt() throws Exception {
Properties kdcConf = MiniKdc.createConf();
kdcConf.setProperty(MiniKdc.MAX_TICKET_LIFETIME, "3");
kdcConf.setProperty(MiniKdc.MIN_TICKET_LIFETIME, "3");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/44] hadoop git commit: HDFS-13436. Fix javadoc of
package-info.java
Posted by xy...@apache.org.
HDFS-13436. Fix javadoc of package-info.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b0b9f2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b0b9f2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b0b9f2a
Branch: refs/heads/HDFS-7240
Commit: 9b0b9f2af2f6827d7430f995d3203c4cb7ef7e48
Parents: 53b3e59
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Apr 13 13:23:44 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Apr 13 13:23:51 2018 +0900
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/datatransfer/package-info.java | 8 ++++----
.../hdfs/server/common/blockaliasmap/package-info.java | 9 +++++----
.../hdfs/server/diskbalancer/connectors/package-info.java | 3 +--
.../hdfs/server/diskbalancer/datamodel/package-info.java | 4 ++--
.../hadoop/hdfs/server/diskbalancer/package-info.java | 2 +-
.../hdfs/server/diskbalancer/planner/package-info.java | 2 +-
6 files changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java
index a13c7d8..13c0c59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java
@@ -15,10 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.protocol.datatransfer;
-import org.apache.hadoop.classification.InterfaceStability;
/**
* This package contains classes related to hdfs data transfer protocol.
- */
\ No newline at end of file
+ */
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.protocol.datatransfer;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java
index b906791..d088945 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java
@@ -15,13 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-package org.apache.hadoop.hdfs.server.common.blockaliasmap;
/**
* The AliasMap defines mapping of PROVIDED HDFS blocks to data in remote
* storage systems.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.hdfs.server.common.blockaliasmap;
+
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java
index b4b4437..f118c2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java
@@ -15,8 +15,6 @@
* the License.
*/
-package org.apache.hadoop.hdfs.server.diskbalancer.connectors;
-
/**
* Connectors package is a set of logical connectors that connect
* to various data sources to read the hadoop cluster information.
@@ -35,3 +33,4 @@ package org.apache.hadoop.hdfs.server.diskbalancer.connectors;
* we can crate dataNodes on the fly and attach to this connector and
* ask the diskBalancer Cluster to read data from this source.
*/
+package org.apache.hadoop.hdfs.server.diskbalancer.connectors;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/package-info.java
index f72e283..212da55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/package-info.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
/**
* Disk Balancer Data Model is the Data Model for the cluster that
* Disk Balancer is working against. This information is read
@@ -28,4 +27,5 @@ package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
* DiskBalancerDataNodes is a collection of DiskBalancerVolumeSets
* DiskBalancerVolumeSets is a collection of DiskBalancerVolumes
* DiskBalancerVolumes represents actual volumes on DataNodes.
- */
\ No newline at end of file
+ */
+package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/package-info.java
index 4bec98f..5ed7db1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/package-info.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.diskbalancer;
/**
* Disk Balancer connects to a {@link org.apache.hadoop.hdfs.server.datanode
* .DataNode} and attempts to spread data across all volumes evenly.
@@ -34,3 +33,4 @@ package org.apache.hadoop.hdfs.server.diskbalancer;
* 3) Disk Balancer operates against data nodes which are live and operational.
*
*/
+package org.apache.hadoop.hdfs.server.diskbalancer;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
index bbcc121..0ad1cd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.server.diskbalancer.planner;
/**
* Planner takes a DiskBalancerVolumeSet, threshold and
* computes a series of steps that lead to an even data
@@ -44,3 +43,4 @@ package org.apache.hadoop.hdfs.server.diskbalancer.planner;
* //when we are done , return the list of steps
* return planner;
*/
+package org.apache.hadoop.hdfs.server.diskbalancer.planner;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/44] hadoop git commit: HDFS-13427. Fix the section titles of
transparent encryption document.
Posted by xy...@apache.org.
HDFS-13427. Fix the section titles of transparent encryption document.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7cd362a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7cd362a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7cd362a
Branch: refs/heads/HDFS-7240
Commit: c7cd362afd21add324c3a82c594b133d41cf8d03
Parents: 0d898b7
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Apr 12 10:44:56 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Apr 12 10:44:56 2018 +0900
----------------------------------------------------------------------
.../src/site/markdown/TransparentEncryption.md | 57 ++++++++++----------
1 file changed, 29 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7cd362a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
index 3454265..d7a70b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
@@ -17,12 +17,12 @@ Transparent Encryption in HDFS
<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
-<a name="Overview"></a>Overview
+Overview
--------
HDFS implements *transparent*, *end-to-end* encryption. Once configured, data read from and written to special HDFS directories is *transparently* encrypted and decrypted without requiring changes to user application code. This encryption is also *end-to-end*, which means the data can only be encrypted and decrypted by the client. HDFS never stores or has access to unencrypted data or unencrypted data encryption keys. This satisfies two typical requirements for encryption: *at-rest encryption* (meaning data on persistent media, such as a disk) as well as *in-transit encryption* (e.g. when data is travelling over the network).
-<a name="Background"></a>Background
+Background
----------
Encryption can be done at different layers in a traditional data management software/hardware stack. Choosing to encrypt at a given layer comes with different advantages and disadvantages.
@@ -39,17 +39,17 @@ HDFS-level encryption fits between database-level and filesystem-level encryptio
HDFS-level encryption also prevents attacks at the filesystem-level and below (so-called "OS-level attacks"). The operating system and disk only interact with encrypted bytes, since the data is already encrypted by HDFS.
-<a name="Use_Cases"></a>Use Cases
+Use Cases
---------
Data encryption is required by a number of different government, financial, and regulatory entities. For example, the health-care industry has HIPAA regulations, the card payment industry has PCI DSS regulations, and the US government has FISMA regulations. Having transparent encryption built into HDFS makes it easier for organizations to comply with these regulations.
Encryption can also be performed at the application-level, but by integrating it into HDFS, existing applications can operate on encrypted data without changes. This integrated architecture implies stronger encrypted file semantics and better coordination with other HDFS functions.
-<a name="Architecture"></a>Architecture
+Architecture
------------
-### <a name="Architecture_overview"></a>Overview
+### Overview
For transparent encryption, we introduce a new abstraction to HDFS: the *encryption zone*. An encryption zone is a special directory whose contents will be transparently encrypted upon write and transparently decrypted upon read. Each encryption zone is associated with a single *encryption zone key* which is specified when the zone is created. Each file within an encryption zone has its own unique *data encryption key (DEK)*. DEKs are never handled directly by HDFS. Instead, HDFS only ever handles an *encrypted data encryption key (EDEK)*. Clients decrypt an EDEK, and then use the subsequent DEK to read and write data. HDFS datanodes simply see a stream of encrypted bytes.
@@ -65,7 +65,7 @@ A new cluster service is required to manage encryption keys: the Hadoop Key Mana
The KMS will be described in more detail below.
-### <a name="Accessing_data_within_an_encryption_zone"></a>Accessing data within an encryption zone
+### Accessing data within an encryption zone
When creating a new file in an encryption zone, the NameNode asks the KMS to generate a new EDEK encrypted with the encryption zone's key. The EDEK is then stored persistently as part of the file's metadata on the NameNode.
@@ -75,7 +75,7 @@ All of the above steps for the read and write path happen automatically through
Access to encrypted file data and metadata is controlled by normal HDFS filesystem permissions. This means that if HDFS is compromised (for example, by gaining unauthorized access to an HDFS superuser account), a malicious user only gains access to ciphertext and encrypted keys. However, since access to encryption zone keys is controlled by a separate set of permissions on the KMS and key store, this does not pose a security threat.
-### <a name="Key_Management_Server_KeyProvider_EDEKs"></a>Key Management Server, KeyProvider, EDEKs
+### Key Management Server, KeyProvider, EDEKs
The KMS is a proxy that interfaces with a backing key store on behalf of HDFS daemons and clients. Both the backing key store and the KMS implement the Hadoop KeyProvider API. See the [KMS documentation](../../hadoop-kms/index.html) for more information.
@@ -85,21 +85,21 @@ The KMS implements additional functionality which enables creation and decryptio
In the context of HDFS encryption, EEKs are *encrypted data encryption keys (EDEKs)*, where a *data encryption key (DEK)* is what is used to encrypt and decrypt file data. Typically, the key store is configured to only allow end users access to the keys used to encrypt DEKs. This means that EDEKs can be safely stored and handled by HDFS, since the HDFS user will not have access to unencrypted encryption keys.
-<a name="Configuration"></a>Configuration
+Configuration
-------------
A necessary prerequisite is an instance of the KMS, as well as a backing key store for the KMS. See the [KMS documentation](../../hadoop-kms/index.html) for more information.
Once a KMS has been set up and the NameNode and HDFS clients have been correctly configured, an admin can use the `hadoop key` and `hdfs crypto` command-line tools to create encryption keys and set up new encryption zones. Existing data can be encrypted by copying it into the new encryption zones using tools like distcp.
-### <a name="Configuring_the_cluster_KeyProvider"></a>Configuring the cluster KeyProvider
+### Configuring the cluster KeyProvider
#### hadoop.security.key.provider.path
The KeyProvider to use when interacting with encryption keys used when reading and writing to an encryption zone.
HDFS clients will use the provider path returned from Namenode via getServerDefaults. If namenode doesn't support returning key provider uri then client's conf will be used.
-### <a name="Selecting_an_encryption_algorithm_and_codec"></a>Selecting an encryption algorithm and codec
+### Selecting an encryption algorithm and codec
#### hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE
@@ -129,7 +129,7 @@ Default: `8192`
The buffer size used by CryptoInputStream and CryptoOutputStream.
-### <a name="Namenode_configuration"></a>Namenode configuration
+### Namenode configuration
#### dfs.namenode.list.encryption.zones.num.responses
@@ -137,10 +137,10 @@ Default: `100`
When listing encryption zones, the maximum number of zones that will be returned in a batch. Fetching the list incrementally in batches improves namenode performance.
-<a name="crypto_command-line_interface"></a>`crypto` command-line interface
+`crypto` command-line interface
-------------------------------
-### <a name="createZone"></a>createZone
+### createZone
Usage: `[-createZone -keyName <keyName> -path <path>]`
@@ -151,13 +151,13 @@ Create a new encryption zone.
| *path* | The path of the encryption zone to create. It must be an empty directory. A trash directory is provisioned under this path.|
| *keyName* | Name of the key to use for the encryption zone. Uppercase key names are unsupported. |
-### <a name="listZones"></a>listZones
+### listZones
Usage: `[-listZones]`
List all encryption zones. Requires superuser permissions.
-### <a name="provisionTrash"></a>provisionTrash
+### provisionTrash
Usage: `[-provisionTrash -path <path>]`
@@ -167,7 +167,7 @@ Provision a trash directory for an encryption zone.
|:---- |:---- |
| *path* | The path to the root of the encryption zone. |
-### <a name="getFileEncryptionInfo"></a>getFileEncryptionInfo
+### getFileEncryptionInfo
Usage: `[-getFileEncryptionInfo -path <path>]`
@@ -177,7 +177,7 @@ Get encryption information from a file. This can be used to find out whether a f
|:---- |:---- |
| *path* | The path of the file to get encryption information. |
-### <a name="reencryptZone"></a>reencryptZone
+### reencryptZone
Usage: `[-reencryptZone <action> -path <zone>]`
@@ -198,13 +198,13 @@ Re-encryption is a NameNode-only operation in HDFS, so could potentially put int
| *dfs.namenode.reencrypt.throttle.limit.handler.ratio* | Ratio of read locks to be held during re-encryption. 1.0 means no throttling. 0.5 means re-encryption can hold the readlock at most 50% of its total processing time. Negative value or 0 are invalid. |
| *dfs.namenode.reencrypt.throttle.limit.updater.ratio* | Ratio of write locks to be held during re-encryption. 1.0 means no throttling. 0.5 means re-encryption can hold the writelock at most 50% of its total processing time. Negative value or 0 are invalid. |
-### <a name="listReencryptionStatus"></a>listReencryptionStatus
+### listReencryptionStatus
Usage: `[-listReencryptionStatus]`
List re-encryption information for all encryption zones. Requires superuser permissions.
-<a name="Example_usage"></a>Example usage
+Example usage
-------------
These instructions assume that you are running as the normal user or HDFS superuser as is appropriate. Use `sudo` as needed for your environment.
@@ -227,10 +227,10 @@ These instructions assume that you are running as the normal user or HDFS superu
hdfs crypto -getFileEncryptionInfo -path /zone/helloWorld
# console output: {cipherSuite: {name: AES/CTR/NoPadding, algorithmBlockSize: 16}, cryptoProtocolVersion: CryptoProtocolVersion{description='Encryption zones', version=1, unknownValue=null}, edek: 2010d301afbd43b58f10737ce4e93b39, iv: ade2293db2bab1a2e337f91361304cb3, keyName: mykey, ezKeyVersionName: mykey@0}
-<a name="Distcp_considerations"></a>Distcp considerations
+Distcp considerations
---------------------
-### <a name="Running_as_the_superuser"></a>Running as the superuser
+### Running as the superuser
One common usecase for distcp is to replicate data between clusters for backup and disaster recovery purposes. This is typically performed by the cluster administrator, who is an HDFS superuser.
@@ -238,11 +238,11 @@ To enable this same workflow when using HDFS encryption, we introduced a new vir
When using `/.reserved/raw` to distcp encrypted data, it's important to preserve extended attributes with the [-px](../../hadoop-distcp/DistCp.html#Command_Line_Options) flag. This is because encrypted file attributes (such as the EDEK) are exposed through extended attributes within `/.reserved/raw`, and must be preserved to be able to decrypt the file. This means that if the distcp is initiated at or above the encryption zone root, it will automatically create an encryption zone at the destination if it does not already exist. However, it's still recommended that the admin first create identical encryption zones on the destination cluster to avoid any potential mishaps.
-### <a name="Copying_into_encrypted_locations"></a>Copying into encrypted locations
+### Copying into encrypted locations
By default, distcp compares checksums provided by the filesystem to verify that the data was successfully copied to the destination. When copying from unencrypted or encrypted location into an encrypted location, the filesystem checksums will not match since the underlying block data is different because a new EDEK will be used to encrypt at destination. In this case, specify the [-skipcrccheck](../../hadoop-distcp/DistCp.html#Command_Line_Options) and [-update](../../hadoop-distcp/DistCp.html#Command_Line_Options) distcp flags to avoid verifying checksums.
-<a name="Rename_and_Trash_considerations"></a>Rename and Trash considerations
+Rename and Trash considerations
---------------------
HDFS restricts file and directory renames across encryption zone boundaries. This includes renaming an encrypted file / directory into an unencrypted directory (e.g., `hdfs dfs mv /zone/encryptedFile /home/bob`), renaming an unencrypted file or directory into an encryption zone (e.g., `hdfs dfs mv /home/bob/unEncryptedFile /zone`), and renaming between two different encryption zones (e.g., `hdfs dfs mv /home/alice/zone1/foo /home/alice/zone2`). In these examples, `/zone`, `/home/alice/zone1`, and `/home/alice/zone2` are encryption zones, while `/home/bob` is not. A rename is only allowed if the source and destination paths are in the same encryption zone, or both paths are unencrypted (not in any encryption zone).
@@ -254,10 +254,11 @@ To comply with the above rule, each encryption zone has its own `.Trash` directo
If the encryption zone is the root directory (e.g., `/` directory), the trash path of root directory is `/.Trash`, not the `.Trash` directory under the user's home directory, and the behavior of renaming sub-directories or sub-files in root directory will keep consistent with the behavior in a general encryption zone, such as `/zone` which is mentioned at the top of this section.
The `crypto` command before Hadoop 2.8.0 does not provision the `.Trash` directory automatically. If an encryption zone is created before Hadoop 2.8.0, and then the cluster is upgraded to Hadoop 2.8.0 or above, the trash directory can be provisioned using `-provisionTrash` option (e.g., `hdfs crypto -provisionTrash -path /zone`).
-<a name="Attack_vectors"></a>Attack vectors
+
+Attack vectors
--------------
-### <a name="Hardware_access_exploits"></a>Hardware access exploits
+### Hardware access exploits
These exploits assume that attacker has gained physical access to hard drives from cluster machines, i.e. datanodes and namenodes.
@@ -271,7 +272,7 @@ These exploits assume that attacker has gained physical access to hard drives fr
* By itself, this does not expose cleartext, as it also requires access to DEKs.
-### <a name="Root_access_exploits"></a>Root access exploits
+### Root access exploits
These exploits assume that attacker has gained root shell access to cluster machines, i.e. datanodes and namenodes. Many of these exploits cannot be addressed in HDFS, since a malicious root user has access to the in-memory state of processes holding encryption keys and cleartext. For these exploits, the only mitigation technique is carefully restricting and monitoring root shell access.
@@ -295,7 +296,7 @@ These exploits assume that attacker has gained root shell access to cluster mach
* By itself, insufficient to read cleartext without the EDEK's encryption key and encrypted block files.
-### <a name="HDFS_admin_exploits"></a>HDFS admin exploits
+### HDFS admin exploits
These exploits assume that the attacker has compromised HDFS, but does not have root or `hdfs` user shell access.
@@ -307,7 +308,7 @@ These exploits assume that the attacker has compromised HDFS, but does not have
* By itself, insufficient to read cleartext without EDEK encryption keys.
-### <a name="Rogue_user_exploits"></a>Rogue user exploits
+### Rogue user exploits
A rogue user can collect keys of files they have access to, and use them later to decrypt the encrypted data of those files. As the user had access to those files, they already had access to the file contents. This can be mitigated through periodic key rolling policies. The [reencryptZone](#reencryptZone) command is usually required after key rolling, to make sure the EDEKs on existing files use the new version key.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/44] hadoop git commit: HADOOP-12502 SetReplication
OutOfMemoryError. Contributed by Vinayakumar B.
Posted by xy...@apache.org.
HADOOP-12502 SetReplication OutOfMemoryError. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d898b7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d898b7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d898b7b
Branch: refs/heads/HDFS-7240
Commit: 0d898b7bb8d8d616133236da979a4316be4c1a6f
Parents: 18de6f2
Author: Aaron Fabbri <fa...@apache.org>
Authored: Wed Apr 11 17:19:56 2018 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Wed Apr 11 17:19:56 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/ChecksumFileSystem.java | 9 ++-
.../java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../org/apache/hadoop/fs/shell/Command.java | 69 ++++++++++++++++++--
.../apache/hadoop/fs/shell/CopyCommands.java | 6 ++
.../java/org/apache/hadoop/fs/shell/Ls.java | 26 +++++++-
.../org/apache/hadoop/fs/shell/PathData.java | 27 ++++++++
.../apache/hadoop/fs/shell/find/TestFind.java | 34 +++++++++-
7 files changed, 161 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 14c1905..663c910 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -677,7 +677,14 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
public FileStatus[] listStatus(Path f) throws IOException {
return fs.listStatus(f, DEFAULT_FILTER);
}
-
+
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(final Path p)
+ throws IOException {
+ // Not-using fs#listStatusIterator() since it includes crc files as well
+ return new DirListingIterator<>(p);
+ }
+
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index facfe03..707b921 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2147,7 +2147,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Generic iterator for implementing {@link #listStatusIterator(Path)}.
*/
- private class DirListingIterator<T extends FileStatus> implements
+ protected class DirListingIterator<T extends FileStatus> implements
RemoteIterator<T> {
private final Path path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index c292cf6..a4746cf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -325,14 +326,9 @@ abstract public class Command extends Configured {
*/
protected void processPaths(PathData parent, PathData ... items)
throws IOException {
- // TODO: this really should be iterative
for (PathData item : items) {
try {
- processPath(item);
- if (recursive && isPathRecursable(item)) {
- recursePath(item);
- }
- postProcessPath(item);
+ processPathInternal(item);
} catch (IOException e) {
displayError(e);
}
@@ -340,6 +336,59 @@ abstract public class Command extends Configured {
}
/**
+ * Iterates over the given expanded paths and invokes
+ * {@link #processPath(PathData)} on each element. If "recursive" is true,
+ * will do a post-visit DFS on directories.
+ * @param parent if called via a recurse, will be the parent dir, else null
+ * @param itemsIterator a iterator of {@link PathData} objects to process
+ * @throws IOException if anything goes wrong...
+ */
+ protected void processPaths(PathData parent,
+ RemoteIterator<PathData> itemsIterator) throws IOException {
+ int groupSize = getListingGroupSize();
+ if (groupSize == 0) {
+ // No grouping of contents required.
+ while (itemsIterator.hasNext()) {
+ processPaths(parent, itemsIterator.next());
+ }
+ } else {
+ List<PathData> items = new ArrayList<PathData>(groupSize);
+ while (itemsIterator.hasNext()) {
+ items.add(itemsIterator.next());
+ if (!itemsIterator.hasNext() || items.size() == groupSize) {
+ processPaths(parent, items.toArray(new PathData[items.size()]));
+ items.clear();
+ }
+ }
+ }
+ }
+
+ private void processPathInternal(PathData item) throws IOException {
+ processPath(item);
+ if (recursive && isPathRecursable(item)) {
+ recursePath(item);
+ }
+ postProcessPath(item);
+ }
+
+ /**
+ * Whether the directory listing for a path should be sorted.?
+ * @return true/false.
+ */
+ protected boolean isSorted() {
+ return false;
+ }
+
+ /**
+ * While using iterator method for listing for a path, whether to group items
+ * and process as array? If so what is the size of array?
+ * @return size of the grouping array.
+ */
+ protected int getListingGroupSize() {
+ return 0;
+ }
+
+ /**
* Determines whether a {@link PathData} item is recursable. Default
* implementation is to recurse directories but can be overridden to recurse
* through symbolic links.
@@ -384,7 +433,13 @@ abstract public class Command extends Configured {
protected void recursePath(PathData item) throws IOException {
try {
depth++;
- processPaths(item, item.getDirectoryContents());
+ if (isSorted()) {
+ // use the non-iterative method for listing because explicit sorting is
+ // required. Iterators not guaranteed to return sorted elements
+ processPaths(item, item.getDirectoryContents());
+ } else {
+ processPaths(item, item.getDirectoryContentsIterator());
+ }
} finally {
depth--;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 11cb3d6..da7a2b2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -142,6 +142,12 @@ class CopyCommands {
srcs.add(src);
}
}
+
+ @Override
+ protected boolean isSorted() {
+ //Sort the children for merge
+ return true;
+ }
}
static class Cp extends CommandWithDestination {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index a2d5017..56504e4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -230,8 +230,30 @@ class Ls extends FsCommand {
}
@Override
- protected void processPaths(PathData parent, PathData ... items)
- throws IOException {
+ protected boolean isSorted() {
+ // use the non-iterative method for listing because explicit sorting is
+ // required based on time/size/reverse or Total number of entries
+ // required to print summary first when non-recursive.
+ return !isRecursive() || isOrderTime() || isOrderSize() || isOrderReverse();
+ }
+
+ @Override
+ protected int getListingGroupSize() {
+ if (pathOnly) {
+ // If there is a need of printing only paths, then no grouping required
+ return 0;
+ }
+ /*
+ * LS output should be formatted properly. Grouping 100 items and formatting
+ * the output to reduce the creation of huge sized arrays. This method will
+ * be called only when recursive is set.
+ */
+ return 100;
+ }
+
+ @Override
+ protected void processPaths(PathData parent, PathData... items)
+ throws IOException {
if (parent != null && !isRecursive() && items.length != 0) {
if (!pathOnly) {
out.println("Found " + items.length + " items");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
index 3cf3273..adf17df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.PathIsDirectoryException;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.fs.RemoteIterator;
/**
* Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
@@ -277,6 +278,32 @@ public class PathData implements Comparable<PathData> {
}
/**
+ * Returns a RemoteIterator for PathData objects of the items contained in the
+ * given directory.
+ * @return remote iterator of PathData objects for its children
+ * @throws IOException if anything else goes wrong...
+ */
+ public RemoteIterator<PathData> getDirectoryContentsIterator()
+ throws IOException {
+ checkIfExists(FileTypeRequirement.SHOULD_BE_DIRECTORY);
+ final RemoteIterator<FileStatus> stats = this.fs.listStatusIterator(path);
+ return new RemoteIterator<PathData>() {
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return stats.hasNext();
+ }
+
+ @Override
+ public PathData next() throws IOException {
+ FileStatus file = stats.next();
+ String child = getStringForChildPath(file.getPath());
+ return new PathData(fs, child, file);
+ }
+ };
+ }
+
+ /**
* Creates a new object for a child entry in this directory
* @param child the basename will be appended to this object's path
* @return PathData for the child
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java
index 716230a..de0e512 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java
@@ -19,18 +19,19 @@ package org.apache.hadoop.fs.shell.find;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
-import static org.mockito.Matchers.*;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
+import java.util.NoSuchElementException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.shell.PathData;
import org.apache.hadoop.fs.shell.find.BaseExpression;
import org.apache.hadoop.fs.shell.find.Expression;
@@ -42,6 +43,9 @@ import org.junit.Rule;
import org.junit.rules.Timeout;
import org.junit.Test;
import org.mockito.InOrder;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
public class TestFind {
@@ -861,6 +865,34 @@ public class TestFind {
when(mockFs.listStatus(eq(item5c.path))).thenReturn(
new FileStatus[] { item5ca.stat });
+ when(mockFs.listStatusIterator(Mockito.any(Path.class)))
+ .thenAnswer(new Answer<RemoteIterator<FileStatus>>() {
+
+ @Override
+ public RemoteIterator<FileStatus> answer(InvocationOnMock invocation)
+ throws Throwable {
+ final Path p = (Path) invocation.getArguments()[0];
+ final FileStatus[] stats = mockFs.listStatus(p);
+
+ return new RemoteIterator<FileStatus>() {
+ private int i = 0;
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return i < stats.length;
+ }
+
+ @Override
+ public FileStatus next() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException("No more entry in " + p);
+ }
+ return stats[i++];
+ }
+ };
+ }
+ });
+
when(item1.stat.isSymlink()).thenReturn(false);
when(item1a.stat.isSymlink()).thenReturn(false);
when(item1aa.stat.isSymlink()).thenReturn(false);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/44] hadoop git commit: Revert "HDFS-13388.
RequestHedgingProxyProvider calls multiple configured NNs all the time.
Contributed by Jinglun."
Posted by xy...@apache.org.
Revert "HDFS-13388. RequestHedgingProxyProvider calls multiple configured NNs all the time. Contributed by Jinglun."
This reverts commit ac32b3576da4cc463dff85118163ccfff02215fc.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5353c75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5353c75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5353c75
Branch: refs/heads/HDFS-7240
Commit: b5353c75d90b9299f04dba255b9e9af5a8cc19eb
Parents: 4571351
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Apr 12 09:28:23 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Apr 12 09:28:23 2018 -0700
----------------------------------------------------------------------
.../ha/RequestHedgingProxyProvider.java | 3 --
.../ha/TestRequestHedgingProxyProvider.java | 34 --------------------
2 files changed, 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5353c75/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
index 1c38791..7b9cd64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
@@ -79,9 +79,6 @@ public class RequestHedgingProxyProvider<T> extends
public Object
invoke(Object proxy, final Method method, final Object[] args)
throws Throwable {
- if (currentUsedProxy != null) {
- return method.invoke(currentUsedProxy.proxy, args);
- }
Map<Future<Object>, ProxyInfo<T>> proxyMap = new HashMap<>();
int numAttempts = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5353c75/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
index 4b3fdf9..8d6b02d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
@@ -43,13 +43,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
import org.mockito.Matchers;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.mock;
import com.google.common.collect.Lists;
@@ -103,37 +100,6 @@ public class TestRequestHedgingProxyProvider {
}
@Test
- public void testRequestNNAfterOneSuccess() throws Exception {
- final AtomicInteger count = new AtomicInteger(0);
- final ClientProtocol goodMock = mock(ClientProtocol.class);
- when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {
- @Override
- public long[] answer(InvocationOnMock invocation) throws Throwable {
- count.incrementAndGet();
- Thread.sleep(1000);
- return new long[]{1};
- }
- });
- final ClientProtocol badMock = mock(ClientProtocol.class);
- when(badMock.getStats()).thenAnswer(new Answer<long[]>() {
- @Override
- public long[] answer(InvocationOnMock invocation) throws Throwable {
- count.incrementAndGet();
- throw new IOException("Bad mock !!");
- }
- });
-
- RequestHedgingProxyProvider<ClientProtocol> provider =
- new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class,
- createFactory(badMock, goodMock, goodMock, badMock));
- ClientProtocol proxy = provider.getProxy().proxy;
- proxy.getStats();
- assertEquals(2, count.get());
- proxy.getStats();
- assertEquals(3, count.get());
- }
-
- @Test
public void testHedgingWhenOneIsSlow() throws Exception {
final ClientProtocol goodMock = Mockito.mock(ClientProtocol.class);
Mockito.when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org