You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/01/26 21:12:47 UTC
[53/57] [abbrv] hadoop git commit: YARN-3637. Handle localization
sym-linking correctly at the YARN level. Contributed by Chris Trezzo.
YARN-3637. Handle localization sym-linking correctly at the YARN level. Contributed by Chris Trezzo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/425a7e50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/425a7e50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/425a7e50
Branch: refs/heads/YARN-5734
Commit: 425a7e502869c4250aba927ecc3c6f3c561c6ff2
Parents: cd59b9c
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed Jan 25 15:51:36 2017 -0800
Committer: Sangjin Lee <sj...@apache.org>
Committed: Wed Jan 25 15:51:36 2017 -0800
----------------------------------------------------------------------
.../yarn/client/api/SharedCacheClient.java | 23 +++++++++---
.../client/api/impl/SharedCacheClientImpl.java | 31 ++++++++++++++--
.../api/impl/TestSharedCacheClientImpl.java | 37 +++++++++++++++++---
3 files changed, 81 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/425a7e50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
index 7cbe0e1..60c1bd98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
@@ -55,22 +55,37 @@ public abstract class SharedCacheClient extends AbstractService {
* {@link ApplicationId} to identify which application will be using the
* resource.
* </p>
- *
+ *
* <p>
* The <code>SharedCacheManager</code> responds with whether or not the
* resource exists in the cache. If the resource exists, a <code>Path</code>
* to the resource in the shared cache is returned. If the resource does not
* exist, null is returned instead.
* </p>
- *
+ *
+ * <p>
+ * Once a path has been returned for a resource, that path is safe to use for
+ * the lifetime of the application that corresponds to the provided
+ * ApplicationId.
+ * </p>
+ *
+ * <p>
+ * Additionally, a name for the resource should be specified. A fragment will
+ * be added to the path with the desired name if the desired name is different
+ * than the name of the provided path from the shared cache. This ensures that
+ * if the returned path is used to create a LocalResource, then the symlink
+ * created during YARN localization will match the name specified.
+ * </p>
+ *
* @param applicationId ApplicationId of the application using the resource
* @param resourceKey the key (i.e. checksum) that identifies the resource
+ * @param resourceName the desired name of the resource
* @return Path to the resource, or null if it does not exist
*/
@Public
@Unstable
- public abstract Path use(ApplicationId applicationId, String resourceKey)
- throws YarnException;
+ public abstract Path use(ApplicationId applicationId, String resourceKey,
+ String resourceName) throws YarnException;
/**
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/425a7e50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
index 0a61ee0..b910c28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -111,8 +113,8 @@ public class SharedCacheClientImpl extends SharedCacheClient {
}
@Override
- public Path use(ApplicationId applicationId, String resourceKey)
- throws YarnException {
+ public Path use(ApplicationId applicationId, String resourceKey,
+ String resourceName) throws YarnException {
Path resourcePath = null;
UseSharedCacheResourceRequest request = Records.newRecord(
UseSharedCacheResourceRequest.class);
@@ -129,6 +131,31 @@ public class SharedCacheClientImpl extends SharedCacheClient {
// We don't handle different exceptions separately at this point.
throw new YarnException(e);
}
+ if (resourcePath != null) {
+ if (resourcePath.getName().equals(resourceName)) {
+ // The preferred name is the same as the name of the item in the cache,
+ // so we skip generating the fragment to save space in the MRconfig.
+ return resourcePath;
+ } else {
+ // We are using the shared cache, and a preferred name has been
+ // specified that is different than the name of the resource in the
+ // shared cache. We need to set the fragment portion of the URI to
+ // preserve the desired name.
+ URI pathURI = resourcePath.toUri();
+ try {
+ // We assume that there is no existing fragment in the URI since the
+ // shared cache manager does not use fragments.
+ pathURI =
+ new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(),
+ resourceName);
+ resourcePath = new Path(pathURI);
+ } catch (URISyntaxException e) {
+ throw new YarnException(
+ "Could not create a new URI due to syntax errors: "
+ + pathURI.toString(), e);
+ }
+ }
+ }
return resourcePath;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/425a7e50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
index 3985e54..16b32db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.api.impl;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -26,6 +27,7 @@ import static org.mockito.Mockito.when;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -106,15 +108,42 @@ public class TestSharedCacheClientImpl {
}
@Test
- public void testUse() throws Exception {
+ public void testUseCacheMiss() throws Exception {
+ UseSharedCacheResourceResponse response =
+ new UseSharedCacheResourceResponsePBImpl();
+ response.setPath(null);
+ when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
+ response);
+ Path newPath = client.use(mock(ApplicationId.class), "key", null);
+ assertNull("The path is not null!", newPath);
+ }
+
+ @Test
+ public void testUseWithResourceName() throws Exception {
+ Path file = new Path("viewfs://test/path");
+ URI useUri = new URI("viewfs://test/path#linkName");
+ Path usePath = new Path(useUri);
+ UseSharedCacheResourceResponse response =
+ new UseSharedCacheResourceResponsePBImpl();
+ response.setPath(file.toString());
+ when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
+ response);
+ Path newPath = client.use(mock(ApplicationId.class), "key", "linkName");
+ assertEquals("The paths are not equal!", usePath, newPath);
+ }
+
+ @Test
+ public void testUseWithSameResourceName() throws Exception {
Path file = new Path("viewfs://test/path");
+ URI useUri = new URI("viewfs://test/path");
+ Path usePath = new Path(useUri);
UseSharedCacheResourceResponse response =
new UseSharedCacheResourceResponsePBImpl();
response.setPath(file.toString());
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
- Path newPath = client.use(mock(ApplicationId.class), "key");
- assertEquals(file, newPath);
+ Path newPath = client.use(mock(ApplicationId.class), "key", "path");
+ assertEquals("The paths are not equal!", usePath, newPath);
}
@Test(expected = YarnException.class)
@@ -122,7 +151,7 @@ public class TestSharedCacheClientImpl {
String message = "Mock IOExcepiton!";
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow(
new IOException(message));
- client.use(mock(ApplicationId.class), "key");
+ client.use(mock(ApplicationId.class), "key", null);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org