You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ct...@apache.org on 2017/09/28 22:29:19 UTC
hadoop git commit: YARN-7250. Update Shared cache client api to use
URLs.
Repository: hadoop
Updated Branches:
refs/heads/trunk 6f789fe05 -> c114da5e6
YARN-7250. Update Shared cache client api to use URLs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c114da5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c114da5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c114da5e
Branch: refs/heads/trunk
Commit: c114da5e64d14b1d9e614081c4171ea0391cb1aa
Parents: 6f789fe
Author: Chris Trezzo <ct...@apache.org>
Authored: Thu Sep 28 15:28:06 2017 -0700
Committer: Chris Trezzo <ct...@apache.org>
Committed: Thu Sep 28 15:28:06 2017 -0700
----------------------------------------------------------------------
.../yarn/client/api/SharedCacheClient.java | 22 ++++--------
.../client/api/impl/SharedCacheClientImpl.java | 36 +++++---------------
.../api/impl/TestSharedCacheClientImpl.java | 31 +++++------------
3 files changed, 23 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
index 60c1bd98..a9c1a07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.impl.SharedCacheClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -58,34 +59,25 @@ public abstract class SharedCacheClient extends AbstractService {
*
* <p>
* The <code>SharedCacheManager</code> responds with whether or not the
- * resource exists in the cache. If the resource exists, a <code>Path</code>
- * to the resource in the shared cache is returned. If the resource does not
+ * resource exists in the cache. If the resource exists, a <code>URL</code> to
+ * the resource in the shared cache is returned. If the resource does not
* exist, null is returned instead.
* </p>
*
* <p>
- * Once a path has been returned for a resource, that path is safe to use for
+ * Once a URL has been returned for a resource, that URL is safe to use for
* the lifetime of the application that corresponds to the provided
* ApplicationId.
* </p>
*
- * <p>
- * Additionally, a name for the resource should be specified. A fragment will
- * be added to the path with the desired name if the desired name is different
- * than the name of the provided path from the shared cache. This ensures that
- * if the returned path is used to create a LocalResource, then the symlink
- * created during YARN localization will match the name specified.
- * </p>
- *
* @param applicationId ApplicationId of the application using the resource
* @param resourceKey the key (i.e. checksum) that identifies the resource
- * @param resourceName the desired name of the resource
- * @return Path to the resource, or null if it does not exist
+ * @return URL to the resource, or null if it does not exist
*/
@Public
@Unstable
- public abstract Path use(ApplicationId applicationId, String resourceKey,
- String resourceName) throws YarnException;
+ public abstract URL use(ApplicationId applicationId, String resourceKey)
+ throws YarnException;
/**
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
index b910c28..3191d36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -38,6 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequ
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.SharedCacheClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -113,8 +112,8 @@ public class SharedCacheClientImpl extends SharedCacheClient {
}
@Override
- public Path use(ApplicationId applicationId, String resourceKey,
- String resourceName) throws YarnException {
+ public URL use(ApplicationId applicationId, String resourceKey)
+ throws YarnException {
Path resourcePath = null;
UseSharedCacheResourceRequest request = Records.newRecord(
UseSharedCacheResourceRequest.class);
@@ -132,31 +131,12 @@ public class SharedCacheClientImpl extends SharedCacheClient {
throw new YarnException(e);
}
if (resourcePath != null) {
- if (resourcePath.getName().equals(resourceName)) {
- // The preferred name is the same as the name of the item in the cache,
- // so we skip generating the fragment to save space in the MRconfig.
- return resourcePath;
- } else {
- // We are using the shared cache, and a preferred name has been
- // specified that is different than the name of the resource in the
- // shared cache. We need to set the fragment portion of the URI to
- // preserve the desired name.
- URI pathURI = resourcePath.toUri();
- try {
- // We assume that there is no existing fragment in the URI since the
- // shared cache manager does not use fragments.
- pathURI =
- new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(),
- resourceName);
- resourcePath = new Path(pathURI);
- } catch (URISyntaxException e) {
- throw new YarnException(
- "Could not create a new URI due to syntax errors: "
- + pathURI.toString(), e);
- }
- }
+ URL pathURL = URL.fromPath(resourcePath);
+ return pathURL;
+ } else {
+ // The resource was not in the cache.
+ return null;
}
- return resourcePath;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
index 16b32db..6a8edd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
@@ -27,7 +27,6 @@ import static org.mockito.Mockito.when;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,6 +39,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
import org.junit.AfterClass;
@@ -114,36 +114,21 @@ public class TestSharedCacheClientImpl {
response.setPath(null);
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
- Path newPath = client.use(mock(ApplicationId.class), "key", null);
- assertNull("The path is not null!", newPath);
+ URL newURL = client.use(mock(ApplicationId.class), "key");
+ assertNull("The path is not null!", newURL);
}
@Test
- public void testUseWithResourceName() throws Exception {
+ public void testUseCacheHit() throws Exception {
Path file = new Path("viewfs://test/path");
- URI useUri = new URI("viewfs://test/path#linkName");
- Path usePath = new Path(useUri);
+ URL useUrl = URL.fromPath(new Path("viewfs://test/path"));
UseSharedCacheResourceResponse response =
new UseSharedCacheResourceResponsePBImpl();
response.setPath(file.toString());
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
- Path newPath = client.use(mock(ApplicationId.class), "key", "linkName");
- assertEquals("The paths are not equal!", usePath, newPath);
- }
-
- @Test
- public void testUseWithSameResourceName() throws Exception {
- Path file = new Path("viewfs://test/path");
- URI useUri = new URI("viewfs://test/path");
- Path usePath = new Path(useUri);
- UseSharedCacheResourceResponse response =
- new UseSharedCacheResourceResponsePBImpl();
- response.setPath(file.toString());
- when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
- response);
- Path newPath = client.use(mock(ApplicationId.class), "key", "path");
- assertEquals("The paths are not equal!", usePath, newPath);
+ URL newURL = client.use(mock(ApplicationId.class), "key");
+ assertEquals("The paths are not equal!", useUrl, newURL);
}
@Test(expected = YarnException.class)
@@ -151,7 +136,7 @@ public class TestSharedCacheClientImpl {
String message = "Mock IOExcepiton!";
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow(
new IOException(message));
- client.use(mock(ApplicationId.class), "key", null);
+ client.use(mock(ApplicationId.class), "key");
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org