You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/02/16 00:23:31 UTC
atlas git commit: ATLAS-2444: fix for IT failures
Repository: atlas
Updated Branches:
refs/heads/master 4152bc6d5 -> c2be0646d
ATLAS-2444: fix for IT failures
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/c2be0646
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/c2be0646
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/c2be0646
Branch: refs/heads/master
Commit: c2be0646d95b7b38ea3b06382091e0e5bcf9988a
Parents: 4152bc6
Author: apoorvnaik <ap...@apache.org>
Authored: Wed Feb 14 21:58:31 2018 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Feb 15 16:23:24 2018 -0800
----------------------------------------------------------------------
.../apache/atlas/hive/bridge/HiveMetaStoreBridge.java | 7 ++++++-
.../src/test/java/org/apache/atlas/hive/HiveITBase.java | 12 ++++++++++--
.../java/org/apache/atlas/hive/hook/HiveHookIT.java | 5 ++---
.../org/apache/atlas/utils/HdfsNameServiceResolver.java | 7 ++++---
4 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 3a775b6..4047c16 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -615,7 +615,12 @@ public class HiveMetaStoreBridge {
ref.set("nameServiceId", nameServiceID);
} else {
ref.set("path", pathUri);
- ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getHdfsPathQualifiedName(clusterName, pathUri));
+ // Only append clusterName for the HDFS path
+ if (pathUri.startsWith(HdfsNameServiceResolver.HDFS_SCHEME)) {
+ ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getHdfsPathQualifiedName(clusterName, pathUri));
+ } else {
+ ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
+ }
}
ref.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
return ref;
http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
index 01418cf..217c6aa 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
@@ -231,13 +231,21 @@ public class HiveITBase {
Referenceable hdfsPathRef = atlasClient.getEntity(hdfsPathId);
Assert.assertEquals(hdfsPathRef.get("path"), testPathNormed);
Assert.assertEquals(hdfsPathRef.get(NAME), Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
- Assert.assertEquals(hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), testPathNormed);
+ if (testPathNormed != null) {
+ Assert.assertTrue(((String)hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)).startsWith(testPathNormed));
+ }
}
}
private String assertHDFSPathIsRegistered(String path) throws Exception {
LOG.debug("Searching for hdfs path {}", path);
- return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
+ // ATLAS-2444 HDFS name node federation adds the cluster name to the qualifiedName
+ if (path.startsWith("hdfs://")) {
+ String pathWithCluster = path + "@" + CLUSTER_NAME;
+ return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathWithCluster, null);
+ } else {
+ return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
+ }
}
protected String assertDatabaseIsRegistered(String dbName) throws Exception {
http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 1b1ebec..f96d47d 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -263,11 +263,10 @@ public class HiveHookIT extends HiveITBase {
Iterator<? extends Entity> iterator = expectedTables.iterator();
for(int i = 0; i < expectedTables.size(); i++) {
Entity hiveEntity = iterator.next();
- if (Entity.Type.TABLE.equals(hiveEntity.getType()) ||
- Entity.Type.DFS_DIR.equals(hiveEntity.getType())) {
+ if (Entity.Type.TABLE.equals(hiveEntity.getType()) || Entity.Type.DFS_DIR.equals(hiveEntity.getType())) {
Referenceable entity = atlasClient.getEntity(tableRef.get(i)._getId());
LOG.debug("Validating output {} {} ", i, entity);
- Assert.assertEquals(entity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), hiveEntity.getName());
+ assertTrue(((String)entity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)).startsWith(hiveEntity.getName()));
}
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java b/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java
index 60e1925..7f2d611 100644
--- a/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java
+++ b/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java
@@ -32,8 +32,9 @@ import java.util.Objects;
public class HdfsNameServiceResolver {
private static final Logger LOG = LoggerFactory.getLogger(HdfsNameServiceResolver.class);
+ public static final String HDFS_SCHEME = "hdfs://";
+
private static final int DEFAULT_PORT = 8020;
- private static final String HDFS_SCHEME = "hdfs://";
private static final String HDFS_NAMESERVICE_PROPERTY_KEY = "dfs.nameservices";
private static final String HDFS_INTERNAL_NAMESERVICE_PROPERTY_KEY = "dfs.internal.nameservices";
private static final String HDFS_NAMENODES_HA_NODES_PREFIX = "dfs.ha.namenodes.";
@@ -83,7 +84,7 @@ public class HdfsNameServiceResolver {
String ret = path;
// Only handle URLs that begin with hdfs://
- if (path.indexOf(HDFS_SCHEME) == 0) {
+ if (path != null && path.indexOf(HDFS_SCHEME) == 0) {
URI uri = new Path(path).toUri();
String nsId;
@@ -114,7 +115,7 @@ public class HdfsNameServiceResolver {
String ret = "";
// Only handle path URLs that begin with hdfs://
- if (path.indexOf(HDFS_SCHEME) == 0) {
+ if (path != null && path.indexOf(HDFS_SCHEME) == 0) {
try {
URI uri = new Path(path).toUri();