You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/03/24 07:04:26 UTC
incubator-atlas git commit: ATLAS-588 import-hive.sh fails while
importing partitions for a non-partitioned table (sumasai via shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master dd66e1580 -> 985465fc7
ATLAS-588 import-hive.sh fails while importing partitions for a non-partitioned table (sumasai via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/985465fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/985465fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/985465fc
Branch: refs/heads/master
Commit: 985465fc73be355986e326ee3d491d3b6a077e05
Parents: dd66e15
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Thu Mar 24 11:34:16 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Thu Mar 24 11:34:16 2016 +0530
----------------------------------------------------------------------
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 7 ++++-
.../hive/bridge/HiveMetaStoreBridgeTest.java | 31 ++++++++++++++++++++
release-log.txt | 1 +
3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index ea38750..0680e65 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -18,6 +18,7 @@
package org.apache.atlas.hive.bridge;
+import com.google.common.base.Joiner;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
@@ -435,7 +436,11 @@ public class HiveMetaStoreBridge {
List<Partition> tableParts = hiveClient.getPartitions(table);
for (Partition hivePart : tableParts) {
- registerPartition(tableReferenceable, sdReferenceable, hivePart);
+ if (hivePart.getValues() != null && hivePart.getValues().size() > 0) {
+ registerPartition(tableReferenceable, sdReferenceable, hivePart);
+ } else {
+ LOG.info("Skipping partition for table {} since partition values are {}", getTableQualifiedName(clusterName, table.getDbName(), table.getTableName()), StringUtils.join(hivePart.getValues(), ","));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
index f8cfb71..c717c0f 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
@@ -35,6 +35,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.mockito.ArgumentMatcher;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
+import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
@@ -163,6 +164,34 @@ public class HiveMetaStoreBridgeTest {
new Integer(lastAccessTime))));
}
+ @Test
+ public void testImportWhenPartitionKeysAreNull() throws Exception {
+ setupDB(hiveClient, TEST_DB_NAME);
+ Table hiveTable = setupTable(hiveClient, TEST_DB_NAME, TEST_TABLE_NAME);
+
+ returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME);
+
+ when(atlasClient.searchByDSL(HiveMetaStoreBridge.getTableDSLQuery(CLUSTER_NAME, TEST_DB_NAME,
+ TEST_TABLE_NAME,
+ HiveDataTypes.HIVE_TABLE.getName()))).thenReturn(
+ getEntityReference("82e06b34-9151-4023-aa9d-b82103a50e77"));
+ when(atlasClient.getEntity("82e06b34-9151-4023-aa9d-b82103a50e77")).thenReturn(createTableReference());
+
+ Partition partition = mock(Partition.class);
+ when(partition.getTable()).thenReturn(hiveTable);
+ List partitionValues = Arrays.asList(new String[]{});
+ when(partition.getValues()).thenReturn(partitionValues);
+
+ when(hiveClient.getPartitions(hiveTable)).thenReturn(Arrays.asList(new Partition[]{partition}));
+
+ HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient);
+ try {
+ bridge.importHiveMetadata();
+ } catch (Exception e) {
+ Assert.fail("Partition with null key caused import to fail with exception ", e);
+ }
+ }
+
private JSONArray getPartitionReference(String id) throws JSONException {
JSONObject resultEntry = new JSONObject();
resultEntry.put(HiveMetaStoreBridge.SEARCH_ENTRY_GUID_ATTR, id);
@@ -203,4 +232,6 @@ public class HiveMetaStoreBridgeTest {
return attrValue.equals(((Referenceable) o).get(attrName));
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 02f2528..4569e55 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-588 import-hive.sh fails while importing partitions for a non-partitioned table (sumasai via shwethags)
ATLAS-575 jetty-maven-plugin fails with ShutdownMonitorThread already started (shwethags)
ATLAS-408 UI : Add a close link (x) on the top right when Tag is added (darshankumar89 via shwethags)
ATLAS-524 Support alter database (sumasai via shwethags)