You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/06/13 17:04:35 UTC
incubator-atlas git commit: ATLAS-837 Enhance Sqoop addon to handle
export operation (venkatnrangan via shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 67a1133a3 -> 5b627b5a1
ATLAS-837 Enhance Sqoop addon to handle export operation (venkatnrangan via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/5b627b5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/5b627b5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/5b627b5a
Branch: refs/heads/master
Commit: 5b627b5a1e3103f750fd4ba981ef569ad2377bd1
Parents: 67a1133
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Mon Jun 13 22:34:28 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Mon Jun 13 22:34:28 2016 +0530
----------------------------------------------------------------------
.../org/apache/atlas/sqoop/hook/SqoopHook.java | 20 +++++++++++++++-----
.../apache/atlas/sqoop/hook/SqoopHookIT.java | 15 +++++++++++++++
release-log.txt | 1 +
3 files changed, 31 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b627b5a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
index 18474ad..626c5a7 100644
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
+++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
@@ -109,8 +109,13 @@ public class SqoopHook extends SqoopJobDataPublisher {
procRef.set(SqoopDataModelGenerator.NAME, sqoopProcessName);
procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName);
procRef.set(SqoopDataModelGenerator.OPERATION, data.getOperation());
- procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef);
- procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef);
+ if (isImportOperation(data)) {
+ procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef);
+ procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef);
+ } else {
+ procRef.set(SqoopDataModelGenerator.INPUTS, hiveTableRef);
+ procRef.set(SqoopDataModelGenerator.OUTPUTS, dbStoreRef);
+ }
procRef.set(SqoopDataModelGenerator.USER, data.getUser());
procRef.set(SqoopDataModelGenerator.START_TIME, new Date(data.getStartTime()));
procRef.set(SqoopDataModelGenerator.END_TIME, new Date(data.getEndTime()));
@@ -126,15 +131,16 @@ public class SqoopHook extends SqoopJobDataPublisher {
}
static String getSqoopProcessName(Data data, String clusterName) {
- StringBuilder name = new StringBuilder(String.format("sqoop import --connect %s", data.getUrl()));
+ StringBuilder name = new StringBuilder(String.format("sqoop %s --connect %s", data.getOperation(),
+ data.getUrl()));
if (StringUtils.isNotEmpty(data.getStoreTable())) {
name.append(" --table ").append(data.getStoreTable());
}
if (StringUtils.isNotEmpty(data.getStoreQuery())) {
name.append(" --query ").append(data.getStoreQuery());
}
- name.append(String.format(" --hive-import --hive-database %s --hive-table %s --hive-cluster %s",
- data.getHiveDB().toLowerCase(), data.getHiveTable().toLowerCase(), clusterName));
+ name.append(String.format(" --hive-%s --hive-database %s --hive-table %s --hive-cluster %s",
+ data.getOperation(), data.getHiveDB().toLowerCase(), data.getHiveTable().toLowerCase(), clusterName));
return name.toString();
}
@@ -149,6 +155,10 @@ public class SqoopHook extends SqoopJobDataPublisher {
return name.toString();
}
+ static boolean isImportOperation(SqoopJobDataPublisher.Data data) {
+ return data.getOperation().toLowerCase().equals("import");
+ }
+
@Override
public void publish(SqoopJobDataPublisher.Data data) throws Exception {
Configuration atlasProperties = ApplicationProperties.get();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b627b5a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
index a81ee15..5256439 100644
--- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
+++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
@@ -88,6 +88,21 @@ public class SqoopHookIT {
assertHiveTableIsRegistered(DEFAULT_DB, "hiveTable");
}
+ @Test
+ public void testSqoopExport() throws Exception {
+ SqoopJobDataPublisher.Data d = new SqoopJobDataPublisher.Data("export", "jdbc:mysql:///localhost/db",
+ "mysqluser", "mysql", "myTable", null, "default", "hiveTable", new Properties(),
+ System.currentTimeMillis() - 100, System.currentTimeMillis());
+ SqoopHook hook = new SqoopHook();
+ hook.publish(d);
+ Thread.sleep(1000);
+ String storeName = SqoopHook.getSqoopDBStoreName(d);
+ assertDBStoreIsRegistered(storeName);
+ String name = SqoopHook.getSqoopProcessName(d, CLUSTER_NAME);
+ assertSqoopProcessIsRegistered(name);
+ assertHiveTableIsRegistered(DEFAULT_DB, "hiveTable");
+ }
+
private String assertDBStoreIsRegistered(String storeName) throws Exception {
LOG.debug("Searching for db store {}", storeName);
String query = String.format(
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b627b5a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 56b3472..8cbdecc 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-837 Enhance Sqoop addon to handle export operation (venkatnrangan via shwethags)
ATLAS-869 Make LDAP/AD properties to be configurable. (nixonrodrigues via yhemanth)
ATLAS-730 Change titan hbase table name (svimal2106 via sumasai )
ATLAS-871 Make audit repository implementation configurable (jnhagelb via shwethags)