You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by bo...@apache.org on 2019/06/05 08:39:09 UTC
[sqoop] branch trunk updated: SQOOP-3438: Sqoop Import with create
hcatalog table for ORC will not work with Hive3 as the table created would
be a ACID table and transactional
This is an automated email from the ASF dual-hosted git repository.
bogi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/sqoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new be260e3 SQOOP-3438: Sqoop Import with create hcatalog table for ORC will not work with Hive3 as the table created would be a ACID table and transactional
be260e3 is described below
commit be260e3096a6a2710e661c7fe33f6b402ce66476
Author: Denes Bodo <bo...@gmail.com>
AuthorDate: Wed Jun 5 10:39:04 2019 +0200
SQOOP-3438: Sqoop Import with create hcatalog table for ORC will not work with Hive3 as the table created would be a ACID table and transactional
(Denes Bodo via Boglarka Egyed)
---
src/docs/user/hcatalog.txt | 7 +++
src/java/org/apache/sqoop/SqoopOptions.java | 10 ++++
.../sqoop/mapreduce/hcat/SqoopHCatUtilities.java | 12 +++--
src/java/org/apache/sqoop/tool/BaseSqoopTool.java | 16 ++++++
.../org/apache/sqoop/hcat/HCatalogImportTest.java | 63 ++++++++++++++++++++++
.../org/apache/sqoop/hcat/HCatalogTestUtils.java | 19 +++++++
6 files changed, 123 insertions(+), 4 deletions(-)
diff --git a/src/docs/user/hcatalog.txt b/src/docs/user/hcatalog.txt
index 2ae1d54..96a90f7 100644
--- a/src/docs/user/hcatalog.txt
+++ b/src/docs/user/hcatalog.txt
@@ -61,6 +61,13 @@ The presence of the +--hcatalog-table+ option signifies that the import
or export job is done using HCatalog tables, and it is a required option for
HCatalog jobs.
++--hcatalog-external-table+::
+Use this flag if you need to create external Hive table for example to store
+data in non-transactional tables. For e.g. with:
+--hcatalog-storage-stanza "stored as orc tblproperties (\"transactional\"=\"false\")"
+This flag can only be used when
+--create-hcatalog-table or --drop-and-create-hcatalog-table is used.
+
+--hcatalog-home+::
The home directory for the HCatalog installation. The directory is
expected to have a +lib+ subdirectory and a +share/hcatalog+ subdirectory
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index a7c19da..cec8103 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -240,6 +240,8 @@ public class SqoopOptions implements Cloneable {
@StoredAsProperty("hive.partition.value") private String hivePartitionValue;
@StoredAsProperty("hcatalog.table.name")
private String hCatTableName;
+ @StoredAsProperty("hcatalog.external.table")
+ private boolean isExternalHCatTable = false;
@StoredAsProperty("hcatalog.database.name")
private String hCatDatabaseName;
@StoredAsProperty("hcatalog.create.table")
@@ -1654,6 +1656,14 @@ public class SqoopOptions implements Cloneable {
return this.hCatTableName;
}
+ public void setExternalHCatTable(boolean value) {
+ this.isExternalHCatTable = value;
+ }
+
+ public boolean isHCatTableExternal() {
+ return this.isExternalHCatTable;
+ }
+
public void setHCatDatabaseName(String hd) {
this.hCatDatabaseName = hd;
}
diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
index 234b7a8..cd93933 100644
--- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
+++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
@@ -377,11 +377,11 @@ public final class SqoopHCatUtilities {
if (options.doCreateHCatalogTable()) {
LOG.info("Creating HCatalog table " + hCatQualifiedTableName
+ " for import");
- createHCatTable(false);
+ createHCatTable(false, options.isHCatTableExternal());
} else if (options.doDropAndCreateHCatalogTable()) {
LOG.info("Dropping and Creating HCatalog table "
+ hCatQualifiedTableName + " for import");
- createHCatTable(true);
+ createHCatTable(true, options.isHCatTableExternal());
}
// For serializing the schema to conf
HCatInputFormat hif = HCatInputFormat.setInput(hCatJob, hCatDatabaseName,
@@ -599,14 +599,18 @@ public final class SqoopHCatUtilities {
return sb;
}
- private void createHCatTable(boolean dropIfExists) throws IOException {
+ private void createHCatTable(boolean dropIfExists, boolean isExternal) throws IOException {
StringBuilder sb = new StringBuilder();
if (dropIfExists) {
sb.append("drop table ").
append(escHCatObj(hCatDatabaseName)).append('.').
append(escHCatObj(hCatTableName)).append(";\n");
}
- sb.append("create table ").
+ sb.append("create ");
+ if(isExternal) {
+ sb.append("external ");
+ }
+ sb.append("table ").
append(escHCatObj(hCatDatabaseName)).append('.');
sb.append(escHCatObj(hCatTableName)).append(" (\n\t");
boolean first = true;
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 96f06de..caf0598 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -132,6 +132,7 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
public static final String CREATE_HIVE_TABLE_ARG =
"create-hive-table";
public static final String HCATALOG_TABLE_ARG = "hcatalog-table";
+ public static final String HCATALOG_EXTERNAL_TABLE_ARG = "hcatalog-external-table";
public static final String HCATALOG_DATABASE_ARG = "hcatalog-database";
public static final String CREATE_HCATALOG_TABLE_ARG =
"create-hcatalog-table";
@@ -662,6 +663,10 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
.withLongOpt(HCATALOG_TABLE_ARG)
.create());
hCatOptions.addOption(OptionBuilder
+ .withDescription("Signing that HCatalog table shall be created as external")
+ .withLongOpt(HCATALOG_EXTERNAL_TABLE_ARG)
+ .create());
+ hCatOptions.addOption(OptionBuilder
.hasArg()
.withDescription("HCatalog database name")
.withLongOpt(HCATALOG_DATABASE_ARG)
@@ -1302,6 +1307,10 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
out.setHCatTableName(in.getOptionValue(HCATALOG_TABLE_ARG));
}
+ if (in.hasOption(HCATALOG_EXTERNAL_TABLE_ARG)) {
+ out.setExternalHCatTable(true);
+ }
+
if (in.hasOption(HCATALOG_DATABASE_ARG)) {
out.setHCatDatabaseName(in.getOptionValue(HCATALOG_DATABASE_ARG));
}
@@ -1588,6 +1597,13 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
+ " option." + HELP_STR);
}
+ if(options.isHCatTableExternal() &&
+ !(options.doCreateHCatalogTable() || options.doDropAndCreateHCatalogTable())) {
+ throw new InvalidOptionsException(String.format(
+ "Using --%s only takes effect when --%s or --%s is present",
+ HCATALOG_EXTERNAL_TABLE_ARG, CREATE_HCATALOG_TABLE_ARG, DROP_AND_CREATE_HCATALOG_TABLE));
+ }
+
if (options.doHiveImport()
&& options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) {
throw new InvalidOptionsException("Hive import is not compatible with "
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
index c7e1ea6..5b5dea9 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
@@ -62,6 +62,10 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import static org.apache.sqoop.tool.BaseSqoopTool.CREATE_HCATALOG_TABLE_ARG;
+import static org.apache.sqoop.tool.BaseSqoopTool.DROP_AND_CREATE_HCATALOG_TABLE;
+import static org.apache.sqoop.tool.BaseSqoopTool.HCATALOG_EXTERNAL_TABLE_ARG;
+import static org.apache.sqoop.tool.BaseSqoopTool.HCATALOG_STORAGE_STANZA_ARG;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -72,6 +76,8 @@ import static org.junit.Assert.fail;
public class HCatalogImportTest extends ImportJobTestCase {
private static final Log LOG =
LogFactory.getLog(HCatalogImportTest.class);
+ public static final String TABLEPROPERTIES_ORC_NONTRANSACTIONAL =
+ "\"stored as orc tblproperties (\"transactional\"=\"false\")\"";
private final HCatalogTestUtils utils = HCatalogTestUtils.instance();
private List<String> extraTestArgs = null;
private List<String> configParams = null;
@@ -771,6 +777,63 @@ public class HCatalogImportTest extends ImportJobTestCase {
}
@Test
+ public void testExternalTableCreation() throws Exception {
+ externalTableCreationCoreTest(true,
+ "--" + CREATE_HCATALOG_TABLE_ARG,
+ "--" + HCATALOG_EXTERNAL_TABLE_ARG,
+ "--" + HCATALOG_STORAGE_STANZA_ARG,
+ TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
+ }
+
+ @Test
+ public void testExternalTableDropAndCreationWithExistingTargetTable() throws Exception {
+ utils.createHCatExternalTable(getTableName().toUpperCase());
+ externalTableCreationCoreTest(false,
+ "--" + DROP_AND_CREATE_HCATALOG_TABLE,
+ "--" + HCATALOG_EXTERNAL_TABLE_ARG,
+ "--" + HCATALOG_STORAGE_STANZA_ARG,
+ TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
+ }
+
+ @Test (expected = IOException.class)
+ public void testExternalTableCreationFailsDueToExistingTable() throws Exception {
+ utils.createHCatExternalTable(getTableName().toUpperCase());
+ externalTableCreationCoreTest(false,
+ "--" + CREATE_HCATALOG_TABLE_ARG,
+ "--" + HCATALOG_EXTERNAL_TABLE_ARG,
+ "--" + HCATALOG_STORAGE_STANZA_ARG,
+ TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
+ }
+
+ @Test(expected = IOException.class)
+ public void testExternalTableCreationFailsIfNoCreateOrDropTablePresent() throws Exception {
+ externalTableCreationCoreTest(true,
+ "--" + HCATALOG_EXTERNAL_TABLE_ARG,
+ "--" + HCATALOG_STORAGE_STANZA_ARG,
+ TABLEPROPERTIES_ORC_NONTRANSACTIONAL);
+ }
+
+ private void externalTableCreationCoreTest(boolean dropHCatTableIfExists, String... lArgs) throws Exception {
+ final int TOTAL_RECORDS = 1 * 10;
+ String table = getTableName().toUpperCase();
+ ColumnGenerator[] cols = new ColumnGenerator[] {
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+ "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+ new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
+ HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+ "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+ new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
+ };
+ List<String> addlArgsArray = new ArrayList(Arrays.asList(lArgs));
+ setExtraArgs(addlArgsArray);
+ if (dropHCatTableIfExists) {
+ utils.dropHCatTableIfExists(table, SqoopHCatUtilities.DEFHCATDB);
+ }
+ runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
+ null, true, false);
+ }
+
+ @Test
public void testTableCreationWithPartition() throws Exception {
final int TOTAL_RECORDS = 1 * 10;
String table = getTableName().toUpperCase();
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
index 28b42dc..df660f5 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java
@@ -752,6 +752,25 @@ public final class HCatalogTestUtils {
return hCatFullSchema;
}
+ HCatSchema createHCatExternalTable(String table, ColumnGenerator... extraCols)
+ throws Exception {
+ HCatSchema hCatTblSchema = generateHCatTableSchema(extraCols);
+ HCatSchema hCatPartSchema = generateHCatPartitionSchema(extraCols);
+ HCatSchema hCatFullSchema = new HCatSchema(hCatTblSchema.getFields());
+ for (HCatFieldSchema hfs : hCatPartSchema.getFields()) {
+ hCatFullSchema.append(hfs);
+ }
+ String databaseName = SqoopHCatUtilities.DEFHCATDB;
+ String createCmd = getHCatCreateTableCmd(databaseName, table,
+ hCatTblSchema.getFields(), hCatPartSchema.getFields())
+ .replaceFirst(
+ "create table",
+ "create external table");
+ utils.launchHCatCli(createCmd);
+ LOG.info("Created HCatalog table " + databaseName + "." + table);
+ return hCatFullSchema;
+ }
+
private void loadHCatTable(HCatSchema hCatSchema, String table,
int count, ColumnGenerator... extraCols)
throws Exception {