You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/18 22:03:32 UTC
[41/67] [abbrv] hive git commit: HIVE-19898: Disable
TransactionalValidationListener when the table is not in the Hive catalog
(Jason Dere, reviewed by Eugene Koifman)
HIVE-19898: Disable TransactionalValidationListener when the table is not in the Hive catalog (Jason Dere, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ebd2c5f8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ebd2c5f8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ebd2c5f8
Branch: refs/heads/master-txnstats
Commit: ebd2c5f8a82b35eabca146520ffcd87605084618
Parents: 766c3dc
Author: Jason Dere <jd...@hortonworks.com>
Authored: Sun Jun 17 21:53:23 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Sun Jun 17 21:53:23 2018 -0700
----------------------------------------------------------------------
.../TestTransactionalValidationListener.java | 127 +++++++++++++++++++
.../TransactionalValidationListener.java | 23 +++-
.../metastore/client/MetaStoreClientTest.java | 2 +-
3 files changed, 146 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ebd2c5f8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java
new file mode 100644
index 0000000..3aaad22
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java
@@ -0,0 +1,127 @@
+package org.apache.hadoop.hive.metastore;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.MetaStoreClientTest;
+import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class TestTransactionalValidationListener extends MetaStoreClientTest {
+
+ private AbstractMetaStoreService metaStore;
+ private IMetaStoreClient client;
+ private boolean createdCatalogs = false;
+
+ @BeforeClass
+ public static void startMetaStores() {
+ Map<MetastoreConf.ConfVars, String> msConf = new HashMap<MetastoreConf.ConfVars, String>();
+
+ // Enable TransactionalValidationListener + create.as.acid
+ Map<String, String> extraConf = new HashMap<>();
+ extraConf.put("metastore.create.as.acid", "true");
+ extraConf.put("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+ extraConf.put("hive.support.concurrency", "true");
+ startMetaStores(msConf, extraConf);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ // Get new client
+ client = metaStore.getClient();
+ if (!createdCatalogs) {
+ createCatalogs();
+ createdCatalogs = true;
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ if (client != null) {
+ client.close();
+ }
+ } finally {
+ client = null;
+ }
+ }
+
+ public TestTransactionalValidationListener(String name, AbstractMetaStoreService metaStore) throws Exception {
+ this.metaStore = metaStore;
+ }
+
+ private void createCatalogs() throws Exception {
+ String[] catNames = {"spark", "myapp"};
+ String[] location = {MetaStoreTestUtils.getTestWarehouseDir("spark"),
+ MetaStoreTestUtils.getTestWarehouseDir("myapp")};
+
+ for (int i = 0; i < catNames.length; i++) {
+ Catalog cat = new CatalogBuilder()
+ .setName(catNames[i])
+ .setLocation(location[i])
+ .build();
+ client.createCatalog(cat);
+ File dir = new File(cat.getLocationUri());
+ Assert.assertTrue(dir.exists() && dir.isDirectory());
+ }
+ }
+
+ private Table createOrcTable(String catalog) throws Exception {
+ Table table = new Table();
+ StorageDescriptor sd = new StorageDescriptor();
+ List<FieldSchema> cols = new ArrayList<>();
+
+ table.setDbName("default");
+ table.setTableName("test_table");
+ cols.add(new FieldSchema("column_name", "int", null));
+ sd.setCols(cols);
+ sd.setSerdeInfo(new SerDeInfo());
+ sd.setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
+ sd.setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
+ table.setSd(sd);
+ table.setCatName(catalog);
+ table.setTableType("MANAGED_TABLE");
+
+ client.createTable(table);
+ Table createdTable = client.getTable(catalog, table.getDbName(), table.getTableName());
+ return createdTable;
+ }
+
+ @Test
+ public void testCreateAsAcid() throws Exception {
+ // Table created in hive catalog should have been automatically set to transactional
+ Table createdTable = createOrcTable("hive");
+ assertTrue(AcidUtils.isTransactionalTable(createdTable));
+
+ // Non-hive catalogs should not be transactional
+ createdTable = createOrcTable("spark");
+ assertFalse(AcidUtils.isTransactionalTable(createdTable));
+
+ createdTable = createOrcTable("myapp");
+ assertFalse(AcidUtils.isTransactionalTable(createdTable));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ebd2c5f8/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 76069bb..33cf542 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -53,8 +53,11 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default";
public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only";
+ private final Set<String> supportedCatalogs = new HashSet<String>();
+
TransactionalValidationListener(Configuration conf) {
super(conf);
+ supportedCatalogs.add("hive");
}
@Override
@@ -73,11 +76,21 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
}
private void handle(PreAlterTableEvent context) throws MetaException {
- handleAlterTableTransactionalProp(context);
+ if (supportedCatalogs.contains(getTableCatalog(context.getNewTable()))) {
+ handleAlterTableTransactionalProp(context);
+ }
}
private void handle(PreCreateTableEvent context) throws MetaException {
- handleCreateTableTransactionalProp(context);
+ if (supportedCatalogs.contains(getTableCatalog(context.getTable()))) {
+ handleCreateTableTransactionalProp(context);
+ }
+ }
+
+ private String getTableCatalog(Table table) {
+ String catName = table.isSetCatName() ? table.getCatName() :
+ MetaStoreUtils.getDefaultCatalog(getConf());
+ return catName.toLowerCase();
}
/**
@@ -230,7 +243,8 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
newTable.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL));
return;
}
- Configuration conf = MetastoreConf.newMetastoreConf();
+
+ Configuration conf = getConf();
boolean makeAcid =
//no point making an acid table if these other props are not set since it will just throw
//exceptions when someone tries to use the table.
@@ -437,8 +451,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
try {
Warehouse wh = hmsHandler.getWh();
if (table.getSd().getLocation() == null || table.getSd().getLocation().isEmpty()) {
- String catName = table.isSetCatName() ? table.getCatName() :
- MetaStoreUtils.getDefaultCatalog(getConf());
+ String catName = getTableCatalog(table);
tablePath = wh.getDefaultTablePath(hmsHandler.getMS().getDatabase(
catName, table.getDbName()), table);
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/ebd2c5f8/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java
index a0e9d32..dc48fa8 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java
@@ -67,7 +67,7 @@ public abstract class MetaStoreClientTest {
* @param msConf Specific MetaStore configuration values
* @param extraConf Specific other configuration values
*/
- static void startMetaStores(Map<MetastoreConf.ConfVars, String> msConf,
+ public static void startMetaStores(Map<MetastoreConf.ConfVars, String> msConf,
Map<String, String> extraConf) {
for(AbstractMetaStoreService metaStoreService : metaStoreServices) {
try {