You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/05/28 16:38:57 UTC

[1/2] hive git commit: HIVE-19490: Locking on Insert into for non native and managed tables (Slim Bouguerra reviewed by Eugene Koifman)

Repository: hive
Updated Branches:
  refs/heads/branch-3 3c50790f3 -> 89eba2545


HIVE-19490: Locking on Insert into for non native and managed tables (Slim Bouguerra reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a9a75cfc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a9a75cfc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a9a75cfc

Branch: refs/heads/branch-3
Commit: a9a75cfc39ffffc4c12c58bbe7abfacac0d18595
Parents: 3c50790
Author: Slim Bouguerra <sl...@gmail.com>
Authored: Mon May 28 09:30:16 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon May 28 09:37:50 2018 -0700

----------------------------------------------------------------------
 .../hadoop/hive/druid/DruidStorageHandler.java  |  74 +++++-----
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java    | 147 +++++++++++--------
 .../hive/ql/metadata/HiveStorageHandler.java    |  13 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  17 ++-
 .../hive/ql/lockmgr/TestDbTxnManager2.java      |  29 ++++
 .../hive/ql/metadata/StorageHandlerMock.java    | 103 +++++++++++++
 6 files changed, 278 insertions(+), 105 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a9a75cfc/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
index 3e707e3..fc5a5fa 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
@@ -81,9 +81,11 @@ import org.apache.hadoop.hive.metastore.HiveMetaHook;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockType;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
 import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -98,7 +100,6 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.OutputFormat;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hive.common.util.ShutdownHookManager;
-
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.joda.time.DateTime;
@@ -888,6 +889,14 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
     return conf;
   }
 
+  @Override public LockType getLockType(WriteEntity writeEntity
+  ) {
+    if (writeEntity.getWriteType().equals(WriteEntity.WriteType.INSERT)) {
+      return LockType.SHARED_READ;
+    }
+    return LockType.SHARED_WRITE;
+  }
+
   @Override
   public String toString() {
     return Constants.DRUID_HIVE_STORAGE_HANDLER_ID;
@@ -918,55 +927,50 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
   }
 
   private SQLMetadataConnector getConnector() {
+    return Suppliers.memoize(this::buildConnector).get();
+  }
+
+  private SQLMetadataConnector buildConnector() {
+
     if (connector != null) {
       return connector;
     }
 
-    final String dbType = HiveConf
-            .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_TYPE);
-    final String username = HiveConf
-            .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_USERNAME);
-    final String password = HiveConf
-            .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_PASSWORD);
-    final String uri = HiveConf
-            .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_URI);
-
-
-    final Supplier<MetadataStorageConnectorConfig> storageConnectorConfigSupplier = Suppliers.ofInstance(
-            new MetadataStorageConnectorConfig() {
-              @Override
-              public String getConnectURI() {
-                return uri;
-              }
-
-              @Override
-              public String getUser() {
-                return Strings.emptyToNull(username);
-              }
-
-              @Override
-              public String getPassword() {
-                return Strings.emptyToNull(password);
-              }
-            });
+    final String dbType = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_TYPE);
+    final String username = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_USERNAME);
+    final String password = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_PASSWORD);
+    final String uri = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_URI);
+    LOG.debug("Supplying SQL Connector with DB type {}, URI {}, User {}", dbType, uri, username);
+    final Supplier<MetadataStorageConnectorConfig> storageConnectorConfigSupplier =
+        Suppliers.ofInstance(new MetadataStorageConnectorConfig() {
+          @Override public String getConnectURI() {
+            return uri;
+          }
+
+          @Override public String getUser() {
+            return Strings.emptyToNull(username);
+          }
+
+          @Override public String getPassword() {
+            return Strings.emptyToNull(password);
+          }
+        });
     if (dbType.equals("mysql")) {
       connector = new MySQLConnector(storageConnectorConfigSupplier,
-              Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
-          , new MySQLConnectorConfig());
+          Suppliers.ofInstance(getDruidMetadataStorageTablesConfig()), new MySQLConnectorConfig()
+      );
     } else if (dbType.equals("postgresql")) {
       connector = new PostgreSQLConnector(storageConnectorConfigSupplier,
-              Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
+          Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
       );
 
     } else if (dbType.equals("derby")) {
       connector = new DerbyConnector(new DerbyMetadataStorage(storageConnectorConfigSupplier.get()),
-              storageConnectorConfigSupplier, Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
+          storageConnectorConfigSupplier, Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
       );
-    }
-    else {
+    } else {
       throw new IllegalStateException(String.format("Unknown metadata storage type [%s]", dbType));
     }
-
     return connector;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a9a75cfc/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 2486321..4fd1d4e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -18,39 +18,48 @@
 package org.apache.hadoop.hive.ql.lockmgr;
 
 import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.curator.shaded.com.google.common.collect.Lists;
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.plan.HiveOperation;
-import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.LockTableDesc;
-import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.common.util.ShutdownHookManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.LockComponentBuilder;
 import org.apache.hadoop.hive.metastore.LockRequestBuilder;
-import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.hooks.Entity;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
 import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hive.common.util.ShutdownHookManager;
 import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
@@ -503,60 +512,80 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
         /* base this on HiveOperation instead?  this and DDL_NO_LOCK is peppered all over the code...
          Seems much cleaner if each stmt is identified as a particular HiveOperation (which I'd think
          makes sense everywhere).  This however would be problematic for merge...*/
-        case DDL_EXCLUSIVE:
+      case DDL_EXCLUSIVE:
+        compBuilder.setExclusive();
+        compBuilder.setOperationType(DataOperationType.NO_TXN);
+        break;
+      case INSERT_OVERWRITE:
+        t = getTable(output);
+        if (AcidUtils.isTransactionalTable(t)) {
+          if (conf.getBoolVar(HiveConf.ConfVars.TXN_OVERWRITE_X_LOCK)) {
+            compBuilder.setExclusive();
+          } else {
+            compBuilder.setSemiShared();
+          }
+          compBuilder.setOperationType(DataOperationType.UPDATE);
+        } else {
           compBuilder.setExclusive();
           compBuilder.setOperationType(DataOperationType.NO_TXN);
-          break;
-        case INSERT_OVERWRITE:
-          t = getTable(output);
-          if (AcidUtils.isTransactionalTable(t)) {
-            if(conf.getBoolVar(HiveConf.ConfVars.TXN_OVERWRITE_X_LOCK)) {
-              compBuilder.setExclusive();
-            } else {
-              compBuilder.setSemiShared();
-            }
-            compBuilder.setOperationType(DataOperationType.UPDATE);
-          } else {
+        }
+        break;
+      case INSERT:
+        assert t != null;
+        if (AcidUtils.isTransactionalTable(t)) {
+          compBuilder.setShared();
+        } else if (MetaStoreUtils.isNonNativeTable(t.getTTable())) {
+          final HiveStorageHandler storageHandler = Preconditions.checkNotNull(t.getStorageHandler(),
+              "Thought all the non native tables have an instance of storage handler"
+          );
+          LockType lockType = storageHandler.getLockType(output);
+          switch (lockType) {
+          case EXCLUSIVE:
             compBuilder.setExclusive();
-            compBuilder.setOperationType(DataOperationType.NO_TXN);
-          }
-          break;
-        case INSERT:
-          assert t != null;
-          if(AcidUtils.isTransactionalTable(t)) {
+            break;
+          case SHARED_READ:
             compBuilder.setShared();
+            break;
+          case SHARED_WRITE:
+            compBuilder.setSemiShared();
+            break;
+          default:
+            throw new IllegalArgumentException(String
+                .format("Lock type [%s] for Database.Table [%s.%s] is unknown", lockType, t.getDbName(),
+                    t.getTableName()
+                ));
           }
-          else {
-            if (conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE)) {
-              compBuilder.setExclusive();
-            } else {  // this is backward compatible for non-ACID resources, w/o ACID semantics
-              compBuilder.setShared();
-            }
+
+        } else {
+          if (conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE)) {
+            compBuilder.setExclusive();
+          } else {  // this is backward compatible for non-ACID resources, w/o ACID semantics
+            compBuilder.setShared();
           }
-          compBuilder.setOperationType(DataOperationType.INSERT);
-          break;
-        case DDL_SHARED:
-          compBuilder.setShared();
-          compBuilder.setOperationType(DataOperationType.NO_TXN);
-          break;
+        }
+        compBuilder.setOperationType(DataOperationType.INSERT);
+        break;
+      case DDL_SHARED:
+        compBuilder.setShared();
+        compBuilder.setOperationType(DataOperationType.NO_TXN);
+        break;
 
-        case UPDATE:
-          compBuilder.setSemiShared();
-          compBuilder.setOperationType(DataOperationType.UPDATE);
-          break;
-        case DELETE:
-          compBuilder.setSemiShared();
-          compBuilder.setOperationType(DataOperationType.DELETE);
-          break;
+      case UPDATE:
+        compBuilder.setSemiShared();
+        compBuilder.setOperationType(DataOperationType.UPDATE);
+        break;
+      case DELETE:
+        compBuilder.setSemiShared();
+        compBuilder.setOperationType(DataOperationType.DELETE);
+        break;
 
-        case DDL_NO_LOCK:
-          continue; // No lock required here
+      case DDL_NO_LOCK:
+        continue; // No lock required here
 
-        default:
-          throw new RuntimeException("Unknown write type " +
-              output.getWriteType().toString());
+      default:
+        throw new RuntimeException("Unknown write type " + output.getWriteType().toString());
       }
-      if(t != null) {
+      if (t != null) {
         compBuilder.setIsTransactional(AcidUtils.isTransactionalTable(t));
       }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a9a75cfc/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index 1696243..2ebb149 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -18,22 +18,23 @@
 
 package org.apache.hadoop.hive.ql.metadata;
 
-import java.util.Collections;
-import java.util.Map;
-
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
 import org.apache.hadoop.hive.common.classification.InterfaceStability;
 import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.api.LockType;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.OutputFormat;
 
+import java.util.Map;
+
 /**
  * HiveStorageHandler defines a pluggable interface for adding
  * new storage handlers to Hive.  A storage handler consists of
@@ -167,4 +168,8 @@ public interface HiveStorageHandler extends Configurable {
   {
     return null;
   }
+
+  default LockType getLockType(WriteEntity writeEntity){
+    return LockType.EXCLUSIVE;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a9a75cfc/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 312d8dc..263de3a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.parse;
 
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import com.google.common.math.IntMath;
@@ -7297,6 +7298,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         boolean overwrite = !qb.getParseInfo().isInsertIntoTable(
             String.format("%s.%s", dest_tab.getDbName(), dest_tab.getTableName()));
         createPreInsertDesc(dest_tab, overwrite);
+
+        ltd = new LoadTableDesc(queryTmpdir, table_desc, partSpec == null ? ImmutableMap.of() : partSpec);
+        ltd.setInsertOverwrite(overwrite);
+        ltd.setLoadFileType(overwrite ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING);
       }
 
       if (dest_tab.isMaterializedView()) {
@@ -14395,15 +14400,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
   }
 
   private WriteEntity.WriteType determineWriteType(LoadTableDesc ltd, boolean isNonNativeTable, String dest) {
-    // Don't know the characteristics of non-native tables,
-    // and don't have a rational way to guess, so assume the most
-    // conservative case.
-    if (isNonNativeTable) {
+
+    if (ltd == null) {
       return WriteEntity.WriteType.INSERT_OVERWRITE;
-    } else {
-      return ((ltd.getLoadFileType() == LoadFileType.REPLACE_ALL || ltd.isInsertOverwrite())
-          ? WriteEntity.WriteType.INSERT_OVERWRITE : getWriteType(dest));
     }
+    return ((ltd.getLoadFileType() == LoadFileType.REPLACE_ALL || ltd
+        .isInsertOverwrite()) ? WriteEntity.WriteType.INSERT_OVERWRITE : getWriteType(dest));
+
   }
 
   private WriteEntity.WriteType getWriteType(String dest) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a9a75cfc/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index e06f0a4..77fe736 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -771,6 +771,35 @@ public class TestDbTxnManager2 {
     conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true);
   }
 
+  @Test
+  public void testLockingOnInsertIntoNonNativeTables() throws Exception {
+    dropTable(new String[] {"tab_not_acid"});
+    checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int)  " +
+        " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'"));
+    txnMgr.openTxn(ctx, "T1");
+    checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid values(1,2)", true));
+
+    txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
+    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
+    Assert.assertEquals("Unexpected lock count", 2, locks.size());
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
+  }
+
+  @Test
+  public void testLockingOnInsertOverwriteNonNativeTables() throws Exception {
+    dropTable(new String[] {"tab_not_acid"});
+    checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int)  " +
+        " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'"));
+    txnMgr.openTxn(ctx, "T1");
+    checkCmdOnDriver(driver.compileAndRespond("insert overwrite table tab_not_acid values(1,2)", true));
+    txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
+    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
+    Assert.assertEquals("Unexpected lock count", 2, locks.size());
+    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
+  }
+
   /** The list is small, and the object is generated, so we don't use sets/equals/etc. */
   public static ShowLocksResponseElement checkLock(LockType expectedType, LockState expectedState, String expectedDb,
       String expectedTable, String expectedPartition, List<ShowLocksResponseElement> actuals) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a9a75cfc/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java
new file mode 100644
index 0000000..4de7d3e
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.metadata;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+
+import java.io.IOException;
+
+/**
+ * Mock class used for unit testing.
+ * {@link org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2#testLockingOnInsertIntoNonNativeTables()}
+ */
+public class StorageHandlerMock extends DefaultStorageHandler {
+  @Override public HiveMetaHook getMetaHook() {
+    return new HiveMetaHook() {
+      @Override public void preCreateTable(Table table) throws MetaException {
+
+      }
+
+      @Override public void rollbackCreateTable(Table table) throws MetaException {
+
+      }
+
+      @Override public void commitCreateTable(Table table) throws MetaException {
+
+      }
+
+      @Override public void preDropTable(Table table) throws MetaException {
+
+      }
+
+      @Override public void rollbackDropTable(Table table) throws MetaException {
+
+      }
+
+      @Override public void commitDropTable(Table table, boolean deleteData) throws MetaException {
+
+      }
+    };
+  }
+
+  @Override public LockType getLockType(WriteEntity writeEntity
+  ) {
+    if (writeEntity.getWriteType().equals(WriteEntity.WriteType.INSERT)) {
+      return LockType.SHARED_READ;
+    }
+    return LockType.SHARED_WRITE;
+  }
+
+  @Override public Class<? extends OutputFormat> getOutputFormatClass() {
+    return MockOutputFormat.class;
+  }
+
+  /**
+   * Dummy no op output format.
+   */
+  public static class MockOutputFormat implements OutputFormat {
+
+    @Override public RecordWriter getRecordWriter(FileSystem fileSystem, JobConf jobConf, String s,
+        Progressable progressable
+    ) throws IOException {
+      return new RecordWriter() {
+        @Override public void write(Object o, Object o2) throws IOException {
+          //noop
+        }
+
+        @Override public void close(Reporter reporter) throws IOException {
+
+        }
+      };
+    }
+
+    @Override public void checkOutputSpecs(FileSystem fileSystem, JobConf jobConf) throws IOException {
+
+    }
+  }
+}


[2/2] hive git commit: HIVE-19691: Start SessionState in materialized views registry (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

Posted by jc...@apache.org.
HIVE-19691: Start SessionState in materialized views registry (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/89eba254
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/89eba254
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/89eba254

Branch: refs/heads/branch-3
Commit: 89eba25450c6945864c0b024d809284240b3f47b
Parents: a9a75cf
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon May 28 09:31:29 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon May 28 09:37:55 2018 -0700

----------------------------------------------------------------------
 .../hive/ql/metadata/HiveMaterializedViewsRegistry.java       | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/89eba254/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 960ad76..98c9ce9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.conf.Constants;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.QueryState;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
@@ -66,6 +67,7 @@ import org.apache.hadoop.hive.ql.parse.ColumnStatsList;
 import org.apache.hadoop.hive.ql.parse.ParseUtils;
 import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
 import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -126,7 +128,7 @@ public final class HiveMaterializedViewsRegistry {
       // Create a new conf object to bypass metastore authorization, as we need to
       // retrieve all materialized views from all databases
       HiveConf conf = new HiveConf();
-      conf.set(HiveConf.ConfVars.METASTORE_FILTER_HOOK.varname,
+      conf.set(MetastoreConf.ConfVars.FILTER_HOOK.getVarname(),
           DefaultMetaStoreFilterHookImpl.class.getName());
       init(Hive.get(conf));
     } catch (HiveException e) {
@@ -159,6 +161,7 @@ public final class HiveMaterializedViewsRegistry {
     @Override
     public void run() {
       try {
+        SessionState.start(db.getConf());
         for (String dbName : db.getAllDatabases()) {
           for (Table mv : db.getAllMaterializedViewObjects(dbName)) {
             addMaterializedView(db.getConf(), mv, OpType.LOAD);
@@ -413,7 +416,7 @@ public final class HiveMaterializedViewsRegistry {
       return analyzer.genLogicalPlan(node);
     } catch (Exception e) {
       // We could not parse the view
-      LOG.error(e.getMessage());
+      LOG.error("Error parsing original query for materialized view", e);
       return null;
     }
   }