You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/05/28 16:37:16 UTC
[1/4] hive git commit: HIVE-19490: Locking on Insert into for non
native and managed tables (Slim Bouguerra reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master 43945fb19 -> 780b0127f
HIVE-19490: Locking on Insert into for non native and managed tables (Slim Bouguerra reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/619f79d9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/619f79d9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/619f79d9
Branch: refs/heads/master
Commit: 619f79d9c4ffec2f6b9ebad6fd6e73805fb55dbf
Parents: 0240f5d
Author: Slim Bouguerra <sl...@gmail.com>
Authored: Mon May 28 09:30:16 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon May 28 09:35:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/druid/DruidStorageHandler.java | 74 +++++-----
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 147 +++++++++++--------
.../hive/ql/metadata/HiveStorageHandler.java | 13 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 17 ++-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 29 ++++
.../hive/ql/metadata/StorageHandlerMock.java | 103 +++++++++++++
6 files changed, 278 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/619f79d9/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
index 3e707e3..fc5a5fa 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
@@ -81,9 +81,11 @@ import org.apache.hadoop.hive.metastore.HiveMetaHook;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo;
import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -98,7 +100,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.common.util.ShutdownHookManager;
-
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import org.joda.time.DateTime;
@@ -888,6 +889,14 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
return conf;
}
+ @Override public LockType getLockType(WriteEntity writeEntity
+ ) {
+ if (writeEntity.getWriteType().equals(WriteEntity.WriteType.INSERT)) {
+ return LockType.SHARED_READ;
+ }
+ return LockType.SHARED_WRITE;
+ }
+
@Override
public String toString() {
return Constants.DRUID_HIVE_STORAGE_HANDLER_ID;
@@ -918,55 +927,50 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
}
private SQLMetadataConnector getConnector() {
+ return Suppliers.memoize(this::buildConnector).get();
+ }
+
+ private SQLMetadataConnector buildConnector() {
+
if (connector != null) {
return connector;
}
- final String dbType = HiveConf
- .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_TYPE);
- final String username = HiveConf
- .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_USERNAME);
- final String password = HiveConf
- .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_PASSWORD);
- final String uri = HiveConf
- .getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_URI);
-
-
- final Supplier<MetadataStorageConnectorConfig> storageConnectorConfigSupplier = Suppliers.ofInstance(
- new MetadataStorageConnectorConfig() {
- @Override
- public String getConnectURI() {
- return uri;
- }
-
- @Override
- public String getUser() {
- return Strings.emptyToNull(username);
- }
-
- @Override
- public String getPassword() {
- return Strings.emptyToNull(password);
- }
- });
+ final String dbType = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_TYPE);
+ final String username = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_USERNAME);
+ final String password = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_PASSWORD);
+ final String uri = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_URI);
+ LOG.debug("Supplying SQL Connector with DB type {}, URI {}, User {}", dbType, uri, username);
+ final Supplier<MetadataStorageConnectorConfig> storageConnectorConfigSupplier =
+ Suppliers.ofInstance(new MetadataStorageConnectorConfig() {
+ @Override public String getConnectURI() {
+ return uri;
+ }
+
+ @Override public String getUser() {
+ return Strings.emptyToNull(username);
+ }
+
+ @Override public String getPassword() {
+ return Strings.emptyToNull(password);
+ }
+ });
if (dbType.equals("mysql")) {
connector = new MySQLConnector(storageConnectorConfigSupplier,
- Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
- , new MySQLConnectorConfig());
+ Suppliers.ofInstance(getDruidMetadataStorageTablesConfig()), new MySQLConnectorConfig()
+ );
} else if (dbType.equals("postgresql")) {
connector = new PostgreSQLConnector(storageConnectorConfigSupplier,
- Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
+ Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
);
} else if (dbType.equals("derby")) {
connector = new DerbyConnector(new DerbyMetadataStorage(storageConnectorConfigSupplier.get()),
- storageConnectorConfigSupplier, Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
+ storageConnectorConfigSupplier, Suppliers.ofInstance(getDruidMetadataStorageTablesConfig())
);
- }
- else {
+ } else {
throw new IllegalStateException(String.format("Unknown metadata storage type [%s]", dbType));
}
-
return connector;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/619f79d9/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 2486321..4fd1d4e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -18,39 +18,48 @@
package org.apache.hadoop.hive.ql.lockmgr;
import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.curator.shaded.com.google.common.collect.Lists;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.plan.HiveOperation;
-import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.LockTableDesc;
-import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.common.util.ShutdownHookManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.LockComponentBuilder;
import org.apache.hadoop.hive.metastore.LockRequestBuilder;
-import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.hooks.Entity;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hive.common.util.ShutdownHookManager;
import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
@@ -503,60 +512,80 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
/* base this on HiveOperation instead? this and DDL_NO_LOCK is peppered all over the code...
Seems much cleaner if each stmt is identified as a particular HiveOperation (which I'd think
makes sense everywhere). This however would be problematic for merge...*/
- case DDL_EXCLUSIVE:
+ case DDL_EXCLUSIVE:
+ compBuilder.setExclusive();
+ compBuilder.setOperationType(DataOperationType.NO_TXN);
+ break;
+ case INSERT_OVERWRITE:
+ t = getTable(output);
+ if (AcidUtils.isTransactionalTable(t)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.TXN_OVERWRITE_X_LOCK)) {
+ compBuilder.setExclusive();
+ } else {
+ compBuilder.setSemiShared();
+ }
+ compBuilder.setOperationType(DataOperationType.UPDATE);
+ } else {
compBuilder.setExclusive();
compBuilder.setOperationType(DataOperationType.NO_TXN);
- break;
- case INSERT_OVERWRITE:
- t = getTable(output);
- if (AcidUtils.isTransactionalTable(t)) {
- if(conf.getBoolVar(HiveConf.ConfVars.TXN_OVERWRITE_X_LOCK)) {
- compBuilder.setExclusive();
- } else {
- compBuilder.setSemiShared();
- }
- compBuilder.setOperationType(DataOperationType.UPDATE);
- } else {
+ }
+ break;
+ case INSERT:
+ assert t != null;
+ if (AcidUtils.isTransactionalTable(t)) {
+ compBuilder.setShared();
+ } else if (MetaStoreUtils.isNonNativeTable(t.getTTable())) {
+ final HiveStorageHandler storageHandler = Preconditions.checkNotNull(t.getStorageHandler(),
+ "Thought all the non native tables have an instance of storage handler"
+ );
+ LockType lockType = storageHandler.getLockType(output);
+ switch (lockType) {
+ case EXCLUSIVE:
compBuilder.setExclusive();
- compBuilder.setOperationType(DataOperationType.NO_TXN);
- }
- break;
- case INSERT:
- assert t != null;
- if(AcidUtils.isTransactionalTable(t)) {
+ break;
+ case SHARED_READ:
compBuilder.setShared();
+ break;
+ case SHARED_WRITE:
+ compBuilder.setSemiShared();
+ break;
+ default:
+ throw new IllegalArgumentException(String
+ .format("Lock type [%s] for Database.Table [%s.%s] is unknown", lockType, t.getDbName(),
+ t.getTableName()
+ ));
}
- else {
- if (conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE)) {
- compBuilder.setExclusive();
- } else { // this is backward compatible for non-ACID resources, w/o ACID semantics
- compBuilder.setShared();
- }
+
+ } else {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE)) {
+ compBuilder.setExclusive();
+ } else { // this is backward compatible for non-ACID resources, w/o ACID semantics
+ compBuilder.setShared();
}
- compBuilder.setOperationType(DataOperationType.INSERT);
- break;
- case DDL_SHARED:
- compBuilder.setShared();
- compBuilder.setOperationType(DataOperationType.NO_TXN);
- break;
+ }
+ compBuilder.setOperationType(DataOperationType.INSERT);
+ break;
+ case DDL_SHARED:
+ compBuilder.setShared();
+ compBuilder.setOperationType(DataOperationType.NO_TXN);
+ break;
- case UPDATE:
- compBuilder.setSemiShared();
- compBuilder.setOperationType(DataOperationType.UPDATE);
- break;
- case DELETE:
- compBuilder.setSemiShared();
- compBuilder.setOperationType(DataOperationType.DELETE);
- break;
+ case UPDATE:
+ compBuilder.setSemiShared();
+ compBuilder.setOperationType(DataOperationType.UPDATE);
+ break;
+ case DELETE:
+ compBuilder.setSemiShared();
+ compBuilder.setOperationType(DataOperationType.DELETE);
+ break;
- case DDL_NO_LOCK:
- continue; // No lock required here
+ case DDL_NO_LOCK:
+ continue; // No lock required here
- default:
- throw new RuntimeException("Unknown write type " +
- output.getWriteType().toString());
+ default:
+ throw new RuntimeException("Unknown write type " + output.getWriteType().toString());
}
- if(t != null) {
+ if (t != null) {
compBuilder.setIsTransactional(AcidUtils.isTransactionalTable(t));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/619f79d9/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index 1696243..2ebb149 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -18,22 +18,23 @@
package org.apache.hadoop.hive.ql.metadata;
-import java.util.Collections;
-import java.util.Map;
-
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
+import java.util.Map;
+
/**
* HiveStorageHandler defines a pluggable interface for adding
* new storage handlers to Hive. A storage handler consists of
@@ -167,4 +168,8 @@ public interface HiveStorageHandler extends Configurable {
{
return null;
}
+
+ default LockType getLockType(WriteEntity writeEntity){
+ return LockType.EXCLUSIVE;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/619f79d9/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 55e4df3..e23e403 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.parse;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.math.IntMath;
@@ -7298,6 +7299,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
boolean overwrite = !qb.getParseInfo().isInsertIntoTable(
String.format("%s.%s", dest_tab.getDbName(), dest_tab.getTableName()));
createPreInsertDesc(dest_tab, overwrite);
+
+ ltd = new LoadTableDesc(queryTmpdir, table_desc, partSpec == null ? ImmutableMap.of() : partSpec);
+ ltd.setInsertOverwrite(overwrite);
+ ltd.setLoadFileType(overwrite ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING);
}
if (dest_tab.isMaterializedView()) {
@@ -14409,15 +14414,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private WriteEntity.WriteType determineWriteType(LoadTableDesc ltd, boolean isNonNativeTable, String dest) {
- // Don't know the characteristics of non-native tables,
- // and don't have a rational way to guess, so assume the most
- // conservative case.
- if (isNonNativeTable) {
+
+ if (ltd == null) {
return WriteEntity.WriteType.INSERT_OVERWRITE;
- } else {
- return ((ltd.getLoadFileType() == LoadFileType.REPLACE_ALL || ltd.isInsertOverwrite())
- ? WriteEntity.WriteType.INSERT_OVERWRITE : getWriteType(dest));
}
+ return ((ltd.getLoadFileType() == LoadFileType.REPLACE_ALL || ltd
+ .isInsertOverwrite()) ? WriteEntity.WriteType.INSERT_OVERWRITE : getWriteType(dest));
+
}
private WriteEntity.WriteType getWriteType(String dest) {
http://git-wip-us.apache.org/repos/asf/hive/blob/619f79d9/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index e06f0a4..77fe736 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -771,6 +771,35 @@ public class TestDbTxnManager2 {
conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true);
}
+ @Test
+ public void testLockingOnInsertIntoNonNativeTables() throws Exception {
+ dropTable(new String[] {"tab_not_acid"});
+ checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int) " +
+ " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'"));
+ txnMgr.openTxn(ctx, "T1");
+ checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid values(1,2)", true));
+
+ txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
+ List<ShowLocksResponseElement> locks = getLocks(txnMgr);
+ Assert.assertEquals("Unexpected lock count", 2, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
+ }
+
+ @Test
+ public void testLockingOnInsertOverwriteNonNativeTables() throws Exception {
+ dropTable(new String[] {"tab_not_acid"});
+ checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int) " +
+ " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'"));
+ txnMgr.openTxn(ctx, "T1");
+ checkCmdOnDriver(driver.compileAndRespond("insert overwrite table tab_not_acid values(1,2)", true));
+ txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
+ List<ShowLocksResponseElement> locks = getLocks(txnMgr);
+ Assert.assertEquals("Unexpected lock count", 2, locks.size());
+ checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
+ }
+
/** The list is small, and the object is generated, so we don't use sets/equals/etc. */
public static ShowLocksResponseElement checkLock(LockType expectedType, LockState expectedState, String expectedDb,
String expectedTable, String expectedPartition, List<ShowLocksResponseElement> actuals) {
http://git-wip-us.apache.org/repos/asf/hive/blob/619f79d9/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java
new file mode 100644
index 0000000..4de7d3e
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StorageHandlerMock.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.metadata;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+
+import java.io.IOException;
+
+/**
+ * Mock class used for unit testing.
+ * {@link org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2#testLockingOnInsertIntoNonNativeTables()}
+ */
+public class StorageHandlerMock extends DefaultStorageHandler {
+ @Override public HiveMetaHook getMetaHook() {
+ return new HiveMetaHook() {
+ @Override public void preCreateTable(Table table) throws MetaException {
+
+ }
+
+ @Override public void rollbackCreateTable(Table table) throws MetaException {
+
+ }
+
+ @Override public void commitCreateTable(Table table) throws MetaException {
+
+ }
+
+ @Override public void preDropTable(Table table) throws MetaException {
+
+ }
+
+ @Override public void rollbackDropTable(Table table) throws MetaException {
+
+ }
+
+ @Override public void commitDropTable(Table table, boolean deleteData) throws MetaException {
+
+ }
+ };
+ }
+
+ @Override public LockType getLockType(WriteEntity writeEntity
+ ) {
+ if (writeEntity.getWriteType().equals(WriteEntity.WriteType.INSERT)) {
+ return LockType.SHARED_READ;
+ }
+ return LockType.SHARED_WRITE;
+ }
+
+ @Override public Class<? extends OutputFormat> getOutputFormatClass() {
+ return MockOutputFormat.class;
+ }
+
+ /**
+ * Dummy no op output format.
+ */
+ public static class MockOutputFormat implements OutputFormat {
+
+ @Override public RecordWriter getRecordWriter(FileSystem fileSystem, JobConf jobConf, String s,
+ Progressable progressable
+ ) throws IOException {
+ return new RecordWriter() {
+ @Override public void write(Object o, Object o2) throws IOException {
+ //noop
+ }
+
+ @Override public void close(Reporter reporter) throws IOException {
+
+ }
+ };
+ }
+
+ @Override public void checkOutputSpecs(FileSystem fileSystem, JobConf jobConf) throws IOException {
+
+ }
+ }
+}
[2/4] hive git commit: HIVE-19118: Vectorization: Turning on
vectorization in escape_crlf produces wrong results (Haifeng Chen,
reviewed by Matt McCline)
Posted by jc...@apache.org.
HIVE-19118: Vectorization: Turning on vectorization in escape_crlf produces wrong results (Haifeng Chen, reviewed by Matt McCline)
This reverts commit 355cd73b38625d52894d5fa9b19efb9e9b4c105b.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0240f5df
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0240f5df
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0240f5df
Branch: refs/heads/master
Commit: 0240f5df61b9f6f91431ef169206c1375f2de095
Parents: 92eb2b9
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon May 28 09:33:40 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon May 28 09:35:12 2018 -0700
----------------------------------------------------------------------
.../test/queries/clientpositive/escape_crlf.q | 1 -
.../clientpositive/vectorization_escape_crlf.q | 22 +++++
.../vectorization_escape_crlf.q.out | 92 ++++++++++++++++++++
.../lazy/fast/LazySimpleDeserializeRead.java | 5 +-
.../fast/TestLazySimpleDeserializeRead.java | 84 ++++++++++++++++++
5 files changed, 201 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0240f5df/ql/src/test/queries/clientpositive/escape_crlf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/escape_crlf.q b/ql/src/test/queries/clientpositive/escape_crlf.q
index 0b7b77c..a2b7f65 100644
--- a/ql/src/test/queries/clientpositive/escape_crlf.q
+++ b/ql/src/test/queries/clientpositive/escape_crlf.q
@@ -1,4 +1,3 @@
--- Suppress vectorization due to known bug. See HIVE-19118.
set hive.vectorized.execution.enabled=false;
set hive.test.vectorized.execution.enabled.override=disable;
http://git-wip-us.apache.org/repos/asf/hive/blob/0240f5df/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q b/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q
new file mode 100644
index 0000000..e8ae720
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q
@@ -0,0 +1,22 @@
+set hive.vectorized.execution.enabled=true;
+set hive.test.vectorized.execution.enabled.override=enable;
+
+DROP TABLE IF EXISTS base_tab;
+CREATE TABLE base_tab(a STRING, b STRING)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|';
+DESCRIBE EXTENDED base_tab;
+
+LOAD DATA LOCAL INPATH '../../data/files/escape_crlf.txt' OVERWRITE INTO TABLE base_tab;
+-- No crlf escaping
+SELECT * FROM base_tab;
+
+-- Crlf escaping
+ALTER TABLE base_tab SET SERDEPROPERTIES ('escape.delim'='\\', 'serialization.escape.crlf'='true');
+SELECT * FROM base_tab;
+
+SET hive.fetch.task.conversion=none;
+-- Make sure intermediate serde works correctly
+SELECT * FROM base_tab;
+
+DROP TABLE base_tab;
http://git-wip-us.apache.org/repos/asf/hive/blob/0240f5df/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out b/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out
new file mode 100644
index 0000000..8b5df8c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out
@@ -0,0 +1,92 @@
+PREHOOK: query: DROP TABLE IF EXISTS base_tab
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS base_tab
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE base_tab(a STRING, b STRING)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@base_tab
+POSTHOOK: query: CREATE TABLE base_tab(a STRING, b STRING)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@base_tab
+PREHOOK: query: DESCRIBE EXTENDED base_tab
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@base_tab
+POSTHOOK: query: DESCRIBE EXTENDED base_tab
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@base_tab
+a string
+b string
+
+#### A masked pattern was here ####
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/escape_crlf.txt' OVERWRITE INTO TABLE base_tab
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@base_tab
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/escape_crlf.txt' OVERWRITE INTO TABLE base_tab
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@base_tab
+PREHOOK: query: SELECT * FROM base_tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@base_tab
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM base_tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@base_tab
+#### A masked pattern was here ####
+This\nis\rthe first\r\nmulti-line field field1-2
+This\nis\rthe second\r\nmulti-line field field2-2
+PREHOOK: query: ALTER TABLE base_tab SET SERDEPROPERTIES ('escape.delim'='\\', 'serialization.escape.crlf'='true')
+PREHOOK: type: ALTERTABLE_SERDEPROPERTIES
+PREHOOK: Input: default@base_tab
+PREHOOK: Output: default@base_tab
+POSTHOOK: query: ALTER TABLE base_tab SET SERDEPROPERTIES ('escape.delim'='\\', 'serialization.escape.crlf'='true')
+POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES
+POSTHOOK: Input: default@base_tab
+POSTHOOK: Output: default@base_tab
+PREHOOK: query: SELECT * FROM base_tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@base_tab
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM base_tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@base_tab
+#### A masked pattern was here ####
+This
+is
+the first
+multi-line field field1-2
+This
+is
+the second
+multi-line field field2-2
+PREHOOK: query: SELECT * FROM base_tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@base_tab
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM base_tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@base_tab
+#### A masked pattern was here ####
+This
+is
+the first
+multi-line field field1-2
+This
+is
+the second
+multi-line field field2-2
+PREHOOK: query: DROP TABLE base_tab
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@base_tab
+PREHOOK: Output: default@base_tab
+POSTHOOK: query: DROP TABLE base_tab
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@base_tab
+POSTHOOK: Output: default@base_tab
http://git-wip-us.apache.org/repos/asf/hive/blob/0240f5df/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
index dd88da8..1890f18 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
@@ -917,10 +917,11 @@ public final class LazySimpleDeserializeRead extends DeserializeRead {
private void copyToBuffer(byte[] buffer, int bufferStart, int bufferLength) {
final int fieldStart = currentFieldStart;
+ final int fieldLength = currentFieldLength;
int k = 0;
- for (int i = 0; i < bufferLength; i++) {
+ for (int i = 0; i < fieldLength; i++) {
byte b = bytes[fieldStart + i];
- if (b == escapeChar && i < bufferLength - 1) {
+ if (b == escapeChar && i < fieldLength - 1) {
++i;
// Check if it's '\r' or '\n'
if (bytes[fieldStart + i] == 'r') {
http://git-wip-us.apache.org/repos/asf/hive/blob/0240f5df/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
new file mode 100644
index 0000000..a7873f2
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy.fast;
+
+import junit.framework.TestCase;
+
+import java.util.Properties;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Unit tests for LazySimpleDeserializeRead.
+ *
+ */
+public class TestLazySimpleDeserializeRead extends TestCase {
+
+ /**
+ * Test for escaping.
+ *
+ */
+ public void testEscaping() throws Exception {
+ HiveConf hconf = new HiveConf();
+
+ // set the escaping related properties
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.FIELD_DELIM, "|");
+ props.setProperty(serdeConstants.ESCAPE_CHAR, "\\");
+ props.setProperty(serdeConstants.SERIALIZATION_ESCAPE_CRLF, "true");
+
+ LazySerDeParameters lazyParams =
+ new LazySerDeParameters(hconf, props,
+ LazySimpleSerDe.class.getName());
+
+ TypeInfo[] typeInfos = new TypeInfo[2];
+ typeInfos[0] = TypeInfoFactory.getPrimitiveTypeInfo("string");
+ typeInfos[1] = TypeInfoFactory.getPrimitiveTypeInfo("string");
+
+ LazySimpleDeserializeRead deserializeRead =
+ new LazySimpleDeserializeRead(typeInfos, null, true, lazyParams);
+
+ // set and parse the row
+ String s = "This\\nis\\rthe first\\r\\nmulti-line field\\n|field1-2";
+ Text row = new Text(s.getBytes("UTF-8"));
+ deserializeRead.set(row.getBytes(), 0, row.getLength());
+
+ assertTrue(deserializeRead.readNextField());
+ assertTrue(deserializeRead.currentExternalBufferNeeded);
+
+ int externalBufferLen = deserializeRead.currentExternalBufferNeededLen;
+ assertEquals("Wrong external buffer length", externalBufferLen, 36);
+
+ byte[] externalBuffer = new byte[externalBufferLen];
+ deserializeRead.copyToExternalBuffer(externalBuffer, 0);
+
+ Text field = new Text();
+ field.set(externalBuffer, 0, externalBufferLen);
+
+ String f = "This\nis\rthe first\r\nmulti-line field\n";
+ Text escaped = new Text(f.getBytes("UTF-8"));
+
+ assertTrue("The escaped result is incorrect", field.compareTo(escaped) == 0);
+ }
+}
[4/4] hive git commit: HIVE-19691: Start SessionState in materialized
views registry (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
HIVE-19691: Start SessionState in materialized views registry (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/780b0127
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/780b0127
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/780b0127
Branch: refs/heads/master
Commit: 780b0127fd22ec95a6b225a493872bcec364ef76
Parents: 619f79d
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon May 28 09:31:29 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon May 28 09:35:13 2018 -0700
----------------------------------------------------------------------
.../hive/ql/metadata/HiveMaterializedViewsRegistry.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/780b0127/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 960ad76..98c9ce9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
@@ -66,6 +67,7 @@ import org.apache.hadoop.hive.ql.parse.ColumnStatsList;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -126,7 +128,7 @@ public final class HiveMaterializedViewsRegistry {
// Create a new conf object to bypass metastore authorization, as we need to
// retrieve all materialized views from all databases
HiveConf conf = new HiveConf();
- conf.set(HiveConf.ConfVars.METASTORE_FILTER_HOOK.varname,
+ conf.set(MetastoreConf.ConfVars.FILTER_HOOK.getVarname(),
DefaultMetaStoreFilterHookImpl.class.getName());
init(Hive.get(conf));
} catch (HiveException e) {
@@ -159,6 +161,7 @@ public final class HiveMaterializedViewsRegistry {
@Override
public void run() {
try {
+ SessionState.start(db.getConf());
for (String dbName : db.getAllDatabases()) {
for (Table mv : db.getAllMaterializedViewObjects(dbName)) {
addMaterializedView(db.getConf(), mv, OpType.LOAD);
@@ -413,7 +416,7 @@ public final class HiveMaterializedViewsRegistry {
return analyzer.genLogicalPlan(node);
} catch (Exception e) {
// We could not parse the view
- LOG.error(e.getMessage());
+ LOG.error("Error parsing original query for materialized view", e);
return null;
}
}
[3/4] hive git commit: Revert "Vectorization: Turning on
vectorization in escape_crlf produces wrong results (Haifeng Chen,
reviewed by Matt McCline)"
Posted by jc...@apache.org.
Revert "Vectorization: Turning on vectorization in escape_crlf produces wrong results (Haifeng Chen, reviewed by Matt McCline)"
This reverts commit 43945fb191a6ee4f1fb4582a97113e5fe7a7f15e.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92eb2b93
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92eb2b93
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92eb2b93
Branch: refs/heads/master
Commit: 92eb2b93f15eb91db6220eb466982c363ca54ad8
Parents: 43945fb
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon May 28 09:32:56 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon May 28 09:35:12 2018 -0700
----------------------------------------------------------------------
.../test/queries/clientpositive/escape_crlf.q | 1 +
.../clientpositive/vectorization_escape_crlf.q | 22 -----
.../vectorization_escape_crlf.q.out | 92 --------------------
.../lazy/fast/LazySimpleDeserializeRead.java | 5 +-
.../fast/TestLazySimpleDeserializeRead.java | 84 ------------------
5 files changed, 3 insertions(+), 201 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/92eb2b93/ql/src/test/queries/clientpositive/escape_crlf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/escape_crlf.q b/ql/src/test/queries/clientpositive/escape_crlf.q
index a2b7f65..0b7b77c 100644
--- a/ql/src/test/queries/clientpositive/escape_crlf.q
+++ b/ql/src/test/queries/clientpositive/escape_crlf.q
@@ -1,3 +1,4 @@
+-- Suppress vectorization due to known bug. See HIVE-19118.
set hive.vectorized.execution.enabled=false;
set hive.test.vectorized.execution.enabled.override=disable;
http://git-wip-us.apache.org/repos/asf/hive/blob/92eb2b93/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q b/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q
deleted file mode 100644
index e8ae720..0000000
--- a/ql/src/test/queries/clientpositive/vectorization_escape_crlf.q
+++ /dev/null
@@ -1,22 +0,0 @@
-set hive.vectorized.execution.enabled=true;
-set hive.test.vectorized.execution.enabled.override=enable;
-
-DROP TABLE IF EXISTS base_tab;
-CREATE TABLE base_tab(a STRING, b STRING)
-ROW FORMAT DELIMITED
- FIELDS TERMINATED BY '|';
-DESCRIBE EXTENDED base_tab;
-
-LOAD DATA LOCAL INPATH '../../data/files/escape_crlf.txt' OVERWRITE INTO TABLE base_tab;
--- No crlf escaping
-SELECT * FROM base_tab;
-
--- Crlf escaping
-ALTER TABLE base_tab SET SERDEPROPERTIES ('escape.delim'='\\', 'serialization.escape.crlf'='true');
-SELECT * FROM base_tab;
-
-SET hive.fetch.task.conversion=none;
--- Make sure intermediate serde works correctly
-SELECT * FROM base_tab;
-
-DROP TABLE base_tab;
http://git-wip-us.apache.org/repos/asf/hive/blob/92eb2b93/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out b/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out
deleted file mode 100644
index 8b5df8c..0000000
--- a/ql/src/test/results/clientpositive/vectorization_escape_crlf.q.out
+++ /dev/null
@@ -1,92 +0,0 @@
-PREHOOK: query: DROP TABLE IF EXISTS base_tab
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE IF EXISTS base_tab
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE base_tab(a STRING, b STRING)
-ROW FORMAT DELIMITED
- FIELDS TERMINATED BY '|'
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@base_tab
-POSTHOOK: query: CREATE TABLE base_tab(a STRING, b STRING)
-ROW FORMAT DELIMITED
- FIELDS TERMINATED BY '|'
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@base_tab
-PREHOOK: query: DESCRIBE EXTENDED base_tab
-PREHOOK: type: DESCTABLE
-PREHOOK: Input: default@base_tab
-POSTHOOK: query: DESCRIBE EXTENDED base_tab
-POSTHOOK: type: DESCTABLE
-POSTHOOK: Input: default@base_tab
-a string
-b string
-
-#### A masked pattern was here ####
-PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/escape_crlf.txt' OVERWRITE INTO TABLE base_tab
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@base_tab
-POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/escape_crlf.txt' OVERWRITE INTO TABLE base_tab
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@base_tab
-PREHOOK: query: SELECT * FROM base_tab
-PREHOOK: type: QUERY
-PREHOOK: Input: default@base_tab
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM base_tab
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@base_tab
-#### A masked pattern was here ####
-This\nis\rthe first\r\nmulti-line field field1-2
-This\nis\rthe second\r\nmulti-line field field2-2
-PREHOOK: query: ALTER TABLE base_tab SET SERDEPROPERTIES ('escape.delim'='\\', 'serialization.escape.crlf'='true')
-PREHOOK: type: ALTERTABLE_SERDEPROPERTIES
-PREHOOK: Input: default@base_tab
-PREHOOK: Output: default@base_tab
-POSTHOOK: query: ALTER TABLE base_tab SET SERDEPROPERTIES ('escape.delim'='\\', 'serialization.escape.crlf'='true')
-POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES
-POSTHOOK: Input: default@base_tab
-POSTHOOK: Output: default@base_tab
-PREHOOK: query: SELECT * FROM base_tab
-PREHOOK: type: QUERY
-PREHOOK: Input: default@base_tab
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM base_tab
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@base_tab
-#### A masked pattern was here ####
-This
-is
-the first
-multi-line field field1-2
-This
-is
-the second
-multi-line field field2-2
-PREHOOK: query: SELECT * FROM base_tab
-PREHOOK: type: QUERY
-PREHOOK: Input: default@base_tab
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM base_tab
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@base_tab
-#### A masked pattern was here ####
-This
-is
-the first
-multi-line field field1-2
-This
-is
-the second
-multi-line field field2-2
-PREHOOK: query: DROP TABLE base_tab
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@base_tab
-PREHOOK: Output: default@base_tab
-POSTHOOK: query: DROP TABLE base_tab
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@base_tab
-POSTHOOK: Output: default@base_tab
http://git-wip-us.apache.org/repos/asf/hive/blob/92eb2b93/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
index 1890f18..dd88da8 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
@@ -917,11 +917,10 @@ public final class LazySimpleDeserializeRead extends DeserializeRead {
private void copyToBuffer(byte[] buffer, int bufferStart, int bufferLength) {
final int fieldStart = currentFieldStart;
- final int fieldLength = currentFieldLength;
int k = 0;
- for (int i = 0; i < fieldLength; i++) {
+ for (int i = 0; i < bufferLength; i++) {
byte b = bytes[fieldStart + i];
- if (b == escapeChar && i < fieldLength - 1) {
+ if (b == escapeChar && i < bufferLength - 1) {
++i;
// Check if it's '\r' or '\n'
if (bytes[fieldStart + i] == 'r') {
http://git-wip-us.apache.org/repos/asf/hive/blob/92eb2b93/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
deleted file mode 100644
index a7873f2..0000000
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.lazy.fast;
-
-import junit.framework.TestCase;
-
-import java.util.Properties;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
-import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.Text;
-
-/**
- * Unit tests for LazySimpleDeserializeRead.
- *
- */
-public class TestLazySimpleDeserializeRead extends TestCase {
-
- /**
- * Test for escaping.
- *
- */
- public void testEscaping() throws Exception {
- HiveConf hconf = new HiveConf();
-
- // set the escaping related properties
- Properties props = new Properties();
- props.setProperty(serdeConstants.FIELD_DELIM, "|");
- props.setProperty(serdeConstants.ESCAPE_CHAR, "\\");
- props.setProperty(serdeConstants.SERIALIZATION_ESCAPE_CRLF, "true");
-
- LazySerDeParameters lazyParams =
- new LazySerDeParameters(hconf, props,
- LazySimpleSerDe.class.getName());
-
- TypeInfo[] typeInfos = new TypeInfo[2];
- typeInfos[0] = TypeInfoFactory.getPrimitiveTypeInfo("string");
- typeInfos[1] = TypeInfoFactory.getPrimitiveTypeInfo("string");
-
- LazySimpleDeserializeRead deserializeRead =
- new LazySimpleDeserializeRead(typeInfos, null, true, lazyParams);
-
- // set and parse the row
- String s = "This\\nis\\rthe first\\r\\nmulti-line field\\n|field1-2";
- Text row = new Text(s.getBytes("UTF-8"));
- deserializeRead.set(row.getBytes(), 0, row.getLength());
-
- assertTrue(deserializeRead.readNextField());
- assertTrue(deserializeRead.currentExternalBufferNeeded);
-
- int externalBufferLen = deserializeRead.currentExternalBufferNeededLen;
- assertEquals("Wrong external buffer length", externalBufferLen, 36);
-
- byte[] externalBuffer = new byte[externalBufferLen];
- deserializeRead.copyToExternalBuffer(externalBuffer, 0);
-
- Text field = new Text();
- field.set(externalBuffer, 0, externalBufferLen);
-
- String f = "This\nis\rthe first\r\nmulti-line field\n";
- Text escaped = new Text(f.getBytes("UTF-8"));
-
- assertTrue("The escaped result is incorrect", field.compareTo(escaped) == 0);
- }
-}