You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/22 18:11:32 UTC
[7/7] hive git commit: HIVE-19416 : merge master into branch (Sergey
Shelukhin)
HIVE-19416 : merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4743c798
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4743c798
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4743c798
Branch: refs/heads/master-txnstats
Commit: 4743c7984d8a547cab135f9fc0fc550ac32dd61f
Parents: 42a9f3b 6d532e7
Author: sergey <se...@apache.org>
Authored: Fri Jun 22 11:06:12 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Jun 22 11:06:12 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../hcatalog/pig/AbstractHCatStorerTest.java | 2 +-
.../pig/TestHCatLoaderComplexSchema.java | 3 +
.../hive/hcatalog/pig/TestHCatStorer.java | 4 +-
.../listener/DummyRawStoreFailEvent.java | 6 +
.../test/resources/testconfiguration.properties | 1 +
.../apache/hadoop/hive/ql/exec/MoveTask.java | 3 +-
.../apache/hadoop/hive/ql/io/IOConstants.java | 1 +
.../ql/io/JsonFileStorageFormatDescriptor.java | 51 ++
.../apache/hadoop/hive/ql/metadata/Hive.java | 104 ++--
.../rules/HiveFilterProjectTransposeRule.java | 8 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 64 ++-
...he.hadoop.hive.ql.io.StorageFormatDescriptor | 1 +
.../hive/ql/io/TestStorageFormatDescriptor.java | 3 +
.../hive/ql/metadata/TestHiveCopyFiles.java | 12 +-
.../clientpositive/cbo_ppd_non_deterministic.q | 42 ++
.../test/queries/clientpositive/json_serde1.q | 9 +-
.../vector_parquet_nested_two_level_complex.q | 67 +++
.../cbo_ppd_non_deterministic.q.out | 195 +++++++
.../results/clientpositive/json_serde1.q.out | 44 +-
...ector_parquet_nested_two_level_complex.q.out | 540 +++++++++++++++++++
.../results/clientpositive/ppd_udf_col.q.out | 62 ++-
.../results/clientpositive/union_offcbo.q.out | 64 ++-
.../hadoop/hive/metastore/HiveMetaStore.java | 111 ++--
.../hadoop/hive/metastore/ObjectStore.java | 46 ++
.../apache/hadoop/hive/metastore/RawStore.java | 15 +
.../hive/metastore/cache/CachedStore.java | 6 +
.../hadoop/hive/metastore/utils/FileUtils.java | 14 +
.../DummyRawStoreControlledCommit.java | 6 +
.../DummyRawStoreForJdoConnection.java | 6 +
.../client/MetaStoreFactoryForTests.java | 1 +
.../TestTablesCreateDropAlterTruncate.java | 14 +-
32 files changed, 1337 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 3918e62,eab9f4a..f9fab96
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@@ -78,8 -78,13 +78,9 @@@ import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.HiveStatsUtils;
-import org.apache.hadoop.hive.common.ObjectPair;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
+import org.apache.hadoop.hive.common.*;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable;
import org.apache.hadoop.hive.common.log.InPlaceUpdate;
@@@ -1815,15 -1775,14 +1811,15 @@@ public class Hive
Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath);
alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString());
validatePartition(newTPart);
+ setTableSnapshotForTransactionalPartition(conf, newTPart);
+ // If config is set, table is not temporary and partition being inserted exists, capture
+ // the list of files added. For not yet existing partitions (insert overwrite to new partition
+ // or dynamic partition inserts), the add partition event will capture the list of files added.
// Generate an insert event only if inserting into an existing partition
// When inserting into a new partition, the add partition event takes care of insert event
- if ((null != oldPart) && (null != newFiles)) {
+ if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && (null != oldPart)) {
fireInsertEvent(tbl, partSpec, (loadFileType == LoadFileType.REPLACE_ALL), newFiles);
- } else {
- LOG.debug("No new files were created, and is not a replace, or we're inserting into a "
- + "partition that does not exist yet. Skipping generating INSERT event.");
}
// column stats will be inaccurate
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------