You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2018/07/03 12:36:42 UTC
[1/5] hive git commit: HIVE-20051: Skip authorization for temp tables
(Zoltan Haindrich reviewed by Igor Kryvenko, Jason Dere)
Repository: hive
Updated Branches:
refs/heads/branch-3 9e10efc43 -> e965ddc28
HIVE-20051: Skip authorization for temp tables (Zoltan Haindrich reviewed by Igor Kryvenko, Jason Dere)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4d9e3f7f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4d9e3f7f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4d9e3f7f
Branch: refs/heads/branch-3
Commit: 4d9e3f7f24dff12e43c4937849b3c329cb87191d
Parents: 9e10efc
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Tue Jul 3 13:37:08 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jul 3 13:37:08 2018 +0200
----------------------------------------------------------------------
.../TestHiveAuthorizerCheckInvocation.java | 94 +++++++++++++++++++-
.../java/org/apache/hadoop/hive/ql/Driver.java | 10 ++-
.../apache/hadoop/hive/ql/metadata/Table.java | 7 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 13 +--
4 files changed, 113 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4d9e3f7f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
index e3c83d2..b9ef8b7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.security.authorization.plugin;
+import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -27,6 +28,7 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -53,7 +55,6 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
/**
* Test HiveAuthorizer api invocation
@@ -386,6 +387,97 @@ public class TestHiveAuthorizerCheckInvocation {
}
@Test
+ public void testTempTable() throws Exception {
+
+ String tmpTableDir = getDefaultTmp() + File.separator + "THSAC_testTableTable";
+
+ final String tableName = "testTempTable";
+ { // create temp table
+ reset(mockedAuthorizer);
+ int status = driver.run("create temporary table " + tableName + "(i int) location '" + tmpTableDir + "'")
+ .getResponseCode();
+ assertEquals(0, status);
+
+ List<HivePrivilegeObject> inputs = getHivePrivilegeObjectInputs().getLeft();
+ List<HivePrivilegeObject> outputs = getHivePrivilegeObjectInputs().getRight();
+
+ // only the URI should be passed for authorization check
+ assertEquals("input count", 1, inputs.size());
+ assertEquals("input type", HivePrivilegeObjectType.LOCAL_URI, inputs.get(0).getType());
+
+ // only the dbname should be passed authorization check
+ assertEquals("output count", 1, outputs.size());
+ assertEquals("output type", HivePrivilegeObjectType.DATABASE, outputs.get(0).getType());
+
+ status = driver.compile("select * from " + tableName);
+ assertEquals(0, status);
+ }
+ { // select from the temp table
+ reset(mockedAuthorizer);
+ int status = driver.compile("insert into " + tableName + " values(1)");
+ assertEquals(0, status);
+
+ // temp tables should be skipped from authorization
+ List<HivePrivilegeObject> inputs = getHivePrivilegeObjectInputs().getLeft();
+ List<HivePrivilegeObject> outputs = getHivePrivilegeObjectInputs().getRight();
+ System.err.println("inputs " + inputs);
+ System.err.println("outputs " + outputs);
+
+ assertEquals("input count", 0, inputs.size());
+ assertEquals("output count", 0, outputs.size());
+ }
+ { // select from the temp table
+ reset(mockedAuthorizer);
+ int status = driver.compile("select * from " + tableName);
+ assertEquals(0, status);
+
+ // temp tables should be skipped from authorization
+ List<HivePrivilegeObject> inputs = getHivePrivilegeObjectInputs().getLeft();
+ List<HivePrivilegeObject> outputs = getHivePrivilegeObjectInputs().getRight();
+ System.err.println("inputs " + inputs);
+ System.err.println("outputs " + outputs);
+
+ assertEquals("input count", 0, inputs.size());
+ assertEquals("output count", 0, outputs.size());
+ }
+
+ }
+
+ @Test
+ public void testTempTableImplicit() throws Exception {
+ final String tableName = "testTempTableImplicit";
+ int status = driver.run("create table " + tableName + "(i int)").getResponseCode();
+ assertEquals(0, status);
+
+ reset(mockedAuthorizer);
+ status = driver.compile("insert into " + tableName + " values (1)");
+ assertEquals(0, status);
+
+ List<HivePrivilegeObject> inputs = getHivePrivilegeObjectInputs().getLeft();
+ List<HivePrivilegeObject> outputs = getHivePrivilegeObjectInputs().getRight();
+
+ // only the URI should be passed for authorization check
+ assertEquals("input count", 0, inputs.size());
+
+ reset(mockedAuthorizer);
+ status = driver.compile("select * from " + tableName);
+ assertEquals(0, status);
+
+ inputs = getHivePrivilegeObjectInputs().getLeft();
+ outputs = getHivePrivilegeObjectInputs().getRight();
+
+ // temp tables should be skipped from authorization
+ assertEquals("input count", 1, inputs.size());
+ assertEquals("output count", 0, outputs.size());
+
+ }
+
+ private String getDefaultTmp() {
+ return System.getProperty("test.tmp.dir",
+ "target" + File.separator + "test" + File.separator + "tmp");
+ }
+
+ @Test
public void testUpdateSomeColumnsUsed() throws Exception {
reset(mockedAuthorizer);
int status = driver.compile("update " + acidTableName + " set i = 5 where j = 3");
http://git-wip-us.apache.org/repos/asf/hive/blob/4d9e3f7f/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 4a7131a..0ca8676 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -41,8 +41,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
-import com.google.common.annotations.VisibleForTesting;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
@@ -77,8 +75,8 @@ import org.apache.hadoop.hive.ql.exec.DagUtils;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
-import org.apache.hadoop.hive.ql.exec.FunctionUtils;
import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionType;
+import org.apache.hadoop.hive.ql.exec.FunctionUtils;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -150,6 +148,7 @@ import org.apache.hive.common.util.TxnIdUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
@@ -1331,6 +1330,11 @@ public class Driver implements IDriver {
//do not authorize temporary uris
continue;
}
+ if (privObject.getTyp() == Type.TABLE
+ && (privObject.getT() == null || privObject.getT().isTemporary())) {
+ // skip temporary tables from authorization
+ continue;
+ }
//support for authorization on partitions needs to be added
String dbname = null;
String objName = null;
http://git-wip-us.apache.org/repos/asf/hive/blob/4d9e3f7f/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index f0061c0..14e60f0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -75,6 +74,8 @@ import org.apache.hive.common.util.ReflectionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
/**
* A Hive Table: is a fundamental unit of data in Hive that shares a common schema/DDL.
*
@@ -1010,6 +1011,10 @@ public class Table implements Serializable {
return tTable.isTemporary();
}
+ public void setTemporary(boolean isTemporary) {
+ tTable.setTemporary(isTemporary);
+ }
+
public static boolean hasMetastoreBasedSchema(HiveConf conf, String serdeLib) {
return StringUtils.isEmpty(serdeLib) ||
conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(serdeLib);
http://git-wip-us.apache.org/repos/asf/hive/blob/4d9e3f7f/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 499c5f6..85ecbd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -13166,7 +13166,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
case CREATE_TABLE: // REGULAR CREATE TABLE DDL
tblProps = addDefaultProperties(
tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary);
- addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, tblProps);
+ addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps);
CreateTableDesc crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols,
bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
@@ -13190,7 +13190,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
case CTLT: // create table like <tbl_name>
tblProps = addDefaultProperties(
tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary);
- addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, tblProps);
+ addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps);
if (isTemporary) {
Table likeTable = getTable(likeTableName, false);
@@ -13270,7 +13270,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
tblProps = addDefaultProperties(
tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary);
- addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, tblProps);
+ addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps);
tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim,
@@ -13294,13 +13294,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
/** Adds entities for create table/create view. */
private void addDbAndTabToOutputs(String[] qualifiedTabName, TableType type,
- Map<String, String> tblProps) throws SemanticException {
+ boolean isTemporary, Map<String, String> tblProps) throws SemanticException {
Database database = getDatabase(qualifiedTabName[0]);
outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_SHARED));
Table t = new Table(qualifiedTabName[0], qualifiedTabName[1]);
t.setParameters(tblProps);
t.setTableType(type);
+ t.setTemporary(isTemporary);
outputs.add(new WriteEntity(t, WriteEntity.WriteType.DDL_NO_LOCK));
}
@@ -13401,7 +13402,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
storageFormat.getInputFormat(), storageFormat.getOutputFormat(),
location, storageFormat.getSerde(), storageFormat.getStorageHandler(),
storageFormat.getSerdeProps());
- addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW, tblProps);
+ addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW, false, tblProps);
queryState.setCommandType(HiveOperation.CREATE_MATERIALIZED_VIEW);
} else {
createVwDesc = new CreateViewDesc(
@@ -13410,7 +13411,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
storageFormat.getOutputFormat(), storageFormat.getSerde());
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
createVwDesc)));
- addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW, tblProps);
+ addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW, false, tblProps);
queryState.setCommandType(HiveOperation.CREATEVIEW);
}
qb.setViewDesc(createVwDesc);
[3/5] hive git commit: HIVE-19995: Aggregate row traffic for acid
tables (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Posted by kg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out b/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out
index febcddb..3954be5 100644
--- a/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out
+++ b/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out
@@ -1704,19 +1704,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: insert_into1_n0
- Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (value = 1) (type: boolean)
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), value (type: string), i (type: int)
outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: string), _col3 (type: int)
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
@@ -1726,10 +1726,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 1 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1811,19 +1811,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: insert_into1_n0
- Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (value = 1) (type: boolean)
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), i (type: int)
outputColumnNames: _col0, _col3
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: int)
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
@@ -1833,10 +1833,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 1 (type: int), null (type: string), VALUE._col0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2368,15 +2368,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t
- Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: enforce_constraint(key is not null) (type: boolean)
- Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Map 5
@@ -2404,18 +2404,18 @@ STAGE PLANS:
0 key (type: int)
1 key (type: int)
outputColumnNames: _col0, _col6
- Statistics: Num rows: 11 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is null (type: boolean)
- Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col6 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Map-reduce partition columns: null (type: string)
- Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
Reducer 3
Execution mode: llap
@@ -2423,10 +2423,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: int), 'a1' (type: string), null (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2436,7 +2436,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), 'a1' (type: string), null (type: string)
outputColumnNames: key, a1, value
- Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(key, 'hll'), compute_stats(a1, 'hll'), compute_stats(value, 'hll')
mode: hash
@@ -2565,12 +2565,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t
- Statistics: Num rows: 20 Data size: 3760 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 20 Data size: 3760 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: string), ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
@@ -2597,62 +2597,62 @@ STAGE PLANS:
0 key (type: int)
1 key (type: int)
outputColumnNames: _col0, _col2, _col5, _col6, _col7
- Statistics: Num rows: 22 Data size: 4136 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col0 = _col6) and (_col6 < 3)) (type: boolean)
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col5 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col0 = _col6) and (_col6 > 3) and (_col6 >= 3) and enforce_constraint(_col0 is not null)) (type: boolean)
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col5 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), _col0 (type: int), _col2 (type: string)
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: string)
Filter Operator
predicate: (_col0 = _col6) (type: boolean)
- Statistics: Num rows: 11 Data size: 2068 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col5 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col5
- Statistics: Num rows: 11 Data size: 2068 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
keys: _col5 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 11 Data size: 2068 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
- Statistics: Num rows: 11 Data size: 2068 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Filter Operator
predicate: (_col0 is null and enforce_constraint(_col6 is not null)) (type: boolean)
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col6 (type: int), _col7 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Map-reduce partition columns: null (type: string)
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string)
Reducer 3
Execution mode: vectorized, llap
@@ -2660,10 +2660,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2676,10 +2676,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), 'a1' (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2694,17 +2694,17 @@ STAGE PLANS:
keys: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col1 > 1L) (type: boolean)
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cardinality_violation(_col0) (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2713,7 +2713,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int)
outputColumnNames: val
- Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(val, 'hll')
mode: complete
@@ -2736,10 +2736,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), null (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2749,7 +2749,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), _col1 (type: string), null (type: string)
outputColumnNames: key, a1, value
- Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(key, 'hll'), compute_stats(a1, 'hll'), compute_stats(value, 'hll')
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
index ed8ae5b..e0ed969 100644
--- a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
+++ b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
@@ -170,6 +170,8 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 1
+ numRows 12288
+ rawDataSize 0
totalSize 295638
transactional true
transactional_properties default
@@ -205,9 +207,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 5865 Data size: 2956380 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 2956380 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 5865 Data size: 2956380 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 2956380 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
@@ -376,6 +378,8 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 1
+ numRows 2
+ rawDataSize 0
totalSize 1652
transactional true
transactional_properties default
@@ -411,9 +415,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 32 Data size: 16520 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 16520 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 32 Data size: 16520 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 16520 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
@@ -509,6 +513,8 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 2
+ numRows 4
+ rawDataSize 0
totalSize 3304
transactional true
transactional_properties default
@@ -544,9 +550,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 65 Data size: 33040 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 33040 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 65 Data size: 33040 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 33040 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
@@ -638,6 +644,8 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 3
+ numRows 12292
+ rawDataSize 0
totalSize 298943
transactional true
transactional_properties default
@@ -673,9 +681,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 5931 Data size: 2989430 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12292 Data size: 2989430 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 5931 Data size: 2989430 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12292 Data size: 2989430 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
index 237ff00..909ab0b 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
@@ -749,19 +749,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2
- Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -775,17 +775,17 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: decimal(10,2))
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2))
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 3
Execution mode: llap
Reduce Operator Tree:
@@ -793,14 +793,14 @@ STAGE PLANS:
keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2))
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1027,19 +1027,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2
- Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1053,17 +1053,17 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: decimal(10,2))
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2))
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 3
Execution mode: llap
Reduce Operator Tree:
@@ -1071,14 +1071,14 @@ STAGE PLANS:
keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2))
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
index 8aa6c72..e8adae8 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
@@ -938,6 +938,8 @@ Table Type: MATERIALIZED_VIEW
Table Parameters:
bucketing_version 2
numFiles 3
+ numRows 3
+ rawDataSize 248
totalSize 1508
transactional true
transactional_properties default
@@ -1065,19 +1067,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n2
- Statistics: Num rows: 61 Data size: 7320 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 20 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 20 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 20 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2)), _col2 (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1091,7 +1093,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 33 Data size: 3960 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col3)
keys: _col0 (type: int), _col2 (type: decimal(10,2))
@@ -1284,19 +1286,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n2
- Statistics: Num rows: 75 Data size: 9000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2)), _col2 (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1310,7 +1312,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 41 Data size: 4920 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col3)
keys: _col0 (type: int), _col2 (type: decimal(10,2))
@@ -1533,19 +1535,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n2
- Statistics: Num rows: 91 Data size: 10920 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 10 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 10 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2)), _col2 (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1643,7 +1645,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 16 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col3)
keys: _col0 (type: int), _col2 (type: decimal(10,2))
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
index 2c4ea52..6a07ff9 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
@@ -400,6 +400,8 @@ Table Type: MATERIALIZED_VIEW
Table Parameters:
bucketing_version 2
numFiles 2
+ numRows 5
+ rawDataSize 348
totalSize 1071
transactional true
transactional_properties default
@@ -523,19 +525,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n3
- Statistics: Num rows: 61 Data size: 7076 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 20 Data size: 2320 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 20 Data size: 2320 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 20 Data size: 2320 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -549,14 +551,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col2 (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -566,7 +568,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), _col1 (type: decimal(10,2))
outputColumnNames: a, c
- Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll')
mode: hash
@@ -731,19 +733,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n3
- Statistics: Num rows: 75 Data size: 8700 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -757,14 +759,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col2 (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -774,7 +776,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), _col1 (type: decimal(10,2))
outputColumnNames: a, c
- Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll')
mode: hash
@@ -943,19 +945,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n3
- Statistics: Num rows: 91 Data size: 10556 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 10 Data size: 1160 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -969,14 +971,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col2 (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -986,7 +988,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), _col1 (type: decimal(10,2))
outputColumnNames: a, c
- Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll')
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
index 3b7fb57..295ad7e 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
@@ -749,19 +749,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n0
- Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -775,17 +775,17 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: decimal(10,2))
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2))
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 3
Execution mode: llap
Reduce Operator Tree:
@@ -793,14 +793,14 @@ STAGE PLANS:
keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2))
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1027,19 +1027,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n0
- Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1053,17 +1053,17 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: decimal(10,2))
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2))
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 3
Execution mode: llap
Reduce Operator Tree:
@@ -1071,14 +1071,14 @@ STAGE PLANS:
keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2))
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/results_cache_invalidation.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/results_cache_invalidation.q.out b/ql/src/test/results/clientpositive/llap/results_cache_invalidation.q.out
index 346e0bb..bf4b56d 100644
--- a/ql/src/test/results/clientpositive/llap/results_cache_invalidation.q.out
+++ b/ql/src/test/results/clientpositive/llap/results_cache_invalidation.q.out
@@ -57,12 +57,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (UDFToDouble(key) >= 0.0D) (type: boolean)
- Statistics: Num rows: 30 Data size: 5338 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 166 Data size: 29077 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 30 Data size: 5338 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 166 Data size: 29077 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -127,11 +127,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab2_n5
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: key
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(key)
mode: hash
@@ -197,38 +197,38 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab1_n6
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Map 4
Map Operator Tree:
TableScan
alias: tab2_n5
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -240,7 +240,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- Statistics: Num rows: 95 Data size: 17028 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 522 Data size: 91524 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -393,12 +393,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 111 Data size: 19688 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 501 Data size: 87768 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (UDFToDouble(key) >= 0.0D) (type: boolean)
- Statistics: Num rows: 37 Data size: 6562 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 167 Data size: 29256 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 37 Data size: 6562 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 167 Data size: 29256 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -490,38 +490,38 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab1_n6
- Statistics: Num rows: 111 Data size: 19688 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 501 Data size: 87768 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Map 4
Map Operator Tree:
TableScan
alias: tab2_n5
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -533,7 +533,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- Statistics: Num rows: 116 Data size: 20681 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 523 Data size: 91726 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -634,11 +634,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab2_n5
- Statistics: Num rows: 111 Data size: 19688 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 501 Data size: 87768 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: key
- Statistics: Num rows: 111 Data size: 19688 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 501 Data size: 87768 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(key)
mode: hash
@@ -705,38 +705,38 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab1_n6
- Statistics: Num rows: 111 Data size: 19688 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 501 Data size: 87768 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Map 4
Map Operator Tree:
TableScan
alias: tab2_n5
- Statistics: Num rows: 111 Data size: 19688 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 501 Data size: 87768 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 106 Data size: 18801 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 476 Data size: 83388 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -748,7 +748,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- Statistics: Num rows: 116 Data size: 20681 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 523 Data size: 91726 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/results_cache_transactional.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/results_cache_transactional.q.out b/ql/src/test/results/clientpositive/llap/results_cache_transactional.q.out
index accea78..3740b76 100644
--- a/ql/src/test/results/clientpositive/llap/results_cache_transactional.q.out
+++ b/ql/src/test/results/clientpositive/llap/results_cache_transactional.q.out
@@ -56,11 +56,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab1_n1
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: key
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(key)
mode: hash
@@ -151,38 +151,38 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab1_n1
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Map 4
Map Operator Tree:
TableScan
alias: tab2_n1
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -194,7 +194,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- Statistics: Num rows: 95 Data size: 17028 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 522 Data size: 91524 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -426,19 +426,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab1_n1
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Map 4
@@ -536,19 +536,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab1_n1
- Statistics: Num rows: 91 Data size: 16192 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87 Data size: 15480 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 475 Data size: 83204 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
Map 4
[5/5] hive git commit: HIVE-19979: Backport HIVE-19304 to branch-3
(Daniel Voros via Zoltan Haindrich)
Posted by kg...@apache.org.
HIVE-19979: Backport HIVE-19304 to branch-3 (Daniel Voros via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e965ddc2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e965ddc2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e965ddc2
Branch: refs/heads/branch-3
Commit: e965ddc28e4a66b08f59352c717e690971983c37
Parents: 9fae0d1
Author: Daniel Voros <da...@gmail.com>
Authored: Tue Jul 3 14:08:13 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jul 3 14:08:13 2018 +0200
----------------------------------------------------------------------
llap-server/src/main/resources/templates.py | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e965ddc2/llap-server/src/main/resources/templates.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/templates.py b/llap-server/src/main/resources/templates.py
index 00a84cf..1dc1611 100644
--- a/llap-server/src/main/resources/templates.py
+++ b/llap-server/src/main/resources/templates.py
@@ -6,10 +6,9 @@ yarnfile = """
"configuration": {
"properties": {
"yarn.service.rolling-log.include-pattern": ".*\\\\.done",
- "yarn.component.placement.policy" : "%(placement)d",
- "yarn.container.health.threshold.percent": "%(health_percent)d",
- "yarn.container.health.threshold.window.secs": "%(health_time_window)d",
- "yarn.container.health.threshold.init.delay.secs": "%(health_init_delay)d"%(service_appconfig_global_append)s
+ "yarn.service.container-health-threshold.percent": "%(health_percent)d",
+ "yarn.service.container-health-threshold.window-secs": "%(health_time_window)d",
+ "yarn.service.container-health-threshold.init-delay-secs": "%(health_init_delay)d"%(service_appconfig_global_append)s
}
},
"components": [
@@ -25,6 +24,17 @@ yarnfile = """
"cpus": 1,
"memory": "%(container.mb)d"
},
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "NODE",
+ "target_tags": [
+ "llap"
+ ]
+ }
+ ]
+ },
"configuration": {
"env": {
"JAVA_HOME": "%(java_home)s",
[2/5] hive git commit: HIVE-19995: Aggregate row traffic for acid
tables (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Posted by kg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out b/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out
new file mode 100644
index 0000000..94d0ea3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out
@@ -0,0 +1,511 @@
+PREHOOK: query: create table t(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: create table upd_t(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@upd_t
+POSTHOOK: query: create table upd_t(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@upd_t
+PREHOOK: query: desc formatted t
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@t
+POSTHOOK: query: desc formatted t
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@t
+# col_name data_type comment
+a int
+b int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
+ bucketing_version 2
+ numFiles 0
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: 2
+Bucket Columns: [a]
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: insert into t values (1,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values (1,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a SCRIPT []
+POSTHOOK: Lineage: t.b SCRIPT []
+PREHOOK: query: insert into upd_t values (1,1),(2,2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@upd_t
+POSTHOOK: query: insert into upd_t values (1,1),(2,2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@upd_t
+POSTHOOK: Lineage: upd_t.a SCRIPT []
+POSTHOOK: Lineage: upd_t.b SCRIPT []
+PREHOOK: query: desc formatted t
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@t
+POSTHOOK: query: desc formatted t
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@t
+# col_name data_type comment
+a int
+b int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ bucketing_version 2
+ numFiles 1
+ numRows 1
+ rawDataSize 0
+ totalSize 657
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: 2
+Bucket Columns: [a]
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN UPDATE SET b = 99
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN UPDATE SET b = 99
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-4 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-4
+ Stage-5 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-4
+ Stage-6 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-3
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ value expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
+ Execution mode: vectorized, llap
+ LLAP IO: may be used (ACID table)
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: u
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: b (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Right Outer Join 0 to 1
+ keys:
+ 0 a (type: int)
+ 1 a (type: int)
+ outputColumnNames: _col0, _col4, _col5, _col6
+ Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col0 = _col5) (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), _col0 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int)
+ Filter Operator
+ predicate: (_col0 = _col5) (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
+ outputColumnNames: _col4
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: _col4 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Filter Operator
+ predicate: _col0 is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), 99 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.t
+ Write Type: UPDATE
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col1 > 1L) (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cardinality_violation(_col0) (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.merge_tmp_table
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: val
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: compute_stats(val, 'hll')
+ mode: complete
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 432 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.t
+ Write Type: INSERT
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int)
+ outputColumnNames: a, b
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll')
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 848 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 848 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>)
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-4
+ Dependency Collection
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.t
+ Write Type: UPDATE
+
+ Stage: Stage-5
+ Stats Work
+ Basic Stats Work:
+
+ Stage: Stage-2
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.merge_tmp_table
+
+ Stage: Stage-6
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: val
+ Column Types: int
+ Table: default.merge_tmp_table
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.t
+ Write Type: INSERT
+
+ Stage: Stage-7
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: a, b
+ Column Types: int, int
+ Table: default.t
+
+PREHOOK: query: merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN UPDATE SET b = 99
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+PREHOOK: Input: default@upd_t
+PREHOOK: Output: default@merge_tmp_table
+PREHOOK: Output: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN UPDATE SET b = 99
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+POSTHOOK: Input: default@upd_t
+POSTHOOK: Output: default@merge_tmp_table
+POSTHOOK: Output: default@t
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: merge_tmp_table.val EXPRESSION [(t)t.FieldSchema(name:ROW__ID, type:struct<writeId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: t.a SIMPLE [(upd_t)u.FieldSchema(name:a, type:int, comment:null), ]
+POSTHOOK: Lineage: t.b SIMPLE [(upd_t)u.FieldSchema(name:b, type:int, comment:null), ]
+PREHOOK: query: select assert_true(count(1) = 2) from t group by a>-1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select assert_true(count(1) = 2) from t group by a>-1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: desc formatted t
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@t
+POSTHOOK: query: desc formatted t
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@t
+# col_name data_type comment
+a int
+b int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ bucketing_version 2
+ numFiles 4
+ numRows 2
+ rawDataSize 0
+ totalSize 2690
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: 2
+Bucket Columns: [a]
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN DELETE
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+PREHOOK: Input: default@upd_t
+PREHOOK: Output: default@merge_tmp_table
+PREHOOK: Output: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN DELETE
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+POSTHOOK: Input: default@upd_t
+POSTHOOK: Output: default@merge_tmp_table
+POSTHOOK: Output: default@t
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: merge_tmp_table.val EXPRESSION [(t)t.FieldSchema(name:ROW__ID, type:struct<writeId:bigint,bucketId:int,rowId:bigint>, comment:), ]
+POSTHOOK: Lineage: t.a SIMPLE [(upd_t)u.FieldSchema(name:a, type:int, comment:null), ]
+POSTHOOK: Lineage: t.b SIMPLE [(upd_t)u.FieldSchema(name:b, type:int, comment:null), ]
+PREHOOK: query: select assert_true(count(1) = 0) from t group by a>-1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select assert_true(count(1) = 0) from t group by a>-1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+PREHOOK: query: desc formatted t
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@t
+POSTHOOK: query: desc formatted t
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@t
+# col_name data_type comment
+a int
+b int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ bucketing_version 2
+ numFiles 6
+ numRows 0
+ rawDataSize 0
+ totalSize 4052
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: 2
+Bucket Columns: [a]
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/row__id.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/row__id.q.out b/ql/src/test/results/clientpositive/row__id.q.out
index 7762a58..2fdbc72 100644
--- a/ql/src/test/results/clientpositive/row__id.q.out
+++ b/ql/src/test/results/clientpositive/row__id.q.out
@@ -62,24 +62,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hello_acid
- Statistics: Num rows: 78 Data size: 19860 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 3 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID.writeid (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 78 Data size: 19860 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 3 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
- Statistics: Num rows: 78 Data size: 19860 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 3 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 78 Data size: 19860 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 3 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 78 Data size: 19860 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 3 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -124,17 +124,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hello_acid
- Statistics: Num rows: 78 Data size: 19860 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 3 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (ROW__ID.writeid = 3) (type: boolean)
- Statistics: Num rows: 39 Data size: 9930 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 6620 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID.writeid (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 39 Data size: 9930 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 6620 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 39 Data size: 9930 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 6620 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
index 2531f4a..634b4ea 100644
--- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
+++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
@@ -680,22 +680,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: over10k_orc_bucketed_n0
- Statistics: Num rows: 1234 Data size: 706090 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2098 Data size: 622340 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: ROW__ID
- Statistics: Num rows: 1234 Data size: 706090 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2098 Data size: 622340 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 617 Data size: 51828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1049 Data size: 88116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
- Statistics: Num rows: 617 Data size: 51828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1049 Data size: 88116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
@@ -704,13 +704,13 @@ STAGE PLANS:
keys: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 617 Data size: 51828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1049 Data size: 88116 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (_col1 > 1L) (type: boolean)
- Statistics: Num rows: 205 Data size: 17220 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 349 Data size: 29316 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 205 Data size: 17220 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 349 Data size: 29316 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
index 9369df8..8d71080 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
@@ -295,16 +295,16 @@ Stage-3
Reducer 2
File Output Operator [FS_8]
table:{"name:":"default.acid_uami_n2"}
- Select Operator [SEL_4] (rows=10/2 width=316)
+ Select Operator [SEL_4] (rows=4/2 width=328)
Output:["_col0","_col1","_col2","_col3"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
PartitionCols:UDFToInteger(_col0)
- Select Operator [SEL_2] (rows=10/2 width=316)
+ Select Operator [SEL_2] (rows=4/2 width=328)
Output:["_col0","_col1","_col3"]
- Filter Operator [FIL_9] (rows=10/2 width=316)
+ Filter Operator [FIL_9] (rows=4/2 width=328)
predicate:((de = 109.23) or (de = 119.23))
- TableScan [TS_0] (rows=85/4 width=316)
+ TableScan [TS_0] (rows=4/4 width=328)
default@acid_uami_n2,acid_uami_n2, ACID table,Tbl:COMPLETE,Col:NONE,Output:["i","de","vc"]
PREHOOK: query: select * from acid_uami_n2 order by de
[4/5] hive git commit: HIVE-19995: Aggregate row traffic for acid
tables (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Posted by kg...@apache.org.
HIVE-19995: Aggregate row traffic for acid tables (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9fae0d19
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9fae0d19
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9fae0d19
Branch: refs/heads/branch-3
Commit: 9fae0d19ecebfc2c70f893d92dba9c970af6a8ad
Parents: 4d9e3f7
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Tue Jul 3 13:38:24 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jul 3 13:38:24 2018 +0200
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/stats/BasicStatsTask.java | 10 +-
.../queries/clientpositive/sqlmerge_stats.q | 38 ++
.../results/clientpositive/acid_nullscan.q.out | 8 +-
.../clientpositive/acid_table_stats.q.out | 16 +-
.../clientpositive/autoColumnStats_4.q.out | 4 +
.../clientpositive/druid/druidmini_mv.q.out | 50 +-
.../llap/acid_bucket_pruning.q.out | 6 +-
.../llap/acid_vectorization_original.q.out | 14 +-
.../llap/dynpart_sort_optimization_acid.q.out | 144 +++---
.../llap/enforce_constraint_notnull.q.out | 24 +-
.../llap/insert_into_default_keyword.q.out | 100 ++--
.../insert_values_orig_table_use_metadata.q.out | 24 +-
.../materialized_view_create_rewrite_3.q.out | 40 +-
.../materialized_view_create_rewrite_4.q.out | 32 +-
.../materialized_view_create_rewrite_5.q.out | 50 +-
...ized_view_create_rewrite_rebuild_dummy.q.out | 40 +-
.../llap/results_cache_invalidation.q.out | 74 +--
.../llap/results_cache_transactional.q.out | 38 +-
.../clientpositive/llap/sqlmerge_stats.q.out | 511 +++++++++++++++++++
.../test/results/clientpositive/row__id.q.out | 18 +-
.../tez/acid_vectorization_original_tez.q.out | 14 +-
.../clientpositive/tez/explainanalyze_5.q.out | 8 +-
23 files changed, 918 insertions(+), 346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index e677493..5e7c917 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -699,6 +699,7 @@ minillaplocal.query.files=\
smb_cache.q,\
special_character_in_tabnames_1.q,\
sqlmerge.q,\
+ sqlmerge_stats.q,\
stats_based_fetch_decision.q,\
stats_only_external.q,\
strict_managed_tables_sysdb.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
index 8c23887..f31c170 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
@@ -169,7 +169,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
// though we are marking stats as not being accurate.
if (StatsSetupConst.areBasicStatsUptoDate(parameters) || p.isTransactionalTable()) {
String prefix = getAggregationPrefix(p.getTable(), p.getPartition());
- updateStats(statsAggregator, parameters, prefix, p.isAcid());
+ updateStats(statsAggregator, parameters, prefix);
}
}
@@ -206,14 +206,8 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
}
private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters,
- String aggKey, boolean isFullAcid) throws HiveException {
+ String aggKey) throws HiveException {
for (String statType : StatsSetupConst.statsRequireCompute) {
- if (isFullAcid && !work.isTargetRewritten()) {
- // Don't bother with aggregation in this case, it will probably be invalid.
- parameters.remove(statType);
- continue;
- }
-
String value = statsAggregator.aggregateStats(aggKey, statType);
if (value != null && !value.isEmpty()) {
long longValue = Long.parseLong(value);
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/queries/clientpositive/sqlmerge_stats.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/sqlmerge_stats.q b/ql/src/test/queries/clientpositive/sqlmerge_stats.q
new file mode 100644
index 0000000..c480eb6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/sqlmerge_stats.q
@@ -0,0 +1,38 @@
+set hive.mapred.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.explain.user=false;
+set hive.merge.cardinality.check=true;
+
+create table t(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');
+create table upd_t(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false');
+
+desc formatted t;
+
+insert into t values (1,1);
+insert into upd_t values (1,1),(2,2);
+
+desc formatted t;
+
+explain merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN UPDATE SET b = 99
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b);
+
+merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN UPDATE SET b = 99
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b);
+
+-- merge could keep track of inserts
+select assert_true(count(1) = 2) from t group by a>-1;
+-- rownum is 2
+desc formatted t;
+
+merge into t as t using upd_t as u ON t.a = u.a
+WHEN MATCHED THEN DELETE
+WHEN NOT MATCHED THEN INSERT VALUES(u.a, u.b);
+
+
+select assert_true(count(1) = 0) from t group by a>-1;
+-- rownum is 0; because the orc writer can keep track of delta
+desc formatted t;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/acid_nullscan.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_nullscan.q.out b/ql/src/test/results/clientpositive/acid_nullscan.q.out
index c9684dd..19fcc8c 100644
--- a/ql/src/test/results/clientpositive/acid_nullscan.q.out
+++ b/ql/src/test/results/clientpositive/acid_nullscan.q.out
@@ -42,12 +42,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_vectorized_n1
- Statistics: Num rows: 90 Data size: 25960 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 25960 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: false (type: boolean)
- Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 2360 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(a)
mode: hash
@@ -79,6 +79,8 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.acid_vectorized_n1
numFiles 3
+ numRows 11
+ rawDataSize 0
serialization.ddl struct acid_vectorized_n1 { i32 a, string b}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
@@ -101,6 +103,8 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.acid_vectorized_n1
numFiles 3
+ numRows 11
+ rawDataSize 0
serialization.ddl struct acid_vectorized_n1 { i32 a, string b}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/acid_table_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out
index 3d3fcb9..9a9617a 100644
--- a/ql/src/test/results/clientpositive/acid_table_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out
@@ -94,6 +94,8 @@ Table: acid
#### A masked pattern was here ####
Partition Parameters:
numFiles 2
+ numRows 2000
+ rawDataSize 0
totalSize 4063
#### A masked pattern was here ####
@@ -132,17 +134,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
- Statistics: Num rows: 83 Data size: 40630 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 40630 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 83 Data size: 40630 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 40630 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
Execution mode: vectorized
Reduce Operator Tree:
@@ -150,10 +152,10 @@ STAGE PLANS:
aggregations: count(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -385,6 +387,8 @@ Table: acid
#### A masked pattern was here ####
Partition Parameters:
numFiles 4
+ numRows 3000
+ rawDataSize 208000
totalSize 8118
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
index def65ff..5e52092 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
@@ -200,6 +200,8 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 2
+ numRows 10
+ rawDataSize 0
totalSize 1899
transactional true
transactional_properties default
@@ -243,6 +245,8 @@ Table Parameters:
COLUMN_STATS_ACCURATE {}
bucketing_version 2
numFiles 4
+ numRows 8
+ rawDataSize 0
totalSize 3275
transactional true
transactional_properties default
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
index 470f132..3d351d8 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
@@ -342,33 +342,33 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_n2
- Statistics: Num rows: 31 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (a = 3) (type: boolean)
- Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double)
Map 3
Map Operator Tree:
TableScan
alias: cmv_basetable_n2
- Statistics: Num rows: 31 Data size: 496 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double)
Reducer 2
Reduce Operator Tree:
@@ -379,14 +379,14 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1
- Statistics: Num rows: 5 Data size: 145 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 49 Data size: 1421 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 3 (type: int), _col0 (type: double), 3 (type: int), _col1 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 145 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 49 Data size: 1421 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 145 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 49 Data size: 1421 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -459,34 +459,34 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_n2
- Statistics: Num rows: 31 Data size: 22692 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (a = 3) (type: boolean)
- Statistics: Num rows: 5 Data size: 3660 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: CAST( t AS timestamp with local time zone) (type: timestamp with local time zone), 3 (type: int), b (type: varchar(256)), c (type: double), userid (type: varchar(256))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 5 Data size: 3660 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256)), floor_hour(CAST( GenericUDFEpochMilli(_col0) AS TIMESTAMP)) (type: timestamp)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, __time_granularity
- Statistics: Num rows: 5 Data size: 3660 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: __time_granularity (type: timestamp)
sort order: +
Map-reduce partition columns: __time_granularity (type: timestamp)
- Statistics: Num rows: 5 Data size: 3660 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: timestamp with local time zone), _col1 (type: int), _col2 (type: varchar(256)), _col3 (type: double), _col4 (type: varchar(256))
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: int), VALUE._col2 (type: varchar(256)), VALUE._col3 (type: double), VALUE._col4 (type: varchar(256)), KEY.__time_granularity (type: timestamp)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, __time_granularity
- Statistics: Num rows: 5 Data size: 3660 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Dp Sort State: PARTITION_SORTED
- Statistics: Num rows: 5 Data size: 3660 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 5124 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
@@ -550,17 +550,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_n2
- Statistics: Num rows: 31 Data size: 496 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: c (type: double)
outputColumnNames: _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double)
Map 3
Map Operator Tree:
@@ -585,14 +585,14 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col6
- Statistics: Num rows: 3 Data size: 87 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 21 Data size: 609 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: double), _col0 (type: int), _col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 3 Data size: 87 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 21 Data size: 609 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 3 Data size: 87 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 21 Data size: 609 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
index b856b99..29a05ae 100644
--- a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
@@ -45,7 +45,7 @@ STAGE PLANS:
alias: acidtbldefault
filterExpr: (a = 1) (type: boolean)
buckets included: [13,] of 16
- Statistics: Num rows: 1850 Data size: 7036 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9174 Data size: 34868 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
@@ -99,6 +99,8 @@ STAGE PLANS:
location hdfs://### HDFS PATH ###
name default.acidtbldefault
numFiles 17
+ numRows 9174
+ rawDataSize 0
serialization.ddl struct acidtbldefault { i32 a}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
@@ -122,6 +124,8 @@ STAGE PLANS:
location hdfs://### HDFS PATH ###
name default.acidtbldefault
numFiles 17
+ numRows 9174
+ rawDataSize 0
serialization.ddl struct acidtbldefault { i32 a}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
index 028c3ca..57ff575 100644
--- a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
+++ b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
@@ -665,22 +665,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: over10k_orc_bucketed
- Statistics: Num rows: 1234 Data size: 706090 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2098 Data size: 622340 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: ROW__ID
- Statistics: Num rows: 1234 Data size: 706090 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2098 Data size: 622340 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 617 Data size: 51828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1049 Data size: 88116 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
- Statistics: Num rows: 617 Data size: 51828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1049 Data size: 88116 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -692,13 +692,13 @@ STAGE PLANS:
keys: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 617 Data size: 51828 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1049 Data size: 88116 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (_col1 > 1L) (type: boolean)
- Statistics: Num rows: 205 Data size: 17220 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 349 Data size: 29316 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 205 Data size: 17220 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 349 Data size: 29316 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
index 40b73fb..bb1d41c 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
@@ -94,19 +94,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part
- Statistics: Num rows: 160 Data size: 61001 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1601 Data size: 150414 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 1906 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
- Statistics: Num rows: 5 Data size: 1906 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 1906 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -115,10 +115,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 1906 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 1906 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -189,19 +189,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part
- Statistics: Num rows: 159 Data size: 104317 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 1601 Data size: 444998 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 3280 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), ds (type: string)
outputColumnNames: _col0, _col3
- Statistics: Num rows: 5 Data size: 3280 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 3280 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col3 (type: string)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -211,10 +211,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 3280 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 3280 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -380,19 +380,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part_sdpo
- Statistics: Num rows: 176 Data size: 67063 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1601 Data size: 150414 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 1905 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
- Statistics: Num rows: 5 Data size: 1905 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 1905 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -401,10 +401,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 1905 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 1905 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -475,19 +475,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part_sdpo
- Statistics: Num rows: 171 Data size: 112152 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 1601 Data size: 444998 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 3279 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), ds (type: string)
outputColumnNames: _col0, _col3
- Statistics: Num rows: 5 Data size: 3279 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col3 (type: string), '_bucket_number' (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +++
Map-reduce partition columns: _col3 (type: string)
- Statistics: Num rows: 5 Data size: 3279 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -496,11 +496,11 @@ STAGE PLANS:
Select Operator
expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), KEY._col3 (type: string), KEY.'_bucket_number' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, '_bucket_number'
- Statistics: Num rows: 5 Data size: 3279 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
Dp Sort State: PARTITION_BUCKET_SORTED
- Statistics: Num rows: 5 Data size: 3279 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -675,19 +675,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part
- Statistics: Num rows: 157 Data size: 60527 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1601 Data size: 150414 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -696,10 +696,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -771,19 +771,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part
- Statistics: Num rows: 1804 Data size: 235871 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 3201 Data size: 313458 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 455 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), hr (type: int)
outputColumnNames: _col0, _col4
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col4 (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -793,10 +793,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), VALUE._col2 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -897,19 +897,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part
- Statistics: Num rows: 2015 Data size: 726272 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 4200 Data size: 1253037 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (value = 'bar') (type: boolean)
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1375 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), ds (type: string), hr (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string), _col2 (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -919,10 +919,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1095,19 +1095,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part_sdpo
- Statistics: Num rows: 157 Data size: 60527 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1601 Data size: 150414 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -1116,10 +1116,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 1927 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 5 Data size: 469 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1191,19 +1191,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part_sdpo
- Statistics: Num rows: 1804 Data size: 235871 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 3201 Data size: 313458 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 455 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), hr (type: int)
outputColumnNames: _col0, _col4
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: '2008-04-08' (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: ++++
Map-reduce partition columns: '2008-04-08' (type: string), _col4 (type: int)
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -1212,11 +1212,11 @@ STAGE PLANS:
Select Operator
expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number'
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1360 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
Dp Sort State: PARTITION_BUCKET_SORTED
- Statistics: Num rows: 5 Data size: 653 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1360 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1317,19 +1317,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part_sdpo
- Statistics: Num rows: 2015 Data size: 726272 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 4952 Data size: 1456618 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (value = 'bar') (type: boolean)
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1375 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), ds (type: string), hr (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col1 (type: string), _col2 (type: int), '_bucket_number' (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: ++++
Map-reduce partition columns: _col1 (type: string), _col2 (type: int)
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -1338,11 +1338,11 @@ STAGE PLANS:
Select Operator
expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), KEY._col1 (type: string), KEY._col2 (type: int), KEY.'_bucket_number' (type: string)
outputColumnNames: _col0, _col1, _col2, '_bucket_number'
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1810 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
Dp Sort State: PARTITION_BUCKET_SORTED
- Statistics: Num rows: 5 Data size: 1802 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1810 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1515,19 +1515,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part_sdpo_no_cp
- Statistics: Num rows: 97 Data size: 82922 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 1601 Data size: 599036 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 4274 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1860 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), key (type: string), ds (type: string), hr (type: int)
outputColumnNames: _col0, _col1, _col3, _col4
- Statistics: Num rows: 5 Data size: 4274 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: ++++
Map-reduce partition columns: _col3 (type: string), _col4 (type: int)
- Statistics: Num rows: 5 Data size: 4274 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string), 'bar' (type: string)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1537,11 +1537,11 @@ STAGE PLANS:
Select Operator
expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number'
- Statistics: Num rows: 5 Data size: 4274 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 3165 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
Dp Sort State: PARTITION_BUCKET_SORTED
- Statistics: Num rows: 5 Data size: 4274 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 3165 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1613,19 +1613,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part_sdpo_no_cp
- Statistics: Num rows: 1725 Data size: 705510 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 3201 Data size: 1197516 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
- Statistics: Num rows: 5 Data size: 2044 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 1860 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), key (type: string), ds (type: string), hr (type: int)
outputColumnNames: _col0, _col1, _col3, _col4
- Statistics: Num rows: 5 Data size: 2044 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: ++++
Map-reduce partition columns: _col3 (type: string), _col4 (type: int)
- Statistics: Num rows: 5 Data size: 2044 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string), 'bar' (type: string)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1635,11 +1635,11 @@ STAGE PLANS:
Select Operator
expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number'
- Statistics: Num rows: 5 Data size: 2044 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 3165 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
Dp Sort State: PARTITION_BUCKET_SORTED
- Statistics: Num rows: 5 Data size: 2044 Basic stats: PARTIAL Column stats: PARTIAL
+ Statistics: Num rows: 5 Data size: 3165 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9fae0d19/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out
index 7a880dd..ab8c7a1 100644
--- a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out
+++ b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out
@@ -3233,19 +3233,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_uami_n1
- Statistics: Num rows: 281 Data size: 87904 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1002 Data size: 312584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (((de = 109.23) or (de = 119.23)) and enforce_constraint(vc is not null)) (type: boolean)
- Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), i (type: int), vc (type: varchar(128))
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: varchar(128))
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
@@ -3255,10 +3255,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -3326,19 +3326,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_uami_n1
- Statistics: Num rows: 320 Data size: 100040 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1002 Data size: 312584 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((de = 3.14) and enforce_constraint((i is not null and vc is not null))) (type: boolean)
- Statistics: Num rows: 2 Data size: 625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 623 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), i (type: int), vc (type: varchar(128))
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 2 Data size: 625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 623 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
- Statistics: Num rows: 2 Data size: 625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 623 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: varchar(128))
Execution mode: vectorized, llap
LLAP IO: may be used (ACID table)
@@ -3348,10 +3348,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 623 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 623 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat