You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by za...@apache.org on 2022/06/09 10:09:25 UTC
[hive] branch master updated (c55318eb586 -> efae863fe01)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
from c55318eb586 HIVE-26293: Migrate remaining exclusive DDL operations to EXCL_WRITE lock & bug fixes (Denys Kuzmenko, reviewed by Peter Vary)
new d781701d268 HIVE-26278: Add unit tests for Hive#getPartitionsByNames using batching (Stamatis Zampetakis, reviewed by Zoltan Haindrich, Krisztian Kasa, Ayush Saxena)
new 798d25c6126 HIVE-26290: Remove useless calls to DateTimeFormatter#withZone without assignment (Stamatis Zampetakis, reviewed by Ayush Saxena)
new efae863fe01 HIVE-26296: RuntimeException when executing EXPLAIN CBO JOINCOST on query with JDBC tables (Stamatis Zampetakis, reviewed by Alessandro Solimando, Krisztian Kasa)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
...hor_table.sql => q_test_author_book_tables.sql} | 11 +++
.../calcite/stats/HiveRelMdSelectivity.java | 5 +-
.../ql/udf/generic/GenericUDFFromUnixTime.java | 2 -
...TestHiveMetaStoreClientApiArgumentsChecker.java | 25 ++++++
.../queries/clientpositive/cbo_jdbc_joincost.q | 34 ++++++++
.../clientpositive/llap/cbo_jdbc_joincost.q.out | 93 ++++++++++++++++++++++
6 files changed, 164 insertions(+), 6 deletions(-)
copy data/scripts/{q_test_author_table.sql => q_test_author_book_tables.sql} (50%)
create mode 100644 ql/src/test/queries/clientpositive/cbo_jdbc_joincost.q
create mode 100644 ql/src/test/results/clientpositive/llap/cbo_jdbc_joincost.q.out
[hive] 02/03: HIVE-26290: Remove useless calls to DateTimeFormatter#withZone without assignment (Stamatis Zampetakis, reviewed by Ayush Saxena)
Posted by za...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 798d25c61262d872d756b5c73d38172fe1293207
Author: Stamatis Zampetakis <za...@gmail.com>
AuthorDate: Fri Jun 3 18:46:47 2022 +0200
HIVE-26290: Remove useless calls to DateTimeFormatter#withZone without assignment (Stamatis Zampetakis, reviewed by Ayush Saxena)
Closes #3342
---
.../org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java
index fb634bc7c97..21081cf7c11 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java
@@ -88,7 +88,6 @@ public class GenericUDFFromUnixTime extends GenericUDF {
if (timeZone == null) {
timeZone = SessionState.get() == null ? new HiveConf().getLocalTimeZone() : SessionState.get().getConf()
.getLocalTimeZone();
- FORMATTER.withZone(timeZone);
}
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
@@ -99,7 +98,6 @@ public class GenericUDFFromUnixTime extends GenericUDF {
if (context != null) {
String timeZoneStr = HiveConf.getVar(context.getJobConf(), HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE);
timeZone = TimestampTZUtil.parseTimeZone(timeZoneStr);
- FORMATTER.withZone(timeZone);
}
}
[hive] 03/03: HIVE-26296: RuntimeException when executing EXPLAIN CBO JOINCOST on query with JDBC tables (Stamatis Zampetakis, reviewed by Alessandro Solimando, Krisztian Kasa)
Posted by za...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit efae863fe010ed5c4b7de1874a336ed93b3c60b8
Author: Stamatis Zampetakis <za...@gmail.com>
AuthorDate: Tue Jun 7 17:02:12 2022 +0200
HIVE-26296: RuntimeException when executing EXPLAIN CBO JOINCOST on query with JDBC tables (Stamatis Zampetakis, reviewed by Alessandro Solimando, Krisztian Kasa)
Compute selectivity for all types of joins in the same way. There is no
particular reason to throw an exception when the Join operator is not
an instance of HiveJoin.
Closes #3349
---
data/scripts/q_test_author_book_tables.sql | 19 +++++
.../calcite/stats/HiveRelMdSelectivity.java | 5 +-
.../queries/clientpositive/cbo_jdbc_joincost.q | 34 ++++++++
.../clientpositive/llap/cbo_jdbc_joincost.q.out | 93 ++++++++++++++++++++++
4 files changed, 147 insertions(+), 4 deletions(-)
diff --git a/data/scripts/q_test_author_book_tables.sql b/data/scripts/q_test_author_book_tables.sql
new file mode 100644
index 00000000000..9b5ff99266b
--- /dev/null
+++ b/data/scripts/q_test_author_book_tables.sql
@@ -0,0 +1,19 @@
+create table author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+);
+insert into author values (1, 'Victor', 'Hugo');
+insert into author values (2, 'Alexandre', 'Dumas');
+
+create table book
+(
+ id int,
+ title varchar(100),
+ author int
+);
+insert into book
+values (1, 'Les Miserables', 1);
+insert into book
+values (2, 'The Count Of Monte Cristo', 2);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
index 2c36d8f14e6..19bd13de9a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
@@ -149,11 +149,8 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
if (j.isSemiJoin() || (j.getJoinType().equals(JoinRelType.ANTI))) {
ndvEstimate = Math.min(mq.getRowCount(j.getLeft()),
ndvEstimate);
- } else if (j instanceof HiveJoin) {
- ndvEstimate = Math.min(mq.getRowCount(j.getLeft())
- * mq.getRowCount(j.getRight()), ndvEstimate);
} else {
- throw new RuntimeException("Unexpected Join type: " + j.getClass().getName());
+ ndvEstimate = Math.min(mq.getRowCount(j.getLeft()) * mq.getRowCount(j.getRight()), ndvEstimate);
}
}
diff --git a/ql/src/test/queries/clientpositive/cbo_jdbc_joincost.q b/ql/src/test/queries/clientpositive/cbo_jdbc_joincost.q
new file mode 100644
index 00000000000..7255f3b87b0
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cbo_jdbc_joincost.q
@@ -0,0 +1,34 @@
+--!qt:database:mysql:q_test_author_book_tables.sql
+CREATE EXTERNAL TABLE author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "author"
+ );
+
+CREATE EXTERNAL TABLE book
+(
+ id int,
+ title varchar(100),
+ author int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "book"
+ );
+
+EXPLAIN CBO JOINCOST SELECT a.lname, b.title FROM author a JOIN book b ON a.id=b.author;
diff --git a/ql/src/test/results/clientpositive/llap/cbo_jdbc_joincost.q.out b/ql/src/test/results/clientpositive/llap/cbo_jdbc_joincost.q.out
new file mode 100644
index 00000000000..0dc3effcef3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/cbo_jdbc_joincost.q.out
@@ -0,0 +1,93 @@
+PREHOOK: query: CREATE EXTERNAL TABLE author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "author"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@author
+POSTHOOK: query: CREATE EXTERNAL TABLE author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "author"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@author
+PREHOOK: query: CREATE EXTERNAL TABLE book
+(
+ id int,
+ title varchar(100),
+ author int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "book"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@book
+POSTHOOK: query: CREATE EXTERNAL TABLE book
+(
+ id int,
+ title varchar(100),
+ author int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "book"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@book
+PREHOOK: query: EXPLAIN CBO JOINCOST SELECT a.lname, b.title FROM author a JOIN book b ON a.id=b.author
+PREHOOK: type: QUERY
+PREHOOK: Input: default@author
+PREHOOK: Input: default@book
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO JOINCOST SELECT a.lname, b.title FROM author a JOIN book b ON a.id=b.author
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@author
+POSTHOOK: Input: default@book
+#### A masked pattern was here ####
+CBO PLAN:
+HiveJdbcConverter(convention=[JDBC.MYSQL]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcProject(lname=[$1], title=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcJoin(condition=[=($0, $3)], joinType=[inner]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcProject(id=[$0], lname=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcFilter(condition=[IS NOT NULL($0)]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcHiveTableScan(table=[[default, author]], table:alias=[a]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcProject(title=[$1], author=[$2]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcFilter(condition=[IS NOT NULL($2)]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+ JdbcHiveTableScan(table=[[default, book]], table:alias=[b]): rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###
+
[hive] 01/03: HIVE-26278: Add unit tests for Hive#getPartitionsByNames using batching (Stamatis Zampetakis, reviewed by Zoltan Haindrich, Krisztian Kasa, Ayush Saxena)
Posted by za...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit d781701d26859b78161514ac237119243f9bd1e3
Author: Stamatis Zampetakis <za...@gmail.com>
AuthorDate: Tue Feb 8 16:56:56 2022 +0100
HIVE-26278: Add unit tests for Hive#getPartitionsByNames using batching (Stamatis Zampetakis, reviewed by Zoltan Haindrich, Krisztian Kasa, Ayush Saxena)
Ensure that ValidWriteIdList is set when batching is involved in
getPartitionByNames.
Closes #3335
---
...TestHiveMetaStoreClientApiArgumentsChecker.java | 25 ++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java
index 6aefc44c563..175b47c47d8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreClientApiArgumentsChecker.java
@@ -38,10 +38,13 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.thrift.TException;
+
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -80,6 +83,8 @@ public class TestHiveMetaStoreClientApiArgumentsChecker {
hive.getConf().set(ValidTxnList.VALID_TXNS_KEY, "1:");
hive.getConf().set(ValidWriteIdList.VALID_WRITEIDS_KEY, TABLE_NAME + ":1:");
hive.getConf().setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.TestTxnManager");
+ // Pick a small number for the batch size to easily test code with multiple batches.
+ hive.getConf().setIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX, 2);
SessionState.start(hive.getConf());
SessionState.get().initTxnMgr(hive.getConf());
Context ctx = new Context(hive.getConf());
@@ -140,6 +145,26 @@ public class TestHiveMetaStoreClientApiArgumentsChecker {
hive.getPartitionsByNames(t, new ArrayList<>(), true);
}
+ @Test
+ public void testGetPartitionsByNamesWithSingleBatch() throws HiveException {
+ hive.getPartitionsByNames(t, Arrays.asList("Greece", "Italy"), true);
+ }
+
+ @Test
+ public void testGetPartitionsByNamesWithMultipleEqualSizeBatches()
+ throws HiveException {
+ List<String> names = Arrays.asList("Greece", "Italy", "France", "Spain");
+ hive.getPartitionsByNames(t, names, true);
+ }
+
+ @Test
+ public void testGetPartitionsByNamesWithMultipleUnequalSizeBatches()
+ throws HiveException {
+ List<String> names =
+ Arrays.asList("Greece", "Italy", "France", "Spain", "Hungary");
+ hive.getPartitionsByNames(t, names, true);
+ }
+
@Test
public void testGetPartitionsByExpr() throws HiveException, TException {
List<Partition> partitions = new ArrayList<>();