You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2024/09/23 11:46:01 UTC
(hive) branch branch-4.0 updated (8def11266ab -> 84c4de4f88f)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a change to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
from 8def11266ab HIVE-28225: Iceberg: Delete on entire table fails on COW mode. (#5219). (Ayush Saxena, reviewed by Stamatis Zampetakis)
new a7d0a186014 HIVE-28451: JDBC: TableName matcher fix in GenericJdbcDatabaseAccessor#addBoundaryToQuery (Denys Kuzmenko, reviewed by Simhadri Govindappa)
new 84c4de4f88f HIVE-26047: Vectorized LIKE UDF optimization (Ryu Kobayashi, reviewed by Denys Kuzmenko)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../jdbc/dao/GenericJdbcDatabaseAccessor.java | 23 +-
.../AbstractFilterStringColLikeStringScalar.java | 14 +-
.../FilterStringColLikeStringScalar.java | 164 ++++++------
.../expressions/TestVectorStringExpressions.java | 68 +++++
.../queries/clientpositive/external_jdbc_view.q | 89 +++++++
.../clientpositive/llap/external_jdbc_view.q.out | 292 +++++++++++++++++++++
6 files changed, 539 insertions(+), 111 deletions(-)
create mode 100644 ql/src/test/queries/clientpositive/external_jdbc_view.q
create mode 100644 ql/src/test/results/clientpositive/llap/external_jdbc_view.q.out
(hive) 01/02: HIVE-28451: JDBC: TableName matcher fix in GenericJdbcDatabaseAccessor#addBoundaryToQuery (Denys Kuzmenko, reviewed by Simhadri Govindappa)
Posted by dk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
commit a7d0a18601451836dede4bcc6adece5075403f83
Author: Denys Kuzmenko <dk...@apache.org>
AuthorDate: Fri Aug 23 14:01:59 2024 +0200
HIVE-28451: JDBC: TableName matcher fix in GenericJdbcDatabaseAccessor#addBoundaryToQuery (Denys Kuzmenko, reviewed by Simhadri Govindappa)
Closes #5396
(cherry picked from commit 407ae674bf3786bc9312d7a5d59cd1db7d0abd0f)
---
.../jdbc/dao/GenericJdbcDatabaseAccessor.java | 23 +-
.../queries/clientpositive/external_jdbc_view.q | 89 +++++++
.../clientpositive/llap/external_jdbc_view.q.out | 292 +++++++++++++++++++++
3 files changed, 386 insertions(+), 18 deletions(-)
diff --git a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java
index d6a36f0b729..1b98075e859 100644
--- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java
+++ b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java
@@ -62,7 +62,7 @@ public class GenericJdbcDatabaseAccessor implements DatabaseAccessor {
protected static final int DEFAULT_FETCH_SIZE = 1000;
protected static final Logger LOGGER = LoggerFactory.getLogger(GenericJdbcDatabaseAccessor.class);
private DataSource dbcpDataSource = null;
- static final Pattern fromPattern = Pattern.compile("(.*?\\sfrom\\s)(.*+)", Pattern.CASE_INSENSITIVE|Pattern.DOTALL);
+ static final Pattern fromPattern = Pattern.compile("(.*?\\sfrom\\s+)([^\\s]+)(.*?)", Pattern.CASE_INSENSITIVE|Pattern.DOTALL);
private static ColumnMetadataAccessor<TypeInfo> typeInfoTranslator = (meta, col) -> {
JDBCType type = JDBCType.valueOf(meta.getColumnType(col));
@@ -196,7 +196,7 @@ public class GenericJdbcDatabaseAccessor implements DatabaseAccessor {
return rs.getInt(1);
}
else {
- LOGGER.warn("The count query did not return any results.", countQuery);
+ LOGGER.warn("The count query {} did not return any results.", countQuery);
throw new HiveJdbcDatabaseAccessException("Count query did not return any results.");
}
}
@@ -361,26 +361,13 @@ public class GenericJdbcDatabaseAccessor implements DatabaseAccessor {
String result;
if (tableName != null) {
// Looking for table name in from clause, replace with the boundary query
- // TODO consolidate this
- // Currently only use simple string match, this should be improved by looking
- // for only table name in from clause
- String tableString = null;
Matcher m = fromPattern.matcher(sql);
Preconditions.checkArgument(m.matches());
- String queryBeforeFrom = m.group(1);
- String queryAfterFrom = " " + m.group(2) + " ";
-
- Character[] possibleDelimits = new Character[] {'`', '\"', ' '};
- for (Character possibleDelimit : possibleDelimits) {
- if (queryAfterFrom.contains(possibleDelimit + tableName + possibleDelimit)) {
- tableString = possibleDelimit + tableName + possibleDelimit;
- break;
- }
- }
- if (tableString == null) {
+
+ if (!tableName.equals(m.group(2).replaceAll("[`\"]", ""))) {
throw new RuntimeException("Cannot find " + tableName + " in sql query " + sql);
}
- result = queryBeforeFrom + queryAfterFrom.replace(tableString, " (" + boundaryQuery + ") " + tableName + " ");
+ result = String.format("%s (%s) tmptable %s", m.group(1), boundaryQuery, m.group(3));
} else {
result = boundaryQuery;
}
diff --git a/ql/src/test/queries/clientpositive/external_jdbc_view.q b/ql/src/test/queries/clientpositive/external_jdbc_view.q
new file mode 100644
index 00000000000..9575cabaac0
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/external_jdbc_view.q
@@ -0,0 +1,89 @@
+CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput';
+
+SELECT
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_jdbc_view;create=true','','',
+'CREATE TABLE person ("id" INTEGER, "name" VARCHAR(25), "jid" INTEGER, "cid" INTEGER)' );
+
+SELECT
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_jdbc_view;create=true','','',
+'CREATE TABLE country ("id" INTEGER, "name" VARCHAR(25))' );
+
+SELECT
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_jdbc_view;create=true','','',
+'CREATE TABLE job ("id" INTEGER, "title" VARCHAR(20))' );
+
+
+CREATE EXTERNAL TABLE person
+(
+ id int,
+ name varchar(25),
+ jid int,
+ cid int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+ "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_jdbc_view;create=true;collation=TERRITORY_BASED:PRIMARY",
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.table" = "PERSON",
+ "hive.sql.dbcp.maxActive" = "1"
+);
+INSERT INTO `person` VALUES (1, 'Laszlo', 1, 1);
+
+CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(25)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+ "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_jdbc_view;create=true;collation=TERRITORY_BASED:PRIMARY",
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.table" = "COUNTRY",
+ "hive.sql.dbcp.maxActive" = "1",
+ "hive.sql.numPartitions"="2",
+ "hive.sql.partitionColumn"="id"
+);
+INSERT INTO `country` VALUES (1, 'Hungary');
+INSERT INTO `country` VALUES (2, 'Poland');
+INSERT INTO `country` VALUES (3, 'Germany');
+
+CREATE EXTERNAL TABLE job
+(
+ id int,
+ title varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+ "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_jdbc_view;create=true;collation=TERRITORY_BASED:PRIMARY",
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.schema" = "APP",
+ "hive.sql.table" = "JOB",
+ "hive.sql.dbcp.maxActive" = "1",
+ "hive.sql.numPartitions"="2",
+ "hive.sql.partitionColumn"="id"
+);
+INSERT INTO `job` VALUES (1, 'Software Engineer');
+INSERT INTO `job` VALUES (2, 'Architect');
+INSERT INTO `job` VALUES (3, 'Quality Assurance');
+
+CREATE VIEW v1 AS
+SELECT id FROM job WHERE title = 'Software Engineer';
+
+select * from v1;
+
+CREATE VIEW v2 AS
+SELECT person.* FROM person
+ JOIN v1 job on person.jid = job.id
+ JOIN country on person.cid = country.id
+WHERE country.name = 'Hungary';
+
+select * from v2;
diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_view.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_view.q.out
new file mode 100644
index 00000000000..4094a8a80a9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/external_jdbc_view.q.out
@@ -0,0 +1,292 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: dboutput
+POSTHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: dboutput
+PREHOOK: query: SELECT
+#### A masked pattern was here ####
+'CREATE TABLE person ("id" INTEGER, "name" VARCHAR(25), "jid" INTEGER, "cid" INTEGER)' )
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+#### A masked pattern was here ####
+'CREATE TABLE person ("id" INTEGER, "name" VARCHAR(25), "jid" INTEGER, "cid" INTEGER)' )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+0
+PREHOOK: query: SELECT
+#### A masked pattern was here ####
+'CREATE TABLE country ("id" INTEGER, "name" VARCHAR(25))' )
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+#### A masked pattern was here ####
+'CREATE TABLE country ("id" INTEGER, "name" VARCHAR(25))' )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+0
+PREHOOK: query: SELECT
+#### A masked pattern was here ####
+'CREATE TABLE job ("id" INTEGER, "title" VARCHAR(20))' )
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+#### A masked pattern was here ####
+'CREATE TABLE job ("id" INTEGER, "title" VARCHAR(20))' )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+0
+PREHOOK: query: CREATE EXTERNAL TABLE person
+(
+ id int,
+ name varchar(25),
+ jid int,
+ cid int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.table" = "PERSON",
+ "hive.sql.dbcp.maxActive" = "1"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@person
+POSTHOOK: query: CREATE EXTERNAL TABLE person
+(
+ id int,
+ name varchar(25),
+ jid int,
+ cid int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.table" = "PERSON",
+ "hive.sql.dbcp.maxActive" = "1"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@person
+PREHOOK: query: INSERT INTO `person` VALUES (1, 'Laszlo', 1, 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@person
+POSTHOOK: query: INSERT INTO `person` VALUES (1, 'Laszlo', 1, 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@person
+PREHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(25)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.table" = "COUNTRY",
+ "hive.sql.dbcp.maxActive" = "1",
+ "hive.sql.numPartitions"="2",
+ "hive.sql.partitionColumn"="id"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@country
+POSTHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(25)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.table" = "COUNTRY",
+ "hive.sql.dbcp.maxActive" = "1",
+ "hive.sql.numPartitions"="2",
+ "hive.sql.partitionColumn"="id"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@country
+PREHOOK: query: INSERT INTO `country` VALUES (1, 'Hungary')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@country
+POSTHOOK: query: INSERT INTO `country` VALUES (1, 'Hungary')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@country
+PREHOOK: query: INSERT INTO `country` VALUES (2, 'Poland')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@country
+POSTHOOK: query: INSERT INTO `country` VALUES (2, 'Poland')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@country
+PREHOOK: query: INSERT INTO `country` VALUES (3, 'Germany')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@country
+POSTHOOK: query: INSERT INTO `country` VALUES (3, 'Germany')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@country
+PREHOOK: query: CREATE EXTERNAL TABLE job
+(
+ id int,
+ title varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.schema" = "APP",
+ "hive.sql.table" = "JOB",
+ "hive.sql.dbcp.maxActive" = "1",
+ "hive.sql.numPartitions"="2",
+ "hive.sql.partitionColumn"="id"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@job
+POSTHOOK: query: CREATE EXTERNAL TABLE job
+(
+ id int,
+ title varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "APP",
+ "hive.sql.dbcp.password" = "mine",
+ "hive.sql.schema" = "APP",
+ "hive.sql.table" = "JOB",
+ "hive.sql.dbcp.maxActive" = "1",
+ "hive.sql.numPartitions"="2",
+ "hive.sql.partitionColumn"="id"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@job
+PREHOOK: query: INSERT INTO `job` VALUES (1, 'Software Engineer')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@job
+POSTHOOK: query: INSERT INTO `job` VALUES (1, 'Software Engineer')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@job
+PREHOOK: query: INSERT INTO `job` VALUES (2, 'Architect')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@job
+POSTHOOK: query: INSERT INTO `job` VALUES (2, 'Architect')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@job
+PREHOOK: query: INSERT INTO `job` VALUES (3, 'Quality Assurance')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@job
+POSTHOOK: query: INSERT INTO `job` VALUES (3, 'Quality Assurance')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@job
+PREHOOK: query: CREATE VIEW v1 AS
+SELECT id FROM job WHERE title = 'Software Engineer'
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@job
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v1
+POSTHOOK: query: CREATE VIEW v1 AS
+SELECT id FROM job WHERE title = 'Software Engineer'
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@job
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v1
+POSTHOOK: Lineage: v1.id SIMPLE [(job)job.FieldSchema(name:id, type:int, comment:from deserializer), ]
+PREHOOK: query: select * from v1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@job
+PREHOOK: Input: default@v1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@job
+POSTHOOK: Input: default@v1
+#### A masked pattern was here ####
+1
+PREHOOK: query: CREATE VIEW v2 AS
+SELECT person.* FROM person
+ JOIN v1 job on person.jid = job.id
+ JOIN country on person.cid = country.id
+WHERE country.name = 'Hungary'
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@country
+PREHOOK: Input: default@job
+PREHOOK: Input: default@person
+PREHOOK: Input: default@v1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v2
+POSTHOOK: query: CREATE VIEW v2 AS
+SELECT person.* FROM person
+ JOIN v1 job on person.jid = job.id
+ JOIN country on person.cid = country.id
+WHERE country.name = 'Hungary'
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@country
+POSTHOOK: Input: default@job
+POSTHOOK: Input: default@person
+POSTHOOK: Input: default@v1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v2
+POSTHOOK: Lineage: v2.cid SIMPLE [(person)person.FieldSchema(name:cid, type:int, comment:from deserializer), ]
+POSTHOOK: Lineage: v2.id SIMPLE [(person)person.FieldSchema(name:id, type:int, comment:from deserializer), ]
+POSTHOOK: Lineage: v2.jid SIMPLE [(person)person.FieldSchema(name:jid, type:int, comment:from deserializer), ]
+POSTHOOK: Lineage: v2.name SIMPLE [(person)person.FieldSchema(name:name, type:varchar(25), comment:from deserializer), ]
+PREHOOK: query: select * from v2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@country
+PREHOOK: Input: default@job
+PREHOOK: Input: default@person
+PREHOOK: Input: default@v1
+PREHOOK: Input: default@v2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@country
+POSTHOOK: Input: default@job
+POSTHOOK: Input: default@person
+POSTHOOK: Input: default@v1
+POSTHOOK: Input: default@v2
+#### A masked pattern was here ####
+1 Laszlo 1 1
(hive) 02/02: HIVE-26047: Vectorized LIKE UDF optimization (Ryu Kobayashi, reviewed by Denys Kuzmenko)
Posted by dk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 84c4de4f88f62559d94971194adbb08b22b6beff
Author: Ryu Kobayashi <be...@gmail.com>
AuthorDate: Fri Apr 19 21:37:33 2024 +0900
HIVE-26047: Vectorized LIKE UDF optimization (Ryu Kobayashi, reviewed by Denys Kuzmenko)
Closes #4998
(cherry picked from commit 2134e3dafaf95d56ec8531a1185ac0170199b218)
---
.../AbstractFilterStringColLikeStringScalar.java | 14 +-
.../FilterStringColLikeStringScalar.java | 164 ++++++++++-----------
.../expressions/TestVectorStringExpressions.java | 68 +++++++++
3 files changed, 153 insertions(+), 93 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/AbstractFilterStringColLikeStringScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/AbstractFilterStringColLikeStringScalar.java
index 85c07b6dc51..542c6b38149 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/AbstractFilterStringColLikeStringScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/AbstractFilterStringColLikeStringScalar.java
@@ -226,7 +226,7 @@ public abstract class AbstractFilterStringColLikeStringScalar extends VectorExpr
protected static final class NoneChecker implements Checker {
final byte [] byteSub;
- NoneChecker(String pattern) {
+ public NoneChecker(String pattern) {
byteSub = pattern.getBytes(StandardCharsets.UTF_8);
}
@@ -250,7 +250,7 @@ public abstract class AbstractFilterStringColLikeStringScalar extends VectorExpr
protected static final class BeginChecker implements Checker {
final byte[] byteSub;
- BeginChecker(String pattern) {
+ public BeginChecker(String pattern) {
byteSub = pattern.getBytes(StandardCharsets.UTF_8);
}
@@ -269,7 +269,7 @@ public abstract class AbstractFilterStringColLikeStringScalar extends VectorExpr
protected static final class EndChecker implements Checker {
final byte[] byteSub;
- EndChecker(String pattern) {
+ public EndChecker(String pattern) {
byteSub = pattern.getBytes(StandardCharsets.UTF_8);
}
@@ -288,7 +288,7 @@ public abstract class AbstractFilterStringColLikeStringScalar extends VectorExpr
protected static final class MiddleChecker implements Checker {
final StringExpr.Finder finder;
- MiddleChecker(String pattern) {
+ public MiddleChecker(String pattern) {
finder = StringExpr.compile(pattern.getBytes(StandardCharsets.UTF_8));
}
@@ -324,7 +324,7 @@ public abstract class AbstractFilterStringColLikeStringScalar extends VectorExpr
final int beginLen;
final int endLen;
- ChainedChecker(String pattern) {
+ public ChainedChecker(String pattern) {
final StringTokenizer tokens = new StringTokenizer(pattern, "%");
final boolean leftAnchor = pattern.startsWith("%") == false;
final boolean rightAnchor = pattern.endsWith("%") == false;
@@ -413,12 +413,12 @@ public abstract class AbstractFilterStringColLikeStringScalar extends VectorExpr
/**
* Matches each string to a pattern with Java regular expression package.
*/
- protected static class ComplexChecker implements Checker {
+ protected static final class ComplexChecker implements Checker {
Pattern compiledPattern;
Matcher matcher;
FastUTF8Decoder decoder;
- ComplexChecker(String pattern) {
+ public ComplexChecker(String pattern) {
compiledPattern = Pattern.compile(pattern);
matcher = compiledPattern.matcher("");
decoder = new FastUTF8Decoder();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColLikeStringScalar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColLikeStringScalar.java
index 46cc4300413..88f12a2a9fa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColLikeStringScalar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColLikeStringScalar.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,11 +20,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import org.apache.hadoop.hive.ql.udf.UDFLike;
+import com.google.common.collect.ImmutableList;
+
import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
/**
* Evaluate LIKE filter on a batch for a vector of strings.
@@ -32,13 +31,16 @@ import java.util.regex.Pattern;
public class FilterStringColLikeStringScalar extends AbstractFilterStringColLikeStringScalar {
private static final long serialVersionUID = 1L;
- private transient final static List<CheckerFactory> checkerFactories = Arrays.asList(
- new BeginCheckerFactory(),
- new EndCheckerFactory(),
- new MiddleCheckerFactory(),
- new NoneCheckerFactory(),
- new ChainedCheckerFactory(),
- new ComplexCheckerFactory());
+ private static final List<CheckerFactory> CHECKER_FACTORIES = ImmutableList.of(
+ pattern -> {
+ UDFLikePattern udfLike = UDFLikePattern.matcher(pattern);
+ try {
+ return udfLike.checker.getConstructor(String.class).newInstance(
+ udfLike.format(pattern));
+ } catch (Exception e) {
+ throw new IllegalArgumentException("unable to initialize Checker");
+ }
+ });
public FilterStringColLikeStringScalar() {
super();
@@ -51,93 +53,83 @@ public class FilterStringColLikeStringScalar extends AbstractFilterStringColLike
@Override
protected List<CheckerFactory> getCheckerFactories() {
- return checkerFactories;
+ return CHECKER_FACTORIES;
}
- /**
- * Accepts simple LIKE patterns like "abc%" and creates corresponding checkers.
- */
- private static class BeginCheckerFactory implements CheckerFactory {
- private static final Pattern BEGIN_PATTERN = Pattern.compile("([^_%]+)%");
-
- public Checker tryCreate(String pattern) {
- Matcher matcher = BEGIN_PATTERN.matcher(pattern);
- if (matcher.matches()) {
- return new BeginChecker(matcher.group(1));
+ private enum UDFLikePattern {
+ // Accepts simple LIKE patterns like "abc%" and creates corresponding checkers.
+ BEGIN(BeginChecker.class) {
+ @Override
+ String format(String pattern) {
+ return pattern.substring(0, pattern.length() - 1);
}
- return null;
- }
- }
-
- /**
- * Accepts simple LIKE patterns like "%abc" and creates a corresponding checkers.
- */
- private static class EndCheckerFactory implements CheckerFactory {
- private static final Pattern END_PATTERN = Pattern.compile("%([^_%]+)");
-
- public Checker tryCreate(String pattern) {
- Matcher matcher = END_PATTERN.matcher(pattern);
- if (matcher.matches()) {
- return new EndChecker(matcher.group(1));
+ },
+ // Accepts simple LIKE patterns like "%abc" and creates a corresponding checkers.
+ END(EndChecker.class) {
+ @Override
+ String format(String pattern) {
+ return pattern.substring(1);
}
- return null;
- }
- }
-
- /**
- * Accepts simple LIKE patterns like "%abc%" and creates a corresponding checkers.
- */
- private static class MiddleCheckerFactory implements CheckerFactory {
- private static final Pattern MIDDLE_PATTERN = Pattern.compile("%([^_%]+)%");
-
- public Checker tryCreate(String pattern) {
- Matcher matcher = MIDDLE_PATTERN.matcher(pattern);
- if (matcher.matches()) {
- return new MiddleChecker(matcher.group(1));
+ },
+ // Accepts simple LIKE patterns like "%abc%" and creates a corresponding checkers.
+ MIDDLE(MiddleChecker.class) {
+ @Override
+ String format(String pattern) {
+ return pattern.substring(1, pattern.length() - 1);
}
- return null;
- }
- }
+ },
+ // Accepts any LIKE patterns and creates corresponding checkers.
+ COMPLEX(ComplexChecker.class) {
+ @Override
+ String format(String pattern) {
+ return "^" + UDFLike.likePatternToRegExp(pattern) + "$";
+ }
+ },
+ // Accepts chained LIKE patterns without escaping like "abc%def%ghi%" and
+ // creates corresponding checkers.
+ CHAINED(ChainedChecker.class),
+ // Accepts simple LIKE patterns like "abc" and creates corresponding checkers.
+ NONE(NoneChecker.class);
- /**
- * Accepts simple LIKE patterns like "abc" and creates corresponding checkers.
- */
- private static class NoneCheckerFactory implements CheckerFactory {
- private static final Pattern NONE_PATTERN = Pattern.compile("[^%_]+");
+ Class<? extends Checker> checker;
- public Checker tryCreate(String pattern) {
- Matcher matcher = NONE_PATTERN.matcher(pattern);
- if (matcher.matches()) {
- return new NoneChecker(pattern);
- }
- return null;
+ UDFLikePattern(Class<? extends Checker> checker) {
+ this.checker = checker;
}
- }
- /**
- * Accepts chained LIKE patterns without escaping like "abc%def%ghi%" and creates corresponding
- * checkers.
- *
- */
- private static class ChainedCheckerFactory implements CheckerFactory {
- private static final Pattern CHAIN_PATTERN = Pattern.compile("(%?[^%_\\\\]+%?)+");
-
- public Checker tryCreate(String pattern) {
- Matcher matcher = CHAIN_PATTERN.matcher(pattern);
- if (matcher.matches()) {
- return new ChainedChecker(pattern);
+ private static UDFLikePattern matcher(String pattern) {
+ UDFLikePattern lastType = NONE;
+ int length = pattern.length();
+ char lastChar = 0;
+
+ for (int i = 0; i < length; i++) {
+ char n = pattern.charAt(i);
+ if (n == '_' && lastChar != '\\') { // such as "a_bc"
+ return COMPLEX;
+ } else if (n == '%') {
+ if (i == 0) { // such as "%abc"
+ lastType = END;
+ } else if (i < length - 1) {
+ if (lastChar != '\\') { // such as "a%bc"
+ lastType = CHAINED;
+ }
+ } else {
+ if (lastChar != '\\') {
+ if (lastType == END) { // such as "%abc%"
+ lastType = MIDDLE;
+ } else if (lastType != CHAINED) {
+ lastType = BEGIN; // such as "abc%"
+ }
+ }
+ }
+ }
+ lastChar = n;
}
- return null;
+ return lastType;
}
- }
- /**
- * Accepts any LIKE patterns and creates corresponding checkers.
- */
- private static class ComplexCheckerFactory implements CheckerFactory {
- public Checker tryCreate(String pattern) {
- // anchor the pattern to the start:end of the whole string.
- return new ComplexChecker("^" + UDFLike.likePatternToRegExp(pattern) + "$");
+ String format(String pattern) {
+ return pattern;
}
}
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java
index 62c262e3409..22ac562c43a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java
@@ -4303,6 +4303,12 @@ public class TestVectorStringExpressions {
Assert.assertEquals(FilterStringColLikeStringScalar.BeginChecker.class,
expr.checker.getClass());
+ expr = new FilterStringColLikeStringScalar(0, "abc\\%def%".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.BeginChecker.class,
+ expr.checker.getClass());
+
// END pattern
expr = new FilterStringColLikeStringScalar(0, "%abc".getBytes(StandardCharsets.UTF_8));
expr.transientInit(hiveConf);
@@ -4310,6 +4316,12 @@ public class TestVectorStringExpressions {
Assert.assertEquals(FilterStringColLikeStringScalar.EndChecker.class,
expr.checker.getClass());
+ expr = new FilterStringColLikeStringScalar(0, "%abc\\%def".getBytes(StandardCharsets.UTF_8));
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.EndChecker.class,
+ expr.checker.getClass());
+
// MIDDLE pattern
expr = new FilterStringColLikeStringScalar(0, "%abc%".getBytes());
expr.transientInit(hiveConf);
@@ -4317,6 +4329,12 @@ public class TestVectorStringExpressions {
Assert.assertEquals(FilterStringColLikeStringScalar.MiddleChecker.class,
expr.checker.getClass());
+ expr = new FilterStringColLikeStringScalar(0, "%abc\\%def%".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.MiddleChecker.class,
+ expr.checker.getClass());
+
// CHAIN pattern
expr = new FilterStringColLikeStringScalar(0, "%abc%de".getBytes());
expr.transientInit(hiveConf);
@@ -4331,6 +4349,56 @@ public class TestVectorStringExpressions {
Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
expr.checker.getClass());
+ expr = new FilterStringColLikeStringScalar(0, "abc_".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
+ expr = new FilterStringColLikeStringScalar(0, "abc\\_def_".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
+ expr = new FilterStringColLikeStringScalar(0, "_abc".getBytes(StandardCharsets.UTF_8));
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
+ expr = new FilterStringColLikeStringScalar(0, "_abc\\_def".getBytes(StandardCharsets.UTF_8));
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
+ expr = new FilterStringColLikeStringScalar(0, "_abc_".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
+ expr = new FilterStringColLikeStringScalar(0, "_abc\\_def_".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
+ expr = new FilterStringColLikeStringScalar(0, "_abc_de".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
+
+ expr = new FilterStringColLikeStringScalar(0,
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_b".getBytes());
+ expr.transientInit(hiveConf);
+ expr.evaluate(vrb);
+ Assert.assertEquals(FilterStringColLikeStringScalar.ComplexChecker.class,
+ expr.checker.getClass());
+
// NONE pattern
expr = new FilterStringColLikeStringScalar(0, "abc".getBytes());
expr.transientInit(hiveConf);