You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by de...@apache.org on 2024/10/09 04:58:56 UTC
(hive) branch master updated: HIVE-21614: Derby/Oracle does not support CLOB comparisons when listTableNamesByFilter using the parameter (#3720) (Zhihua Deng, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f583467f608 HIVE-21614: Derby/Oracle does not support CLOB comparisons when listTableNamesByFilter using the parameter (#3720) (Zhihua Deng, reviewed by Denys Kuzmenko)
f583467f608 is described below
commit f583467f6080f127d8c4c65ba3c407af7eba6873
Author: dengzh <de...@gmail.com>
AuthorDate: Wed Oct 9 12:58:49 2024 +0800
HIVE-21614: Derby/Oracle does not support CLOB comparisons when listTableNamesByFilter using the parameter (#3720) (Zhihua Deng, reviewed by Denys Kuzmenko)
---
.../hive/metastore/TestTablesListByFilter.java | 171 +++++++++++++++++++++
.../hadoop/hive/metastore/DatabaseProduct.java | 5 +
.../hive/metastore/parser/ExpressionTree.java | 6 +
3 files changed, 182 insertions(+)
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/TestTablesListByFilter.java b/ql/src/test/org/apache/hadoop/hive/metastore/TestTablesListByFilter.java
new file mode 100644
index 00000000000..9a84236b4d6
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/TestTablesListByFilter.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.client.MetaStoreClientTest;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+@Category(MetastoreCheckinTest.class)
+public class TestTablesListByFilter extends MetaStoreClientTest {
+ private static final String DEFAULT_DATABASE = "test_tables_list_by_filter";
+ private AbstractMetaStoreService metaStore;
+ private HiveMetaStoreClient client;
+ private Table[] testTables = new Table[6];
+
+ public TestTablesListByFilter(String name, AbstractMetaStoreService metaStore) {
+ this.metaStore = metaStore;
+ }
+
+ @Before
+ public void setupDb() throws Exception {
+ client = metaStore.getClient();
+ // Clean up the database
+ client.dropDatabase(DEFAULT_DATABASE, true, true, true);
+ Configuration conf = metaStore.getConf();
+ new DatabaseBuilder().setName(DEFAULT_DATABASE).create(client, conf);
+
+ testTables[0] =
+ new TableBuilder()
+ .setDbName(DEFAULT_DATABASE)
+ .setTableName("filter_test_table_0")
+ .addCol("test_col", "int")
+ .setOwner("Owner1")
+ .setLastAccessTime(1000)
+ .addTableParam("param1", "value1")
+ .create(client, conf);
+
+ testTables[1] =
+ new TableBuilder()
+ .setDbName(DEFAULT_DATABASE)
+ .setTableName("filter_test_table_1")
+ .addCol("test_col", "int")
+ .setOwner("Owner1")
+ .setLastAccessTime(2000)
+ .addTableParam("param1", "value2")
+ .create(client, conf);
+
+ testTables[2] =
+ new TableBuilder()
+ .setDbName(DEFAULT_DATABASE)
+ .setTableName("filter_test_table_2")
+ .addCol("test_col", "int")
+ .setOwner("Owner2")
+ .setLastAccessTime(1000)
+ .addTableParam("param1", "value2")
+ .create(client, conf);
+
+ testTables[3] =
+ new TableBuilder()
+ .setDbName(DEFAULT_DATABASE)
+ .setTableName("filter_test_table_3")
+ .addCol("test_col", "int")
+ .setOwner("Owner3")
+ .setLastAccessTime(3000)
+ .addTableParam("param1", "value2")
+ .create(client, conf);
+
+ testTables[4] =
+ new TableBuilder()
+ .setDbName(DEFAULT_DATABASE)
+ .setTableName("filter_test_table_4")
+ .addCol("test_col", "int")
+ .setOwner("Tester")
+ .setLastAccessTime(2500)
+ .addTableParam("param1", "value4")
+ .create(client, conf);
+
+ testTables[5] =
+ new TableBuilder()
+ .setDbName(DEFAULT_DATABASE)
+ .setTableName("filter_test_table_5")
+ .addCol("test_col", "int")
+ .create(client, conf);
+
+ // Reload tables from the MetaStore
+ for(int i=0; i < testTables.length; i++) {
+ testTables[i] = client.getTable(testTables[i].getCatName(), testTables[i].getDbName(),
+ testTables[i].getTableName());
+ }
+ }
+
+ @After
+ public void tearDownDb() throws Exception {
+ try {
+ if (client != null) {
+ // Clean up the database
+ client.dropDatabase(DEFAULT_DATABASE, true, true, true);
+ }
+ } finally {
+ if (client != null) {
+ client.close();
+ }
+ }
+ }
+
+ @Test
+ public void testListTableNamesByFilterCheckParameter() throws Exception {
+ String filter = hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + "param1=\"value2\"";
+ List<String> tableNames = client.listTableNamesByFilter(DEFAULT_DATABASE, filter, (short)-1);
+ Assert.assertEquals("Found tables", 3, tableNames.size());
+ Assert.assertTrue(tableNames.contains(testTables[1].getTableName()));
+ Assert.assertTrue(tableNames.contains(testTables[2].getTableName()));
+ Assert.assertTrue(tableNames.contains(testTables[3].getTableName()));
+ }
+
+ @Test
+ public void testListTableNamesByFilterCheckNotEquals() throws Exception {
+ String filter = hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + "param1<>\"value2\"";
+ List<String> tableNames = client.listTableNamesByFilter(DEFAULT_DATABASE, filter, (short)-1);
+ filter = hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + "param1 != \"value2\"";
+ List<String> tableNames1 = client.listTableNamesByFilter(DEFAULT_DATABASE, filter, (short) -1);
+ Assert.assertEquals(tableNames, tableNames1);
+ Assert.assertEquals("Found tables", 2, tableNames.size());
+ Assert.assertTrue(tableNames.contains(testTables[0].getTableName()));
+ Assert.assertTrue(tableNames.contains(testTables[4].getTableName()));
+ }
+
+ @Test
+ public void testListTableNamesByFilterCheckCombined() throws Exception {
+ // Combined: last_access<=3000 and (Owner="Tester" or param1="param2")
+ String filter = hive_metastoreConstants.HIVE_FILTER_FIELD_LAST_ACCESS + "<3000 and ("
+ + hive_metastoreConstants.HIVE_FILTER_FIELD_OWNER + "=\"Tester\" or "
+ + hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + "param1=\"value2\")";
+ List<String> tableNames = client.listTableNamesByFilter(DEFAULT_DATABASE, filter, (short)-1);
+ Assert.assertEquals("Found tables", 3, tableNames.size());
+ Assert.assertTrue(tableNames.contains(testTables[1].getTableName()));
+ Assert.assertTrue(tableNames.contains(testTables[2].getTableName()));
+ Assert.assertTrue(tableNames.contains(testTables[4].getTableName()));
+ }
+
+}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
index e18feedbf56..7a5f2059589 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
@@ -812,4 +812,9 @@ public class DatabaseProduct implements Configurable {
derbyLock.unlock();
}
}
+
+ public static boolean isDerbyOracle() {
+ return dbType == DbType.DERBY || dbType == DbType.ORACLE;
+ }
+
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
index 07acf2adcd6..b6baa3333c1 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
@@ -27,6 +27,7 @@ import java.util.Stack;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.ColumnType;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -342,6 +343,11 @@ public class ExpressionTree {
operator = node.operator;
value = node.value;
isReverseOrder = node.isReverseOrder;
+ if (node.keyName.startsWith(hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS)
+ && DatabaseProduct.isDerbyOracle() && node.operator == Operator.EQUALS) {
+ // Rewrite the EQUALS operator to LIKE
+ operator = Operator.LIKE;
+ }
if (partitionKeys != null) {
generateJDOFilterOverPartitions(conf, params, filterBuilder, partitionKeys);
} else {