You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2019/03/01 20:11:59 UTC
[hive] branch master updated: HIVE-21255: Remove
QueryConditionBuilder in JdbcStorageHandler (Daniel Dai,
reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
daijy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f51f108 HIVE-21255: Remove QueryConditionBuilder in JdbcStorageHandler (Daniel Dai, reviewed by Jesus Camacho Rodriguez)
f51f108 is described below
commit f51f108b761f0c88647f48f30447dae12b308f31
Author: Daniel Dai <da...@gmail.com>
AuthorDate: Fri Mar 1 11:52:18 2019 -0800
HIVE-21255: Remove QueryConditionBuilder in JdbcStorageHandler (Daniel Dai, reviewed by Jesus Camacho Rodriguez)
Signed-off-by: Jesus Camacho Rodriguez <jc...@apache.org>
---
.../hive/storage/jdbc/QueryConditionBuilder.java | 186 ---------------------
.../jdbc/conf/JdbcStorageConfigManager.java | 11 --
.../storage/jdbc/TestQueryConditionBuilder.java | 150 -----------------
3 files changed, 347 deletions(-)
diff --git a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java
deleted file mode 100644
index 194fad8..0000000
--- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.storage.jdbc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig;
-
-import java.beans.XMLDecoder;
-import java.io.ByteArrayInputStream;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Translates the hive query condition into a condition that can be run on the underlying database
- */
-public class QueryConditionBuilder {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(QueryConditionBuilder.class);
- private static final String EMPTY_STRING = "";
- private static QueryConditionBuilder instance = null;
-
-
- public static QueryConditionBuilder getInstance() {
- if (instance == null) {
- instance = new QueryConditionBuilder();
- }
-
- return instance;
- }
-
-
- private QueryConditionBuilder() {
-
- }
-
-
- public String buildCondition(Configuration conf) {
- if (conf == null) {
- return EMPTY_STRING;
- }
-
- String filterXml = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
- String hiveColumns = conf.get(serdeConstants.LIST_COLUMNS);
- String columnMapping = conf.get(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName());
-
- if ((filterXml == null) || ((columnMapping == null) && (hiveColumns == null))) {
- return EMPTY_STRING;
- }
-
- if (hiveColumns == null) {
- hiveColumns = "";
- }
-
- Map<String, String> columnMap = buildColumnMapping(columnMapping, hiveColumns);
- String condition = createConditionString(filterXml, columnMap);
- return condition;
- }
-
-
- /*
- * Build a Hive-to-X column mapping,
- *
- */
- private Map<String, String> buildColumnMapping(String columnMapping, String hiveColumns) {
- if ((columnMapping == null) || (columnMapping.trim().isEmpty())) {
- return createIdentityMap(hiveColumns);
- }
-
- Map<String, String> columnMap = new HashMap<String, String>();
- String[] mappingPairs = columnMapping.toLowerCase().split(",");
- for (String mapPair : mappingPairs) {
- String[] columns = mapPair.split("=");
- columnMap.put(columns[0].trim(), columns[1].trim());
- }
-
- return columnMap;
- }
-
-
- /*
- * When no mapping is defined, it is assumed that the hive column names are equivalent to the column names in the
- * underlying table
- */
- private Map<String, String> createIdentityMap(String hiveColumns) {
- Map<String, String> columnMap = new HashMap<String, String>();
- String[] columns = hiveColumns.toLowerCase().split(",");
-
- for (String col : columns) {
- columnMap.put(col.trim(), col.trim());
- }
-
- return columnMap;
- }
-
-
- /*
- * Walk to Hive AST and translate the hive column names to their equivalent mappings. This is basically a cheat.
- *
- */
- private String createConditionString(String filterXml, Map<String, String> columnMap) {
- if ((filterXml == null) || (filterXml.trim().isEmpty())) {
- return EMPTY_STRING;
- }
-
- try (XMLDecoder decoder = new XMLDecoder(new ByteArrayInputStream(filterXml.getBytes("UTF-8")))) {
- Object object = decoder.readObject();
- if (!(object instanceof ExprNodeDesc)) {
- LOGGER.error("Deserialized filter expression is not of the expected type");
- throw new RuntimeException("Deserialized filter expression is not of the expected type");
- }
-
- ExprNodeDesc conditionNode = (ExprNodeDesc) object;
- walkTreeAndTranslateColumnNames(conditionNode, columnMap);
- return conditionNode.getExprString();
- }
- catch (Exception e) {
- LOGGER.error("Error during condition build", e);
- return EMPTY_STRING;
- }
- }
-
-
- /*
- * Translate column names by walking the AST
- */
- private void walkTreeAndTranslateColumnNames(ExprNodeDesc node, Map<String, String> columnMap) {
- if (node == null) {
- return;
- }
-
- if (node instanceof ExprNodeColumnDesc) {
- ExprNodeColumnDesc column = (ExprNodeColumnDesc) node;
- String hiveColumnName = column.getColumn().toLowerCase();
- if (columnMap.containsKey(hiveColumnName)) {
- String dbColumnName = columnMap.get(hiveColumnName);
- String finalName = formatColumnName(dbColumnName);
- column.setColumn(finalName);
- }
- }
- else {
- if (node.getChildren() != null) {
- for (ExprNodeDesc childNode : node.getChildren()) {
- walkTreeAndTranslateColumnNames(childNode, columnMap);
- }
- }
- }
- }
-
-
- /**
- * This is an ugly hack for handling date column types because Hive doesn't have a built-in type for dates
- */
- private String formatColumnName(String dbColumnName) {
- if (dbColumnName.contains(":")) {
- String[] typeSplit = dbColumnName.split(":");
-
- if (typeSplit[1].equalsIgnoreCase("date")) {
- return "{d " + typeSplit[0] + "}";
- }
-
- return typeSplit[0];
- }
- else {
- return dbColumnName;
- }
- }
-}
diff --git a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java
index 5679f1b..2039144 100644
--- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java
+++ b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java
@@ -24,8 +24,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.storage.jdbc.QueryConditionBuilder;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,10 +128,6 @@ public class JdbcStorageConfigManager {
if (tableName != null) {
// We generate query as select *
query = "select * from " + tableName;
- String hiveFilterCondition = QueryConditionBuilder.getInstance().buildCondition(config);
- if ((hiveFilterCondition != null) && (!hiveFilterCondition.trim().isEmpty())) {
- query = query + " WHERE " + hiveFilterCondition;
- }
} else {
query = config.get(Constants.JDBC_QUERY);
}
@@ -152,11 +146,6 @@ public class JdbcStorageConfigManager {
String tableName = config.get(JdbcStorageConfig.TABLE.getPropertyName());
query = "select * from " + tableName;
- String hiveFilterCondition = QueryConditionBuilder.getInstance().buildCondition(config);
- if ((hiveFilterCondition != null) && (!hiveFilterCondition.trim().isEmpty())) {
- query = query + " WHERE " + hiveFilterCondition;
- }
-
return query;
}
diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java
deleted file mode 100644
index a59645d..0000000
--- a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.storage.jdbc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Scanner;
-
-import static org.hamcrest.Matchers.equalToIgnoringWhiteSpace;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.junit.Assert.assertThat;
-
-public class TestQueryConditionBuilder {
-
- private static String condition1;
- private static String condition2;
-
-
- @BeforeClass
- public static void setup() throws IOException {
- condition1 = readFileContents("condition1.xml");
- condition2 = readFileContents("condition2.xml");
- }
-
-
- private static String readFileContents(String name) throws IOException {
- try (Scanner s = new Scanner(TestQueryConditionBuilder.class.getClassLoader().getResourceAsStream(name))) {
- return s.useDelimiter("\\Z").next();
- }
- }
-
-
- @Test
- public void testSimpleCondition_noTranslation() {
- Configuration conf = new Configuration();
- conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1);
- conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id");
- String condition = QueryConditionBuilder.getInstance().buildCondition(conf);
-
- assertThat(condition, is(notNullValue()));
- assertThat(condition, is(equalToIgnoringWhiteSpace("(visitor_id = 'x')")));
- }
-
-
- @Test
- public void testSimpleCondition_withTranslation() {
- Configuration conf = new Configuration();
- conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1);
- conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id");
- conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(),
- "visitor_id=vid, sentiment=sentiment, tracking_id=tracking_id");
- String condition = QueryConditionBuilder.getInstance().buildCondition(conf);
-
- assertThat(condition, is(notNullValue()));
- assertThat(condition, is(equalToIgnoringWhiteSpace("(vid = 'x')")));
- }
-
-
- @Test
- public void testSimpleCondition_withDateType() {
- Configuration conf = new Configuration();
- conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1);
- conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id");
- conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(),
- "visitor_id=vid:date, sentiment=sentiment, tracking_id=tracking_id");
- String condition = QueryConditionBuilder.getInstance().buildCondition(conf);
-
- assertThat(condition, is(notNullValue()));
- assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')")));
- }
-
-
- @Test
- public void testSimpleCondition_withVariedCaseMappings() {
- Configuration conf = new Configuration();
- conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1);
- conf.set(serdeConstants.LIST_COLUMNS, "visitor_ID,sentiment,tracking_id");
- conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(),
- "visitor_id=VID:date, sentiment=sentiment, tracking_id=tracking_id");
- String condition = QueryConditionBuilder.getInstance().buildCondition(conf);
-
- assertThat(condition, is(notNullValue()));
- assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')")));
- }
-
-
- @Test
- public void testMultipleConditions_noTranslation() {
- Configuration conf = new Configuration();
- conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2);
- conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id");
- String condition = QueryConditionBuilder.getInstance().buildCondition(conf);
-
- assertThat(condition, is(notNullValue()));
- assertThat(condition, is(equalToIgnoringWhiteSpace("((visitor_id = 'x') and (sentiment = 'y'))")));
- }
-
-
- @Test
- public void testMultipleConditions_withTranslation() {
- Configuration conf = new Configuration();
- conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2);
- conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id");
- conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t");
- String condition = QueryConditionBuilder.getInstance().buildCondition(conf);
-
- assertThat(condition, is(notNullValue()));
- assertThat(condition, is(equalToIgnoringWhiteSpace("((v = 'x') and (s = 'y'))")));
- }
-
-
- @Test
- public void testWithNullConf() {
- String condition = QueryConditionBuilder.getInstance().buildCondition(null);
- assertThat(condition, is(notNullValue()));
- assertThat(condition.trim().isEmpty(), is(true));
- }
-
-
- @Test
- public void testWithUndefinedFilterExpr() {
- Configuration conf = new Configuration();
- conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id");
- conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t");
- String condition = QueryConditionBuilder.getInstance().buildCondition(conf);
-
- assertThat(condition, is(notNullValue()));
- assertThat(condition.trim().isEmpty(), is(true));
- }
-
-}