You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/01/09 02:43:31 UTC
(incubator-paimon) branch master updated: [hive] Fix hive to be compatible with uppercase fields when using predicates to query paimon external tables. (#2659)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c250e09e6 [hive] Fix hive to be compatible with uppercase fields when using predicates to query paimon external tables. (#2659)
c250e09e6 is described below
commit c250e09e6400e202fccfd5c3733b8381464e35a3
Author: Kerwin <37...@users.noreply.github.com>
AuthorDate: Tue Jan 9 10:43:26 2024 +0800
[hive] Fix hive to be compatible with uppercase fields when using predicates to query paimon external tables. (#2659)
---
.../hive/SearchArgumentToPredicateConverter.java | 20 +++++++++++++-------
.../java/org/apache/paimon/hive/utils/HiveUtils.java | 6 +++---
.../java/org/apache/paimon/hive/HiveReadITCase.java | 7 +++++++
3 files changed, 23 insertions(+), 10 deletions(-)
diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java
index 67d74243b..fa7881771 100644
--- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java
+++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java
@@ -41,7 +41,7 @@ import java.util.stream.Collectors;
import static org.apache.paimon.predicate.PredicateBuilder.convertJavaObject;
-/** Converts {@link SearchArgument} to {@link Predicate} with best effort. */
+/** Converts {@link SearchArgument} to {@link Predicate} with the best effort. */
public class SearchArgumentToPredicateConverter {
private static final Logger LOG =
@@ -55,20 +55,26 @@ public class SearchArgumentToPredicateConverter {
private final PredicateBuilder builder;
public SearchArgumentToPredicateConverter(
- SearchArgument sarg,
+ SearchArgument searchArgument,
List<String> columnNames,
List<DataType> columnTypes,
@Nullable Set<String> readColumnNames) {
- this.root = sarg.getExpression();
- this.leaves = sarg.getLeaves();
- this.columnNames = columnNames;
+ this.root = searchArgument.getExpression();
+ this.leaves = searchArgument.getLeaves();
+ this.columnNames =
+ columnNames.stream().map(String::toLowerCase).collect(Collectors.toList());
this.columnTypes = columnTypes;
+ if (readColumnNames != null) {
+ readColumnNames =
+ readColumnNames.stream().map(String::toLowerCase).collect(Collectors.toSet());
+ }
this.readColumnNames = readColumnNames;
+
this.builder =
new PredicateBuilder(
RowType.of(
- columnTypes.toArray(new DataType[0]),
- columnNames.toArray(new String[0])));
+ this.columnTypes.toArray(new DataType[0]),
+ this.columnNames.toArray(new String[0])));
}
public Optional<Predicate> convert() {
diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java
index 20b7cd017..e2959ea6a 100644
--- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java
+++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java
@@ -57,8 +57,8 @@ public class HiveUtils {
public static Optional<Predicate> createPredicate(
TableSchema tableSchema, JobConf jobConf, boolean limitToReadColumnNames) {
- SearchArgument sarg = ConvertAstToSearchArg.createFromConf(jobConf);
- if (sarg == null) {
+ SearchArgument searchArgument = ConvertAstToSearchArg.createFromConf(jobConf);
+ if (searchArgument == null) {
return Optional.empty();
}
Set<String> readColumnNames = null;
@@ -79,7 +79,7 @@ public class HiveUtils {
}
SearchArgumentToPredicateConverter converter =
new SearchArgumentToPredicateConverter(
- sarg,
+ searchArgument,
tableSchema.fieldNames(),
tableSchema.logicalRowType().getFieldTypes(),
readColumnNames);
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java
index bdeff0acd..8e4ea57c3 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java
@@ -91,5 +91,12 @@ public class HiveReadITCase extends HiveTestBase {
assertThat(result).containsExactly("Hello", "Paimon");
result = hiveShell.executeQuery("SELECT Col2 FROM " + tableName);
assertThat(result).containsExactly("Hello", "Paimon");
+
+ result = hiveShell.executeQuery("SELECT * FROM " + tableName + " WHERE col2 = 'Hello'");
+ assertThat(result).containsExactly("1\tHello");
+ result =
+ hiveShell.executeQuery(
+ "SELECT * FROM " + tableName + " WHERE Col2 in ('Hello', 'Paimon')");
+ assertThat(result).containsExactly("1\tHello", "2\tPaimon");
}
}