You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2025/01/27 02:22:28 UTC
(hudi) branch master updated: [MINOR] Removed `RecordLevelIndexSupport::fetchQueryWithAttribute`, which duplicates `filterQueryWithRecordKey` (#12701)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a7456c68ba2 [MINOR] Removed `RecordLevelIndexSupport::fetchQueryWithAttribute`, which duplicates `filterQueryWithRecordKey` (#12701)
a7456c68ba2 is described below
commit a7456c68ba284fdc4308d8702397133575f573c1
Author: Geser Dugarov <ge...@gmail.com>
AuthorDate: Mon Jan 27 09:22:23 2025 +0700
[MINOR] Removed `RecordLevelIndexSupport::fetchQueryWithAttribute`, which duplicates `filterQueryWithRecordKey` (#12701)
---
.../org/apache/hudi/RecordLevelIndexSupport.scala | 74 ----------------------
1 file changed, 74 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
index ae4d6afef8e..40bb888770b 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
@@ -229,80 +229,6 @@ object RecordLevelIndexSupport {
}
}
- def fetchQueryWithAttribute(queryFilter: Expression, recordKeyOpt: Option[String], literalGenerator: Function2[AttributeReference, Literal, String],
- attributeFetcher: Function1[Expression, Expression]): (Option[(Expression, List[String])], Boolean) = {
- queryFilter match {
- case equalToQuery: EqualTo =>
- val attributeLiteralTuple = getAttributeLiteralTuple(attributeFetcher.apply(equalToQuery.left), attributeFetcher.apply(equalToQuery.right)).orNull
- if (attributeLiteralTuple != null) {
- val attribute = attributeLiteralTuple._1
- val literal = attributeLiteralTuple._2
- if (attribute != null && attribute.name != null && attributeMatchesRecordKey(attribute.name, recordKeyOpt)) {
- val recordKeyLiteral = literalGenerator.apply(attribute, literal)
- (Option.apply(EqualTo(attribute, literal), List.apply(recordKeyLiteral)), true)
- } else {
- (Option.empty, true)
- }
- } else {
- (Option.empty, true)
- }
-
- case inQuery: In =>
- var validINQuery = true
- val attributeOpt = Option.apply(
- attributeFetcher.apply(inQuery.value) match {
- case attribute: AttributeReference =>
- if (!attributeMatchesRecordKey(attribute.name, recordKeyOpt)) {
- validINQuery = false
- null
- } else {
- attribute
- }
- case _ =>
- validINQuery = false
- null
- })
- var literals: List[String] = List.empty
- inQuery.list.foreach {
- case literal: Literal if attributeOpt.isDefined =>
- val recordKeyLiteral = literalGenerator.apply(attributeOpt.get, literal)
- literals = literals :+ recordKeyLiteral
- case _ => validINQuery = false
- }
- if (validINQuery) {
- (Option.apply(In(attributeOpt.get, inQuery.list), literals), true)
- } else {
- (Option.empty, true)
- }
-
- // Handle And expression (composite filter)
- case andQuery: And =>
- val leftResult = filterQueryWithRecordKey(andQuery.left, recordKeyOpt, literalGenerator, attributeFetcher)
- val rightResult = filterQueryWithRecordKey(andQuery.right, recordKeyOpt, literalGenerator, attributeFetcher)
-
- val isSupported = leftResult._2 && rightResult._2
- if (!isSupported) {
- (Option.empty, false)
- } else {
- // If both left and right filters are valid, concatenate their results
- (leftResult._1, rightResult._1) match {
- case (Some((leftExp, leftKeys)), Some((rightExp, rightKeys))) =>
- // Return concatenated expressions and record keys
- (Option.apply(And(leftExp, rightExp), leftKeys ++ rightKeys), true)
- case (Some((leftExp, leftKeys)), None) =>
- // Return concatenated expressions and record keys
- (Option.apply(leftExp, leftKeys), true)
- case (None, Some((rightExp, rightKeys))) =>
- // Return concatenated expressions and record keys
- (Option.apply(rightExp, rightKeys), true)
- case _ => (Option.empty, true)
- }
- }
-
- case _ => (Option.empty, false)
- }
- }
-
/**
* Returns the list of storage paths from the pruned partitions and file slices.
*