You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/08/23 00:10:53 UTC
spark git commit: [SPARK-25127] DataSourceV2: Remove
SupportsPushDownCatalystFilters
Repository: spark
Updated Branches:
refs/heads/master 2bc7b7553 -> 0295ad40d
[SPARK-25127] DataSourceV2: Remove SupportsPushDownCatalystFilters
## What changes were proposed in this pull request?
They depend on internal Expression APIs. Let's see how far we can get without it.
## How was this patch tested?
Just some code removal. There's no existing tests as far as I can tell so it's easy to remove.
Closes #22185 from rxin/SPARK-25127.
Authored-by: Reynold Xin <rx...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0295ad40
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0295ad40
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0295ad40
Branch: refs/heads/master
Commit: 0295ad40def41b9a8ccefaaa1a7658899fb632a4
Parents: 2bc7b75
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu Aug 23 08:10:45 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Aug 23 08:10:45 2018 +0800
----------------------------------------------------------------------
.../reader/SupportsPushDownCatalystFilters.java | 57 --------------------
.../v2/reader/SupportsPushDownFilters.java | 4 --
.../datasources/v2/DataSourceV2Strategy.scala | 5 --
3 files changed, 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0295ad40/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownCatalystFilters.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownCatalystFilters.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownCatalystFilters.java
deleted file mode 100644
index 9d79a18..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownCatalystFilters.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.catalyst.expressions.Expression;
-
-/**
- * A mix-in interface for {@link ScanConfigBuilder}. Data source readers can implement this
- * interface to push down arbitrary expressions as predicates to the data source.
- * This is an experimental and unstable interface as {@link Expression} is not public and may get
- * changed in the future Spark versions.
- *
- * Note that, if data source readers implement both this interface and
- * {@link SupportsPushDownFilters}, Spark will ignore {@link SupportsPushDownFilters} and only
- * process this interface.
- */
-@InterfaceStability.Unstable
-public interface SupportsPushDownCatalystFilters extends ScanConfigBuilder {
-
- /**
- * Pushes down filters, and returns filters that need to be evaluated after scanning.
- */
- Expression[] pushCatalystFilters(Expression[] filters);
-
- /**
- * Returns the catalyst filters that are pushed to the data source via
- * {@link #pushCatalystFilters(Expression[])}.
- *
- * There are 3 kinds of filters:
- * 1. pushable filters which don't need to be evaluated again after scanning.
- * 2. pushable filters which still need to be evaluated after scanning, e.g. parquet
- * row group filter.
- * 3. non-pushable filters.
- * Both case 1 and 2 should be considered as pushed filters and should be returned by this method.
- *
- * It's possible that there is no filters in the query and
- * {@link #pushCatalystFilters(Expression[])} is never called, empty array should be returned for
- * this case.
- */
- Expression[] pushedCatalystFilters();
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/0295ad40/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java
index 5d32a8a..5e7985f 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java
@@ -23,10 +23,6 @@ import org.apache.spark.sql.sources.Filter;
/**
* A mix-in interface for {@link ScanConfigBuilder}. Data sources can implement this interface to
* push down filters to the data source and reduce the size of the data to be read.
- *
- * Note that, if data source readers implement both this interface and
- * {@link SupportsPushDownCatalystFilters}, Spark will ignore this interface and only process
- * {@link SupportsPushDownCatalystFilters}.
*/
@InterfaceStability.Evolving
public interface SupportsPushDownFilters extends ScanConfigBuilder {
http://git-wip-us.apache.org/repos/asf/spark/blob/0295ad40/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index fe713ff..9a3109e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -40,11 +40,6 @@ object DataSourceV2Strategy extends Strategy {
configBuilder: ScanConfigBuilder,
filters: Seq[Expression]): (Seq[Expression], Seq[Expression]) = {
configBuilder match {
- case r: SupportsPushDownCatalystFilters =>
- val postScanFilters = r.pushCatalystFilters(filters.toArray)
- val pushedFilters = r.pushedCatalystFilters()
- (pushedFilters, postScanFilters)
-
case r: SupportsPushDownFilters =>
// A map from translated data source filters to original catalyst filter expressions.
val translatedFilterToExpr = mutable.HashMap.empty[sources.Filter, Expression]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org