You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/15 02:30:16 UTC
[doris] branch master updated: [improvement](tvf)Support hdfs and s3 tvf for nereids (#20829)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 31c17f1088 [improvement](tvf)Support hdfs and s3 tvf for nereids (#20829)
31c17f1088 is described below
commit 31c17f108895c9009909ae27e490f20936c6fc55
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Thu Jun 15 10:30:09 2023 +0800
[improvement](tvf)Support hdfs and s3 tvf for nereids (#20829)
Support hdfs and s3 tvf for nereids.
---
.../doris/catalog/BuiltinTableValuedFunctions.java | 6 +-
.../glue/translator/PhysicalPlanTranslator.java | 1 +
.../trees/expressions/functions/table/Hdfs.java | 71 ++++++++++++++++++++++
.../trees/expressions/functions/table/S3.java | 71 ++++++++++++++++++++++
.../visitor/TableValuedFunctionVisitor.java | 10 +++
5 files changed, 158 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
index 16f6735852..66bdce4c09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
@@ -17,7 +17,9 @@
package org.apache.doris.catalog;
+import org.apache.doris.nereids.trees.expressions.functions.table.Hdfs;
import org.apache.doris.nereids.trees.expressions.functions.table.Numbers;
+import org.apache.doris.nereids.trees.expressions.functions.table.S3;
import com.google.common.collect.ImmutableList;
@@ -29,7 +31,9 @@ import com.google.common.collect.ImmutableList;
*/
public class BuiltinTableValuedFunctions implements FunctionHelper {
public final ImmutableList<TableValuedFunc> tableValuedFunctions = ImmutableList.of(
- tableValued(Numbers.class, "numbers")
+ tableValued(Numbers.class, "numbers"),
+ tableValued(Hdfs.class, "hdfs"),
+ tableValued(S3.class, "s3")
);
public static final BuiltinTableValuedFunctions INSTANCE = new BuiltinTableValuedFunctions();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index cf51a955e1..2a961d70e3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -756,6 +756,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction();
ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor);
+ Utils.execWithUncheckedException(scanNode::init);
context.getRuntimeTranslator().ifPresent(
runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(tvfRelation.getId()).forEach(
expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
new file mode 100644
index 0000000000..8ea2d18414
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.table;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.TVFProperties;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.coercion.AnyDataType;
+import org.apache.doris.statistics.Statistics;
+import org.apache.doris.tablefunction.HdfsTableValuedFunction;
+import org.apache.doris.tablefunction.TableValuedFunctionIf;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** hdfs */
+public class Hdfs extends TableValuedFunction {
+ public Hdfs(TVFProperties properties) {
+ super("hdfs", properties);
+ }
+
+ @Override
+ public FunctionSignature customSignature() {
+ return FunctionSignature.of(AnyDataType.INSTANCE, (List) getArgumentsTypes());
+ }
+
+ @Override
+ protected TableValuedFunctionIf toCatalogFunction() {
+ try {
+ Map<String, String> arguments = getTVFProperties().getMap();
+ return new HdfsTableValuedFunction(arguments);
+ } catch (Throwable t) {
+ throw new AnalysisException("Can not build HdfsTableValuedFunction by "
+ + this + ": " + t.getMessage(), t);
+ }
+ }
+
+ @Override
+ public Statistics computeStats(List<Slot> slots) {
+ return new Statistics(0, new HashMap<>());
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitHdfs(this, context);
+ }
+
+ @Override
+ public PhysicalProperties getPhysicalProperties() {
+ return PhysicalProperties.ANY;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
new file mode 100644
index 0000000000..3d995c6fca
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.table;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.TVFProperties;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.coercion.AnyDataType;
+import org.apache.doris.statistics.Statistics;
+import org.apache.doris.tablefunction.S3TableValuedFunction;
+import org.apache.doris.tablefunction.TableValuedFunctionIf;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** s3 */
+public class S3 extends TableValuedFunction {
+ public S3(TVFProperties properties) {
+ super("s3", properties);
+ }
+
+ @Override
+ public FunctionSignature customSignature() {
+ return FunctionSignature.of(AnyDataType.INSTANCE, (List) getArgumentsTypes());
+ }
+
+ @Override
+ protected TableValuedFunctionIf toCatalogFunction() {
+ try {
+ Map<String, String> arguments = getTVFProperties().getMap();
+ return new S3TableValuedFunction(arguments);
+ } catch (Throwable t) {
+ throw new AnalysisException("Can not build S3TableValuedFunction by "
+ + this + ": " + t.getMessage(), t);
+ }
+ }
+
+ @Override
+ public Statistics computeStats(List<Slot> slots) {
+ return new Statistics(0, new HashMap<>());
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitS3(this, context);
+ }
+
+ @Override
+ public PhysicalProperties getPhysicalProperties() {
+ return PhysicalProperties.ANY;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
index b904e3cf02..0c4adcabe1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
@@ -17,7 +17,9 @@
package org.apache.doris.nereids.trees.expressions.visitor;
+import org.apache.doris.nereids.trees.expressions.functions.table.Hdfs;
import org.apache.doris.nereids.trees.expressions.functions.table.Numbers;
+import org.apache.doris.nereids.trees.expressions.functions.table.S3;
import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
/** TableValuedFunctionVisitor */
@@ -27,4 +29,12 @@ public interface TableValuedFunctionVisitor<R, C> {
default R visitNumbers(Numbers numbers, C context) {
return visitTableValuedFunction(numbers, context);
}
+
+ default R visitHdfs(Hdfs hdfs, C context) {
+ return visitTableValuedFunction(hdfs, context);
+ }
+
+ default R visitS3(S3 s3, C context) {
+ return visitTableValuedFunction(s3, context);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org