You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/12/27 04:38:38 UTC
(doris) branch branch-2.0 updated: [fix](paimon)Remove the static attribute of the source for paimon for branch_2.0 #29032 (#29054)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3edac2f753f [fix](paimon)Remove the static attribute of the source for paimon for branch_2.0 #29032 (#29054)
3edac2f753f is described below
commit 3edac2f753f690f89fda39ddfe5d8c37c31e26a7
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Wed Dec 27 12:38:31 2023 +0800
[fix](paimon)Remove the static attribute of the source for paimon for branch_2.0 #29032 (#29054)
---
.../doris/planner/external/FileQueryScanNode.java | 23 ++++++----------------
.../doris/planner/external/MaxComputeScanNode.java | 9 ++++++++-
.../doris/planner/external/hudi/HudiScanNode.java | 9 ++++++++-
.../planner/external/iceberg/IcebergScanNode.java | 9 ++++++++-
.../planner/external/paimon/PaimonScanNode.java | 13 +++++++++---
.../paimon/test_paimon_catalog.groovy | 18 +++++++++++++++++
6 files changed, 58 insertions(+), 23 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
index e11299df876..c2cc969cd6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -39,12 +39,6 @@ import org.apache.doris.datasource.hive.AcidInfo;
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.planner.PlanNodeId;
-import org.apache.doris.planner.external.hudi.HudiScanNode;
-import org.apache.doris.planner.external.hudi.HudiSplit;
-import org.apache.doris.planner.external.iceberg.IcebergScanNode;
-import org.apache.doris.planner.external.iceberg.IcebergSplit;
-import org.apache.doris.planner.external.paimon.PaimonScanNode;
-import org.apache.doris.planner.external.paimon.PaimonSplit;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.spi.Split;
import org.apache.doris.statistics.StatisticalType;
@@ -261,6 +255,10 @@ public abstract class FileQueryScanNode extends FileScanNode {
return params;
}
+ // Set some parameters of scan to support different types of file data sources
+ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+ }
+
@Override
public void createScanRangeLocations() throws UserException {
long start = System.currentTimeMillis();
@@ -330,17 +328,7 @@ public abstract class FileQueryScanNode extends FileScanNode {
rangeDesc.setTableFormatParams(tableFormatFileDesc);
}
- // external data lake table
- if (fileSplit instanceof IcebergSplit) {
- // TODO: extract all data lake split to factory
- IcebergScanNode.setIcebergParams(rangeDesc, (IcebergSplit) fileSplit);
- } else if (fileSplit instanceof PaimonSplit) {
- PaimonScanNode.setPaimonParams(rangeDesc, (PaimonSplit) fileSplit);
- } else if (fileSplit instanceof HudiSplit) {
- HudiScanNode.setHudiParams(rangeDesc, (HudiSplit) fileSplit);
- } else if (fileSplit instanceof MaxComputeSplit) {
- MaxComputeScanNode.setScanParams(rangeDesc, (MaxComputeSplit) fileSplit);
- }
+ setScanParams(rangeDesc, fileSplit);
curLocations.getScanRange().getExtScanRange().getFileScanRange().addToRanges(rangeDesc);
TScanRangeLocation location = new TScanRangeLocation();
@@ -450,6 +438,7 @@ public abstract class FileQueryScanNode extends FileScanNode {
protected abstract TFileFormatType getFileFormatType() throws UserException;
+
protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws UserException {
return Util.inferFileCompressTypeByPath(fileSplit.getPath().toString());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
index ae0b424ad81..8fe7c8dff22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
@@ -63,7 +63,14 @@ public class MaxComputeScanNode extends FileQueryScanNode {
catalog = (MaxComputeExternalCatalog) table.getCatalog();
}
- public static void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) {
+ @Override
+ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+ if (split instanceof MaxComputeSplit) {
+ setScanParams(rangeDesc, (MaxComputeSplit) split);
+ }
+ }
+
+ public void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(TableFormatType.MAX_COMPUTE.value());
TMaxComputeFileDesc fileDesc = new TMaxComputeFileDesc();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
index 3921eb5cb89..4da4e85453b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
@@ -133,7 +133,14 @@ public class HudiScanNode extends HiveScanNode {
}
}
- public static void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) {
+ @Override
+ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+ if (split instanceof HudiSplit) {
+ setHudiParams(rangeDesc, (HudiSplit) split);
+ }
+ }
+
+ public void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(hudiSplit.getTableFormatType().value());
THudiFileDesc fileDesc = new THudiFileDesc();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
index c8a9437e243..d343ac9d30e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
@@ -129,7 +129,14 @@ public class IcebergScanNode extends FileQueryScanNode {
super.doInitialize();
}
- public static void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) {
+ @Override
+ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+ if (split instanceof IcebergSplit) {
+ setIcebergParams(rangeDesc, (IcebergSplit) split);
+ }
+ }
+
+ public void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value());
TIcebergFileDesc fileDesc = new TIcebergFileDesc();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
index 36e58ffb79d..4ba87c878e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
@@ -54,8 +54,8 @@ import java.util.Set;
import java.util.stream.Collectors;
public class PaimonScanNode extends FileQueryScanNode {
- private static PaimonSource source = null;
- private static List<Predicate> predicates;
+ private PaimonSource source = null;
+ private List<Predicate> predicates;
public PaimonScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, needCheckColumnPriv);
@@ -91,7 +91,14 @@ public class PaimonScanNode extends FileQueryScanNode {
}
}
- public static void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) {
+ @Override
+ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+ if (split instanceof PaimonSplit) {
+ setPaimonParams(rangeDesc, (PaimonSplit) split);
+ }
+ }
+
+ public void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(paimonSplit.getTableFormatType().value());
TPaimonFileDesc fileDesc = new TPaimonFileDesc();
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index 70bbb40078c..c271099dc5d 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -284,5 +284,23 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_
qt_c98 c98
qt_c99 c99
qt_c100 c100
+
+ // test view from jion paimon
+ sql """ switch internal """
+ String view_db = "test_view_for_paimon"
+ sql """ drop database if exists ${view_db}"""
+ sql """ create database if not exists ${view_db}"""
+ sql """use ${view_db}"""
+ sql """ create view test_tst_1 as select * from ${catalog_name}.`db1`.all_table; """
+ sql """ create view test_tst_2 as select * from ${catalog_name}.`db1`.all_table_with_parquet; """
+ sql """ create view test_tst_5 as select * from ${catalog_name}.`db1`.array_nested; """
+ sql """ create table test_tst_6 properties ("replication_num" = "1") as
+ select f.c2,f.c3,c.c1 from
+ (select a.c2,b.c3 from test_tst_1 a inner join test_tst_2 b on a.c2=b.c2) f
+ inner join test_tst_5 c on f.c2=c.c1;
+ """
+ def view1 = """select * from test_tst_6 order by c1"""
+
+ // qt_view1 view1
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org