You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/14 06:47:37 UTC
[doris] branch branch-1.2-lts updated (b970188b5f -> a028e81f49)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from b970188b5f [Bug](DictoryColumn) reverse the _codes.size() replace _reserve_size (#14984)
new 16ba1c53f3 [Fix](Table Valued function) fix the problem that can not catchthe exception thrown by the TableValuedFunctionRef constructor (#14983)
new df5e8ee5d3 [fix](multi-catalog)fix page index thrift deserialize (#15001)
new 6828111c71 [fix](multi catalog)Fix convert_to_doris_type missing break for some cases (#14992)
new 36e504ca43 [fix](multi catalog)Check orc file reader is not null before using it. (#14988)
new a028e81f49 [fix](fe)fix select from temporary partition bug (#14809)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/vec/exec/format/orc/vorc_reader.cpp | 10 ++--
be/src/vec/exec/format/parquet/schema_desc.cpp | 2 +
.../exec/format/parquet/vparquet_page_index.cpp | 6 +--
.../vec/exec/format/parquet/vparquet_page_index.h | 2 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 18 ++++---
.../java/org/apache/doris/analysis/SelectStmt.java | 1 +
.../doris/analysis/TableValuedFunctionRef.java | 15 ++++--
.../java/org/apache/doris/catalog/OlapTable.java | 3 ++
.../ExternalFileTableValuedFunction.java | 12 +++--
.../tablefunction/HdfsTableValuedFunction.java | 3 +-
.../tablefunction/NumbersTableValuedFunction.java | 11 ++---
.../doris/tablefunction/S3TableValuedFunction.java | 8 +++-
.../doris/tablefunction/TableValuedFunctionIf.java | 5 +-
.../test_temp_partition.out} | 1 +
.../correctness_p0/test_temp_partition.groovy | 56 ++++++++++++++++++++++
15 files changed, 117 insertions(+), 36 deletions(-)
copy regression-test/data/{correctness/test_constant_push_down_through_outer_join.out => correctness_p0/test_temp_partition.out} (78%)
create mode 100644 regression-test/suites/correctness_p0/test_temp_partition.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/05: [fix](fe)fix select from temporary partition bug (#14809)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a028e81f49a25a196f0133baffa78b23b6e0d30e
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Tue Dec 6 14:32:35 2022 +0800
[fix](fe)fix select from temporary partition bug (#14809)
---
.../java/org/apache/doris/catalog/OlapTable.java | 3 ++
.../data/correctness_p0/test_temp_partition.out | 4 ++
.../correctness_p0/test_temp_partition.groovy | 56 ++++++++++++++++++++++
3 files changed, 63 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 91cd500ddf..71659b6cb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -388,6 +388,9 @@ public class OlapTable extends Table {
public List<MaterializedIndex> getVisibleIndex() {
Optional<Partition> partition = idToPartition.values().stream().findFirst();
+ if (!partition.isPresent()) {
+ partition = tempPartitions.getAllPartitions().stream().findFirst();
+ }
return partition.isPresent() ? partition.get().getMaterializedIndices(IndexExtState.VISIBLE)
: Collections.emptyList();
}
diff --git a/regression-test/data/correctness_p0/test_temp_partition.out b/regression-test/data/correctness_p0/test_temp_partition.out
new file mode 100644
index 0000000000..94b0002ac2
--- /dev/null
+++ b/regression-test/data/correctness_p0/test_temp_partition.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+0 a 10 asddfa 2022-11-11T00:00
+
diff --git a/regression-test/suites/correctness_p0/test_temp_partition.groovy b/regression-test/suites/correctness_p0/test_temp_partition.groovy
new file mode 100644
index 0000000000..4c6047b54b
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_temp_partition.groovy
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_temp_partition") {
+ sql """
+ drop table if exists t_temp_partition;
+ """
+
+ sql """
+ create table t_temp_partition (
+ id bigint(20) not null ,
+ name varchar(64) not null ,
+ age varchar(64) not null ,
+ address varchar(300) not null,
+ create_time datetime not null
+ ) engine = olap
+ duplicate key (`id`)
+ partition by range (`create_time`) ()
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ properties (
+ "replication_num" = "1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+ sql """
+ alter table t_temp_partition add temporary partition tp1 values[("2022-11-01 00:00:00"), ("2022-11-17 00:00:00"));
+ """
+
+ sql """
+ insert into t_temp_partition temporary partition(tp1) values (0, "a", "10", "asddfa","2022-11-11 00:00:00");
+ """
+
+ qt_select """
+ select * from t_temp_partition temporary partition(tp1)
+ """
+
+ sql """
+ drop table if exists t_temp_partition;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/05: [fix](multi catalog)Check orc file reader is not null before using it. (#14988)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 36e504ca435bc403c843960e2b4998adbd5b1486
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Tue Dec 13 11:27:51 2022 +0800
[fix](multi catalog)Check orc file reader is not null before using it. (#14988)
The external table file path cache may out of date, which will cause orc reader to visit non-exist files.
In this case, orc file reader is nullptr.
This pr is to check the reader before using it to avoid core dump of visiting nullptr.
---
be/src/vec/exec/format/orc/vorc_reader.cpp | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 216aa24ec9..538f91e732 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -96,10 +96,12 @@ OrcReader::~OrcReader() {
void OrcReader::close() {
if (!_closed) {
if (_profile != nullptr) {
- auto& fst = _file_reader->statistics();
- COUNTER_UPDATE(_orc_profile.read_time, fst.read_time);
- COUNTER_UPDATE(_orc_profile.read_calls, fst.read_calls);
- COUNTER_UPDATE(_orc_profile.read_bytes, fst.read_bytes);
+ if (_file_reader != nullptr) {
+ auto& fst = _file_reader->statistics();
+ COUNTER_UPDATE(_orc_profile.read_time, fst.read_time);
+ COUNTER_UPDATE(_orc_profile.read_calls, fst.read_calls);
+ COUNTER_UPDATE(_orc_profile.read_bytes, fst.read_bytes);
+ }
COUNTER_UPDATE(_orc_profile.column_read_time, _statistics.column_read_time);
COUNTER_UPDATE(_orc_profile.get_batch_time, _statistics.get_batch_time);
COUNTER_UPDATE(_orc_profile.parse_meta_time, _statistics.parse_meta_time);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/05: [fix](multi catalog)Fix convert_to_doris_type missing break for some cases (#14992)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6828111c710ffab1814e16db05e2ca335fad58da
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Tue Dec 13 13:34:55 2022 +0800
[fix](multi catalog)Fix convert_to_doris_type missing break for some cases (#14992)
---
be/src/vec/exec/format/parquet/schema_desc.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp
index 529c5d5b78..6d07886e89 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.cpp
+++ b/be/src/vec/exec/format/parquet/schema_desc.cpp
@@ -228,6 +228,7 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::t
switch (convertedType) {
case tparquet::ConvertedType::type::UTF8:
type.type = TYPE_STRING;
+ break;
case tparquet::ConvertedType::type::DECIMAL:
type.type = TYPE_DECIMALV2;
type.precision = 27;
@@ -250,6 +251,7 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::t
case tparquet::ConvertedType::type::INT_16:
case tparquet::ConvertedType::type::INT_32:
type.type = TYPE_INT;
+ break;
case tparquet::ConvertedType::type::UINT_32:
case tparquet::ConvertedType::type::UINT_64:
case tparquet::ConvertedType::type::INT_64:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/05: [Fix](Table Valued function) fix the problem that can not catchthe exception thrown by the TableValuedFunctionRef constructor (#14983)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 16ba1c53f301dd25f0861f897f54742bd20d9cb6
Author: Tiewei Fang <43...@users.noreply.github.com>
AuthorDate: Tue Dec 13 11:26:04 2022 +0800
[Fix](Table Valued function) fix the problem that can not catchthe exception thrown by the TableValuedFunctionRef constructor (#14983)
Put the generation of TableValuedFunctionIf in the analyze function, instead of the generation in the
TableValuedFunctionRef constructor.
---
.../main/java/org/apache/doris/analysis/SelectStmt.java | 1 +
.../org/apache/doris/analysis/TableValuedFunctionRef.java | 15 +++++++++++----
.../tablefunction/ExternalFileTableValuedFunction.java | 12 ++++++++----
.../doris/tablefunction/HdfsTableValuedFunction.java | 3 +--
.../doris/tablefunction/NumbersTableValuedFunction.java | 11 +++++------
.../apache/doris/tablefunction/S3TableValuedFunction.java | 8 ++++++--
.../apache/doris/tablefunction/TableValuedFunctionIf.java | 5 ++---
7 files changed, 34 insertions(+), 21 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 5e1cb19a73..b715014149 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -304,6 +304,7 @@ public class SelectStmt extends QueryStmt {
inlineStmt.getTables(analyzer, expandView, tableMap, parentViewNameSet);
} else if (tblRef instanceof TableValuedFunctionRef) {
TableValuedFunctionRef tblFuncRef = (TableValuedFunctionRef) tblRef;
+ tblFuncRef.analyze(analyzer);
tableMap.put(tblFuncRef.getTableFunction().getTable().getId(),
tblFuncRef.getTableFunction().getTable());
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
index 328b1819fa..2bbf5e00be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
@@ -18,7 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Table;
-import org.apache.doris.common.UserException;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -29,10 +29,13 @@ public class TableValuedFunctionRef extends TableRef {
private Table table;
private TableValuedFunctionIf tableFunction;
+ private String funcName;
+ private Map<String, String> params;
- public TableValuedFunctionRef(String funcName, String alias, Map<String, String> params) throws UserException {
+ public TableValuedFunctionRef(String funcName, String alias, Map<String, String> params) {
super(new TableName(null, null, "_table_valued_function_" + funcName), alias);
- this.tableFunction = TableValuedFunctionIf.getTableFunction(funcName, params);
+ this.funcName = funcName;
+ this.params = params;
if (hasExplicitAlias()) {
return;
}
@@ -41,7 +44,10 @@ public class TableValuedFunctionRef extends TableRef {
public TableValuedFunctionRef(TableValuedFunctionRef other) {
super(other);
+ this.funcName = other.funcName;
+ this.params = other.params;
this.tableFunction = other.tableFunction;
+ this.table = other.table;
}
@Override
@@ -60,12 +66,13 @@ public class TableValuedFunctionRef extends TableRef {
* Register this table ref and then analyze the Join clause.
*/
@Override
- public void analyze(Analyzer analyzer) throws UserException {
+ public void analyze(Analyzer analyzer) throws AnalysisException {
if (isAnalyzed) {
return;
}
// Table function could generate a table which will has columns
// Maybe will call be during this process
+ this.tableFunction = TableValuedFunctionIf.getTableFunction(funcName, params);
this.table = tableFunction.getTable();
desc = analyzer.registerTableRef(this);
isAnalyzed = true; // true that we have assigned desc
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 17b3364294..b7d13f071a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -134,13 +134,17 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
return "";
}
- protected void parseFile() throws UserException {
+ protected void parseFile() throws AnalysisException {
String path = getFilePath();
BrokerDesc brokerDesc = getBrokerDesc();
- BrokerUtil.parseFile(path, brokerDesc, fileStatuses);
+ try {
+ BrokerUtil.parseFile(path, brokerDesc, fileStatuses);
+ } catch (UserException e) {
+ throw new AnalysisException("parse file failed, path = " + path);
+ }
}
- protected void parseProperties(Map<String, String> validParams) throws UserException {
+ protected void parseProperties(Map<String, String> validParams) throws AnalysisException {
String formatString = validParams.getOrDefault(FORMAT, "").toLowerCase();
switch (formatString) {
case "csv":
@@ -295,7 +299,7 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
break;
}
if (firstFile == null) {
- throw new AnalysisException("Can not get first file, please check s3 uri.");
+ throw new AnalysisException("Can not get first file, please check uri.");
}
// set TFileRangeDesc
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
index 175c9e501a..b555a34e12 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
@@ -21,7 +21,6 @@ import org.apache.doris.analysis.BrokerDesc;
import org.apache.doris.analysis.ExportStmt;
import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.UserException;
import org.apache.doris.common.util.URI;
import org.apache.doris.thrift.TFileType;
@@ -65,7 +64,7 @@ public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction {
private URI hdfsUri;
private String filePath;
- public HdfsTableValuedFunction(Map<String, String> params) throws UserException {
+ public HdfsTableValuedFunction(Map<String, String> params) throws AnalysisException {
Map<String, String> fileFormatParams = new CaseInsensitiveMap();
locationProperties = Maps.newHashMap();
for (String key : params.keySet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java
index 9e1a6ed181..3e0e92b5b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java
@@ -22,7 +22,6 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.UserException;
import org.apache.doris.planner.DataGenScanNode;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.ScanNode;
@@ -64,9 +63,9 @@ public class NumbersTableValuedFunction extends DataGenTableValuedFunction {
/**
* Constructor.
* @param params params from user
- * @throws UserException exception
+ * @throws AnalysisException exception
*/
- public NumbersTableValuedFunction(Map<String, String> params) throws UserException {
+ public NumbersTableValuedFunction(Map<String, String> params) throws AnalysisException {
Map<String, String> validParams = Maps.newHashMap();
for (String key : params.keySet()) {
if (!PROPERTIES_SET.contains(key.toLowerCase())) {
@@ -78,17 +77,17 @@ public class NumbersTableValuedFunction extends DataGenTableValuedFunction {
try {
tabletsNum = Integer.parseInt(validParams.getOrDefault(BACKEND_NUM, "1"));
} catch (NumberFormatException e) {
- throw new UserException("can not parse `backend_num` param to natural number");
+ throw new AnalysisException("can not parse `backend_num` param to natural number");
}
String numberStr = validParams.get(NUMBER);
if (!Strings.isNullOrEmpty(numberStr)) {
try {
totalNumbers = Long.parseLong(numberStr);
} catch (NumberFormatException e) {
- throw new UserException("can not parse `number` param to natural number");
+ throw new AnalysisException("can not parse `number` param to natural number");
}
} else {
- throw new UserException(
+ throw new AnalysisException(
"can not find `number` param, please specify `number`, like: numbers(\"number\" = \"10\")");
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
index 6af05f3374..e0b4e310f5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
@@ -64,7 +64,7 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction {
private String virtualBucket;
private boolean forceVirtualHosted;
- public S3TableValuedFunction(Map<String, String> params) throws UserException {
+ public S3TableValuedFunction(Map<String, String> params) throws AnalysisException {
Map<String, String> validParams = new CaseInsensitiveMap();
for (String key : params.keySet()) {
if (!PROPERTIES_SET.contains(key.toLowerCase()) && !FILE_FORMAT_PROPERTIES.contains(key.toLowerCase())) {
@@ -82,7 +82,11 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction {
forceVirtualHosted = !Boolean.valueOf(validParams.get(USE_PATH_STYLE)).booleanValue();
}
- s3uri = S3URI.create(validParams.get(S3_URI), forceVirtualHosted);
+ try {
+ s3uri = S3URI.create(validParams.get(S3_URI), forceVirtualHosted);
+ } catch (UserException e) {
+ throw new AnalysisException("parse s3 uri failed, uri = " + validParams.get(S3_URI));
+ }
if (forceVirtualHosted) {
// s3uri.getVirtualBucket() is: virtualBucket.endpoint, Eg:
// uri: http://my_bucket.cos.ap-beijing.myqcloud.com/file.txt
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
index 56167d1b87..862b986e97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
@@ -22,7 +22,6 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionGenTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.UserException;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.ScanNode;
@@ -42,7 +41,7 @@ public abstract class TableValuedFunctionIf {
// All table functions should be registered here
public static TableValuedFunctionIf getTableFunction(String funcName, Map<String, String> params)
- throws UserException {
+ throws AnalysisException {
switch (funcName.toLowerCase()) {
case NumbersTableValuedFunction.NAME:
return new NumbersTableValuedFunction(params);
@@ -51,7 +50,7 @@ public abstract class TableValuedFunctionIf {
case HdfsTableValuedFunction.NAME:
return new HdfsTableValuedFunction(params);
default:
- throw new UserException("Could not find table function " + funcName);
+ throw new AnalysisException("Could not find table function " + funcName);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/05: [fix](multi-catalog)fix page index thrift deserialize (#15001)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit df5e8ee5d3bf86f2a80b28ebb3eb4ada7ac73fa1
Author: slothever <18...@users.noreply.github.com>
AuthorDate: Tue Dec 13 13:33:19 2022 +0800
[fix](multi-catalog)fix page index thrift deserialize (#15001)
fix the err when parse page index: Couldn't deserialize thrift msg.
use two buffer to store column index and offset index msg, avoid parse them in a buffer
---
be/src/vec/exec/format/parquet/vparquet_page_index.cpp | 6 +++---
be/src/vec/exec/format/parquet/vparquet_page_index.h | 2 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 18 +++++++++++-------
3 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_index.cpp b/be/src/vec/exec/format/parquet/vparquet_page_index.cpp
index a5673833a8..acb56d0c2a 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_index.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_page_index.cpp
@@ -94,11 +94,11 @@ Status PageIndex::parse_column_index(const tparquet::ColumnChunk& chunk, const u
}
Status PageIndex::parse_offset_index(const tparquet::ColumnChunk& chunk, const uint8_t* buff,
- int64_t buffer_size, tparquet::OffsetIndex* offset_index) {
- int64_t buffer_offset = chunk.offset_index_offset - _offset_index_start + _column_index_size;
+ tparquet::OffsetIndex* offset_index) {
+ int64_t buffer_offset = chunk.offset_index_offset - _offset_index_start;
uint32_t length = chunk.offset_index_length;
DCHECK_GE(buffer_offset, 0);
- DCHECK_LE(buffer_offset + length, buffer_size);
+ DCHECK_LE(buffer_offset + length, _offset_index_size);
RETURN_IF_ERROR(deserialize_thrift_msg(buff + buffer_offset, &length, true, offset_index));
return Status::OK();
}
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_index.h b/be/src/vec/exec/format/parquet/vparquet_page_index.h
index c5f8183e35..978a798bf4 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_index.h
+++ b/be/src/vec/exec/format/parquet/vparquet_page_index.h
@@ -38,7 +38,7 @@ public:
Status parse_column_index(const tparquet::ColumnChunk& chunk, const uint8_t* buff,
tparquet::ColumnIndex* _column_index);
Status parse_offset_index(const tparquet::ColumnChunk& chunk, const uint8_t* buff,
- int64_t buffer_size, tparquet::OffsetIndex* _offset_index);
+ tparquet::OffsetIndex* _offset_index);
private:
friend class ParquetReader;
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index cb22f9d36e..01c226b1e4 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -545,14 +545,18 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group) {
if (!_has_page_index(row_group.columns, page_index)) {
return Status::OK();
}
- int64_t buffer_size = page_index._column_index_size + page_index._offset_index_size;
- uint8_t buff[buffer_size];
+ // int64_t buffer_size = page_index._column_index_size;
+ uint8_t col_index_buff[page_index._column_index_size];
int64_t bytes_read = 0;
- RETURN_IF_ERROR(
- _file_reader->readat(page_index._column_index_start, buffer_size, &bytes_read, buff));
-
+ RETURN_IF_ERROR(_file_reader->readat(page_index._column_index_start,
+ page_index._column_index_size, &bytes_read,
+ col_index_buff));
auto& schema_desc = _file_metadata->schema();
std::vector<RowRange> skipped_row_ranges;
+ uint8_t off_index_buff[page_index._offset_index_size];
+ RETURN_IF_ERROR(_file_reader->readat(page_index._offset_index_start,
+ page_index._offset_index_size, &bytes_read,
+ off_index_buff));
for (auto& read_col : _read_columns) {
auto conjunct_iter = _colname_to_value_range->find(read_col._file_slot_name);
if (_colname_to_value_range->end() == conjunct_iter) {
@@ -563,7 +567,7 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group) {
if (chunk.column_index_offset == 0 && chunk.column_index_length == 0) {
return Status::OK();
}
- RETURN_IF_ERROR(page_index.parse_column_index(chunk, buff, &column_index));
+ RETURN_IF_ERROR(page_index.parse_column_index(chunk, col_index_buff, &column_index));
const int num_of_pages = column_index.null_pages.size();
if (num_of_pages <= 0) {
break;
@@ -577,7 +581,7 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group) {
continue;
}
tparquet::OffsetIndex offset_index;
- RETURN_IF_ERROR(page_index.parse_offset_index(chunk, buff, buffer_size, &offset_index));
+ RETURN_IF_ERROR(page_index.parse_offset_index(chunk, off_index_buff, &offset_index));
for (int page_id : skipped_page_range) {
RowRange skipped_row_range;
page_index.create_skipped_row_range(offset_index, row_group.num_rows, page_id,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org