You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/23 14:44:12 UTC
[doris] branch branch-1.2-lts updated (d078112585 -> c61be0cb8a)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from d078112585 (improvement)[bucket] Add auto bucket implement (#15250)
new d016aef2bc [fix](planner) fix bugs in uncheckedCastChild (#15905)
new 7095f4db39 [fix](daemon) should use GetMonoTimeMicros() (#16070)
new 42ab0c39de [fix](privilege)fix grant resource bug (#16045)
new a071827534 [vectorzied](jdbc) fix jdbc executor for get result by batch and memo… (#15843)
new 82f923ee1f [debug](ParquetReader) print file path if failed to read parquet file (#16118)
new c61be0cb8a [vectorized](analytic) fix analytic node of window function get wrong… (#16074)
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/daemon.cpp | 4 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 8 +-
be/src/vec/exec/vanalytic_eval_node.cpp | 28 +++++--
be/src/vec/exec/vanalytic_eval_node.h | 14 +++-
be/src/vec/exec/vjdbc_connector.cpp | 6 +-
be/src/vec/exec/vjdbc_connector.h | 1 +
.../main/java/org/apache/doris/catalog/Type.java | 7 ++
.../main/java/org/apache/doris/analysis/Expr.java | 7 +-
.../java/org/apache/doris/analysis/IntLiteral.java | 47 +++++++----
.../java/org/apache/doris/catalog/ResourceMgr.java | 2 +-
.../doris/mysql/privilege/PrivPredicate.java | 7 +-
.../java/org/apache/doris/analysis/ExprTest.java | 10 +++
.../org/apache/doris/analysis/InsertStmtTest.java | 5 +-
.../org/apache/doris/analysis/IntLiteralTest.java} | 21 ++---
.../org/apache/doris/mysql/privilege/AuthTest.java | 4 +
fe/java-udf/pom.xml | 5 --
.../main/java/org/apache/doris/udf/FakeDriver.java | 70 ++++++++++++++++
.../java/org/apache/doris/udf/JdbcExecutor.java | 96 ++++++++++++----------
18 files changed, 248 insertions(+), 94 deletions(-)
copy fe/fe-core/src/{main/java/org/apache/doris/analysis/ShowStmt.java => test/java/org/apache/doris/analysis/IntLiteralTest.java} (63%)
create mode 100644 fe/java-udf/src/main/java/org/apache/doris/udf/FakeDriver.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/06: [vectorzied](jdbc) fix jdbc executor for get result by batch and memo… (#15843)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a071827534cca966b87d1ea412ceb1c3f11058bb
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Sat Jan 21 08:22:22 2023 +0800
[vectorzied](jdbc) fix jdbc executor for get result by batch and memo… (#15843)
result set should be get by batch size2.
fix memory leak3.
---
be/src/vec/exec/vjdbc_connector.cpp | 6 +-
be/src/vec/exec/vjdbc_connector.h | 1 +
fe/java-udf/pom.xml | 5 --
.../main/java/org/apache/doris/udf/FakeDriver.java | 70 ++++++++++++++++
.../java/org/apache/doris/udf/JdbcExecutor.java | 96 ++++++++++++----------
5 files changed, 129 insertions(+), 49 deletions(-)
diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp
index 519ccbbc77..0eb6d6f2a3 100644
--- a/be/src/vec/exec/vjdbc_connector.cpp
+++ b/be/src/vec/exec/vjdbc_connector.cpp
@@ -370,8 +370,8 @@ Status JdbcConnector::get_next(bool* eos, std::vector<MutableColumnPtr>& columns
const std::string& column_name = slot_desc->col_name();
jobject column_data =
env->CallObjectMethod(block_obj, _executor_get_list_id, materialized_column_index);
- jint num_rows = env->CallIntMethod(column_data, _executor_get_list_size_id);
-
+ jint num_rows = env->CallNonvirtualIntMethod(_executor_obj, _executor_clazz,
+ _executor_block_rows_id);
for (int row = 0; row < num_rows; ++row) {
jobject cur_data = env->CallObjectMethod(column_data, _executor_get_list_id, row);
RETURN_IF_ERROR(_convert_column_data(env, cur_data, slot_desc,
@@ -413,6 +413,8 @@ Status JdbcConnector::_register_func_id(JNIEnv* env) {
_executor_close_id));
RETURN_IF_ERROR(register_id(_executor_clazz, "hasNext", JDBC_EXECUTOR_HAS_NEXT_SIGNATURE,
_executor_has_next_id));
+ RETURN_IF_ERROR(
+ register_id(_executor_clazz, "getCurBlockRows", "()I", _executor_block_rows_id));
RETURN_IF_ERROR(register_id(_executor_clazz, "getBlock", JDBC_EXECUTOR_GET_BLOCK_SIGNATURE,
_executor_get_blocks_id));
RETURN_IF_ERROR(register_id(_executor_clazz, "convertDateToLong",
diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h
index c1d416783c..4f05253d64 100644
--- a/be/src/vec/exec/vjdbc_connector.h
+++ b/be/src/vec/exec/vjdbc_connector.h
@@ -114,6 +114,7 @@ private:
jmethodID _executor_write_id;
jmethodID _executor_read_id;
jmethodID _executor_has_next_id;
+ jmethodID _executor_block_rows_id;
jmethodID _executor_get_blocks_id;
jmethodID _executor_get_types_id;
jmethodID _executor_get_arr_list_id;
diff --git a/fe/java-udf/pom.xml b/fe/java-udf/pom.xml
index 493015d49c..b37e9c4696 100644
--- a/fe/java-udf/pom.xml
+++ b/fe/java-udf/pom.xml
@@ -104,11 +104,6 @@ under the License.
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.zaxxer</groupId>
- <artifactId>HikariCP</artifactId>
- <version>${hikaricp.version}</version>
- </dependency>
</dependencies>
<build>
<finalName>java-udf</finalName>
diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/FakeDriver.java b/fe/java-udf/src/main/java/org/apache/doris/udf/FakeDriver.java
new file mode 100644
index 0000000000..94fbde6217
--- /dev/null
+++ b/fe/java-udf/src/main/java/org/apache/doris/udf/FakeDriver.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.udf;
+
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.DriverPropertyInfo;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.Properties;
+import java.util.logging.Logger;
+
+
+public class FakeDriver implements Driver {
+ private Driver driver;
+
+ FakeDriver(Driver driver) {
+ this.driver = driver;
+ }
+
+ @Override
+ public Connection connect(String url, Properties info) throws SQLException {
+ return this.driver.connect(url, info);
+ }
+
+ @Override
+ public boolean acceptsURL(String url) throws SQLException {
+ return this.driver.acceptsURL(url);
+ }
+
+ @Override
+ public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
+ return this.driver.getPropertyInfo(url, info);
+ }
+
+ @Override
+ public int getMajorVersion() {
+ return this.driver.getMajorVersion();
+ }
+
+ @Override
+ public int getMinorVersion() {
+ return this.driver.getMinorVersion();
+ }
+
+ @Override
+ public boolean jdbcCompliant() {
+ return this.driver.jdbcCompliant();
+ }
+
+ @Override
+ public Logger getParentLogger() throws SQLFeatureNotSupportedException {
+ return null;
+ }
+}
diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
index d90aa4055a..e76e9e78e3 100644
--- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
+++ b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
@@ -21,17 +21,20 @@ import org.apache.doris.thrift.TJdbcExecutorCtorParams;
import org.apache.doris.thrift.TJdbcOperation;
import com.google.common.base.Preconditions;
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
import org.apache.log4j.Logger;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
-import java.io.FileNotFoundException;
+import java.io.File;
+import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
import java.sql.Connection;
import java.sql.Date;
+import java.sql.Driver;
+import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
@@ -50,10 +53,12 @@ public class JdbcExecutor {
private Statement stmt = null;
private ResultSet resultSet = null;
private ResultSetMetaData resultSetMetaData = null;
- // Use HikariDataSource to help us manage the JDBC connections.
- private HikariDataSource dataSource = null;
private List<String> resultColumnTypeNames = null;
private int baseTypeInt = 0;
+ private URLClassLoader classLoader = null;
+ private List<List<Object>> block = null;
+ private int bacthSizeNum = 0;
+ private int curBlockRows = 0;
public JdbcExecutor(byte[] thriftParams) throws Exception {
TJdbcExecutorCtorParams request = new TJdbcExecutorCtorParams();
@@ -75,22 +80,31 @@ public class JdbcExecutor {
stmt.close();
}
if (conn != null) {
+ conn.clearWarnings();
conn.close();
}
- if (dataSource != null) {
- dataSource.close();
+ if (classLoader != null) {
+ classLoader.clearAssertionStatus();
+ classLoader.close();
}
resultSet = null;
stmt = null;
conn = null;
- dataSource = null;
+ classLoader = null;
}
public int read() throws UdfRuntimeException {
try {
resultSet = ((PreparedStatement) stmt).executeQuery();
resultSetMetaData = resultSet.getMetaData();
- return resultSetMetaData.getColumnCount();
+ int columnCount = resultSetMetaData.getColumnCount();
+ resultColumnTypeNames = new ArrayList<>(columnCount);
+ block = new ArrayList<>(columnCount);
+ for (int i = 0; i < columnCount; ++i) {
+ resultColumnTypeNames.add(resultSetMetaData.getColumnClassName(i + 1));
+ block.add(Arrays.asList(new Object[bacthSizeNum]));
+ }
+ return columnCount;
} catch (SQLException e) {
throw new UdfRuntimeException("JDBC executor sql has error: ", e);
}
@@ -111,17 +125,8 @@ public class JdbcExecutor {
}
}
- public List<String> getResultColumnTypeNames() throws UdfRuntimeException {
- try {
- int count = resultSetMetaData.getColumnCount();
- resultColumnTypeNames = new ArrayList<>(count);
- for (int i = 0; i < count; ++i) {
- resultColumnTypeNames.add(resultSetMetaData.getColumnClassName(i + 1));
- }
- return resultColumnTypeNames;
- } catch (SQLException e) {
- throw new UdfRuntimeException("JDBC executor getResultColumnTypeNames has error: ", e);
- }
+ public List<String> getResultColumnTypeNames() {
+ return resultColumnTypeNames;
}
public List<Object> getArrayColumnData(Object object) throws UdfRuntimeException {
@@ -169,20 +174,15 @@ public class JdbcExecutor {
}
public List<List<Object>> getBlock(int batchSize) throws UdfRuntimeException {
- List<List<Object>> block = null;
try {
int columnCount = resultSetMetaData.getColumnCount();
- block = new ArrayList<>(columnCount);
- for (int i = 0; i < columnCount; ++i) {
- block.add(new ArrayList<>(batchSize));
- }
- int numRows = 0;
+ curBlockRows = 0;
do {
for (int i = 0; i < columnCount; ++i) {
- block.get(i).add(resultSet.getObject(i + 1));
+ block.get(i).set(curBlockRows, resultSet.getObject(i + 1));
}
- numRows++;
- } while (numRows < batchSize && resultSet.next());
+ curBlockRows++;
+ } while (curBlockRows < batchSize && resultSet.next());
} catch (SQLException e) {
throw new UdfRuntimeException("get next block failed: ", e);
} catch (Exception e) {
@@ -191,6 +191,10 @@ public class JdbcExecutor {
return block;
}
+ public int getCurBlockRows() {
+ return curBlockRows;
+ }
+
public boolean hasNext() throws UdfRuntimeException {
try {
if (resultSet == null) {
@@ -242,33 +246,41 @@ public class JdbcExecutor {
private void init(String driverUrl, String sql, int batchSize, String driverClass, String jdbcUrl, String jdbcUser,
String jdbcPassword, TJdbcOperation op) throws UdfRuntimeException {
try {
- ClassLoader parent = getClass().getClassLoader();
- ClassLoader classLoader = UdfUtils.getClassLoader(driverUrl, parent);
- Thread.currentThread().setContextClassLoader(classLoader);
- HikariConfig config = new HikariConfig();
- config.setDriverClassName(driverClass);
- config.setJdbcUrl(jdbcUrl);
- config.setUsername(jdbcUser);
- config.setPassword(jdbcPassword);
- config.setMaximumPoolSize(1);
+ File file = new File(driverUrl);
+ URL url = file.toURI().toURL();
+ classLoader = new URLClassLoader(new URL[] {url});
+ Driver driver = (Driver) Class.forName(driverClass, true, classLoader).getDeclaredConstructor()
+ .newInstance();
+ // in jdk11 cann't call addURL function by reflect to load class. so use this way
+ // But DriverManager can't find the driverClass correctly, so add a faker driver
+ // https://www.kfu.com/~nsayer/Java/dyn-jdbc.html
+ DriverManager.registerDriver(new FakeDriver(driver));
+ conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword);
- dataSource = new HikariDataSource(config);
- conn = dataSource.getConnection();
if (op == TJdbcOperation.READ) {
conn.setAutoCommit(false);
Preconditions.checkArgument(sql != null);
stmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY,
ResultSet.FETCH_FORWARD);
stmt.setFetchSize(batchSize);
+ bacthSizeNum = batchSize;
} else {
stmt = conn.createStatement();
}
- } catch (FileNotFoundException e) {
- throw new UdfRuntimeException("Can not find driver file: " + driverUrl, e);
+ } catch (ClassNotFoundException e) {
+ throw new UdfRuntimeException("ClassNotFoundException: " + driverClass, e);
} catch (MalformedURLException e) {
throw new UdfRuntimeException("MalformedURLException to load class about " + driverUrl, e);
} catch (SQLException e) {
throw new UdfRuntimeException("Initialize datasource failed: ", e);
+ } catch (InstantiationException e) {
+ throw new UdfRuntimeException("InstantiationException failed: ", e);
+ } catch (IllegalAccessException e) {
+ throw new UdfRuntimeException("IllegalAccessException failed: ", e);
+ } catch (InvocationTargetException e) {
+ throw new UdfRuntimeException("InvocationTargetException new instance failed: ", e);
+ } catch (NoSuchMethodException e) {
+ throw new UdfRuntimeException("NoSuchMethodException Load class failed: ", e);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/06: [vectorized](analytic) fix analytic node of window function get wrong… (#16074)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c61be0cb8a85cda1b61b1bc0535020488207b46c
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Mon Jan 23 16:09:46 2023 +0800
[vectorized](analytic) fix analytic node of window function get wrong… (#16074)
[Bug] 基础函数rank()开窗排序结果错误 #15951
---
be/src/vec/exec/vanalytic_eval_node.cpp | 28 ++++++++++++++++++++++------
be/src/vec/exec/vanalytic_eval_node.h | 14 +++++++++++++-
2 files changed, 35 insertions(+), 7 deletions(-)
diff --git a/be/src/vec/exec/vanalytic_eval_node.cpp b/be/src/vec/exec/vanalytic_eval_node.cpp
index 74625c14a6..55d98d47ab 100644
--- a/be/src/vec/exec/vanalytic_eval_node.cpp
+++ b/be/src/vec/exec/vanalytic_eval_node.cpp
@@ -395,8 +395,8 @@ BlockRowPos VAnalyticEvalNode::_get_partition_by_end() {
}
//_partition_by_columns,_order_by_columns save in blocks, so if need to calculate the boundary, may find in which blocks firstly
-BlockRowPos VAnalyticEvalNode::_compare_row_to_find_end(int idx, BlockRowPos start,
- BlockRowPos end) {
+BlockRowPos VAnalyticEvalNode::_compare_row_to_find_end(int idx, BlockRowPos start, BlockRowPos end,
+ bool need_check_first) {
int64_t start_init_row_num = start.row_num;
ColumnPtr start_column = _input_blocks[start.block_num].get_by_position(idx).column;
ColumnPtr start_next_block_column = start_column;
@@ -406,10 +406,20 @@ BlockRowPos VAnalyticEvalNode::_compare_row_to_find_end(int idx, BlockRowPos sta
int64_t start_block_num = start.block_num;
int64_t end_block_num = end.block_num;
int64_t mid_blcok_num = end.block_num;
+ // To fix this problem: https://github.com/apache/doris/issues/15951
+ // in this case, the partition by column is last row of block, so it's pointed to a new block at row = 0, range is: [left, right)
+ // From the perspective of order by column, the two values are exactly equal.
+ // so the range will be get wrong because it's compare_at == 0 with next block at row = 0
+ if (need_check_first && end.block_num > 0 && end.row_num == 0) {
+ end.block_num--;
+ end_block_num--;
+ end.row_num = _input_blocks[end_block_num].rows();
+ }
//binary search find in which block
while (start_block_num < end_block_num) {
mid_blcok_num = (start_block_num + end_block_num + 1) >> 1;
start_next_block_column = _input_blocks[mid_blcok_num].get_by_position(idx).column;
+ //Compares (*this)[n] and rhs[m], this: start[init_row] rhs: mid[0]
if (start_column->compare_at(start_init_row_num, 0, *start_next_block_column, 1) == 0) {
start_block_num = mid_blcok_num;
} else {
@@ -417,6 +427,8 @@ BlockRowPos VAnalyticEvalNode::_compare_row_to_find_end(int idx, BlockRowPos sta
}
}
+ // have check the start.block_num: start_column[start_init_row_num] with mid_blcok_num start_next_block_column[0]
+ // now next block must not be result, so need check with end_block_num: start_next_block_column[last_row]
if (end_block_num == mid_blcok_num - 1) {
start_next_block_column = _input_blocks[end_block_num].get_by_position(idx).column;
int64_t block_size = _input_blocks[end_block_num].rows();
@@ -429,6 +441,7 @@ BlockRowPos VAnalyticEvalNode::_compare_row_to_find_end(int idx, BlockRowPos sta
}
//check whether need get column again, maybe same as first init
+ // if the start_block_num have move to forword, so need update start block num and compare it from row_num=0
if (start_column.get() != start_next_block_column.get()) {
start_init_row_num = 0;
start.block_num = start_block_num;
@@ -437,15 +450,18 @@ BlockRowPos VAnalyticEvalNode::_compare_row_to_find_end(int idx, BlockRowPos sta
//binary search, set start and end pos
int64_t start_pos = start_init_row_num;
int64_t end_pos = _input_blocks[start.block_num].rows() - 1;
+ //if end_block_num haven't moved, only start_block_num go to the end block
+ //so could used the end.row_num for binary search
if (start.block_num == end.block_num) {
end_pos = end.row_num;
}
while (start_pos < end_pos) {
int64_t mid_pos = (start_pos + end_pos) >> 1;
- if (start_column->compare_at(start_init_row_num, mid_pos, *start_column, 1))
+ if (start_column->compare_at(start_init_row_num, mid_pos, *start_column, 1)) {
end_pos = mid_pos;
- else
+ } else {
start_pos = mid_pos + 1;
+ }
}
start.row_num = start_pos; //upadte row num, return the find end
return start;
@@ -615,8 +631,8 @@ void VAnalyticEvalNode::_update_order_by_range() {
_order_by_start = _order_by_end;
_order_by_end = _partition_by_end;
for (size_t i = 0; i < _order_by_eq_expr_ctxs.size(); ++i) {
- _order_by_end =
- _compare_row_to_find_end(_ordey_by_column_idxs[i], _order_by_start, _order_by_end);
+ _order_by_end = _compare_row_to_find_end(_ordey_by_column_idxs[i], _order_by_start,
+ _order_by_end, true);
}
_order_by_start.pos =
input_block_first_row_positions[_order_by_start.block_num] + _order_by_start.row_num;
diff --git a/be/src/vec/exec/vanalytic_eval_node.h b/be/src/vec/exec/vanalytic_eval_node.h
index 9597606f6c..e4eff3df99 100644
--- a/be/src/vec/exec/vanalytic_eval_node.h
+++ b/be/src/vec/exec/vanalytic_eval_node.h
@@ -19,6 +19,8 @@
#include <thrift/protocol/TDebugProtocol.h>
+#include <string>
+
#include "exec/exec_node.h"
#include "exprs/expr.h"
#include "runtime/tuple.h"
@@ -34,6 +36,15 @@ struct BlockRowPos {
int64_t block_num; //the pos at which block
int64_t row_num; //the pos at which row
int64_t pos; //pos = all blocks size + row_num
+ std::string debug_string() {
+ std::string res = "\t block_num: ";
+ res += std::to_string(block_num);
+ res += "\t row_num: ";
+ res += std::to_string(row_num);
+ res += "\t pos: ";
+ res += std::to_string(pos);
+ return res;
+ }
};
class AggFnEvaluator;
@@ -73,7 +84,8 @@ private:
void _insert_result_info(int64_t current_block_rows);
Status _output_current_block(Block* block);
BlockRowPos _get_partition_by_end();
- BlockRowPos _compare_row_to_find_end(int idx, BlockRowPos start, BlockRowPos end);
+ BlockRowPos _compare_row_to_find_end(int idx, BlockRowPos start, BlockRowPos end,
+ bool need_check_first = false);
Status _fetch_next_block_data(RuntimeState* state);
Status _consumed_block_and_init_partition(RuntimeState* state, bool* next_partition, bool* eos);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/06: [fix](privilege)fix grant resource bug (#16045)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 42ab0c39de0a87f14d0860dc32a3c38a40231aec
Author: zhangdong <49...@qq.com>
AuthorDate: Fri Jan 20 19:00:44 2023 +0800
[fix](privilege)fix grant resource bug (#16045)
GRANT USAGE_PRIV ON RESOURCE * TO user;
user will see all database
Describe your changes.
Set a PrivPredicate for show resources and remove USAGE under PrivPredicate in SHOW_ PRIV
---
fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java | 2 +-
.../main/java/org/apache/doris/mysql/privilege/PrivPredicate.java | 7 +++++--
.../src/test/java/org/apache/doris/mysql/privilege/AuthTest.java | 4 ++++
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
index 3805b5848d..d9a79b8616 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
@@ -229,7 +229,7 @@ public class ResourceMgr implements Writable {
Resource resource = entry.getValue();
// check resource privs
if (!Env.getCurrentEnv().getAuth().checkResourcePriv(ConnectContext.get(), resource.getName(),
- PrivPredicate.SHOW)) {
+ PrivPredicate.SHOW_RESOURCES)) {
continue;
}
resource.getProcNodeData(result);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PrivPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PrivPredicate.java
index 7a805eefc2..3bf80cd149 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PrivPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PrivPredicate.java
@@ -27,8 +27,11 @@ public class PrivPredicate {
PaloPrivilege.LOAD_PRIV,
PaloPrivilege.ALTER_PRIV,
PaloPrivilege.CREATE_PRIV,
- PaloPrivilege.DROP_PRIV,
- PaloPrivilege.USAGE_PRIV),
+ PaloPrivilege.DROP_PRIV),
+ Operator.OR);
+ //show resources
+ public static final PrivPredicate SHOW_RESOURCES = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV,
+ PaloPrivilege.USAGE_PRIV),
Operator.OR);
// create/drop/alter/show user
public static final PrivPredicate GRANT = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
index 3df1b4d1ed..07169777f6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
@@ -1530,6 +1530,8 @@ public class AuthTest {
}
Assert.assertTrue(auth.checkResourcePriv(userIdentity, resourceName, PrivPredicate.USAGE));
Assert.assertTrue(auth.checkGlobalPriv(userIdentity, PrivPredicate.USAGE));
+ Assert.assertTrue(auth.checkGlobalPriv(userIdentity, PrivPredicate.SHOW_RESOURCES));
+ Assert.assertFalse(auth.checkGlobalPriv(userIdentity, PrivPredicate.SHOW));
// 3. revoke usage_priv on resource '*' from 'testUser'@'%'
revokeStmt = new RevokeStmt(userIdentity, null, anyResourcePattern, usagePrivileges);
@@ -1542,6 +1544,8 @@ public class AuthTest {
}
Assert.assertFalse(auth.checkResourcePriv(userIdentity, resourceName, PrivPredicate.USAGE));
Assert.assertFalse(auth.checkGlobalPriv(userIdentity, PrivPredicate.USAGE));
+ Assert.assertFalse(auth.checkGlobalPriv(userIdentity, PrivPredicate.SHOW_RESOURCES));
+ Assert.assertFalse(auth.checkGlobalPriv(userIdentity, PrivPredicate.SHOW));
// 4. drop user
dropUserStmt = new DropUserStmt(userIdentity);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/06: [fix](daemon) should use GetMonoTimeMicros() (#16070)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7095f4db3997964a825fc58f493dcf2dca83c8ef
Author: Xin Huang <57...@users.noreply.github.com>
AuthorDate: Thu Jan 19 10:44:06 2023 +0800
[fix](daemon) should use GetMonoTimeMicros() (#16070)
---
be/src/common/daemon.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index 054dbdc533..aabb3eb9b9 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -275,13 +275,13 @@ void Daemon::calculate_metrics_thread() {
DorisMetrics::instance()->metric_registry()->trigger_all_hooks(true);
if (last_ts == -1L) {
- last_ts = GetCurrentTimeMicros() / 1000;
+ last_ts = GetMonoTimeMicros() / 1000;
lst_query_bytes = DorisMetrics::instance()->query_scan_bytes->value();
DorisMetrics::instance()->system_metrics()->get_disks_io_time(&lst_disks_io_time);
DorisMetrics::instance()->system_metrics()->get_network_traffic(&lst_net_send_bytes,
&lst_net_receive_bytes);
} else {
- int64_t current_ts = GetCurrentTimeMicros() / 1000;
+ int64_t current_ts = GetMonoTimeMicros() / 1000;
long interval = (current_ts - last_ts) / 1000;
last_ts = current_ts;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/06: [fix](planner) fix bugs in uncheckedCastChild (#15905)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d016aef2bc44f20242e1a9561e5f9fb352b8f653
Author: minghong <en...@gmail.com>
AuthorDate: Thu Jan 19 15:51:08 2023 +0800
[fix](planner) fix bugs in uncheckedCastChild (#15905)
1. `uncheckedCastChild` may generate redundant `CastExpr` like `cast( cast(XXX as Date) as Date)`
2. generate DateLiteral to replace cast(IntLiteral as Date)
---
.../main/java/org/apache/doris/catalog/Type.java | 7 ++++
.../main/java/org/apache/doris/analysis/Expr.java | 7 +++-
.../java/org/apache/doris/analysis/IntLiteral.java | 47 ++++++++++++++--------
.../java/org/apache/doris/analysis/ExprTest.java | 10 +++++
.../org/apache/doris/analysis/InsertStmtTest.java | 5 ++-
.../org/apache/doris/analysis/IntLiteralTest.java | 35 ++++++++++++++++
6 files changed, 90 insertions(+), 21 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 74c9182f1a..184bafb313 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -273,6 +273,13 @@ public abstract class Type {
return isScalarType(PrimitiveType.DATETIMEV2);
}
+ public boolean isDateLike() {
+ return isScalarType(PrimitiveType.DATETIME)
+ || isScalarType(PrimitiveType.DATETIMEV2)
+ || isScalarType(PrimitiveType.DATE)
+ || isScalarType(PrimitiveType.DATEV2);
+ }
+
public boolean isTimeV2() {
return isScalarType(PrimitiveType.TIMEV2);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index 79df8f3395..6329d51d62 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -1440,8 +1440,11 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
public void uncheckedCastChild(Type targetType, int childIndex)
throws AnalysisException {
Expr child = getChild(childIndex);
- Expr newChild = child.uncheckedCastTo(targetType);
- setChild(childIndex, newChild);
+ //avoid to generate Expr like cast (cast(... as date) as date)
+ if (!child.getType().equals(targetType)) {
+ Expr newChild = child.uncheckedCastTo(targetType);
+ setChild(childIndex, newChild);
+ }
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java
index 0d3749fa86..8933ec496e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java
@@ -294,28 +294,41 @@ public class IntLiteral extends LiteralExpr {
@Override
protected Expr uncheckedCastTo(Type targetType) throws AnalysisException {
- if (!targetType.isNumericType()) {
- return super.uncheckedCastTo(targetType);
- }
- if (targetType.isFixedPointType()) {
- if (!targetType.isScalarType(PrimitiveType.LARGEINT)) {
- if (!type.equals(targetType)) {
- IntLiteral intLiteral = new IntLiteral(this);
- intLiteral.setType(targetType);
- return intLiteral;
+ if (targetType.isNumericType()) {
+ if (targetType.isFixedPointType()) {
+ if (!targetType.isScalarType(PrimitiveType.LARGEINT)) {
+ if (!type.equals(targetType)) {
+ IntLiteral intLiteral = new IntLiteral(this);
+ intLiteral.setType(targetType);
+ return intLiteral;
+ }
+ return this;
+ } else {
+ return new LargeIntLiteral(Long.toString(value));
}
- return this;
- } else {
- return new LargeIntLiteral(Long.toString(value));
+ } else if (targetType.isFloatingPointType()) {
+ return new FloatLiteral(new Double(value), targetType);
+ } else if (targetType.isDecimalV2() || targetType.isDecimalV3()) {
+ DecimalLiteral res = new DecimalLiteral(new BigDecimal(value));
+ res.setType(targetType);
+ return res;
+ }
+ return this;
+ } else if (targetType.isDateLike()) {
+ try {
+ //int like 20200101 can be cast to date(2020,01,01)
+ DateLiteral res = new DateLiteral("" + value, targetType);
+ res.setType(targetType);
+ return res;
+ } catch (AnalysisException e) {
+ //invalid date format. leave it to BE to cast it as NULL
}
- } else if (targetType.isFloatingPointType()) {
- return new FloatLiteral(new Double(value), targetType);
- } else if (targetType.isDecimalV2() || targetType.isDecimalV3()) {
- DecimalLiteral res = new DecimalLiteral(new BigDecimal(value));
+ } else if (targetType.isStringType()) {
+ StringLiteral res = new StringLiteral("" + value);
res.setType(targetType);
return res;
}
- return this;
+ return super.uncheckedCastTo(targetType);
}
@Override
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java
index aa63a1c933..9cf8ed7ae0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java
@@ -32,6 +32,7 @@ import mockit.Injectable;
import mockit.Mocked;
import org.junit.Assert;
import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
import java.io.DataInputStream;
import java.io.DataOutputStream;
@@ -206,6 +207,15 @@ public class ExprTest {
Assert.assertFalse(Expr.equalSets(list1, list2));
}
+ @Test
+ public void testUncheckedCastChildAvoidDoubleCast() throws AnalysisException {
+ Expr cast = new CastExpr(Type.DATETIME, new IntLiteral(10000101));
+ FunctionCallExpr call = new FunctionCallExpr("leap", Lists.newArrayList(cast));
+ call.uncheckedCastChild(Type.DATETIME, 0);
+ //do not cast a castExpr
+ Assertions.assertTrue(call.getChild(0).getChild(0) instanceof IntLiteral);
+ }
+
@Test
public void testSrcSlotRef(@Injectable SlotDescriptor slotDescriptor) {
TableName tableName = new TableName(internalCtl, "db1", "table1");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertStmtTest.java
index 898a949952..72c9e67d8a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertStmtTest.java
@@ -202,9 +202,10 @@ public class InsertStmtTest {
FunctionCallExpr expr4 = (FunctionCallExpr) queryStmtSubstitute.getResultExprs().get(4);
Assert.assertTrue(expr4.getFnName().getFunction().equals("to_bitmap"));
List<Expr> slots = Lists.newArrayList();
- expr4.collect(IntLiteral.class, slots);
+ expr4.collect(StringLiteral.class, slots);
Assert.assertEquals(1, slots.size());
- Assert.assertEquals(queryStmtSubstitute.getResultExprs().get(0), slots.get(0));
+ Assert.assertEquals(queryStmtSubstitute.getResultExprs().get(0).getStringValue(),
+ slots.get(0).getStringValue());
Assert.assertTrue(queryStmtSubstitute.getResultExprs().get(5) instanceof FunctionCallExpr);
FunctionCallExpr expr5 = (FunctionCallExpr) queryStmtSubstitute.getResultExprs().get(5);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/IntLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/IntLiteralTest.java
new file mode 100644
index 0000000000..a3c9b48282
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IntLiteralTest.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class IntLiteralTest {
+ @Test
+ public void testUncheckedCastTo() throws AnalysisException {
+ IntLiteral intLiteral = new IntLiteral(20200101);
+ Expr expr = intLiteral.uncheckedCastTo(Type.DATETIME);
+ Assertions.assertTrue(expr instanceof DateLiteral);
+ expr = intLiteral.uncheckedCastTo(Type.STRING);
+ Assertions.assertTrue(expr instanceof StringLiteral);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/06: [debug](ParquetReader) print file path if failed to read parquet file (#16118)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 82f923ee1f44dc1538c4f8446cbaec5493bd2d50
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Sat Jan 21 08:05:17 2023 +0800
[debug](ParquetReader) print file path if failed to read parquet file (#16118)
---
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 26ab574486..1d2575a70e 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -385,8 +385,12 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
DCHECK(_current_group_reader != nullptr);
{
SCOPED_RAW_TIMER(&_statistics.column_read_time);
- RETURN_IF_ERROR(
- _current_group_reader->next_batch(block, _batch_size, read_rows, &_row_group_eof));
+ Status batch_st =
+ _current_group_reader->next_batch(block, _batch_size, read_rows, &_row_group_eof);
+ if (!batch_st.ok()) {
+ return Status::InternalError("Read parquet file {} failed, reason = {}",
+ _scan_range.path, batch_st.to_string());
+ }
}
if (_row_group_eof) {
auto column_st = _current_group_reader->statistics();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org