You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/04/08 02:31:14 UTC
[incubator-doris] branch dev-1.0.1 updated (8928954865 -> 075f9e6420)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
from 8928954865 [fix](ut)(compile) Fix ut failure at functions_geo and compilation bug (#8843)
new 7b989c0c97 Window funnel (#8485)
new b77baafcac min and max window function bug fix (#8822)
new 7aace71325 [improvement](hll) Optimize Hyperloglog (#8829)
new 28892f2807 [fix](storage) Fix core bug of convert to predicate column (#8833)
new 1eebb658e1 [fix](ut) fix fe run CreateTableAsSelectStmtTest ,UserPropertyTest, ProjectPlannerFunctionTest and AggregateTest failed (#8838)
new cf94d9fe0f [feature](diagnose) support "ADMIN DIAGNOSE TABLET" stmt (#8839)
new e882eecbee [Refactor][httpv2]remove http v1 code (#8848)
new 5b6f9329c8 [Fix](Lateral View) The Error expr type when exploding a function result of inline view (#8851)
new 8c4c4b237f [improvement](restore) set table property 'dynamic_partition.enable' to false after restore (#8852)
new 930b458901 [fix] access parallel_flat_hash_map via thread safely methods (#8854)
new b86f290a7c [fix] Disable cast operation of object type (#8882)
new 32ccf29b86 [feature](vectorized) support vexplode_bitmap (#8890)
new f7ee301995 [fix](datax) Fix the problem of keyword error when importing datax (#8893)
new 075f9e6420 [fix](ut)(memory-leak) Fix be asan ut failed and hdfs file reader memory leak (#8905)
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/hdfs_file_reader.cpp | 1 +
be/src/exprs/agg_fn_evaluator.cpp | 2 +-
be/src/exprs/aggregate_functions.cpp | 213 +++++++++-
be/src/exprs/aggregate_functions.h | 10 +
be/src/exprs/table_function/explode_bitmap.h | 5 +-
.../table_function/table_function_factory.cpp | 13 +-
be/src/olap/hll.cpp | 143 ++-----
be/src/olap/hll.h | 118 ++----
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 2 +-
be/src/util/brpc_client_cache.h | 17 +-
be/src/vec/CMakeLists.txt | 2 +
.../aggregate_function_simple_factory.cpp | 5 +-
...ader.h => aggregate_function_window_funnel.cpp} | 24 +-
.../aggregate_function_window_funnel.h | 214 ++++++++++
be/src/vec/columns/column.h | 2 +-
be/src/vec/columns/column_dictionary.h | 14 +-
be/src/vec/columns/column_nullable.h | 6 +-
.../{vexplode_numbers.cpp => vexplode_bitmap.cpp} | 54 ++-
.../{vexplode_numbers.h => vexplode_bitmap.h} | 17 +-
be/src/vec/functions/function_fake.cpp | 1 +
be/src/vec/functions/function_fake.h | 8 +
be/test/exprs/CMakeLists.txt | 2 +-
be/test/exprs/window_funnel_test.cpp | 431 +++++++++++++++++++++
be/test/vec/aggregate_functions/CMakeLists.txt | 1 +
.../aggregate_functions/vec_window_funnel_test.cpp | 425 ++++++++++++++++++++
docs/.vuepress/sidebar/en.js | 1 +
docs/.vuepress/sidebar/zh-CN.js | 1 +
.../Administration/ADMIN-DIAGNOSE-TABLET.md | 59 +++
.../Administration/ADMIN-DIAGNOSE-TABLET.md | 59 +++
.../writer/doriswriter/DorisWriterEmitter.java | 3 +-
fe/fe-core/pom.xml | 4 +-
fe/fe-core/src/main/cup/sql_parser.cup | 8 +-
.../src/main/java/org/apache/doris/PaloFe.java | 29 +-
...tionsStmt.java => AdminDiagnoseTabletStmt.java} | 51 ++-
.../java/org/apache/doris/analysis/Analyzer.java | 1 -
.../java/org/apache/doris/analysis/CastExpr.java | 28 +-
.../apache/doris/analysis/FunctionCallExpr.java | 72 +++-
.../org/apache/doris/analysis/LateralViewRef.java | 12 +-
.../java/org/apache/doris/backup/RestoreJob.java | 3 +
.../apache/doris/catalog/AggregateFunction.java | 2 +-
.../java/org/apache/doris/catalog/Catalog.java | 5 +-
.../main/java/org/apache/doris/catalog/Column.java | 5 +-
.../java/org/apache/doris/catalog/FunctionSet.java | 31 ++
.../java/org/apache/doris/catalog/OlapTable.java | 10 +
.../org/apache/doris/catalog/PrimitiveType.java | 8 -
.../java/org/apache/doris/catalog/ScalarType.java | 12 +-
.../org/apache/doris/catalog/TableProperty.java | 12 +
.../main/java/org/apache/doris/catalog/Type.java | 4 +
.../main/java/org/apache/doris/common/Config.java | 17 -
.../org/apache/doris/http/ActionController.java | 88 -----
.../java/org/apache/doris/http/BaseAction.java | 375 ------------------
.../java/org/apache/doris/http/BaseRequest.java | 158 --------
.../java/org/apache/doris/http/BaseResponse.java | 99 -----
.../org/apache/doris/http/HttpAuthManager.java | 64 ---
.../java/org/apache/doris/http/HttpServer.java | 279 -------------
.../org/apache/doris/http/HttpServerHandler.java | 117 ------
.../main/java/org/apache/doris/http/IAction.java | 25 --
.../apache/doris/http/UnauthorizedException.java | 26 --
.../main/java/org/apache/doris/http/WebUtils.java | 230 -----------
.../apache/doris/http/action/BackendAction.java | 101 -----
.../org/apache/doris/http/action/HaAction.java | 174 ---------
.../org/apache/doris/http/action/HelpAction.java | 223 -----------
.../org/apache/doris/http/action/IndexAction.java | 275 -------------
.../org/apache/doris/http/action/LogAction.java | 168 --------
.../apache/doris/http/action/NotFoundAction.java | 52 ---
.../org/apache/doris/http/action/QueryAction.java | 127 ------
.../doris/http/action/QueryProfileAction.java | 68 ----
.../apache/doris/http/action/SessionAction.java | 85 ----
.../doris/http/action/StaticResourceAction.java | 242 ------------
.../org/apache/doris/http/action/SystemAction.java | 184 ---------
.../apache/doris/http/action/VariableAction.java | 84 ----
.../apache/doris/http/action/WebBaseAction.java | 374 ------------------
.../http/common/DorisHttpPostObjectAggregator.java | 61 ---
.../doris/http/meta/ColocateMetaService.java | 229 -----------
.../doris/http/meta/InvalidClientException.java | 26 --
.../org/apache/doris/http/meta/MetaBaseAction.java | 108 ------
.../org/apache/doris/http/meta/MetaService.java | 356 -----------------
.../doris/http/rest/BootstrapFinishAction.java | 162 --------
.../apache/doris/http/rest/CancelStreamLoad.java | 83 ----
.../doris/http/rest/CheckDecommissionAction.java | 93 -----
.../apache/doris/http/rest/ConnectionAction.java | 62 ---
.../apache/doris/http/rest/GetDdlStmtAction.java | 108 ------
.../apache/doris/http/rest/GetLoadInfoAction.java | 88 -----
.../apache/doris/http/rest/GetLogFileAction.java | 131 -------
.../apache/doris/http/rest/GetSmallFileAction.java | 92 -----
.../apache/doris/http/rest/GetStreamLoadState.java | 86 ----
.../org/apache/doris/http/rest/HealthAction.java | 47 ---
.../org/apache/doris/http/rest/LoadAction.java | 141 -------
.../doris/http/rest/MetaReplayerCheckAction.java | 70 ----
.../org/apache/doris/http/rest/MetricsAction.java | 65 ----
.../apache/doris/http/rest/MigrationAction.java | 164 --------
.../org/apache/doris/http/rest/MultiAbort.java | 72 ----
.../org/apache/doris/http/rest/MultiCommit.java | 77 ----
.../java/org/apache/doris/http/rest/MultiDesc.java | 85 ----
.../java/org/apache/doris/http/rest/MultiList.java | 82 ----
.../org/apache/doris/http/rest/MultiStart.java | 92 -----
.../org/apache/doris/http/rest/MultiUnload.java | 76 ----
.../org/apache/doris/http/rest/ProfileAction.java | 60 ---
.../apache/doris/http/rest/QueryDetailAction.java | 56 ---
.../org/apache/doris/http/rest/RestBaseAction.java | 127 ------
.../org/apache/doris/http/rest/RowCountAction.java | 115 ------
.../apache/doris/http/rest/SetConfigAction.java | 136 -------
.../org/apache/doris/http/rest/ShowDataAction.java | 91 -----
.../apache/doris/http/rest/ShowMetaInfoAction.java | 203 ----------
.../org/apache/doris/http/rest/ShowProcAction.java | 136 -------
.../doris/http/rest/ShowRuntimeInfoAction.java | 69 ----
.../doris/http/rest/StorageTypeCheckAction.java | 98 -----
.../doris/http/rest/TableQueryPlanAction.java | 306 ---------------
.../doris/http/rest/TableRowCountAction.java | 113 ------
.../apache/doris/http/rest/TableSchemaAction.java | 135 -------
.../{http => httpv2}/IllegalArgException.java | 25 +-
.../{http/rest => httpv2/entity}/ActionStatus.java | 2 +-
.../rest => httpv2/entity}/RestBaseResult.java | 4 +-
.../{http/rest => httpv2/entity}/RestResult.java | 5 +-
.../doris/httpv2/rest/GetLoadInfoAction.java | 2 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 2 +-
.../org/apache/doris/httpv2/rest/MultiAction.java | 2 +-
.../apache/doris/journal/bdbje/BDBDebugger.java | 22 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 11 +
.../java/org/apache/doris/system/Diagnoser.java | 163 ++++++++
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../org/apache/doris/analysis/AggregateTest.java | 93 ++++-
.../analysis/CreateTableAsSelectStmtTest.java | 25 +-
.../org/apache/doris/catalog/OlapTableTest.java | 42 ++
.../org/apache/doris/catalog/UserPropertyTest.java | 41 ++
.../doris/clone/TabletReplicaTooSlowTest.java | 8 +
.../org/apache/doris/common/path/PathTrieTest.java | 19 -
.../org/apache/doris/http/DorisHttpTestCase.java | 23 +-
.../org/apache/doris/http/HttpAuthManagerTest.java | 18 +-
.../doris/planner/ProjectPlannerFunctionTest.java | 2 +-
.../org/apache/doris/planner/QueryPlanTest.java | 31 +-
.../doris/planner/TableFunctionPlanTest.java | 23 +-
132 files changed, 2238 insertions(+), 8412 deletions(-)
copy be/src/vec/aggregate_functions/{aggregate_function_reader.h => aggregate_function_window_funnel.cpp} (53%)
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
copy be/src/vec/exprs/table_function/{vexplode_numbers.cpp => vexplode_bitmap.cpp} (61%)
copy be/src/vec/exprs/table_function/{vexplode_numbers.h => vexplode_bitmap.h} (68%)
create mode 100644 be/test/exprs/window_funnel_test.cpp
create mode 100644 be/test/vec/aggregate_functions/vec_window_funnel_test.cpp
create mode 100644 docs/en/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md
create mode 100644 docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md
copy fe/fe-core/src/main/java/org/apache/doris/analysis/{ShowMigrationsStmt.java => AdminDiagnoseTabletStmt.java} (69%)
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/IAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java
rename fe/fe-core/src/main/java/org/apache/doris/{http => httpv2}/IllegalArgException.java (72%)
rename fe/fe-core/src/main/java/org/apache/doris/{http/rest => httpv2/entity}/ActionStatus.java (95%)
rename fe/fe-core/src/main/java/org/apache/doris/{http/rest => httpv2/entity}/RestBaseResult.java (97%)
rename fe/fe-core/src/main/java/org/apache/doris/{http/rest => httpv2/entity}/RestResult.java (94%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/system/Diagnoser.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 13/14: [fix](datax) Fix the problem of keyword error when importing datax (#8893)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit f7ee3019956495dad1e4c90b897026833e5f4d5e
Author: wudi <67...@qq.com>
AuthorDate: Fri Apr 8 09:20:54 2022 +0800
[fix](datax) Fix the problem of keyword error when importing datax (#8893)
---
.../alibaba/datax/plugin/writer/doriswriter/DorisWriterEmitter.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterEmitter.java b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterEmitter.java
index fb4e46b0ac..5ac5bd33a4 100644
--- a/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterEmitter.java
+++ b/extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisWriterEmitter.java
@@ -53,6 +53,7 @@ import java.net.URL;
import java.util.Base64;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
// Used to load batch of rows to Doris using stream load
public class DorisWriterEmitter {
@@ -181,7 +182,7 @@ public class DorisWriterEmitter {
final HttpPut httpPut = new HttpPut(loadUrl);
final List<String> cols = this.keys.getColumns();
if (null != cols && !cols.isEmpty()) {
- httpPut.setHeader("columns", String.join(",", cols));
+ httpPut.setHeader("columns", String.join(",", cols.stream().map(item -> String.format("`%s`", item.trim().replace("`", ""))).collect(Collectors.toList())));
}
// put loadProps to http header
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 14/14: [fix](ut)(memory-leak) Fix be asan ut failed and hdfs file reader memory leak (#8905)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 075f9e6420641289e62250b51e0080ab31acd4b2
Author: morningman <mo...@163.com>
AuthorDate: Fri Apr 8 10:29:19 2022 +0800
[fix](ut)(memory-leak) Fix be asan ut failed and hdfs file reader memory leak (#8905)
---
be/src/exec/hdfs_file_reader.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp
index 4399e639f6..a047df6236 100644
--- a/be/src/exec/hdfs_file_reader.cpp
+++ b/be/src/exec/hdfs_file_reader.cpp
@@ -65,6 +65,7 @@ Status HdfsFileReader::connect() {
}
}
_hdfs_fs = hdfsBuilderConnect(hdfs_builder);
+ hdfsFreeBuilder(hdfs_builder);
if (_hdfs_fs == nullptr) {
std::stringstream ss;
ss << "connect failed. " << _namenode;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/14: Window funnel (#8485)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 7b989c0c9725e2051e4e154daa16264fce8aa35f
Author: dataroaring <98...@users.noreply.github.com>
AuthorDate: Sat Apr 2 22:08:50 2022 +0800
Window funnel (#8485)
Add new feature window funnel
---
be/src/exprs/aggregate_functions.cpp | 207 ++++++++++
be/src/exprs/aggregate_functions.h | 10 +
be/src/vec/CMakeLists.txt | 1 +
.../aggregate_function_simple_factory.cpp | 5 +-
.../aggregate_function_window_funnel.cpp | 37 ++
.../aggregate_function_window_funnel.h | 214 ++++++++++
be/test/exprs/CMakeLists.txt | 2 +-
be/test/exprs/window_funnel_test.cpp | 431 +++++++++++++++++++++
be/test/vec/aggregate_functions/CMakeLists.txt | 1 +
.../aggregate_functions/vec_window_funnel_test.cpp | 425 ++++++++++++++++++++
.../java/org/apache/doris/analysis/Analyzer.java | 1 -
.../apache/doris/analysis/FunctionCallExpr.java | 72 +++-
.../apache/doris/catalog/AggregateFunction.java | 2 +-
.../java/org/apache/doris/catalog/FunctionSet.java | 31 ++
.../org/apache/doris/analysis/AggregateTest.java | 93 ++++-
15 files changed, 1508 insertions(+), 24 deletions(-)
diff --git a/be/src/exprs/aggregate_functions.cpp b/be/src/exprs/aggregate_functions.cpp
index 42661e8f3d..09d9b1b68f 100644
--- a/be/src/exprs/aggregate_functions.cpp
+++ b/be/src/exprs/aggregate_functions.cpp
@@ -2254,6 +2254,212 @@ void AggregateFunctions::offset_fn_update(FunctionContext* ctx, const T& src, co
*dst = src;
}
+// Refer to AggregateFunctionWindowFunnel.h in https://github.com/ClickHouse/ClickHouse.git
+struct WindowFunnelState {
+ std::vector<std::pair<DateTimeValue, int>> events;
+ int max_event_level;
+ bool sorted;
+ int64_t window;
+
+ WindowFunnelState() {
+ sorted = true;
+ max_event_level = 0;
+ window = 0;
+ }
+
+ void add(DateTimeValue& timestamp, int event_idx, int event_num) {
+ max_event_level = event_num;
+ if (sorted && events.size() > 0) {
+ if (events.back().first == timestamp) {
+ sorted = events.back().second <= event_idx;
+ } else {
+ sorted = events.back().first < timestamp;
+ }
+ }
+ events.emplace_back(timestamp, event_idx);
+ }
+
+ void sort() {
+ if (sorted) {
+ return;
+ }
+ std::stable_sort(events.begin(), events.end());
+ }
+
+ int get_event_level() {
+ std::vector<std::optional<DateTimeValue>> events_timestamp(max_event_level);
+ for (int64_t i = 0; i < events.size(); i++) {
+ int& event_idx = events[i].second;
+ DateTimeValue& timestamp = events[i].first;
+ if (event_idx == 0) {
+ events_timestamp[0] = timestamp;
+ continue;
+ }
+ if (events_timestamp[event_idx - 1].has_value()) {
+ DateTimeValue& first_timestamp = events_timestamp[event_idx - 1].value();
+ DateTimeValue last_timestamp = first_timestamp;
+ TimeInterval interval(SECOND, window, false);
+ last_timestamp.date_add_interval(interval, SECOND);
+
+ if (timestamp <= last_timestamp) {
+ events_timestamp[event_idx] = first_timestamp;
+ if (event_idx + 1 == max_event_level) {
+ // Usually, max event level is small.
+ return max_event_level;
+ }
+ }
+ }
+ }
+
+ for (int64_t i = events_timestamp.size() - 1; i >= 0; i--) {
+ if (events_timestamp[i].has_value()) {
+ return i + 1;
+ }
+ }
+
+ return 0;
+ }
+
+ void merge(WindowFunnelState *other) {
+ if (other->events.empty()) {
+ return;
+ }
+
+ int64_t orig_size = events.size();
+ events.insert(std::end(events), std::begin(other->events), std::end(other->events));
+ const auto begin = std::begin(events);
+ const auto middle = std::next(events.begin(), orig_size);
+ const auto end = std::end(events);
+ if (!other->sorted) {
+ std::stable_sort(middle, end);
+ }
+
+ if (!sorted) {
+ std::stable_sort(begin, middle);
+ }
+ std::inplace_merge(begin, middle, end);
+ max_event_level = max_event_level > 0 ? max_event_level : other->max_event_level;
+ window = window > 0 ? window : other->window;
+
+ sorted = true;
+ }
+
+ int64_t serialized_size() {
+ return sizeof(int) + sizeof(int64_t) + sizeof(uint64_t) +
+ events.size() * (sizeof(int64_t) + sizeof(int));
+ }
+
+ void serialize(uint8_t *buf) {
+ memcpy(buf, &max_event_level, sizeof(int));
+ buf += sizeof(int);
+ memcpy(buf, &window, sizeof(int64_t));
+ buf += sizeof(int64_t);
+
+ uint64_t event_num = events.size();
+ memcpy(buf, &event_num, sizeof(uint64_t));
+ buf += sizeof(uint64_t);
+ for (int64_t i = 0; i < events.size(); i++) {
+ int64_t timestamp = events[i].first;
+ int event_idx = events[i].second;
+ memcpy(buf, ×tamp, sizeof(int64_t));
+ buf += sizeof(int64_t);
+ memcpy(buf, &event_idx, sizeof(int));
+ buf += sizeof(int);
+ }
+ }
+
+ void deserialize(uint8_t *buf) {
+ uint64_t size;
+
+ memcpy(&max_event_level, buf, sizeof(int));
+ buf += sizeof(int);
+ memcpy(&window, buf, sizeof(int64_t));
+ buf += sizeof(int64_t);
+ memcpy(&size, buf, sizeof(uint64_t));
+ buf += sizeof(uint64_t);
+ for (int64_t i = 0; i < size; i++) {
+ int64_t timestamp;
+ int event_idx;
+
+ memcpy(×tamp, buf, sizeof(int64_t));
+ buf += sizeof(int64_t);
+ memcpy(&event_idx, buf, sizeof(int));
+ buf += sizeof(int);
+ DateTimeValue time_value;
+ time_value.from_date_int64(timestamp);
+ add(time_value, event_idx, max_event_level);
+ }
+ }
+};
+
+void AggregateFunctions::window_funnel_init(FunctionContext* ctx, StringVal* dst) {
+ dst->is_null = false;
+ dst->len = sizeof(WindowFunnelState);
+ WindowFunnelState* state = new WindowFunnelState();
+ dst->ptr = (uint8_t*)state;
+ // constant args at index 0 and 1
+ DCHECK(ctx->is_arg_constant(0));
+ BigIntVal* window = reinterpret_cast<BigIntVal*>(ctx->get_constant_arg(0));
+ state->window = window->val;
+ // TODO handle mode in the future
+}
+
+void AggregateFunctions::window_funnel_update(FunctionContext* ctx, const BigIntVal& window,
+ const StringVal& mode, const DateTimeVal& timestamp,
+ int num_cond, const BooleanVal* conds, StringVal* dst) {
+ DCHECK(dst->ptr != nullptr);
+ DCHECK_EQ(sizeof(WindowFunnelState), dst->len);
+
+ if (timestamp.is_null) {
+ return;
+ }
+
+ WindowFunnelState* state = reinterpret_cast<WindowFunnelState*>(dst->ptr);
+ for (int i = 0; i < num_cond; i++) {
+ if (conds[i].is_null) {
+ continue;
+ }
+ if (conds[i].val) {
+ DateTimeValue time_value = DateTimeValue::from_datetime_val(timestamp);
+ state->add(time_value, i, num_cond);
+ }
+ }
+}
+
+StringVal AggregateFunctions::window_funnel_serialize(FunctionContext* ctx,
+ const StringVal& src) {
+ WindowFunnelState* state = reinterpret_cast<WindowFunnelState*>(src.ptr);
+ int64_t serialized_size = state->serialized_size();
+ StringVal result(ctx, sizeof(double) + serialized_size);
+ state->serialize(result.ptr);
+
+ delete state;
+ return result;
+}
+
+void AggregateFunctions::window_funnel_merge(FunctionContext* ctx, const StringVal& src,
+ StringVal* dst) {
+ DCHECK(dst->ptr != nullptr);
+ DCHECK_EQ(sizeof(WindowFunnelState), dst->len);
+ WindowFunnelState* dst_state = reinterpret_cast<WindowFunnelState*>(dst->ptr);
+
+ WindowFunnelState* src_state = new WindowFunnelState;
+
+ src_state->deserialize(src.ptr);
+ dst_state->merge(src_state);
+ delete src_state;
+}
+
+IntVal AggregateFunctions::window_funnel_finalize(FunctionContext* ctx, const StringVal& src) {
+ DCHECK(!src.is_null);
+
+ WindowFunnelState* state = reinterpret_cast<WindowFunnelState*>(src.ptr);
+ state->sort();
+ int val = state->get_event_level();
+ delete state;
+ return doris_udf::IntVal(val);
+}
+
// Stamp out the templates for the types we need.
template void AggregateFunctions::init_zero_null<BigIntVal>(FunctionContext*, BigIntVal* dst);
template void AggregateFunctions::init_zero_null<LargeIntVal>(FunctionContext*, LargeIntVal* dst);
@@ -2729,4 +2935,5 @@ template void AggregateFunctions::percentile_approx_update<doris_udf::DoubleVal>
template void AggregateFunctions::percentile_approx_update<doris_udf::DoubleVal>(
FunctionContext* ctx, const doris_udf::DoubleVal&, const doris_udf::DoubleVal&,
const doris_udf::DoubleVal&, doris_udf::StringVal*);
+
} // namespace doris
diff --git a/be/src/exprs/aggregate_functions.h b/be/src/exprs/aggregate_functions.h
index b3b19ab9ce..b010692079 100644
--- a/be/src/exprs/aggregate_functions.h
+++ b/be/src/exprs/aggregate_functions.h
@@ -347,6 +347,16 @@ public:
static void offset_fn_update(doris_udf::FunctionContext*, const T& src,
const doris_udf::BigIntVal&, const T&, T* dst);
+ // windowFunnel
+ static void window_funnel_init(FunctionContext* ctx, StringVal* dst);
+ static void window_funnel_update(FunctionContext* ctx, const BigIntVal& window,
+ const StringVal& mode, const DateTimeVal& timestamp,
+ int num_cond, const BooleanVal* conds, StringVal* dst);
+ static void window_funnel_merge(FunctionContext* ctx, const StringVal& src,
+ StringVal* dst);
+ static StringVal window_funnel_serialize(FunctionContext* ctx, const StringVal& src);
+ static IntVal window_funnel_finalize(FunctionContext* ctx, const StringVal& src);
+
// todo(kks): keep following HLL methods only for backward compatibility, we should remove these methods
// when doris 0.12 release
static void hll_init(doris_udf::FunctionContext*, doris_udf::StringVal* slot);
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index ab319c5d71..f0781f83db 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -20,6 +20,7 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/vec")
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/vec")
set(VEC_FILES
+ aggregate_functions/aggregate_function_window_funnel.cpp
aggregate_functions/aggregate_function_avg.cpp
aggregate_functions/aggregate_function_count.cpp
aggregate_functions/aggregate_function_distinct.cpp
diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
index 3be7d184c2..fcf333c1bd 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
@@ -43,6 +43,7 @@ void register_aggregate_function_topn(AggregateFunctionSimpleFactory& factory);
void register_aggregate_function_approx_count_distinct(AggregateFunctionSimpleFactory& factory);
void register_aggregate_function_group_concat(AggregateFunctionSimpleFactory& factory);
void register_aggregate_function_percentile(AggregateFunctionSimpleFactory& factory);
+void register_aggregate_function_window_funnel(AggregateFunctionSimpleFactory& factory);
void register_aggregate_function_percentile_approx(AggregateFunctionSimpleFactory& factory);
AggregateFunctionSimpleFactory& AggregateFunctionSimpleFactory::instance() {
static std::once_flag oc;
@@ -62,6 +63,8 @@ AggregateFunctionSimpleFactory& AggregateFunctionSimpleFactory::instance() {
register_aggregate_function_approx_count_distinct(instance);
register_aggregate_function_group_concat(instance);
register_aggregate_function_percentile(instance);
+ register_aggregate_function_percentile_approx(instance);
+ register_aggregate_function_window_funnel(instance);
// if you only register function with no nullable, and wants to add nullable automatically, you should place function above this line
register_aggregate_function_combinator_null(instance);
@@ -75,4 +78,4 @@ AggregateFunctionSimpleFactory& AggregateFunctionSimpleFactory::instance() {
return instance;
}
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
new file mode 100644
index 0000000000..a889299f33
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/aggregate_functions/aggregate_function_window_funnel.h"
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/factory_helpers.h"
+#include "vec/aggregate_functions/helpers.h"
+
+namespace doris::vectorized {
+
+AggregateFunctionPtr create_aggregate_function_window_funnel(const std::string& name,
+ const DataTypes& argument_types,
+ const Array& parameters,
+ const bool result_is_nullable) {
+ return std::make_shared<AggregateFunctionWindowFunnel>(argument_types);
+}
+
+void register_aggregate_function_window_funnel(AggregateFunctionSimpleFactory& factory) {
+ factory.register_function("window_funnel", create_aggregate_function_window_funnel, false);
+}
+} // namespace doris::vectorized
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
new file mode 100644
index 0000000000..f4364eebb4
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
@@ -0,0 +1,214 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/AggregateFunctionWindowFunnel.h
+// and modified by Doris
+
+#pragma once
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/columns_number.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/io/var_int.h"
+
+namespace doris::vectorized {
+
+struct WindowFunnelState {
+ std::vector<std::pair<VecDateTimeValue, int>> events;
+ int max_event_level;
+ bool sorted;
+ int64_t window;
+
+ WindowFunnelState() {
+ sorted = true;
+ max_event_level = 0;
+ window = 0;
+ }
+
+ void reset() {
+ sorted = true;
+ max_event_level = 0;
+ window = 0;
+ events.shrink_to_fit();
+ }
+
+ void add(const VecDateTimeValue& timestamp, int event_idx, int event_num, int64_t win) {
+ window = win;
+ max_event_level = event_num;
+ if (sorted && events.size() > 0) {
+ if (events.back().first == timestamp) {
+ sorted = events.back().second <= event_idx;
+ } else {
+ sorted = events.back().first < timestamp;
+ }
+ }
+ events.emplace_back(timestamp, event_idx);
+ }
+
+ void sort() {
+ if (sorted) {
+ return;
+ }
+ std::stable_sort(events.begin(), events.end());
+ }
+
+ int get() const {
+ std::vector<std::optional<VecDateTimeValue>> events_timestamp(max_event_level);
+ for (int64_t i = 0; i < events.size(); i++) {
+ const int& event_idx = events[i].second;
+ const VecDateTimeValue& timestamp = events[i].first;
+ if (event_idx == 0) {
+ events_timestamp[0] = timestamp;
+ continue;
+ }
+ if (events_timestamp[event_idx - 1].has_value()) {
+ const VecDateTimeValue& first_timestamp = events_timestamp[event_idx - 1].value();
+ VecDateTimeValue last_timestamp = first_timestamp;
+ TimeInterval interval(SECOND, window, false);
+ last_timestamp.date_add_interval(interval, SECOND);
+
+ if (timestamp <= last_timestamp) {
+ events_timestamp[event_idx] = first_timestamp;
+ if (event_idx + 1 == max_event_level) {
+ // Usually, max event level is small.
+ return max_event_level;
+ }
+ }
+ }
+ }
+
+ for (int64_t i = events_timestamp.size() - 1; i >= 0; i--) {
+ if (events_timestamp[i].has_value()) {
+ return i + 1;
+ }
+ }
+
+ return 0;
+ }
+
+ void merge(const WindowFunnelState& other) {
+ if (other.events.empty()) {
+ return;
+ }
+
+ int64_t orig_size = events.size();
+ events.insert(std::end(events), std::begin(other.events), std::end(other.events));
+ const auto begin = std::begin(events);
+ const auto middle = std::next(events.begin(), orig_size);
+ const auto end = std::end(events);
+ if (!other.sorted) {
+ std::stable_sort(middle, end);
+ }
+
+ if (!sorted) {
+ std::stable_sort(begin, middle);
+ }
+ std::inplace_merge(begin, middle, end);
+ max_event_level = max_event_level > 0 ? max_event_level : other.max_event_level;
+ window = window > 0 ? window : other.window;
+
+ sorted = true;
+ }
+
+ void write(BufferWritable &out) const {
+ write_var_int(max_event_level, out);
+ write_var_int(window, out);
+ write_var_int(events.size(), out);
+
+ for (int64_t i = 0; i < events.size(); i++) {
+ int64_t timestamp = events[i].first;
+ int event_idx = events[i].second;
+ write_var_int(timestamp, out);
+ write_var_int(event_idx, out);
+ }
+ }
+
+ void read(BufferReadable& in) {
+ int64_t event_level;
+ read_var_int(event_level, in);
+ max_event_level = (int)event_level;
+ read_var_int(window, in);
+ int64_t size = 0;
+ read_var_int(size, in);
+ for (int64_t i = 0; i < size; i++) {
+ int64_t timestamp;
+ int64_t event_idx;
+
+ read_var_int(timestamp, in);
+ read_var_int(event_idx, in);
+ VecDateTimeValue time_value(timestamp);
+ add(time_value, (int)event_idx, max_event_level, window);
+ }
+ }
+};
+
+class AggregateFunctionWindowFunnel
+ : public IAggregateFunctionDataHelper<WindowFunnelState,
+ AggregateFunctionWindowFunnel> {
+public:
+ AggregateFunctionWindowFunnel(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<WindowFunnelState,
+ AggregateFunctionWindowFunnel>(argument_types_, {}) {
+ }
+
+ String get_name() const override { return "window_funnel"; }
+
+ DataTypePtr get_return_type() const override {
+ return std::make_shared<DataTypeInt32>();
+ }
+
+ void reset(AggregateDataPtr __restrict place) const override { this->data(place).reset(); }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
+ Arena*) const override {
+ const auto& window = static_cast<const ColumnVector<Int64>&>(*columns[0]).get_data()[row_num];
+ // TODO: handle mode in the future.
+ // be/src/olap/row_block2.cpp copy_data_to_column
+ const auto& timestamp = static_cast<const ColumnVector<VecDateTimeValue>&>(*columns[2]).get_data()[row_num];
+ const int NON_EVENT_NUM = 3;
+ for (int i = NON_EVENT_NUM; i < get_argument_types().size(); i++) {
+ const auto& is_set = static_cast<const ColumnVector<UInt8>&>(*columns[i]).get_data()[row_num];
+ if (is_set) {
+ this->data(place).add(timestamp, i - NON_EVENT_NUM,
+ get_argument_types().size() - NON_EVENT_NUM, window);
+ }
+ }
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs,
+ Arena*) const override {
+ this->data(place).merge(this->data(rhs));
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
+ this->data(place).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf,
+ Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
+ this->data(const_cast<AggregateDataPtr>(place)).sort();
+ assert_cast<ColumnInt32&>(to).get_data().push_back(data(place).get());
+ }
+};
+
+} // namespace doris::vectorized
diff --git a/be/test/exprs/CMakeLists.txt b/be/test/exprs/CMakeLists.txt
index 9f4702bb8f..329c9e1100 100644
--- a/be/test/exprs/CMakeLists.txt
+++ b/be/test/exprs/CMakeLists.txt
@@ -39,4 +39,4 @@ ADD_BE_TEST(topn_function_test)
ADD_BE_TEST(runtime_filter_test)
ADD_BE_TEST(bloom_filter_predicate_test)
ADD_BE_TEST(array_functions_test)
-
+ADD_BE_TEST(window_funnel_test)
diff --git a/be/test/exprs/window_funnel_test.cpp b/be/test/exprs/window_funnel_test.cpp
new file mode 100644
index 0000000000..f8f190ae9b
--- /dev/null
+++ b/be/test/exprs/window_funnel_test.cpp
@@ -0,0 +1,431 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include "common/logging.h"
+#include "exprs/aggregate_functions.h"
+#include "runtime/datetime_value.h"
+#include "testutil/function_utils.h"
+
+namespace doris {
+
+class WindowFunnelTest : public testing::Test {
+public:
+ WindowFunnelTest() {}
+};
+
+TEST_F(WindowFunnelTest, testMax4SortedNoMerge) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ const int NUM_CONDS = 4;
+ for (int i = -1; i < NUM_CONDS + 4; i++) {
+ StringVal stringVal1;
+ BigIntVal window(i);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp;
+ DateTimeValue time_value;
+ time_value.set_time(2020, 2, 28, 0, 0, 1, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 2, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds1[NUM_CONDS] = {false, true, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds1, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 3, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds2[NUM_CONDS] = {false, false, true, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds2, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 4, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds3[NUM_CONDS] = {false, false, false, true};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds3, &stringVal1);
+
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal1);
+ LOG(INFO) << "event num: " << NUM_CONDS << " window: " << window.val;
+ ASSERT_EQ(v.val, i < 0 ? 1 : (i < NUM_CONDS ? i + 1 : NUM_CONDS));
+ }
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testMax4SortedMerge) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ const int NUM_CONDS = 4;
+ for (int i = -1; i < NUM_CONDS + 4; i++) {
+ StringVal stringVal1;
+ BigIntVal window(i);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp;
+ DateTimeValue time_value;
+ time_value.set_time(2020, 2, 28, 0, 0, 1, 0);
+ time_value.to_datetime_val(×tamp);
+
+ BooleanVal conds[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 2, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds1[NUM_CONDS] = {false, true, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds1, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 3, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds2[NUM_CONDS] = {false, false, true, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds2, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 4, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds3[NUM_CONDS] = {false, false, false, true};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds3, &stringVal1);
+
+ StringVal s = AggregateFunctions::window_funnel_serialize(context, stringVal1);
+
+ StringVal stringVal2;
+ AggregateFunctions::window_funnel_init(context, &stringVal2);
+ AggregateFunctions::window_funnel_merge(context, s, &stringVal2);
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal2);
+ LOG(INFO) << "event num: " << NUM_CONDS << " window: " << window.val;
+ ASSERT_EQ(v.val, i < 0 ? 1 : (i < NUM_CONDS ? i + 1 : NUM_CONDS));
+ }
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testMax4ReverseSortedNoMerge) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ const int NUM_CONDS = 4;
+ for (int i = -1; i < NUM_CONDS + 4; i++) {
+ StringVal stringVal1;
+ BigIntVal window(i);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp;
+ DateTimeValue time_value;
+ time_value.set_time(2020, 2, 28, 0, 0, 3, 0);
+ time_value.to_datetime_val(×tamp);
+
+ BooleanVal conds[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 2, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds1[NUM_CONDS] = {false, true, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds1, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 1, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds2[NUM_CONDS] = {false, false, true, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds2, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 0, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds3[NUM_CONDS] = {false, false, false, true};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds3, &stringVal1);
+
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal1);
+ LOG(INFO) << "event num: " << NUM_CONDS << " window: " << window.val;
+ ASSERT_EQ(v.val, 1);
+ }
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testMax4ReverseSortedMerge) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ const int NUM_CONDS = 4;
+ for (int i = -1; i < NUM_CONDS + 4; i++) {
+ StringVal stringVal1;
+ BigIntVal window(i);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp;
+ DateTimeValue time_value;
+ time_value.set_time(2020, 2, 28, 0, 0, 3, 0);
+ time_value.to_datetime_val(×tamp);
+
+ BooleanVal conds[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 2, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds1[NUM_CONDS] = {false, true, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds1, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 1, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds2[NUM_CONDS] = {false, false, true, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds2, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 0, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds3[NUM_CONDS] = {false, false, false, true};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds3, &stringVal1);
+
+ StringVal s = AggregateFunctions::window_funnel_serialize(context, stringVal1);
+
+ StringVal stringVal2;
+ AggregateFunctions::window_funnel_init(context, &stringVal2);
+ AggregateFunctions::window_funnel_merge(context, s, &stringVal2);
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal2);
+ LOG(INFO) << "event num: " << NUM_CONDS << " window: " << window.val;
+ ASSERT_EQ(v.val, 1);
+ }
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testMax4DuplicateSortedNoMerge) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ const int NUM_CONDS = 4;
+ for (int i = -1; i < NUM_CONDS + 4; i++) {
+ StringVal stringVal1;
+ BigIntVal window(i);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp;
+ DateTimeValue time_value;
+ time_value.set_time(2020, 2, 28, 0, 0, 0, 0);
+ time_value.to_datetime_val(×tamp);
+
+ BooleanVal conds[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 1, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds1[NUM_CONDS] = {false, true, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds1, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 2, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds2[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds2, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 3, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds3[NUM_CONDS] = {false, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds3, &stringVal1);
+
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal1);
+ LOG(INFO) << "event num: " << NUM_CONDS << " window: " << window.val;
+ ASSERT_EQ(v.val, i < 0 ? 1 : (i < 2 ? i + 1 : 2));
+ }
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testMax4DuplicateSortedMerge) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ const int NUM_CONDS = 4;
+ for (int i = -1; i < NUM_CONDS + 4; i++) {
+ StringVal stringVal1;
+ BigIntVal window(i);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp;
+ DateTimeValue time_value;
+ time_value.set_time(2020, 2, 28, 0, 0, 0, 0);
+ time_value.to_datetime_val(×tamp);
+
+ BooleanVal conds[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 1, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds1[NUM_CONDS] = {false, true, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds1, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 2, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds2[NUM_CONDS] = {true, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds2, &stringVal1);
+
+ time_value.set_time(2020, 2, 28, 0, 0, 3, 0);
+ time_value.to_datetime_val(×tamp);
+ BooleanVal conds3[NUM_CONDS] = {false, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, NUM_CONDS,
+ conds3, &stringVal1);
+
+ StringVal s = AggregateFunctions::window_funnel_serialize(context, stringVal1);
+
+ StringVal stringVal2;
+ AggregateFunctions::window_funnel_init(context, &stringVal2);
+ AggregateFunctions::window_funnel_merge(context, s, &stringVal2);
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal2);
+ LOG(INFO) << "event num: " << NUM_CONDS << " window: " << window.val;
+ ASSERT_EQ(v.val, i < 0 ? 1 : (i < 2 ? i + 1 : 2));
+ }
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testNoMatchedEvent) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ StringVal stringVal1;
+ BigIntVal window(0);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp;
+ DateTimeValue time_value;
+ time_value.set_time(2020, 2, 28, 0, 0, 0, 0);
+ time_value.to_datetime_val(×tamp);
+
+ BooleanVal conds[4] = {false, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, 4,
+ conds, &stringVal1);
+
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal1);
+ ASSERT_EQ(v.val, 0);
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testNoEvent) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ StringVal stringVal1;
+ BigIntVal window(0);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal1);
+ ASSERT_EQ(v.val, 0);
+
+ StringVal stringVal2;
+ AggregateFunctions::window_funnel_init(context, &stringVal2);
+
+ v = AggregateFunctions::window_funnel_finalize(context, stringVal2);
+ ASSERT_EQ(v.val, 0);
+
+ delete futil;
+}
+
+TEST_F(WindowFunnelTest, testInputNull) {
+ FunctionUtils* futil = new FunctionUtils();
+ doris_udf::FunctionContext* context = futil->get_fn_ctx();
+
+ BigIntVal window(0);
+ StringVal mode("default");
+ std::vector<doris_udf::AnyVal*> constant_args;
+ constant_args.emplace_back(&window);
+ constant_args.emplace_back(&mode);
+ context->impl()->set_constant_args(std::move(constant_args));
+
+ StringVal stringVal1;
+ AggregateFunctions::window_funnel_init(context, &stringVal1);
+
+ DateTimeVal timestamp = DateTimeVal::null();
+ BooleanVal conds[4] = {false, false, false, false};
+ AggregateFunctions::window_funnel_update(context, window, mode, timestamp, 4,
+ conds, &stringVal1);
+
+
+ IntVal v = AggregateFunctions::window_funnel_finalize(context, stringVal1);
+ ASSERT_EQ(v.val, 0);
+
+ delete futil;
+}
+
+} // namespace doris
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/be/test/vec/aggregate_functions/CMakeLists.txt b/be/test/vec/aggregate_functions/CMakeLists.txt
index 5c7c278997..13a1315bbe 100644
--- a/be/test/vec/aggregate_functions/CMakeLists.txt
+++ b/be/test/vec/aggregate_functions/CMakeLists.txt
@@ -20,3 +20,4 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test/vec/aggregate_functions")
ADD_BE_TEST(agg_test)
ADD_BE_TEST(agg_min_max_test)
+ADD_BE_TEST(vec_window_funnel_test)
diff --git a/be/test/vec/aggregate_functions/vec_window_funnel_test.cpp b/be/test/vec/aggregate_functions/vec_window_funnel_test.cpp
new file mode 100644
index 0000000000..8bcd255dc6
--- /dev/null
+++ b/be/test/vec/aggregate_functions/vec_window_funnel_test.cpp
@@ -0,0 +1,425 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include "common/logging.h"
+#include "gtest/gtest.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/aggregate_function_topn.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+
+namespace doris::vectorized {
+
+void register_aggregate_function_window_funnel(AggregateFunctionSimpleFactory& factory);
+
+class WindowFunnelTest : public testing::Test {
+public:
+ AggregateFunctionPtr agg_function;
+
+ WindowFunnelTest() {}
+
+ void SetUp() {
+ AggregateFunctionSimpleFactory factory = AggregateFunctionSimpleFactory::instance();
+ DataTypes data_types = {
+ std::make_shared<DataTypeInt64>(),
+ std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeDateTime>(),
+ std::make_shared<DataTypeUInt8>(),
+ std::make_shared<DataTypeUInt8>(),
+ std::make_shared<DataTypeUInt8>(),
+ std::make_shared<DataTypeUInt8>(),
+ };
+ Array array;
+ agg_function = factory.get("window_funnel", data_types, array, false);
+ ASSERT_NE(agg_function, nullptr);
+ }
+
+ void TearDown() {
+ }
+};
+
+TEST_F(WindowFunnelTest, testEmpty) {
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+
+ ColumnString buf;
+ VectorBufferWriter buf_writer(buf);
+ agg_function->serialize(place, buf_writer);
+ buf_writer.commit();
+ LOG(INFO) << "buf size : " << buf.size();
+ VectorBufferReader buf_reader(buf.get_data_at(0));
+ agg_function->deserialize(place, buf_reader, nullptr);
+
+ std::unique_ptr<char[]> memory2(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place2 = memory2.get();
+ agg_function->create(place2);
+
+ agg_function->merge(place, place2, nullptr);
+ ColumnVector<Int32> column_result;
+ agg_function->insert_result_into(place, column_result);
+ ASSERT_EQ(column_result.get_data()[0], 0);
+
+ ColumnVector<Int32> column_result2;
+ agg_function->insert_result_into(place2, column_result2);
+ ASSERT_EQ(column_result2.get_data()[0], 0);
+
+ agg_function->destroy(place);
+ agg_function->destroy(place2);
+}
+
+TEST_F(WindowFunnelTest, testSerialize) {
+ const int NUM_CONDS = 4;
+ auto column_mode = ColumnString::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_mode->insert("mode");
+ }
+
+ auto column_timestamp = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ VecDateTimeValue time_value;
+ time_value.set_time(2022, 2, 28, 0, 0, i);
+ column_timestamp->insert_data((char *)&time_value, 0);
+ }
+ auto column_event1 = ColumnVector<UInt8>::create();
+ column_event1->insert(1);
+ column_event1->insert(0);
+ column_event1->insert(0);
+ column_event1->insert(0);
+
+ auto column_event2 = ColumnVector<UInt8>::create();
+ column_event2->insert(0);
+ column_event2->insert(1);
+ column_event2->insert(0);
+ column_event2->insert(0);
+
+ auto column_event3 = ColumnVector<UInt8>::create();
+ column_event3->insert(0);
+ column_event3->insert(0);
+ column_event3->insert(1);
+ column_event3->insert(0);
+
+ auto column_event4 = ColumnVector<UInt8>::create();
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(1);
+
+ auto column_window = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_window->insert(2);
+ }
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+ const IColumn* column[7] = { column_window.get(), column_mode.get(),
+ column_timestamp.get(),
+ column_event1.get(), column_event2.get(),
+ column_event3.get(), column_event4.get() };
+ for (int i = 0; i < NUM_CONDS; i++) {
+ agg_function->add(place, column, i, nullptr);
+ }
+
+ ColumnString buf;
+ VectorBufferWriter buf_writer(buf);
+ agg_function->serialize(place, buf_writer);
+ buf_writer.commit();
+
+ std::unique_ptr<char[]> memory2(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place2 = memory2.get();
+ agg_function->create(place2);
+
+ VectorBufferReader buf_reader(buf.get_data_at(0));
+ agg_function->deserialize(place2, buf_reader, nullptr);
+
+ ColumnVector<Int32> column_result;
+ agg_function->insert_result_into(place, column_result);
+ ASSERT_EQ(column_result.get_data()[0], 3);
+ agg_function->destroy(place);
+
+ ColumnVector<Int32> column_result2;
+ agg_function->insert_result_into(place2, column_result2);
+ ASSERT_EQ(column_result2.get_data()[0], 3);
+ agg_function->destroy(place2);
+}
+
+
+TEST_F(WindowFunnelTest, testMax4SortedNoMerge) {
+ const int NUM_CONDS = 4;
+ auto column_mode = ColumnString::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_mode->insert("mode");
+ }
+ auto column_timestamp = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ VecDateTimeValue time_value;
+ time_value.set_time(2022, 2, 28, 0, 0, i);
+ column_timestamp->insert_data((char *)&time_value, 0);
+ }
+ auto column_event1 = ColumnVector<UInt8>::create();
+ column_event1->insert(1);
+ column_event1->insert(0);
+ column_event1->insert(0);
+ column_event1->insert(0);
+
+ auto column_event2 = ColumnVector<UInt8>::create();
+ column_event2->insert(0);
+ column_event2->insert(1);
+ column_event2->insert(0);
+ column_event2->insert(0);
+
+ auto column_event3 = ColumnVector<UInt8>::create();
+ column_event3->insert(0);
+ column_event3->insert(0);
+ column_event3->insert(1);
+ column_event3->insert(0);
+
+ auto column_event4 = ColumnVector<UInt8>::create();
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(1);
+
+ for(int win = -1; win < NUM_CONDS + 1; win++) {
+ auto column_window = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_window->insert(win);
+ }
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+ const IColumn* column[7] = { column_window.get(), column_mode.get(),
+ column_timestamp.get(),
+ column_event1.get(), column_event2.get(),
+ column_event3.get(), column_event4.get() };
+ for (int i = 0; i < NUM_CONDS; i++) {
+ agg_function->add(place, column, i, nullptr);
+ }
+
+ ColumnVector<Int32> column_result;
+ agg_function->insert_result_into(place, column_result);
+ ASSERT_EQ(column_result.get_data()[0], win < 0 ? 1 : (win < NUM_CONDS ? win + 1 : NUM_CONDS));
+ agg_function->destroy(place);
+ }
+}
+
+TEST_F(WindowFunnelTest, testMax4SortedMerge) {
+ const int NUM_CONDS = 4;
+ auto column_mode = ColumnString::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_mode->insert("mode");
+ }
+ auto column_timestamp = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ VecDateTimeValue time_value;
+ time_value.set_time(2022, 2, 28, 0, 0, i);
+ column_timestamp->insert_data((char *)&time_value, 0);
+ }
+ auto column_event1 = ColumnVector<UInt8>::create();
+ column_event1->insert(1);
+ column_event1->insert(0);
+ column_event1->insert(0);
+ column_event1->insert(0);
+
+ auto column_event2 = ColumnVector<UInt8>::create();
+ column_event2->insert(0);
+ column_event2->insert(1);
+ column_event2->insert(0);
+ column_event2->insert(0);
+
+ auto column_event3 = ColumnVector<UInt8>::create();
+ column_event3->insert(0);
+ column_event3->insert(0);
+ column_event3->insert(1);
+ column_event3->insert(0);
+
+ auto column_event4 = ColumnVector<UInt8>::create();
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(1);
+
+ for(int win = -1; win < NUM_CONDS + 1; win++) {
+ auto column_window = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_window->insert(win);
+ }
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+ const IColumn* column[7] = { column_window.get(), column_mode.get(),
+ column_timestamp.get(),
+ column_event1.get(), column_event2.get(),
+ column_event3.get(), column_event4.get() };
+ for (int i = 0; i < NUM_CONDS; i++) {
+ agg_function->add(place, column, i, nullptr);
+ }
+
+ std::unique_ptr<char[]> memory2(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place2 = memory2.get();
+ agg_function->create(place2);
+
+ agg_function->merge(place2, place, nullptr);
+ ColumnVector<Int32> column_result;
+ agg_function->insert_result_into(place2, column_result);
+ ASSERT_EQ(column_result.get_data()[0], win < 0 ? 1 : (win < NUM_CONDS ? win + 1 : NUM_CONDS));
+ agg_function->destroy(place);
+ agg_function->destroy(place2);
+ }
+}
+
+TEST_F(WindowFunnelTest, testMax4ReverseSortedNoMerge) {
+ const int NUM_CONDS = 4;
+ auto column_mode = ColumnString::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_mode->insert("mode");
+ }
+ auto column_timestamp = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ VecDateTimeValue time_value;
+ time_value.set_time(2022, 2, 28, 0, 0, NUM_CONDS - i);
+ column_timestamp->insert_data((char *)&time_value, 0);
+ }
+ auto column_event1 = ColumnVector<UInt8>::create();
+ column_event1->insert(0);
+ column_event1->insert(0);
+ column_event1->insert(0);
+ column_event1->insert(1);
+
+ auto column_event2 = ColumnVector<UInt8>::create();
+ column_event2->insert(0);
+ column_event2->insert(0);
+ column_event2->insert(1);
+ column_event2->insert(0);
+
+ auto column_event3 = ColumnVector<UInt8>::create();
+ column_event3->insert(0);
+ column_event3->insert(1);
+ column_event3->insert(0);
+ column_event3->insert(0);
+
+ auto column_event4 = ColumnVector<UInt8>::create();
+ column_event4->insert(1);
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(0);
+
+ for(int win = -1; win < NUM_CONDS + 1; win++) {
+ auto column_window = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_window->insert(win);
+ }
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+ const IColumn* column[7] = { column_window.get(), column_mode.get(),
+ column_timestamp.get(),
+ column_event1.get(), column_event2.get(),
+ column_event3.get(), column_event4.get() };
+ for (int i = 0; i < NUM_CONDS; i++) {
+ agg_function->add(place, column, i, nullptr);
+ }
+
+ LOG(INFO) << "win " << win;
+ ColumnVector<Int32> column_result;
+ agg_function->insert_result_into(place, column_result);
+ ASSERT_EQ(column_result.get_data()[0], win < 0 ? 1 : (win < NUM_CONDS ? win + 1 : NUM_CONDS));
+ agg_function->destroy(place);
+ }
+}
+
+TEST_F(WindowFunnelTest, testMax4ReverseSortedMerge) {
+ const int NUM_CONDS = 4;
+ auto column_mode = ColumnString::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_mode->insert("mode");
+ }
+ auto column_timestamp = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ VecDateTimeValue time_value;
+ time_value.set_time(2022, 2, 28, 0, 0, NUM_CONDS - i);
+ column_timestamp->insert_data((char *)&time_value, 0);
+ }
+ auto column_event1 = ColumnVector<UInt8>::create();
+ column_event1->insert(0);
+ column_event1->insert(0);
+ column_event1->insert(0);
+ column_event1->insert(1);
+
+ auto column_event2 = ColumnVector<UInt8>::create();
+ column_event2->insert(0);
+ column_event2->insert(0);
+ column_event2->insert(1);
+ column_event2->insert(0);
+
+ auto column_event3 = ColumnVector<UInt8>::create();
+ column_event3->insert(0);
+ column_event3->insert(1);
+ column_event3->insert(0);
+ column_event3->insert(0);
+
+ auto column_event4 = ColumnVector<UInt8>::create();
+ column_event4->insert(1);
+ column_event4->insert(0);
+ column_event4->insert(0);
+ column_event4->insert(0);
+
+ for(int win = -1; win < NUM_CONDS + 1; win++) {
+ auto column_window = ColumnVector<Int64>::create();
+ for (int i = 0; i < NUM_CONDS; i++) {
+ column_window->insert(win);
+ }
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+ const IColumn* column[7] = { column_window.get(), column_mode.get(),
+ column_timestamp.get(),
+ column_event1.get(), column_event2.get(),
+ column_event3.get(), column_event4.get() };
+ for (int i = 0; i < NUM_CONDS; i++) {
+ agg_function->add(place, column, i, nullptr);
+ }
+
+ std::unique_ptr<char[]> memory2(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place2 = memory2.get();
+ agg_function->create(place2);
+
+ agg_function->merge(place2, place, NULL);
+ ColumnVector<Int32> column_result;
+ agg_function->insert_result_into(place2, column_result);
+ ASSERT_EQ(column_result.get_data()[0], win < 0 ? 1 : (win < NUM_CONDS ? win + 1 : NUM_CONDS));
+ agg_function->destroy(place);
+ agg_function->destroy(place2);
+ }
+}
+
+} // namespace doris::vectorized
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index 9bcb5a1379..8b5ebab306 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -731,7 +731,6 @@ public class Analyzer {
result.setMultiRef(true);
return result;
}
-
result = addSlotDescriptor(tupleDescriptor);
Column col = new Column(colName, type);
result.setColumn(col);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index eb94fcdfec..e64a2d121c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -125,7 +125,7 @@ public class FunctionCallExpr extends Expr {
}
public FunctionCallExpr(String fnName, FunctionParams params) {
- this(new FunctionName(fnName), params);
+ this(new FunctionName(fnName), params, false);
}
public FunctionCallExpr(FunctionName fnName, FunctionParams params) {
@@ -140,8 +140,8 @@ public class FunctionCallExpr extends Expr {
this.isMergeAggFn = isMergeAggFn;
if (params.exprs() != null) {
children.addAll(params.exprs());
- originChildSize = children.size();
}
+ originChildSize = children.size();
}
// Constructs the same agg function with new params.
@@ -236,29 +236,21 @@ public class FunctionCallExpr extends Expr {
&& fnParams.isStar() == o.fnParams.isStar();
}
- @Override
- public String toSqlImpl() {
- Expr expr;
- if (originStmtFnExpr != null) {
- expr = originStmtFnExpr;
- } else {
- expr = this;
- }
+ private String paramsToSql(FunctionParams params) {
StringBuilder sb = new StringBuilder();
- sb.append(((FunctionCallExpr) expr).fnName).append("(");
- if (((FunctionCallExpr) expr).fnParams.isStar()) {
+ sb.append("(");
+
+ if (params.isStar()) {
sb.append("*");
}
- if (((FunctionCallExpr) expr).fnParams.isDistinct()) {
+ if (params.isDistinct()) {
sb.append("DISTINCT ");
}
- boolean isJsonFunction = false;
- int len = children.size();
+ int len = params.exprs().size();
List<String> result = Lists.newArrayList();
if (fnName.getFunction().equalsIgnoreCase("json_array") ||
fnName.getFunction().equalsIgnoreCase("json_object")) {
len = len - 1;
- isJsonFunction = true;
}
if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
@@ -273,11 +265,27 @@ public class FunctionCallExpr extends Expr {
fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) {
result.add("\'***\'");
} else {
- result.add(children.get(i).toSql());
+ result.add(params.exprs().get(i).toSql());
}
}
sb.append(Joiner.on(", ").join(result)).append(")");
- if (fnName.getFunction().equalsIgnoreCase("json_quote") || isJsonFunction) {
+ return sb.toString();
+ }
+
+ @Override
+ public String toSqlImpl() {
+ Expr expr;
+ if (originStmtFnExpr != null) {
+ expr = originStmtFnExpr;
+ } else {
+ expr = this;
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append(((FunctionCallExpr) expr).fnName);
+ sb.append(paramsToSql(fnParams));
+ if (fnName.getFunction().equalsIgnoreCase("json_quote") ||
+ fnName.getFunction().equalsIgnoreCase("json_array") ||
+ fnName.getFunction().equalsIgnoreCase("json_object")) {
return forJSON(sb.toString());
}
return sb.toString();
@@ -765,6 +773,34 @@ public class FunctionCallExpr extends Expr {
fn = getBuiltinFunction(analyzer, fnName.getFunction(), new Type[]{compatibleType},
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ } else if (fnName.getFunction().equalsIgnoreCase(FunctionSet.WINDOW_FUNNEL)) {
+ if (fnParams.exprs() == null || fnParams.exprs().size() < 4) {
+ throw new AnalysisException("The " + fnName + " function must have at least four params");
+ }
+
+ if (!children.get(0).type.isIntegerType()) {
+ throw new AnalysisException("The window params of " + fnName + " function must be integer");
+ }
+ if (!children.get(1).type.isStringType()) {
+ throw new AnalysisException("The mode params of " + fnName + " function must be integer");
+ }
+ if (!children.get(2).type.isDateType()) {
+ throw new AnalysisException("The 3rd param of " + fnName + " function must be DATE or DATETIME");
+ }
+
+ Type[] childTypes = new Type[children.size()];
+ for (int i = 0; i < 3; i++) {
+ childTypes[i] = children.get(i).type;
+ }
+ for (int i = 3; i < children.size(); i++) {
+ if (children.get(i).type != Type.BOOLEAN) {
+ throw new AnalysisException("The 4th and subsequent params of " + fnName + " function must be boolean");
+ }
+ childTypes[i] = children.get(i).type;
+ }
+
+ fn = getBuiltinFunction(analyzer, fnName.getFunction(), childTypes,
+ Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
} else {
// now first find table function in table function sets
if (isTableFnCall) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
index a695a9c4b4..bc59188867 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
@@ -49,7 +49,7 @@ public class AggregateFunction extends Function {
private static final Logger LOG = LogManager.getLogger(AggregateFunction.class);
public static ImmutableSet<String> NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET =
- ImmutableSet.of("row_number", "rank", "dense_rank", "multi_distinct_count", "multi_distinct_sum", "hll_union_agg", "hll_union", "bitmap_union", "bitmap_intersect", FunctionSet.COUNT, "approx_count_distinct", "ndv", FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT, "ndv_no_finalize");
+ ImmutableSet.of("row_number", "rank", "dense_rank", "multi_distinct_count", "multi_distinct_sum", "hll_union_agg", "hll_union", "bitmap_union", "bitmap_intersect", FunctionSet.COUNT, "approx_count_distinct", "ndv", FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT, "ndv_no_finalize", FunctionSet.WINDOW_FUNNEL);
public static ImmutableSet<String> ALWAYS_NULLABLE_AGGREGATE_FUNCTION_NAME_SET =
ImmutableSet.of("stddev_samp", "variance_samp", "var_samp", "percentile_approx");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index afe79833f0..9ec3cfef5c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -1209,6 +1209,7 @@ public class FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
public static final String COUNT = "count";
+ public static final String WINDOW_FUNNEL = "window_funnel";
// Populate all the aggregate builtins in the catalog.
// null symbols indicate the function does not need that step of the evaluation.
// An empty symbol indicates a TODO for the BE to implement the function.
@@ -1242,6 +1243,36 @@ public class FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
prefix + "17count_star_removeEPN9doris_udf15FunctionContextEPNS1_9BigIntValE",
null, false, true, true, true));
+ // windowFunnel
+ addBuiltin(AggregateFunction.createBuiltin(FunctionSet.WINDOW_FUNNEL,
+ Lists.newArrayList(Type.BIGINT, Type.STRING, Type.DATETIME, Type.BOOLEAN),
+ Type.INT,
+ Type.VARCHAR,
+ true,
+ prefix + "18window_funnel_initEPN9doris_udf15FunctionContextEPNS1_9StringValE",
+ prefix + "20window_funnel_updateEPN9doris_udf15FunctionContextERKNS1_9BigIntValERKNS1_9StringValERKNS1_11DateTimeValEiPKNS1_10BooleanValEPS7_",
+ prefix + "19window_funnel_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_",
+ prefix + "23window_funnel_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE",
+ "",
+ "",
+ prefix + "22window_funnel_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE",
+ true, false, true));
+
+ // Vectorization does not need symbol any more, we should clean it in the future.
+ addBuiltin(AggregateFunction.createBuiltin(FunctionSet.WINDOW_FUNNEL,
+ Lists.newArrayList(Type.BIGINT, Type.STRING, Type.DATETIME, Type.BOOLEAN),
+ Type.INT,
+ Type.VARCHAR,
+ true,
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ true, false, true, true));
+
for (Type t : Type.getSupportedTypes()) {
if (t.isNull()) {
continue; // NULL is handled through type promotion.
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java
index 3f0028cbff..35225dca12 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java
@@ -46,7 +46,7 @@ public class AggregateTest {
dorisAssert = new DorisAssert();
dorisAssert.withDatabase(DB_NAME).useDatabase(DB_NAME);
String createTableSQL = "create table " + DB_NAME + "." + TABLE_NAME + " (empid int, name varchar, " +
- "deptno int, salary int, commission int) "
+ "deptno int, salary int, commission int, time DATETIME) "
+ "distributed by hash(empid) buckets 3 properties('replication_num' = '1');";
dorisAssert.withTable(createTableSQL);
}
@@ -93,8 +93,97 @@ public class AggregateTest {
} while (false);
}
+ @Test
+ public void testWindowFunnelAnalysisException() throws Exception {
+ ConnectContext ctx = UtFrameUtils.createDefaultCtx();
+
+ // normal.
+ {
+ String query = "select empid, window_funnel(1, 'default', time, empid = 1, empid = 2) from " +
+ DB_NAME + "." + TABLE_NAME + " group by empid";
+ try {
+ UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
+ } catch (Exception e) {
+ Assert.fail("must be AnalysisException.");
+ }
+ }
+
+ // less argument.
+ do {
+ String query = "select empid, window_funnel(1, 'default', time) from " +
+ DB_NAME + "." + TABLE_NAME + " group by empid";
+ try {
+ UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("function must have at least four params"));
+ break;
+ } catch (Exception e) {
+ Assert.fail("must be AnalysisException.");
+ }
+ Assert.fail("must be AnalysisException.");
+ } while(false);
+
+ // argument with wrong type.
+ do {
+ String query = "select empid, window_funnel('xx', 'default', time, empid = 1) from " +
+ DB_NAME + "." + TABLE_NAME + " group by empid";
+ try {
+ UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("The window param of window_funnel function must be integer"));
+ break;
+ } catch (Exception e) {
+ Assert.fail("must be AnalysisException.");
+ }
+ Assert.fail("must be AnalysisException.");
+ } while(false);
+
+ do {
+ String query = "select empid, window_funnel(1, 1, time, empid = 1) from " +
+ DB_NAME + "." + TABLE_NAME + " group by empid";
+ try {
+ UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("The mode param of window_funnel function must be string"));
+ break;
+ } catch (Exception e) {
+ Assert.fail("must be AnalysisException.");
+ }
+ Assert.fail("must be AnalysisException.");
+ } while(false);
+
+
+ do {
+ String query = "select empid, window_funnel(1, '1', empid, '1') from " +
+ DB_NAME + "." + TABLE_NAME + " group by empid";
+ try {
+ UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("The 3rd param of window_funnel function must be DATE or DATETIME"));
+ break;
+ } catch (Exception e) {
+ Assert.fail("must be AnalysisException.");
+ }
+ Assert.fail("must be AnalysisException.");
+ } while(false);
+
+ do {
+ String query = "select empid, window_funnel(1, '1', time, '1') from " +
+ DB_NAME + "." + TABLE_NAME + " group by empid";
+ try {
+ UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("The 4th and subsequent params of window_funnel function must be boolean"));
+ break;
+ } catch (Exception e) {
+ Assert.fail("must be AnalysisException.");
+ }
+ Assert.fail("must be AnalysisException.");
+ } while(false);
+ }
+
@AfterClass
public static void afterClass() throws Exception {
UtFrameUtils.cleanDorisFeDir(baseDir);
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 07/14: [Refactor][httpv2]remove http v1 code (#8848)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit e882eecbee7b703860f47c9b5b17b48b391a47ea
Author: jiafeng.zhang <zh...@gmail.com>
AuthorDate: Thu Apr 7 08:38:29 2022 +0800
[Refactor][httpv2]remove http v1 code (#8848)
http v2 has been actually tested in production, and it is completely replaceable to have http code. In order to simplify code maintenance, remove the previous http part of the code
---
.../src/main/java/org/apache/doris/PaloFe.java | 29 +-
.../java/org/apache/doris/catalog/Catalog.java | 2 +-
.../main/java/org/apache/doris/common/Config.java | 17 -
.../org/apache/doris/http/ActionController.java | 88 -----
.../java/org/apache/doris/http/BaseAction.java | 375 ---------------------
.../java/org/apache/doris/http/BaseRequest.java | 158 ---------
.../java/org/apache/doris/http/BaseResponse.java | 99 ------
.../org/apache/doris/http/HttpAuthManager.java | 64 ----
.../java/org/apache/doris/http/HttpServer.java | 279 ---------------
.../org/apache/doris/http/HttpServerHandler.java | 117 -------
.../main/java/org/apache/doris/http/IAction.java | 25 --
.../apache/doris/http/UnauthorizedException.java | 26 --
.../main/java/org/apache/doris/http/WebUtils.java | 230 -------------
.../apache/doris/http/action/BackendAction.java | 101 ------
.../org/apache/doris/http/action/HaAction.java | 174 ----------
.../org/apache/doris/http/action/HelpAction.java | 223 ------------
.../org/apache/doris/http/action/IndexAction.java | 275 ---------------
.../org/apache/doris/http/action/LogAction.java | 168 ---------
.../apache/doris/http/action/NotFoundAction.java | 52 ---
.../org/apache/doris/http/action/QueryAction.java | 127 -------
.../doris/http/action/QueryProfileAction.java | 68 ----
.../apache/doris/http/action/SessionAction.java | 85 -----
.../doris/http/action/StaticResourceAction.java | 242 -------------
.../org/apache/doris/http/action/SystemAction.java | 184 ----------
.../apache/doris/http/action/VariableAction.java | 84 -----
.../apache/doris/http/action/WebBaseAction.java | 374 --------------------
.../http/common/DorisHttpPostObjectAggregator.java | 61 ----
.../doris/http/meta/ColocateMetaService.java | 229 -------------
.../doris/http/meta/InvalidClientException.java | 26 --
.../org/apache/doris/http/meta/MetaBaseAction.java | 108 ------
.../org/apache/doris/http/meta/MetaService.java | 356 -------------------
.../doris/http/rest/BootstrapFinishAction.java | 162 ---------
.../apache/doris/http/rest/CancelStreamLoad.java | 83 -----
.../doris/http/rest/CheckDecommissionAction.java | 93 -----
.../apache/doris/http/rest/ConnectionAction.java | 62 ----
.../apache/doris/http/rest/GetDdlStmtAction.java | 108 ------
.../apache/doris/http/rest/GetLoadInfoAction.java | 88 -----
.../apache/doris/http/rest/GetLogFileAction.java | 131 -------
.../apache/doris/http/rest/GetSmallFileAction.java | 92 -----
.../apache/doris/http/rest/GetStreamLoadState.java | 86 -----
.../org/apache/doris/http/rest/HealthAction.java | 47 ---
.../org/apache/doris/http/rest/LoadAction.java | 141 --------
.../doris/http/rest/MetaReplayerCheckAction.java | 70 ----
.../org/apache/doris/http/rest/MetricsAction.java | 65 ----
.../apache/doris/http/rest/MigrationAction.java | 164 ---------
.../org/apache/doris/http/rest/MultiAbort.java | 72 ----
.../org/apache/doris/http/rest/MultiCommit.java | 77 -----
.../java/org/apache/doris/http/rest/MultiDesc.java | 85 -----
.../java/org/apache/doris/http/rest/MultiList.java | 82 -----
.../org/apache/doris/http/rest/MultiStart.java | 92 -----
.../org/apache/doris/http/rest/MultiUnload.java | 76 -----
.../org/apache/doris/http/rest/ProfileAction.java | 60 ----
.../apache/doris/http/rest/QueryDetailAction.java | 56 ---
.../org/apache/doris/http/rest/RestBaseAction.java | 127 -------
.../org/apache/doris/http/rest/RowCountAction.java | 115 -------
.../apache/doris/http/rest/SetConfigAction.java | 136 --------
.../org/apache/doris/http/rest/ShowDataAction.java | 91 -----
.../apache/doris/http/rest/ShowMetaInfoAction.java | 203 -----------
.../org/apache/doris/http/rest/ShowProcAction.java | 136 --------
.../doris/http/rest/ShowRuntimeInfoAction.java | 69 ----
.../doris/http/rest/StorageTypeCheckAction.java | 98 ------
.../doris/http/rest/TableQueryPlanAction.java | 306 -----------------
.../doris/http/rest/TableRowCountAction.java | 113 -------
.../apache/doris/http/rest/TableSchemaAction.java | 135 --------
.../{http => httpv2}/IllegalArgException.java | 25 +-
.../{http/rest => httpv2/entity}/ActionStatus.java | 2 +-
.../rest => httpv2/entity}/RestBaseResult.java | 4 +-
.../{http/rest => httpv2/entity}/RestResult.java | 5 +-
.../doris/httpv2/rest/GetLoadInfoAction.java | 2 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 2 +-
.../org/apache/doris/httpv2/rest/MultiAction.java | 2 +-
.../apache/doris/journal/bdbje/BDBDebugger.java | 22 +-
.../org/apache/doris/common/path/PathTrieTest.java | 19 --
.../org/apache/doris/http/DorisHttpTestCase.java | 23 +-
.../org/apache/doris/http/HttpAuthManagerTest.java | 18 +-
75 files changed, 62 insertions(+), 8029 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java
index c954055c5a..c8ee60a4e1 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/PaloFe.java
@@ -26,7 +26,7 @@ import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.Version;
import org.apache.doris.common.util.JdkUtils;
import org.apache.doris.common.util.NetUtils;
-import org.apache.doris.http.HttpServer;
+import org.apache.doris.httpv2.HttpServer;
import org.apache.doris.journal.bdbje.BDBDebugger;
import org.apache.doris.journal.bdbje.BDBTool;
import org.apache.doris.journal.bdbje.BDBToolOptions;
@@ -134,25 +134,14 @@ public class PaloFe {
feServer.start();
- if (!Config.enable_http_server_v2) {
- HttpServer httpServer = new HttpServer(
- Config.http_port,
- Config.http_max_line_length,
- Config.http_max_header_size,
- Config.http_max_chunk_size
- );
- httpServer.setup();
- httpServer.start();
- } else {
- org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer();
- httpServer2.setPort(Config.http_port);
- httpServer2.setMaxHttpPostSize(Config.jetty_server_max_http_post_size);
- httpServer2.setAcceptors(Config.jetty_server_acceptors);
- httpServer2.setSelectors(Config.jetty_server_selectors);
- httpServer2.setWorkers(Config.jetty_server_workers);
- httpServer2.start(dorisHomeDir);
- }
-
+ HttpServer httpServer = new HttpServer();
+ httpServer.setPort(Config.http_port);
+ httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size);
+ httpServer.setAcceptors(Config.jetty_server_acceptors);
+ httpServer.setSelectors(Config.jetty_server_selectors);
+ httpServer.setWorkers(Config.jetty_server_workers);
+ httpServer.start(dorisHomeDir);
+
qeService.start();
ThreadPoolManager.registerAllThreadPoolMetric();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index beaa10d415..87913ccd35 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -156,7 +156,7 @@ import org.apache.doris.ha.BDBHA;
import org.apache.doris.ha.FrontendNodeType;
import org.apache.doris.ha.HAProtocol;
import org.apache.doris.ha.MasterInfo;
-import org.apache.doris.http.meta.MetaBaseAction;
+import org.apache.doris.httpv2.meta.MetaBaseAction;
import org.apache.doris.journal.JournalCursor;
import org.apache.doris.journal.JournalEntity;
import org.apache.doris.journal.bdbje.Timestamp;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index a9a8b9e04f..33b9988e52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -18,7 +18,6 @@
package org.apache.doris.common;
import org.apache.doris.PaloFe;
-import org.apache.doris.http.HttpServer;
public class Config extends ConfigBase {
@@ -327,22 +326,6 @@ public class Config extends ConfigBase {
*/
@ConfField public static int http_port = 8030;
- /*
- * Netty http param
- */
- @ConfField public static int http_max_line_length = HttpServer.DEFAULT_MAX_LINE_LENGTH;
-
- @ConfField public static int http_max_header_size = HttpServer.DEFAULT_MAX_HEADER_SIZE;
-
- @ConfField public static int http_max_chunk_size = HttpServer.DEFAULT_MAX_CHUNK_SIZE;
-
- /**
- * The backlog_num for netty http server
- * When you enlarge this backlog_num, you should enlarge the value in
- * the linux /proc/sys/net/core/somaxconn file at the same time
- */
- @ConfField public static int http_backlog_num = 1024;
-
/**
* Jetty container default configuration
* Jetty's thread architecture model is very simple, divided into three thread pools:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java b/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java
deleted file mode 100644
index 8264f9169f..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/ActionController.java
+++ /dev/null
@@ -1,88 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.common.path.PathTrie;
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class ActionController {
- private final PathTrie<IAction> getHandlers = new PathTrie<>(WebUtils.REST_DECODER);
- private final PathTrie<IAction> postHandlers = new PathTrie<>(WebUtils.REST_DECODER);
- private final PathTrie<IAction> putHandlers = new PathTrie<>(WebUtils.REST_DECODER);
- private final PathTrie<IAction> deleteHandlers = new PathTrie<>(WebUtils.REST_DECODER);
- private final PathTrie<IAction> headHandlers = new PathTrie<>(WebUtils.REST_DECODER);
- private final PathTrie<IAction> optionsHandlers = new PathTrie<>(WebUtils.REST_DECODER);
-
- // Registers a rest handler to be execute when the provided method and path match the request.
- public void registerHandler(HttpMethod method, String path, IAction handler)
- throws IllegalArgException {
- if (method.equals(HttpMethod.GET)) {
- getHandlers.insert(path, handler);
- } else if (method.equals(HttpMethod.POST)) {
- postHandlers.insert(path, handler);
- } else if (method.equals(HttpMethod.HEAD)) {
- headHandlers.insert(path, handler);
- } else if (method.equals(HttpMethod.DELETE)) {
- deleteHandlers.insert(path, handler);
- } else if (method.equals(HttpMethod.OPTIONS)) {
- optionsHandlers.insert(path, handler);
- } else if (method.equals(HttpMethod.PUT)) {
- putHandlers.insert(path, handler);
- } else {
- throw new IllegalArgException(
- "Can't handle [" + method + "] for path [" + path + "]");
- }
- }
-
- public IAction getHandler(BaseRequest request) {
- String path = getPath(request.getRequest().uri());
- HttpMethod method = request.getRequest().method();
- if (method.equals(HttpMethod.GET)) {
- return getHandlers.retrieve(path, request.getParams());
- } else if (method.equals(HttpMethod.POST)) {
- return postHandlers.retrieve(path, request.getParams());
- } else if (method.equals(HttpMethod.PUT)) {
- return putHandlers.retrieve(path, request.getParams());
- } else if (method.equals(HttpMethod.DELETE)) {
- return deleteHandlers.retrieve(path, request.getParams());
- } else if (method.equals(HttpMethod.HEAD)) {
- return headHandlers.retrieve(path, request.getParams());
- } else if (method.equals(HttpMethod.OPTIONS)) {
- return optionsHandlers.retrieve(path, request.getParams());
- } else {
- return null;
- }
- }
-
- // e.g.
- // in: /www/system?path=//jobs
- // out: /www/system
- private String getPath(String uri) {
- if (Strings.isNullOrEmpty(uri)) {
- return "";
- }
- int pathEndIndex = uri.indexOf('?');
- if (pathEndIndex < 0) {
- return uri;
- } else {
- return uri.substring(0, pathEndIndex);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java
deleted file mode 100644
index b3df5e3e33..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java
+++ /dev/null
@@ -1,375 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.analysis.UserIdentity;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.system.SystemInfoService;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelProgressiveFuture;
-import io.netty.channel.ChannelProgressiveFutureListener;
-import io.netty.channel.DefaultFileRegion;
-import io.netty.handler.codec.base64.Base64;
-import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.DefaultHttpResponse;
-import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpChunkedInput;
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpHeaderValues;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http.HttpUtil;
-import io.netty.handler.codec.http.HttpVersion;
-import io.netty.handler.codec.http.LastHttpContent;
-import io.netty.handler.codec.http.cookie.Cookie;
-import io.netty.handler.codec.http.cookie.ServerCookieEncoder;
-import io.netty.handler.ssl.SslHandler;
-import io.netty.handler.stream.ChunkedFile;
-import io.netty.handler.stream.ChunkedInput;
-import io.netty.handler.stream.ChunkedStream;
-import io.netty.util.CharsetUtil;
-
-public abstract class BaseAction implements IAction {
- private static final Logger LOG = LogManager.getLogger(BaseAction.class);
-
- protected ActionController controller;
- protected Catalog catalog;
-
- public BaseAction(ActionController controller) {
- this.controller = controller;
- // TODO(zc): remove this instance
- this.catalog = Catalog.getCurrentCatalog();
- }
-
- @Override
- public void handleRequest(BaseRequest request) throws Exception {
- BaseResponse response = new BaseResponse();
- LOG.debug("receive http request. url={}", request.getRequest().uri());
- try {
- execute(request, response);
- } catch (Exception e) {
- LOG.warn("fail to process url: {}", request.getRequest().uri(), e);
- if (e instanceof UnauthorizedException) {
- response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\"");
- writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED);
- } else {
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- }
- }
- }
-
- public abstract void execute(BaseRequest request, BaseResponse response) throws DdlException;
-
- protected void writeResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status) {
- // if (HttpHeaders.is100ContinueExpected(request.getRequest())) {
- // ctx.write(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
- // HttpResponseStatus.CONTINUE));
- // }
-
- FullHttpResponse responseObj = null;
- try {
- responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status,
- Unpooled.wrappedBuffer(response.getContent().toString().getBytes("UTF-8")));
- } catch (UnsupportedEncodingException e) {
- LOG.warn("get exception.", e);
- responseObj = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status,
- Unpooled.wrappedBuffer(response.getContent().toString().getBytes()));
- }
- Preconditions.checkNotNull(responseObj);
- HttpMethod method = request.getRequest().method();
-
- checkDefaultContentTypeHeader(response, responseObj);
- if (!method.equals(HttpMethod.HEAD)) {
- response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(),
- String.valueOf(responseObj.content().readableBytes()));
- }
- writeCustomHeaders(response, responseObj);
- writeCookies(response, responseObj);
-
- boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest());
- if (!keepAlive) {
- request.getContext().write(responseObj).addListener(ChannelFutureListener.CLOSE);
- } else {
- responseObj.headers().set(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString());
- request.getContext().write(responseObj);
- }
- }
-
- // Object only support File or byte[]
- protected void writeObjectResponse(BaseRequest request, BaseResponse response, HttpResponseStatus status,
- Object obj, String fileName, boolean isOctStream) {
- Preconditions.checkState((obj instanceof File) || (obj instanceof byte[]));
-
- HttpResponse responseObj = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status);
-
- if (HttpUtil.isKeepAlive(request.getRequest())) {
- response.updateHeader(HttpHeaderNames.CONNECTION.toString(), HttpHeaderValues.KEEP_ALIVE.toString());
- }
-
- if (isOctStream) {
- response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_OCTET_STREAM.toString());
- response.updateHeader(HttpHeaderNames.CONTENT_DISPOSITION.toString(),
- HttpHeaderValues.ATTACHMENT.toString() + "; " + HttpHeaderValues.FILENAME.toString() + "=" + fileName);
- }
-
- ChannelFuture sendFileFuture;
- ChannelFuture lastContentFuture;
-
- try {
- Object writable = null;
- long contentLen = 0;
- boolean sslEnable = request.getContext().pipeline().get(SslHandler.class) != null;
- if (obj instanceof File) {
- RandomAccessFile rafFile = new RandomAccessFile((File) obj, "r");
- contentLen = rafFile.length();
- if (!sslEnable) {
- // use zero-copy file transfer.
- writable = new DefaultFileRegion(rafFile.getChannel(), 0, contentLen);
- } else {
- // cannot use zero-copy file transfer.
- writable = new ChunkedFile(rafFile, 0, contentLen, 8192);
- }
- } else if (obj instanceof byte[]) {
- contentLen = ((byte[]) obj).length;
- if (!sslEnable) {
- writable = Unpooled.wrappedBuffer((byte[]) obj);
- } else {
- writable = new ChunkedStream(new ByteArrayInputStream((byte[]) obj));
- }
- }
-
- response.updateHeader(HttpHeaderNames.CONTENT_LENGTH.toString(), String.valueOf(contentLen));
- writeCookies(response, responseObj);
- writeCustomHeaders(response, responseObj);
-
- // Write headers
- request.getContext().write(responseObj);
-
- // Write object
- if (!sslEnable) {
- sendFileFuture = request.getContext().write(writable, request.getContext().newProgressivePromise());
- // Write the end marker.
- lastContentFuture = request.getContext().writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
- } else {
- sendFileFuture = request.getContext().writeAndFlush(
- new HttpChunkedInput((ChunkedInput<ByteBuf>) writable),
- request.getContext().newProgressivePromise());
- // HttpChunkedInput will write the end marker (LastHttpContent) for us.
- lastContentFuture = sendFileFuture;
- }
- } catch (FileNotFoundException ignore) {
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- } catch (IOException e1) {
- writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
- return;
- }
-
- sendFileFuture.addListener(new ChannelProgressiveFutureListener() {
- @Override
- public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {
- if (total < 0) { // total unknown
- LOG.debug("{} Transfer progress: {}", future.channel(), progress);
- } else {
- LOG.debug("{} Transfer progress: {} / {}", future.channel(), progress, total);
- }
- }
-
- @Override
- public void operationComplete(ChannelProgressiveFuture future) {
- LOG.debug("{} Transfer complete.", future.channel());
- if (!future.isSuccess()) {
- Throwable cause = future.cause();
- LOG.error("something wrong. ", cause);
- }
- }
- });
-
- // Decide whether to close the connection or not.
- boolean keepAlive = HttpUtil.isKeepAlive(request.getRequest());
- if (!keepAlive) {
- // Close the connection when the whole content is written out.
- lastContentFuture.addListener(ChannelFutureListener.CLOSE);
- }
- }
-
- // Set 'CONTENT_TYPE' header if it hasn't been set.
- protected void checkDefaultContentTypeHeader(BaseResponse response, Object responseOj) {
- if (!Strings.isNullOrEmpty(response.getContentType())) {
- response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), response.getContentType());
- } else {
- response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html");
- }
- }
-
- protected void writeCustomHeaders(BaseResponse response, HttpResponse responseObj) {
- for (Map.Entry<String, List<String>> entry : response.getCustomHeaders().entrySet()) {
- responseObj.headers().add(entry.getKey(), entry.getValue());
- }
- }
-
- protected void writeCookies(BaseResponse response, HttpResponse responseObj) {
- for (Cookie cookie : response.getCookies()) {
- responseObj.headers().add(HttpHeaderNames.SET_COOKIE.toString(), ServerCookieEncoder.LAX.encode(cookie));
- }
- }
-
- public static class ActionAuthorizationInfo {
- public String fullUserName;
- public String remoteIp;
- public String password;
- public String cluster;
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("user: ").append(fullUserName).append(", remote ip: ").append(remoteIp);
- sb.append(", password: ").append(password).append(", cluster: ").append(cluster);
- return sb.toString();
- }
- }
-
- protected void checkGlobalAuth(UserIdentity currentUser, PrivPredicate predicate) throws UnauthorizedException {
- if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(currentUser, predicate)) {
- throw new UnauthorizedException("Access denied; you need (at least one of) the "
- + predicate.getPrivs().toString() + " privilege(s) for this operation");
- }
- }
-
- protected void checkDbAuth(UserIdentity currentUser, String db, PrivPredicate predicate)
- throws UnauthorizedException {
- if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(currentUser, db, predicate)) {
- throw new UnauthorizedException("Access denied; you need (at least one of) the "
- + predicate.getPrivs().toString() + " privilege(s) for this operation");
- }
- }
-
- protected void checkTblAuth(UserIdentity currentUser, String db, String tbl, PrivPredicate predicate)
- throws UnauthorizedException {
- if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, db, tbl, predicate)) {
- throw new UnauthorizedException("Access denied; you need (at least one of) the "
- + predicate.getPrivs().toString() + " privilege(s) for this operation");
- }
- }
-
- // return currentUserIdentity from Doris auth
- protected UserIdentity checkPassword(ActionAuthorizationInfo authInfo)
- throws UnauthorizedException {
- List<UserIdentity> currentUser = Lists.newArrayList();
- if (!Catalog.getCurrentCatalog().getAuth().checkPlainPassword(authInfo.fullUserName,
- authInfo.remoteIp, authInfo.password, currentUser)) {
- throw new UnauthorizedException("Access denied for "
- + authInfo.fullUserName + "@" + authInfo.remoteIp);
- }
- Preconditions.checkState(currentUser.size() == 1);
- return currentUser.get(0);
- }
-
- public ActionAuthorizationInfo getAuthorizationInfo(BaseRequest request)
- throws UnauthorizedException {
- ActionAuthorizationInfo authInfo = new ActionAuthorizationInfo();
- if (!parseAuthInfo(request, authInfo)) {
- LOG.info("parse auth info failed, Authorization header {}, url {}",
- request.getAuthorizationHeader(), request.getRequest().uri());
- throw new UnauthorizedException("Need auth information.");
- }
- LOG.debug("get auth info: {}", authInfo);
- return authInfo;
- }
-
- private boolean parseAuthInfo(BaseRequest request, ActionAuthorizationInfo authInfo) {
- String encodedAuthString = request.getAuthorizationHeader();
- if (Strings.isNullOrEmpty(encodedAuthString)) {
- return false;
- }
- String[] parts = encodedAuthString.split(" ");
- if (parts.length != 2) {
- return false;
- }
- encodedAuthString = parts[1];
- ByteBuf buf = null;
- ByteBuf decodeBuf = null;
- try {
- buf = Unpooled.copiedBuffer(ByteBuffer.wrap(encodedAuthString.getBytes()));
-
- // The authString is a string connecting user-name and password with
- // a colon(':')
- decodeBuf = Base64.decode(buf);
- String authString = decodeBuf.toString(CharsetUtil.UTF_8);
- // Note that password may contain colon, so can not simply use a
- // colon to split.
- int index = authString.indexOf(":");
- authInfo.fullUserName = authString.substring(0, index);
- final String[] elements = authInfo.fullUserName.split("@");
- if (elements != null && elements.length < 2) {
- authInfo.fullUserName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER,
- authInfo.fullUserName);
- authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER;
- } else if (elements != null && elements.length == 2) {
- authInfo.fullUserName = ClusterNamespace.getFullName(elements[1], elements[0]);
- authInfo.cluster = elements[1];
- }
- authInfo.password = authString.substring(index + 1);
- authInfo.remoteIp = request.getHostString();
- } finally {
- // release the buf and decode buf after using Unpooled.copiedBuffer
- // or it will get memory leak
- if (buf != null) {
- buf.release();
- }
-
- if (decodeBuf != null) {
- decodeBuf.release();
- }
- }
- return true;
- }
-
- protected int checkIntParam(String strParam) {
- return Integer.parseInt(strParam);
- }
-
- protected long checkLongParam(String strParam) {
- return Long.parseLong(strParam);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java
deleted file mode 100644
index 81633a5cdc..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseRequest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.common.DdlException;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-
-import java.net.InetSocketAddress;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Map;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.QueryStringDecoder;
-import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
-import io.netty.handler.codec.http.cookie.Cookie;
-
-public class BaseRequest {
- protected ChannelHandlerContext context;
- protected HttpRequest request;
- protected Map<String, String> params = Maps.newHashMap();
-
- private boolean isAuthorized = false;
- private QueryStringDecoder decoder;
-
- public BaseRequest(ChannelHandlerContext ctx, HttpRequest request) {
- this.context = ctx;
- this.request = request;
- }
-
- public ChannelHandlerContext getContext() {
- return context;
- }
-
- public void setContext(ChannelHandlerContext context) {
- this.context = context;
- }
-
- public HttpRequest getRequest() {
- return request;
- }
-
- public void setRequest(HttpRequest request) {
- this.request = request;
- }
-
- public Map<String, String> getParams() {
- return params;
- }
-
- public void setParams(Map<String, String> params) {
- this.params = params;
- }
-
- public boolean isAuthorized() {
- return isAuthorized;
- }
-
- public void setAuthorized(boolean isAuthorized) {
- this.isAuthorized = isAuthorized;
- }
-
- public Cookie getCookieByName(String cookieName) {
- String cookieString = request.headers().get(HttpHeaderNames.COOKIE.toString());
- if (!Strings.isNullOrEmpty(cookieString)) {
- Cookie cookie = ClientCookieDecoder.STRICT.decode(cookieString);
- if (cookie.name().equalsIgnoreCase(cookieName)) {
- return cookie;
- }
- }
- return null;
- }
-
- public String getCookieValue(String cookieName) {
- Cookie cookie = getCookieByName(cookieName);
- if (cookie != null) {
- return cookie.value();
- }
- return null;
- }
-
- // get a single parameter.
- // return null if key is not exist; return the first value if key is an array
- public String getSingleParameter(String key) {
- String uri = request.uri();
- if (decoder == null) {
- decoder = new QueryStringDecoder(uri);
- }
-
- List<String> values = decoder.parameters().get(key);
- if (values != null && values.size() > 0) {
- return values.get(0);
- }
-
- return params.get(key);
- }
-
- public String getContent() throws DdlException {
- if (request instanceof FullHttpRequest) {
- FullHttpRequest fullHttpRequest = (FullHttpRequest) request;
- return fullHttpRequest.content().toString(Charset.forName("UTF-8"));
- } else {
- throw new DdlException("Invalid request");
- }
- }
-
- // get an array parameter.
- // eg. ?a=1&a=2
- public List<String> getArrayParameter(String key) {
- String uri = request.uri();
- if (decoder == null) {
- decoder = new QueryStringDecoder(uri);
- }
-
- return decoder.parameters().get(key);
- }
-
- public Map<String, List<String>> getAllParameters() {
- String uri = request.uri();
- if (decoder == null) {
- decoder = new QueryStringDecoder(uri);
- }
-
- return decoder.parameters();
- }
-
- public String getAuthorizationHeader() {
- String authString = request.headers().get("Authorization");
- return authString;
- }
-
- public String getHostString() {
- // get client host
- InetSocketAddress clientSocket = (InetSocketAddress) context.channel().remoteAddress();
- String clientIp = clientSocket.getHostString();
- return clientIp;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java b/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java
deleted file mode 100644
index 297c8c7f8c..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/BaseResponse.java
+++ /dev/null
@@ -1,99 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import io.netty.handler.codec.http.cookie.Cookie;
-
-public class BaseResponse {
- private String contentType;
- protected StringBuilder content = new StringBuilder();
-
- protected Map<String, List<String>> customHeaders = Maps.newHashMap();
- private Set<Cookie> cookies = Sets.newHashSet();
-
- public String getContentType() {
- return contentType;
- }
-
- public void setContentType(String contentType) {
- this.contentType = contentType;
- }
-
- public StringBuilder getContent() {
- return content;
- }
-
- public Set<Cookie> getCookies() {
- return cookies;
- }
-
- public Map<String, List<String>> getCustomHeaders() {
- return customHeaders;
- }
-
- // update old key-value mapping of 'name' if Exist, or add new mapping if not exists.
- // It will only change the mapping of 'name', other header will not be changed.
- public void updateHeader(String name, String value) {
- if (customHeaders == null) {
- customHeaders = Maps.newHashMap();
- }
- customHeaders.remove(name);
- addHeader(name, value);
- }
-
- // Add a custom header.
- private void addHeader(String name, String value) {
- if (customHeaders == null) {
- customHeaders = Maps.newHashMap();
- }
- List<String> header = customHeaders.get(name);
- if (header == null) {
- header = Lists.newArrayList();
- customHeaders.put(name, header);
- }
- header.add(value);
- }
-
- public void appendContent(String buffer) {
- if (content == null) {
- content = new StringBuilder();
- }
- content.append(buffer);
- }
-
- public void addCookie(Cookie cookie) {
- cookies.add(cookie);
- }
-
- public void updateCookieAge(BaseRequest request, String cookieName, long age) {
- Cookie cookie = request.getCookieByName(cookieName);
- if (cookie != null) {
- cookies.remove(cookie);
- cookie.setMaxAge(age);
- cookies.add(cookie);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java
deleted file mode 100644
index d0cd55ef80..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpAuthManager.java
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.analysis.UserIdentity;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-
-import java.util.concurrent.TimeUnit;
-
-// We simulate a simplified session here: only store user-name of clients who already logged in,
-// and we only have a default admin user for now.
-public final class HttpAuthManager {
- private static long SESSION_EXPIRE_TIME = 2; // hour
- private static long SESSION_MAX_SIZE = 100; // avoid to store too many
-
- private static HttpAuthManager instance = new HttpAuthManager();
-
- public static class SessionValue {
- public UserIdentity currentUser;
- }
-
- // session_id => session value
- private Cache<String, SessionValue> authSessions = CacheBuilder.newBuilder()
- .maximumSize(SESSION_MAX_SIZE)
- .expireAfterAccess(SESSION_EXPIRE_TIME, TimeUnit.HOURS)
- .build();
-
- private HttpAuthManager() {
- // do nothing
- }
-
- public static HttpAuthManager getInstance() {
- return instance;
- }
-
- public SessionValue getSessionValue(String sessionId) {
- return authSessions.getIfPresent(sessionId);
- }
-
- public void addSessionValue(String key, SessionValue value) {
- authSessions.put(key, value);
- }
-
- public Cache<String, SessionValue> getAuthSessions() {
- return authSessions;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java
deleted file mode 100644
index 1b1df9cf26..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServer.java
+++ /dev/null
@@ -1,279 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.common.Config;
-import org.apache.doris.http.action.BackendAction;
-import org.apache.doris.http.action.HaAction;
-import org.apache.doris.http.action.HelpAction;
-import org.apache.doris.http.action.IndexAction;
-import org.apache.doris.http.action.LogAction;
-import org.apache.doris.http.action.QueryAction;
-import org.apache.doris.http.action.QueryProfileAction;
-import org.apache.doris.http.action.SessionAction;
-import org.apache.doris.http.action.StaticResourceAction;
-import org.apache.doris.http.action.SystemAction;
-import org.apache.doris.http.action.VariableAction;
-import org.apache.doris.http.common.DorisHttpPostObjectAggregator;
-import org.apache.doris.http.meta.ColocateMetaService;
-import org.apache.doris.http.meta.MetaService.CheckAction;
-import org.apache.doris.http.meta.MetaService.DumpAction;
-import org.apache.doris.http.meta.MetaService.ImageAction;
-import org.apache.doris.http.meta.MetaService.InfoAction;
-import org.apache.doris.http.meta.MetaService.JournalIdAction;
-import org.apache.doris.http.meta.MetaService.PutAction;
-import org.apache.doris.http.meta.MetaService.RoleAction;
-import org.apache.doris.http.meta.MetaService.VersionAction;
-import org.apache.doris.http.rest.BootstrapFinishAction;
-import org.apache.doris.http.rest.CancelStreamLoad;
-import org.apache.doris.http.rest.CheckDecommissionAction;
-import org.apache.doris.http.rest.ConnectionAction;
-import org.apache.doris.http.rest.GetDdlStmtAction;
-import org.apache.doris.http.rest.GetLoadInfoAction;
-import org.apache.doris.http.rest.GetLogFileAction;
-import org.apache.doris.http.rest.GetSmallFileAction;
-import org.apache.doris.http.rest.GetStreamLoadState;
-import org.apache.doris.http.rest.HealthAction;
-import org.apache.doris.http.rest.LoadAction;
-import org.apache.doris.http.rest.MetaReplayerCheckAction;
-import org.apache.doris.http.rest.MetricsAction;
-import org.apache.doris.http.rest.MigrationAction;
-import org.apache.doris.http.rest.MultiAbort;
-import org.apache.doris.http.rest.MultiCommit;
-import org.apache.doris.http.rest.MultiDesc;
-import org.apache.doris.http.rest.MultiList;
-import org.apache.doris.http.rest.MultiStart;
-import org.apache.doris.http.rest.MultiUnload;
-import org.apache.doris.http.rest.ProfileAction;
-import org.apache.doris.http.rest.QueryDetailAction;
-import org.apache.doris.http.rest.RowCountAction;
-import org.apache.doris.http.rest.SetConfigAction;
-import org.apache.doris.http.rest.ShowDataAction;
-import org.apache.doris.http.rest.ShowMetaInfoAction;
-import org.apache.doris.http.rest.ShowProcAction;
-import org.apache.doris.http.rest.ShowRuntimeInfoAction;
-import org.apache.doris.http.rest.StorageTypeCheckAction;
-import org.apache.doris.http.rest.TableQueryPlanAction;
-import org.apache.doris.http.rest.TableRowCountAction;
-import org.apache.doris.http.rest.TableSchemaAction;
-import org.apache.doris.master.MetaHelper;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.File;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.codec.http.HttpServerCodec;
-import io.netty.handler.stream.ChunkedWriteHandler;
-
-public class HttpServer {
- private static final Logger LOG = LogManager.getLogger(HttpServer.class);
-
- /**
- * The default netty param, witch is the same as `HttpServerCodec`.
- */
- public static final int DEFAULT_MAX_LINE_LENGTH = 4096;
- public static final int DEFAULT_MAX_HEADER_SIZE = 8192;
- public static final int DEFAULT_MAX_CHUNK_SIZE = 8192;
-
- private final int port;
- private final int maxInitialLineLength;
- private final int maxHeaderSize;
- private final int maxChunkSize;
- private ActionController controller;
-
- private Thread serverThread;
-
- private AtomicBoolean isStarted = new AtomicBoolean(false);
-
- public HttpServer(int port) {
- this(port, DEFAULT_MAX_LINE_LENGTH, DEFAULT_MAX_HEADER_SIZE, DEFAULT_MAX_CHUNK_SIZE);
- }
-
- public HttpServer(int port, int maxInitialLineLength, int maxHeaderSize, int maxChunkSize) {
- this.port = port;
- this.maxInitialLineLength = maxInitialLineLength;
- this.maxHeaderSize = maxHeaderSize;
- this.maxChunkSize = maxChunkSize;
- controller = new ActionController();
- }
-
- public void setup() throws IllegalArgException {
- registerActions();
- }
-
- private void registerActions() throws IllegalArgException {
- // add rest action
- LoadAction.registerAction(controller);
- GetLoadInfoAction.registerAction(controller);
- SetConfigAction.registerAction(controller);
- GetDdlStmtAction.registerAction(controller);
- MigrationAction.registerAction(controller);
- StorageTypeCheckAction.registerAction(controller);
- CancelStreamLoad.registerAction(controller);
- GetStreamLoadState.registerAction(controller);
-
- // add web action
- IndexAction.registerAction(controller);
- SystemAction.registerAction(controller);
- BackendAction.registerAction(controller);
- LogAction.registerAction(controller);
- QueryAction.registerAction(controller);
- QueryProfileAction.registerAction(controller);
- SessionAction.registerAction(controller);
- VariableAction.registerAction(controller);
- HelpAction.registerAction(controller);
- StaticResourceAction.registerAction(controller);
- HaAction.registerAction(controller);
-
- // Add multi action
- MultiStart.registerAction(controller);
- MultiDesc.registerAction(controller);
- MultiCommit.registerAction(controller);
- MultiUnload.registerAction(controller);
- MultiAbort.registerAction(controller);
- MultiList.registerAction(controller);
-
- // rest action
- HealthAction.registerAction(controller);
- MetricsAction.registerAction(controller);
- ShowMetaInfoAction.registerAction(controller);
- ShowProcAction.registerAction(controller);
- ShowRuntimeInfoAction.registerAction(controller);
- GetLogFileAction.registerAction(controller);
- GetSmallFileAction.registerAction(controller);
- RowCountAction.registerAction(controller);
- CheckDecommissionAction.registerAction(controller);
- MetaReplayerCheckAction.registerAction(controller);
- ColocateMetaService.BucketSeqAction.registerAction(controller);
- ColocateMetaService.ColocateMetaAction.registerAction(controller);
- ColocateMetaService.MarkGroupStableAction.registerAction(controller);
- ProfileAction.registerAction(controller);
- QueryDetailAction.registerAction(controller);
- ConnectionAction.registerAction(controller);
- ShowDataAction.registerAction(controller);
-
- // meta service action
- File imageDir = MetaHelper.getMasterImageDir();
- ImageAction.registerAction(controller, imageDir);
- InfoAction.registerAction(controller, imageDir);
- VersionAction.registerAction(controller, imageDir);
- PutAction.registerAction(controller, imageDir);
- JournalIdAction.registerAction(controller, imageDir);
- CheckAction.registerAction(controller, imageDir);
- DumpAction.registerAction(controller, imageDir);
- RoleAction.registerAction(controller, imageDir);
-
- // external usage
- TableRowCountAction.registerAction(controller);
- TableSchemaAction.registerAction(controller);
- TableQueryPlanAction.registerAction(controller);
-
- BootstrapFinishAction.registerAction(controller);
- }
-
- public void start() {
- serverThread = new Thread(new HttpServerThread(), "FE Http Server");
- serverThread.start();
- }
-
- protected class PaloHttpServerInitializer extends ChannelInitializer<SocketChannel> {
- @Override
- protected void initChannel(SocketChannel ch) throws Exception {
- ch.pipeline().addLast(new HttpServerCodec(maxInitialLineLength, maxHeaderSize, maxChunkSize));
- ch.pipeline().addLast(new DorisHttpPostObjectAggregator(100 * 65536));
- ch.pipeline().addLast(new ChunkedWriteHandler());
- ch.pipeline().addLast(new HttpServerHandler(controller));
- }
- }
-
- ServerBootstrap serverBootstrap;
-
- private class HttpServerThread implements Runnable {
- @Override
- public void run() {
- // Configure the server.
- EventLoopGroup bossGroup = new NioEventLoopGroup();
- EventLoopGroup workerGroup = new NioEventLoopGroup();
- try {
- serverBootstrap = new ServerBootstrap();
- serverBootstrap.option(ChannelOption.SO_BACKLOG, Config.http_backlog_num);
- // reused address and port to avoid bind already exception
- serverBootstrap.option(ChannelOption.SO_REUSEADDR, true);
- serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, true);
- serverBootstrap.group(bossGroup, workerGroup)
- .channel(NioServerSocketChannel.class)
- .childHandler(new PaloHttpServerInitializer());
- Channel ch = serverBootstrap.bind(port).sync().channel();
-
- isStarted.set(true);
- LOG.info("HttpServer started with port {}", port);
- // block until server is closed
- ch.closeFuture().sync();
- } catch (Exception e) {
- LOG.error("Fail to start FE query http server[port: " + port + "] ", e);
- System.exit(-1);
- } finally {
- bossGroup.shutdownGracefully();
- workerGroup.shutdownGracefully();
- }
- }
- }
-
- // used for test, release bound port
- public void shutDown() {
- if (serverBootstrap != null) {
- Future future = serverBootstrap.config().group().shutdownGracefully(0, 1, TimeUnit.SECONDS).syncUninterruptibly();
- try {
- future.get();
- isStarted.set(false);
- LOG.info("HttpServer was closed completely");
- } catch (Throwable e) {
- LOG.warn("Exception happened when close HttpServer", e);
- }
- serverBootstrap = null;
- }
- }
-
- public boolean isStarted() {
- return isStarted.get();
- }
-
- public static void main(String[] args) throws Exception {
- HttpServer httpServer = new HttpServer(8080);
- httpServer.setup();
- System.out.println("before start http server.");
- httpServer.start();
- System.out.println("after start http server.");
-
- while (true) {
- Thread.sleep(2000);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java b/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java
deleted file mode 100644
index eecbe60a6e..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/HttpServerHandler.java
+++ /dev/null
@@ -1,117 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.http.action.IndexAction;
-import org.apache.doris.http.action.NotFoundAction;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.net.URISyntaxException;
-
-import io.netty.buffer.Unpooled;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http.HttpVersion;
-import io.netty.util.ReferenceCountUtil;
-
-public class HttpServerHandler extends ChannelInboundHandlerAdapter {
- private static final Logger LOG = LogManager.getLogger(HttpServerHandler.class);
-
- private ActionController controller = null;
- protected FullHttpRequest fullRequest = null;
- protected HttpRequest request = null;
- private BaseAction action = null;
-
- public HttpServerHandler(ActionController controller) {
- super();
- this.controller = controller;
- }
-
- @Override
- public void channelReadComplete(ChannelHandlerContext ctx) {
- ctx.flush();
- }
-
- @Override
- public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
- if (msg instanceof HttpRequest) {
- this.request = (HttpRequest) msg;
- LOG.debug("request: url:[{}]", request.uri());
- if (!isRequestValid(ctx, request)) {
- writeResponse(ctx, HttpResponseStatus.BAD_REQUEST, "this is a bad request.");
- return;
- }
- BaseRequest req = new BaseRequest(ctx, request);
-
- action = getAction(req);
- if (action != null) {
- LOG.debug("action: {} ", action.getClass().getName());
- action.handleRequest(req);
- }
- } else {
- ReferenceCountUtil.release(msg);
- }
- }
-
- private boolean isRequestValid(ChannelHandlerContext ctx, HttpRequest request) throws URISyntaxException {
- return true;
- }
-
- private void writeResponse(ChannelHandlerContext context , HttpResponseStatus status, String content) {
- FullHttpResponse responseObj = new DefaultFullHttpResponse(
- HttpVersion.HTTP_1_1,
- status,
- Unpooled.wrappedBuffer(content.getBytes()));
- responseObj.headers().set(HttpHeaderNames.CONTENT_TYPE.toString(), "text/html");
- responseObj.headers().set(HttpHeaderNames.CONTENT_LENGTH.toString(), responseObj.content().readableBytes());
- context.writeAndFlush(responseObj).addListener(ChannelFutureListener.CLOSE);
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
- LOG.warn("Handle http from " + ctx.channel().remoteAddress() + " caught exception", cause);
- ctx.close();
- }
-
- private BaseAction getAction(BaseRequest request) {
- String uri = request.getRequest().uri();
- // ignore this request, which is a default request from client's browser.
- if (uri.endsWith("/favicon.ico")) {
- return NotFoundAction.getNotFoundAction();
- } else if (uri.equals("/")) {
- return new IndexAction(controller);
- }
-
- // Map<String, String> params = Maps.newHashMap();
- BaseAction action = (BaseAction) controller.getHandler(request);
- if (action == null) {
- action = NotFoundAction.getNotFoundAction();
- }
-
- return action;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java
deleted file mode 100644
index a89d0ee3d0..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/IAction.java
+++ /dev/null
@@ -1,25 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-
-public interface IAction {
-
- public void handleRequest(BaseRequest request) throws Exception;
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java b/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java
deleted file mode 100644
index 4802498ce0..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/UnauthorizedException.java
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.common.DdlException;
-
-public class UnauthorizedException extends DdlException {
- public UnauthorizedException(String msg) {
- super(msg);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java b/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java
deleted file mode 100644
index c9efe86ac0..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/WebUtils.java
+++ /dev/null
@@ -1,230 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http;
-
-import org.apache.doris.common.path.PathTrie;
-
-import com.google.common.base.Charsets;
-
-import java.nio.charset.Charset;
-import java.util.Map;
-
-public class WebUtils {
- public static final String HTTP_CORS_ALLOW_ORIGIN_SETTING = "http.cors.allow-origin";
-
- public static final PathTrie.Decoder REST_DECODER = new PathTrie.Decoder() {
- @Override
- public String decode(String value) {
- return WebUtils.decodeComponent(value);
- }
- };
-
- public static boolean isBrowser(String userAgent) {
- if (userAgent == null) {
- return false;
- }
- // chrome, safari, firefox, ie
- if (userAgent.startsWith("Mozilla")) {
- return true;
- }
- return false;
- }
-
- public static void decodeQueryString(String s, int fromIndex, Map<String, String> params) {
- if (fromIndex < 0) {
- return;
- }
- if (fromIndex >= s.length()) {
- return;
- }
-
- String name = null;
- int pos = fromIndex; // Beginning of the unprocessed region
- int i; // End of the unprocessed region
- char c = 0; // Current character
- for (i = fromIndex; i < s.length(); i++) {
- c = s.charAt(i);
- if (c == '=' && name == null) {
- if (pos != i) {
- name = decodeComponent(s.substring(pos, i));
- }
- pos = i + 1;
- } else if (c == '&') {
- if (name == null && pos != i) {
- // We haven't seen an `=' so far but moved forward.
- // Must be a param of the form '&a&' so add it with
- // an empty value.
- addParam(params, decodeComponent(s.substring(pos, i)), "");
- } else if (name != null) {
- addParam(params, name, decodeComponent(s.substring(pos, i)));
- name = null;
- }
- pos = i + 1;
- }
- }
-
- if (pos != i) { // Are there characters we haven't dealt with?
- if (name == null) { // Yes and we haven't seen any `='.
- addParam(params, decodeComponent(s.substring(pos, i)), "");
- } else { // Yes and this must be the last value.
- addParam(params, name, decodeComponent(s.substring(pos, i)));
- }
- } else if (name != null) { // Have we seen a name without value?
- addParam(params, name, "");
- }
- }
-
- private static void addParam(Map<String, String> params, String name, String value) {
- params.put(name, value);
- }
-
- /**
- * Decodes a bit of an URL encoded by a browser.
- * <p/>
- * This is equivalent to calling {@link #decodeComponent(String, Charset)}
- * with the UTF-8 charset (recommended to comply with RFC 3986, Section 2).
- *
- * @param s The string to decode (can be empty).
- * @return The decoded string, or {@code s} if there's nothing to decode.
- * If the string to decode is {@code null}, returns an empty string.
- * @throws IllegalArgumentException if the string contains a malformed
- * escape sequence.
- */
- public static String decodeComponent(final String s) {
- return decodeComponent(s, Charsets.UTF_8);
- }
-
- /**
- * Decodes a bit of an URL encoded by a browser.
- * <p/>
- * The string is expected to be encoded as per RFC 3986, Section 2.
- * This is the encoding used by JavaScript functions {@code encodeURI}
- * and {@code encodeURIComponent}, but not {@code escape}. For example
- * in this encoding, é (in Unicode {@code U+00E9} or in UTF-8
- * {@code 0xC3 0xA9}) is encoded as {@code %C3%A9} or {@code %c3%a9}.
- * <p/>
- * This is essentially equivalent to calling
- * <code>{@link java.net.URLDecoder URLDecoder}.{@link
- * java.net.URLDecoder#decode(String, String)}</code>
- * except that it's over 2x faster and generates less garbage for the GC.
- * Actually this function doesn't allocate any memory if there's nothing
- * to decode, the argument itself is returned.
- *
- * @param s The string to decode (can be empty).
- * @param charset The charset to use to decode the string (should really
- * be {@link Charsets#UTF_8}.
- * @return The decoded string, or {@code s} if there's nothing to decode.
- * If the string to decode is {@code null}, returns an empty string.
- * @throws IllegalArgumentException if the string contains a malformed
- * escape sequence.
- */
- @SuppressWarnings("fallthrough")
- public static String decodeComponent(final String s, final Charset charset) {
- if (s == null) {
- return "";
- }
- final int size = s.length();
- boolean modified = false;
- for (int i = 0; i < size; i++) {
- final char c = s.charAt(i);
- switch (c) {
- case '%':
- i++; // We can skip at least one char, e.g. `%%'.
- // Fall through.
- case '+':
- modified = true;
- break;
- }
- }
- if (!modified) {
- return s;
- }
- final byte[] buf = new byte[size];
- int pos = 0; // position in `buf'.
- for (int i = 0; i < size; i++) {
- char c = s.charAt(i);
- switch (c) {
- case '+':
- buf[pos++] = ' '; // "+" -> " "
- break;
- case '%':
- if (i == size - 1) {
- throw new IllegalArgumentException("unterminated escape"
- + " sequence at end of string: " + s);
- }
- c = s.charAt(++i);
- if (c == '%') {
- buf[pos++] = '%'; // "%%" -> "%"
- break;
- } else if (i == size - 1) {
- throw new IllegalArgumentException("partial escape"
- + " sequence at end of string: " + s);
- }
- c = decodeHexNibble(c);
- final char c2 = decodeHexNibble(s.charAt(++i));
- if (c == Character.MAX_VALUE || c2 == Character.MAX_VALUE) {
- throw new IllegalArgumentException(
- "invalid escape sequence `%" + s.charAt(i - 1)
- + s.charAt(i) + "' at index " + (i - 2)
- + " of: " + s);
- }
- c = (char) (c * 16 + c2);
- // Fall through.
- default:
- buf[pos++] = (byte) c;
- break;
- }
- }
- return new String(buf, 0, pos, charset);
- }
-
- /**
- * Helper to decode half of a hexadecimal number from a string.
- *
- * @param c The ASCII character of the hexadecimal number to decode.
- * Must be in the range {@code [0-9a-fA-F]}.
- * @return The hexadecimal value represented in the ASCII character
- * given, or {@link Character#MAX_VALUE} if the character is invalid.
- */
- private static char decodeHexNibble(final char c) {
- if ('0' <= c && c <= '9') {
- return (char) (c - '0');
- } else if ('a' <= c && c <= 'f') {
- return (char) (c - 'a' + 10);
- } else if ('A' <= c && c <= 'F') {
- return (char) (c - 'A' + 10);
- } else {
- return Character.MAX_VALUE;
- }
- }
-
- /**
- * Determine if CORS setting is a regex
- */
-// public static Pattern getCorsSettingRegex(Settings settings) {
-// String corsSetting = settings.get(HTTP_CORS_ALLOW_ORIGIN_SETTING, "*");
-// int len = corsSetting.length();
-// boolean isRegex = len > 2 && corsSetting.startsWith("/") && corsSetting.endsWith("/");
-//
-// if (isRegex) {
-// return Pattern.compile(corsSetting.substring(1, corsSetting.length()-1));
-// }
-//
-// return null;
-// }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java
deleted file mode 100644
index 73b4413118..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/BackendAction.java
+++ /dev/null
@@ -1,101 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.util.ListComparator;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.system.Backend;
-import com.google.common.collect.ImmutableMap;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class BackendAction extends WebBaseAction {
- private static final Logger LOG = LogManager.getLogger(BackendAction.class);
-
- public BackendAction(ActionController controller) {
- super(controller);
- // TODO Auto-generated constructor stub
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/backend", new BackendAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- appendKnownBackendsInfo(response.getContent());
-
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- private void appendKnownBackendsInfo(StringBuilder buffer) {
- ImmutableMap<Long, Backend> backendMap = Catalog.getCurrentSystemInfo().getIdToBackend();
-
- List<List<Comparable>> backendInfos = new ArrayList<List<Comparable>>();
- for (Backend backend : backendMap.values()) {
- List<Comparable> backendInfo = new ArrayList<Comparable>();
- InetAddress address = null;
- try {
- address = InetAddress.getByName(backend.getHost());
- } catch (UnknownHostException e) {
- LOG.warn("unknown host: " + backend.getHost(), e);
- continue;
- }
- backendInfo.add(address.getHostName());
- backendInfo.add(backend.getId());
- backendInfo.add("heart_port: " + backend.getHeartbeatPort()
- + ", be_port: " + backend.getBePort()
- + ", http_port: " + backend.getHttpPort());
-
- backendInfos.add(backendInfo);
- }
-
- // sort by id
- ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(1);
- Collections.sort(backendInfos, comparator);
-
- // set result
- buffer.append("<h2>Known Backends(" + backendMap.size() + ")</h2>");
-
- buffer.append("<pre>");
- for (List<Comparable> info : backendInfos) {
- buffer.append(info.get(0));
- buffer.append(" [id: " + info.get(1));
- buffer.append(", " + info.get(2) + "]");
- buffer.append(System.getProperty("line.separator"));
- }
- buffer.append("</pre>");
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java
deleted file mode 100644
index aa759b1da5..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/HaAction.java
+++ /dev/null
@@ -1,174 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.Config;
-import org.apache.doris.ha.HAProtocol;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.persist.Storage;
-import org.apache.doris.system.Frontend;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Date;
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class HaAction extends WebBaseAction {
-
- public HaAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/ha", new HaAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- appendRoleInfo(response.getContent());
- appendJournalInfo(response.getContent());
- appendCanReadInfo(response.getContent());
- appendNodesInfo(response.getContent());
- appendImageInfo(response.getContent());
- appendDbNames(response.getContent());
- appendFe(response.getContent());
- appendRemovedFe(response.getContent());
-
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- private void appendRoleInfo(StringBuilder buffer) {
- buffer.append("<h2>Frontend Role</h2>");
- buffer.append("<pre>");
- buffer.append("<p>" + Catalog.getCurrentCatalog().getFeType() + "</p>");
- buffer.append("</pre>");
- }
-
- private void appendJournalInfo(StringBuilder buffer) {
- buffer.append("<h2>Current Journal Id</h2>");
- buffer.append("<pre>");
- if (Catalog.getCurrentCatalog().isMaster()) {
- buffer.append("<p>" + Catalog.getCurrentCatalog().getEditLog().getMaxJournalId() + "</p>");
- } else {
- buffer.append("<p>" + Catalog.getCurrentCatalog().getReplayedJournalId() + "</p>");
- }
- buffer.append("</pre>");
- }
-
- private void appendNodesInfo(StringBuilder buffer) {
- HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol();
- if (haProtocol == null) {
- return;
- }
- List<InetSocketAddress> electableNodes = haProtocol.getElectableNodes(true);
- if (electableNodes.isEmpty()) {
- return;
- }
- buffer.append("<h2>Electable nodes</h2>");
- buffer.append("<pre>");
- for (InetSocketAddress node : electableNodes) {
- buffer.append("<p>" + node.getAddress() + "</p>");
-
- }
- buffer.append("</pre>");
-
- List<InetSocketAddress> observerNodes = haProtocol.getObserverNodes();
- if (observerNodes == null) {
- return;
- }
- buffer.append("<h2>Observer nodes</h2>");
- buffer.append("<pre>");
- for (InetSocketAddress node : observerNodes) {
- buffer.append("<p>" + node.getHostString() + "</p>");
- }
- buffer.append("</pre>");
- }
-
- private void appendCanReadInfo(StringBuilder buffer) {
- buffer.append("<h2>Can Read</h2>");
- buffer.append("<pre>");
- buffer.append("<p>" + Catalog.getCurrentCatalog().canRead() + "</p>");
-
- buffer.append("</pre>");
- }
-
- private void appendImageInfo(StringBuilder buffer) {
- try {
- Storage storage = new Storage(Config.meta_dir + "/image");
- buffer.append("<h2>Checkpoint Info</h2>");
- buffer.append("<pre>");
- buffer.append("<p>last checkpoint version:" + storage.getImageSeq() + "</p>");
- long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
- Date date = new Date(lastCheckpointTime);
- buffer.append("<p>last checkpoint time: " + date + "</p>");
- buffer.append("</pre>");
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private void appendDbNames(StringBuilder buffer) {
- List<Long> names = Catalog.getCurrentCatalog().getEditLog().getDatabaseNames();
- if (names == null) {
- return;
- }
-
- String msg = "";
- for (long name : names) {
- msg += name + " ";
- }
- buffer.append("<h2>Database names</h2>");
- buffer.append("<pre>");
- buffer.append("<p>" + msg + "</p>");
- buffer.append("</pre>");
- }
-
- private void appendFe(StringBuilder buffer) {
- List<Frontend> fes = Catalog.getCurrentCatalog().getFrontends(null /* all */);
- if (fes == null) {
- return;
- }
-
- buffer.append("<h2>Allowed Frontends</h2>");
- buffer.append("<pre>");
- for (Frontend fe : fes) {
- buffer.append("<p>" + fe.toString() + "</p>");
- }
- buffer.append("</pre>");
- }
-
- private void appendRemovedFe(StringBuilder buffer) {
- List<String> feNames = Catalog.getCurrentCatalog().getRemovedFrontendNames();
- buffer.append("<h2>Removed Frontends</h2>");
- buffer.append("<pre>");
- for (String feName : feNames) {
- buffer.append("<p>" + feName + "</p>");
- }
- buffer.append("</pre>");
- }
-
-}
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java
deleted file mode 100644
index 786426250b..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/HelpAction.java
+++ /dev/null
@@ -1,223 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.HelpModule;
-import org.apache.doris.qe.HelpTopic;
-
-import com.google.common.base.Strings;
-
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class HelpAction extends WebBaseAction {
- private static final String DIV_BACKGROUND_COLOR = "#FCFCFC";
-
- String queryString = null;
-
- public HelpAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/help", new HelpAction(controller));
- }
-
- @Override
- public boolean needAdmin() {
- return false;
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
- appendHelpStyle(response.getContent());
-
- queryString = request.getSingleParameter("query");
- if (Strings.isNullOrEmpty(queryString)) {
- // ATTN: according to Mysql protocol, the default query should be "contents"
- // when you want to get server side help.
- queryString = "contents";
- } else {
- queryString = queryString.trim();
- }
- appendHelpInfo(response.getContent());
-
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- private void appendHelpInfo(StringBuilder buffer) {
- buffer.append("<h2>Help Info</h2>");
- buffer.append("<p>This page lists the help info, "
- + "like 'help contents' in Mysql client.</p>");
-
- appendSearchButton(buffer);
- appendExactMatchTopic(buffer);
- appendFuzzyMatchTopic(buffer);
- appendCategories(buffer);
- }
-
- private void appendSearchButton(StringBuilder buffer) {
- buffer.append("<form class=\"form-search\">"
- + "<div class=\"col-lg-3\" style=\"padding-left: 0px;\">"
- + " <div class=\"input-group\">"
- + " <input name = \"query\" type=\"text\" class=\"form-control\" placeholder=\"input here...\">"
- + " <span class=\"input-group-btn\">"
- + " <button class=\"btn btn-default\" type=\"submit\">Search</button>"
- + " </span>"
- + " </div>"
- + "</div>"
- + "<a href=\"/help\" class=\"btn btn-primary\">Back To Home</a>"
- + "</form>");
- }
-
- private void appendExactMatchTopic(StringBuilder buffer) {
- buffer.append("<h3>Exact Matching Topic</h3>");
- buffer.append("<div style=\"background-color:" + DIV_BACKGROUND_COLOR + ";"
- + "padding:0px, 1px, 1px, 0px;"
- + "\">");
- HelpModule module = HelpModule.getInstance();
- HelpTopic topic = module.getTopic(queryString);
- if (topic == null) {
- buffer.append("<pre>No Exact Matching Topic.</pre>");
- } else {
- appendOneTopicInfo(buffer, topic);
- }
- buffer.append("</div>");
- }
-
- private void appendFuzzyMatchTopic(StringBuilder buffer) {
- buffer.append("<h3>Fuzzy Matching Topic(By Keyword)</h3>");
- buffer.append("<div style=\"background-color:" + DIV_BACKGROUND_COLOR + ";"
- + "padding:0px, 1px, 1px, 0px;"
- + "\">");
- HelpModule module = HelpModule.getInstance();
- List<String> topics = module.listTopicByKeyword(queryString);
- if (topics.isEmpty()) {
- buffer.append("<pre>No Fuzzy Matching Topic.</pre>");
- } else if (topics.size() == 1) {
- buffer.append("<p class=\"text-info\"> "
- + "Find only one topic, show you the detail info below.</p>");
- appendOneTopicInfo(buffer, module.getTopic(topics.get(0)));
- } else {
- buffer.append("<p class=\"text-info\"> Find " + topics.size() + " topics:</p>");
- appendNameList(buffer, topics, "Topics");
- }
- buffer.append("</div>");
- }
-
- private void appendCategories(StringBuilder buffer) {
- buffer.append("<h3>Category Info</h3>");
- buffer.append("<div style=\"background-color:" + DIV_BACKGROUND_COLOR + ";"
- + "padding:0px, 1px, 1px, 0px;"
- + "\">");
- HelpModule module = HelpModule.getInstance();
- List<String> categories = module.listCategoryByName(queryString);
- if (categories.isEmpty()) {
- buffer.append("<pre>No Matching Category.</pre>");
- } else if (categories.size() == 1) {
- buffer.append("<p class=\"text-info\"> "
- + "Find only one category, so show you the detail info below. </p>");
- List<String> topics = module.listTopicByCategory(categories.get(0));
- if (topics.size() > 0) {
- buffer.append("<p class=\"text-info\"> Find "
- + topics.size()
- + " sub topics. </p>");
- appendNameList(buffer, topics, "Sub Topics");
- }
-
- List<String> subCategories = module.listCategoryByCategory(categories.get(0));
- if (subCategories.size() > 0) {
- buffer.append("<p class=\"text-info\"> Find "
- + subCategories.size()
- + " sub categories. </p>");
- appendNameList(buffer, subCategories, "Sub Categories");
- }
- } else {
- buffer.append("<p> Find " + categories.size() + " category: </p>");
- appendNameList(buffer, categories, "Categories");
- }
- buffer.append("</div>");
- }
-
- // The browser will combine continuous whitespace to one, we use <pre> tag to solve this issue.
- private void appendOneTopicInfo(StringBuilder buffer, HelpTopic topic) {
- buffer.append("<div style=\""
- + "padding:9.5px; "
- + "margin: 0 0 10px; "
- + "background-color: #f5f5f5;"
- + "border: 1px solid rgba(0, 0, 0, 0.15);"
- + "-webkit-border-radius: 4px;"
- + "-moz-border-radius: 4px;"
- + "border-radius: 4px;"
- + "font-family: Consolas;"
- + "\">");
- buffer.append("<h4>'" + escapeHtmlInPreTag(topic.getName()) + "'</h4>");
-
- buffer.append("<strong>Description</strong>");
- buffer.append("<pre class=\"topic_text\" style=\"border: 0px;\">"
- + escapeHtmlInPreTag(topic.getDescription())
- + "</pre>");
-
- buffer.append("<strong>Example</strong>");
- buffer.append("<pre class=\"topic_text\" style=\"border: 0px\">"
- + escapeHtmlInPreTag(topic.getExample())
- + "</pre>");
-
- buffer.append("<strong>Keyword</strong>");
- buffer.append("<pre class=\"topic_text\" style=\"border: 0px\">"
- + escapeHtmlInPreTag(topic.getKeywords().toString())
- + "</pre>");
-
- buffer.append("<strong>Url</strong>");
- buffer.append("<pre class=\"topic_text\" style=\"border: 0px\">"
- + escapeHtmlInPreTag(topic.getUrl())
- + "</pre>");
- buffer.append("</div>");
- }
-
- private void appendNameList(StringBuilder buffer, List<String> names, String tableHeadName) {
- buffer.append("<div style=\"padding:0px, 1px, 1px, 0px\">");
- buffer.append("<table "
- + "class=\"table table-hover table-bordered table-striped table-hover\"><tr>");
- buffer.append("<th>" + tableHeadName + "</th>");
-
- final String href = "?query=";
- for (String name : names) {
- buffer.append("<tr><td><a href=\"" + href + name + "\">" + name + "</a><br/></td></tr>");
- }
- buffer.append("</table>");
- buffer.append("</div>");
- }
-
- private void appendHelpStyle(StringBuilder buffer) {
- buffer.append("<style type=\"text/css\">"
- + ".topic_text {"
- + " font-family: \"Consolas\";"
- + " margin-left: 5px;"
- + "}"
- + "</style>");
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java
deleted file mode 100644
index af17b29db5..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/IndexAction.java
+++ /dev/null
@@ -1,275 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.common.Version;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-import oshi.SystemInfo;
-import oshi.hardware.CentralProcessor;
-import oshi.hardware.GlobalMemory;
-import oshi.hardware.HWDiskStore;
-import oshi.hardware.HWPartition;
-import oshi.hardware.HardwareAbstractionLayer;
-import oshi.hardware.NetworkIF;
-import oshi.hardware.VirtualMemory;
-import oshi.software.os.FileSystem;
-import oshi.software.os.NetworkParams;
-import oshi.software.os.OSFileStore;
-import oshi.software.os.OSProcess;
-import oshi.software.os.OperatingSystem;
-import oshi.util.FormatUtil;
-import oshi.util.Util;
-
-public class IndexAction extends WebBaseAction {
-
- public IndexAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/index", new IndexAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
- appendVersionInfo(response.getContent());
- appendHardwareInfo(response.getContent());
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- private void appendVersionInfo(StringBuilder buffer) {
- buffer.append("<h2>Version</h2>");
- buffer.append("<pre>version info<br/>");
- buffer.append("Version: " + Version.DORIS_BUILD_VERSION + "<br/>");
- buffer.append("Git: " + Version.DORIS_BUILD_HASH + "<br/>");
- buffer.append("Build Info: " + Version.DORIS_BUILD_INFO + "<br/>");
- buffer.append("Build Time: " + Version.DORIS_BUILD_TIME + "<br/>");
- buffer.append("</pre>");
- }
-
- private void appendHardwareInfo(StringBuilder buffer) {
- List<String> hardwareInfo = new ArrayList<>();
- SystemInfo si = new SystemInfo();
- OperatingSystem os = si.getOperatingSystem();
- HardwareAbstractionLayer hal = si.getHardware();
- CentralProcessor processor = hal.getProcessor();
- GlobalMemory memory = hal.getMemory();
- buffer.append("<h2>Hardware Info</h2>");
- buffer.append("<pre>");
- buffer.append(String.join("<br/>", getOperatingSystem(os)) + "<hr>");
- buffer.append(String.join("<br/>", getProcessor(processor)) + "<hr>");
- buffer.append(String.join("<br/>", getMemory(memory)) + "<hr>");
- buffer.append(String.join("<br/>", getProcesses(os, memory)) + "<hr>");
- buffer.append(String.join("<br/>", getDisks(hal.getDiskStores())) + "<hr>");
- buffer.append(String.join("<br/>", getFileSystem(os.getFileSystem())) + "<hr>");
- buffer.append(String.join("<br/>", getNetworkInterfaces(hal.getNetworkIFs())) + "<hr>");
- buffer.append(String.join("<br/>", getNetworkParameters(os.getNetworkParams())) + "<hr>");
- buffer.append("</pre>");
- }
- private List<String> getOperatingSystem(OperatingSystem os) {
- List<String> osInfo = new ArrayList<>();
- osInfo.add(String.valueOf(os));
- osInfo.add("Booted: " + Instant.ofEpochSecond(os.getSystemBootTime()));
- osInfo.add("Uptime: " + FormatUtil.formatElapsedSecs(os.getSystemUptime()));
- osInfo.add("Running with" + (os.isElevated() ? "" : "out") + " elevated permissions.");
- return osInfo;
- }
- private List<String> getProcessor(CentralProcessor processor) {
- List<String> processorInfo = new ArrayList<>();
- processorInfo.add(String.valueOf(processor));
- processorInfo.add(" " + processor.getPhysicalPackageCount() + " physical CPU package(s)");
- processorInfo.add(" " + processor.getPhysicalProcessorCount() + " physical CPU core(s)");
- processorInfo.add(" " + processor.getLogicalProcessorCount() + " logical CPU(s)");
-
- processorInfo.add("Identifier: " + processor.getIdentifier());
- processorInfo.add("ProcessorID: " + processor.getProcessorID());
- processorInfo.add("Context Switches/Interrupts: " + processor.getContextSwitches()
- + " / " + processor.getInterrupts() + "\n");
-
- long[] prevTicks = processor.getSystemCpuLoadTicks();
- long[][] prevProcTicks = processor.getProcessorCpuLoadTicks();
- processorInfo.add("CPU, IOWait, and IRQ ticks @ 0 sec:" + Arrays.toString(prevTicks));
- // Wait a second...
- Util.sleep(1000);
- long[] ticks = processor.getSystemCpuLoadTicks();
- processorInfo.add("CPU, IOWait, and IRQ ticks @ 1 sec:" + Arrays.toString(ticks));
- long user = ticks[CentralProcessor.TickType.USER.getIndex()] - prevTicks[CentralProcessor.TickType.USER.getIndex()];
- long nice = ticks[CentralProcessor.TickType.NICE.getIndex()] - prevTicks[CentralProcessor.TickType.NICE.getIndex()];
- long sys = ticks[CentralProcessor.TickType.SYSTEM.getIndex()] - prevTicks[CentralProcessor.TickType.SYSTEM.getIndex()];
- long idle = ticks[CentralProcessor.TickType.IDLE.getIndex()] - prevTicks[CentralProcessor.TickType.IDLE.getIndex()];
- long iowait = ticks[CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[CentralProcessor.TickType.IOWAIT.getIndex()];
- long irq = ticks[CentralProcessor.TickType.IRQ.getIndex()] - prevTicks[CentralProcessor.TickType.IRQ.getIndex()];
- long softirq = ticks[CentralProcessor.TickType.SOFTIRQ.getIndex()] - prevTicks[CentralProcessor.TickType.SOFTIRQ.getIndex()];
- long steal = ticks[CentralProcessor.TickType.STEAL.getIndex()] - prevTicks[CentralProcessor.TickType.STEAL.getIndex()];
- long totalCpu = user + nice + sys + idle + iowait + irq + softirq + steal;
-
- processorInfo.add(String.format(
- "User: %.1f%% Nice: %.1f%% System: %.1f%% Idle: %.1f%% IOwait: %.1f%% IRQ: %.1f%% SoftIRQ: %.1f%% Steal: %.1f%%",
- 100d * user / totalCpu, 100d * nice / totalCpu, 100d * sys / totalCpu, 100d * idle / totalCpu,
- 100d * iowait / totalCpu, 100d * irq / totalCpu, 100d * softirq / totalCpu, 100d * steal / totalCpu));
- processorInfo.add(String.format("CPU load: %.1f%%",
- processor.getSystemCpuLoadBetweenTicks(prevTicks) * 100));
- double[] loadAverage = processor.getSystemLoadAverage(3);
- processorInfo.add("CPU load averages:" + (loadAverage[0] < 0 ? " N/A" : String.format(" %.2f", loadAverage[0]))
- + (loadAverage[1] < 0 ? " N/A" : String.format(" %.2f", loadAverage[1]))
- + (loadAverage[2] < 0 ? " N/A" : String.format(" %.2f", loadAverage[2])));
- // per core CPU
- StringBuilder procCpu = new StringBuilder("CPU load per processor:");
- double[] load = processor.getProcessorCpuLoadBetweenTicks(prevProcTicks);
- for (double avg : load) {
- procCpu.append(String.format(" %.1f%%", avg * 100));
- }
- processorInfo.add(procCpu.toString());
- long freq = processor.getVendorFreq();
- if (freq > 0) {
- processorInfo.add("Vendor Frequency: " + FormatUtil.formatHertz(freq));
- }
- freq = processor.getMaxFreq();
- if (freq > 0) {
- processorInfo.add("Max Frequency: " + FormatUtil.formatHertz(freq));
- }
- long[] freqs = processor.getCurrentFreq();
- if (freqs[0] > 0) {
- StringBuilder sb = new StringBuilder("Current Frequencies: ");
- for (int i = 0; i < freqs.length; i++) {
- if (i > 0) {
- sb.append(", ");
- }
- sb.append(FormatUtil.formatHertz(freqs[i]));
- }
- processorInfo.add(sb.toString());
- }
- return processorInfo;
- }
-
- private List<String> getMemory(GlobalMemory memory) {
- List<String> memoryInfo = new ArrayList<>();
- memoryInfo.add("Memory: " + FormatUtil.formatBytes(memory.getAvailable()) + "/"
- + FormatUtil.formatBytes(memory.getTotal()));
- VirtualMemory vm = memory.getVirtualMemory();
- memoryInfo.add("Swap used: " + FormatUtil.formatBytes(vm.getSwapUsed()) + "/"
- + FormatUtil.formatBytes(vm.getSwapTotal()));
- return memoryInfo;
- }
-
- private List<String> getProcesses(OperatingSystem os, GlobalMemory memory) {
- List<String> processInfo = new ArrayList<>();
- processInfo.add("Processes: " + os.getProcessCount() + ", Threads: " + os.getThreadCount());
- // Sort by highest CPU
- List<OSProcess> procs = Arrays.asList(os.getProcesses(5, OperatingSystem.ProcessSort.CPU));
-
- processInfo.add(" PID %CPU %MEM VSZ RSS Name");
- for (int i = 0; i < procs.size() && i < 5; i++) {
- OSProcess p = procs.get(i);
- processInfo.add(String.format(" %5d %5.1f %4.1f %9s %9s %s", p.getProcessID(),
- 100d * (p.getKernelTime() + p.getUserTime()) / p.getUpTime(),
- 100d * p.getResidentSetSize() / memory.getTotal(), FormatUtil.formatBytes(p.getVirtualSize()),
- FormatUtil.formatBytes(p.getResidentSetSize()), p.getName()));
- }
- return processInfo;
- }
- private List<String> getDisks(HWDiskStore[] diskStores) {
- List<String> diskInfo = new ArrayList<>();
- diskInfo.add("Disks:");
- for (HWDiskStore disk : diskStores) {
- boolean readwrite = disk.getReads() > 0 || disk.getWrites() > 0;
- diskInfo.add(String.format(" %s: (model: %s - S/N: %s) size: %s, reads: %s (%s), writes: %s (%s), xfer: %s ms",
- disk.getName(), disk.getModel(), disk.getSerial(),
- disk.getSize() > 0 ? FormatUtil.formatBytesDecimal(disk.getSize()) : "?",
- readwrite ? disk.getReads() : "?", readwrite ? FormatUtil.formatBytes(disk.getReadBytes()) : "?",
- readwrite ? disk.getWrites() : "?", readwrite ? FormatUtil.formatBytes(disk.getWriteBytes()) : "?",
- readwrite ? disk.getTransferTime() : "?"));
- HWPartition[] partitions = disk.getPartitions();
- for (HWPartition part : partitions) {
- diskInfo.add(String.format(" |-- %s: %s (%s) Maj:Min=%d:%d, size: %s%s", part.getIdentification(),
- part.getName(), part.getType(), part.getMajor(), part.getMinor(),
- FormatUtil.formatBytesDecimal(part.getSize()),
- part.getMountPoint().isEmpty() ? "" : " @ " + part.getMountPoint()));
- }
- }
- return diskInfo;
- }
-
- private List<String> getFileSystem(FileSystem fileSystem) {
- List<String> fsInfo = new ArrayList<>();
- fsInfo.add("File System:");
-
- fsInfo.add(String.format(" File Descriptors: %d/%d", fileSystem.getOpenFileDescriptors(),
- fileSystem.getMaxFileDescriptors()));
-
- OSFileStore[] fsArray = fileSystem.getFileStores();
- for (OSFileStore fs : fsArray) {
- long usable = fs.getUsableSpace();
- long total = fs.getTotalSpace();
- fsInfo.add(String.format(" %s (%s) [%s] %s of %s free (%.1f%%), %s of %s files free (%.1f%%) is %s " +
- (fs.getLogicalVolume() != null && fs.getLogicalVolume().length() > 0 ? "[%s]" : "%s") +
- " and is mounted at %s",
- fs.getName(), fs.getDescription().isEmpty() ? "file system" : fs.getDescription(), fs.getType(),
- FormatUtil.formatBytes(usable), FormatUtil.formatBytes(fs.getTotalSpace()), 100d * usable / total,
- FormatUtil.formatValue(fs.getFreeInodes(), ""), FormatUtil.formatValue(fs.getTotalInodes(), ""),
- 100d * fs.getFreeInodes() / fs.getTotalInodes(), fs.getVolume(), fs.getLogicalVolume(),
- fs.getMount()));
- }
- return fsInfo;
- }
-
- private List<String> getNetworkInterfaces(NetworkIF[] networkIFs) {
- List<String> getNetwork = new ArrayList<>();
- getNetwork.add("Network interfaces:");
- for (NetworkIF net : networkIFs) {
- getNetwork.add(String.format(" Name: %s (%s)", net.getName(), net.getDisplayName()));
- getNetwork.add(String.format(" MAC Address: %s", net.getMacaddr()));
- getNetwork.add(String.format(" MTU: %s, Speed: %s", net.getMTU(), FormatUtil.formatValue(net.getSpeed(), "bps")));
- getNetwork.add(String.format(" IPv4: %s", Arrays.toString(net.getIPv4addr())));
- getNetwork.add(String.format(" IPv6: %s", Arrays.toString(net.getIPv6addr())));
- boolean hasData = net.getBytesRecv() > 0 || net.getBytesSent() > 0 || net.getPacketsRecv() > 0
- || net.getPacketsSent() > 0;
- getNetwork.add(String.format(" Traffic: received %s/%s%s; transmitted %s/%s%s",
- hasData ? net.getPacketsRecv() + " packets" : "?",
- hasData ? FormatUtil.formatBytes(net.getBytesRecv()) : "?",
- hasData ? " (" + net.getInErrors() + " err)" : "",
- hasData ? net.getPacketsSent() + " packets" : "?",
- hasData ? FormatUtil.formatBytes(net.getBytesSent()) : "?",
- hasData ? " (" + net.getOutErrors() + " err)" : ""));
- }
- return getNetwork;
- }
-
- private List<String> getNetworkParameters(NetworkParams networkParams) {
- List<String> networkParameterInfo = new ArrayList<>();
- networkParameterInfo.add("Network parameters:");
- networkParameterInfo.add(String.format(" Host name: %s", networkParams.getHostName()));
- networkParameterInfo.add(String.format(" Domain name: %s", networkParams.getDomainName()));
- networkParameterInfo.add(String.format(" DNS servers: %s", Arrays.toString(networkParams.getDnsServers())));
- networkParameterInfo.add(String.format(" IPv4 Gateway: %s", networkParams.getIpv4DefaultGateway()));
- networkParameterInfo.add(String.format(" IPv6 Gateway: %s", networkParams.getIpv6DefaultGateway()));
- return networkParameterInfo;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java
deleted file mode 100644
index 6185debfc7..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/LogAction.java
+++ /dev/null
@@ -1,168 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.common.Config;
-import org.apache.doris.common.Log4jConfig;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import io.netty.handler.codec.http.HttpMethod;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.List;
-
-public class LogAction extends WebBaseAction {
- private static final Logger LOG = LogManager.getLogger(LogAction.class);
- private static long WEB_LOG_BYTES = 1024 * 1024; // 1MB
-
- private String addVerboseName;
- private String delVerboseName;
-
- public LogAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/log", new LogAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- // get parameters
- addVerboseName = request.getSingleParameter("add_verbose");
- delVerboseName = request.getSingleParameter("del_verbose");
- LOG.info("add verbose name: {}, del verbose name: {}", addVerboseName, delVerboseName);
-
- appendLogConf(response.getContent());
- appendLogInfo(response.getContent());
-
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- private void appendLogConf(StringBuilder buffer) {
- buffer.append("<h2>Log Configuration</h2>");
- try {
- Log4jConfig.Tuple<String, String[], String[]> configs = Log4jConfig.updateLogging(null, null, null);
- if (!Strings.isNullOrEmpty(addVerboseName)) {
- addVerboseName = addVerboseName.trim();
- List<String> verboseNames = Lists.newArrayList(configs.y);
- if (!verboseNames.contains(addVerboseName)) {
- verboseNames.add(addVerboseName);
- configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]),
- null);
- }
- }
- if (!Strings.isNullOrEmpty(delVerboseName)) {
- delVerboseName = delVerboseName.trim();
- List<String> verboseNames = Lists.newArrayList(configs.y);
- if (verboseNames.contains(delVerboseName)) {
- verboseNames.remove(delVerboseName);
- configs = Log4jConfig.updateLogging(null, verboseNames.toArray(new String[verboseNames.size()]),
- null);
- }
- }
-
- buffer.append("Level: " + configs.x + "<br/>");
- buffer.append("Verbose Names: " + StringUtils.join(configs.y, ",") + "<br/>");
- buffer.append("Audit Names: " + StringUtils.join(configs.z, ",") + "<br/>");
- appendUpdateVerboseButton(buffer, "add_verbose");
- appendUpdateVerboseButton(buffer, "del_verbose");
- } catch (IOException e) {
- LOG.error(e);
- e.printStackTrace();
- }
- }
-
- private void appendUpdateVerboseButton(StringBuilder buffer, String type) {
- String placeHolder = "";
- String buttonName = "";
- if (type.equals("add_verbose")) {
- placeHolder = "new verbose name";
- buttonName = "Add";
- } else if (type.equals("del_verbose")) {
- placeHolder = "del verbose name";
- buttonName = "Delete";
- } else {
- return;
- }
-
- buffer.append("<form>"
- + "<div class=\"col-lg-3\" style=\"padding-left: 0px;\">"
- + " <div class=\"input-group\">"
- + " <input name = \"" + type + "\" type=\"text\" class=\"form-control\" placeholder=\""
- + placeHolder + "\">"
- + " <span class=\"input-group-btn\" style=\"padding-left: 0px;\">"
- + " <button class=\"btn btn-default\" type=\"submit\">" + buttonName + "</button>"
- + " </span>\n"
- + " </div>\n"
- + "</div>"
- + "</form>");
- }
-
- private void appendLogInfo(StringBuilder buffer) {
- buffer.append("<br/><h2>Log Contents</h2>");
-
- final String logPath = Config.sys_log_dir + "/fe.warn.log";
- buffer.append("Log path is: " + logPath + "<br/>");
-
- RandomAccessFile raf = null;
- try {
- raf = new RandomAccessFile(logPath, "r");
- long fileSize = raf.length();
- long startPos = fileSize < WEB_LOG_BYTES ? 0L : fileSize - WEB_LOG_BYTES;
- long webContentLength = fileSize < WEB_LOG_BYTES ? fileSize : WEB_LOG_BYTES;
- raf.seek(startPos);
- buffer.append("<p>Showing last " + webContentLength + " bytes of log</p>");
- buffer.append("<pre>");
- String fileBuffer = null;
- while ((fileBuffer = raf.readLine()) != null) {
- buffer.append(fileBuffer).append("\n");
- }
- buffer.append("</pre>");
- } catch (FileNotFoundException e) {
- buffer.append("<p class=\"text-error\">Couldn't open log file: "
- + logPath + "</p>");
- } catch (IOException e) {
- buffer.append("<p class=\"text-error\">Failed to read log file: "
- + logPath + "</p>");
- } finally {
- try {
- if (raf != null) {
- raf.close();
- }
- } catch (IOException e) {
- LOG.warn("fail to close log file: " + logPath, e);
- }
- }
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java
deleted file mode 100644
index 4cb6d28dd3..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/NotFoundAction.java
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-public class NotFoundAction extends WebBaseAction {
-
- public NotFoundAction(ActionController controller) {
- super(controller);
- }
-
- @Override
- public boolean needAdmin() {
- return false;
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- response.appendContent("this is 404 page.");
-
- getPageFooter(response.getContent());
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/404", new NotFoundAction(controller));
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java
deleted file mode 100644
index e3e4f6af1a..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryAction.java
+++ /dev/null
@@ -1,127 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.common.util.ProfileManager;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.List;
-
-public class QueryAction extends WebBaseAction {
- private static final Logger LOG = LogManager.getLogger(QueryAction.class);
-
- public QueryAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/query", new QueryAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- addFinishedQueryInfo(response.getContent());
-
- getPageFooter(response.getContent());
-
- writeResponse(request, response);
- }
-
- // Note: we do not show 'Query ID' column in web page
- private void addFinishedQueryInfo(StringBuilder buffer) {
- buffer.append("<h2>Finished Queries</h2>");
- buffer.append("<p>You need to set session variable by executing 'set is_report_success=true' before executing the SQL, to view the profile</p>");
- buffer.append("<p>This table lists the latest 100 queries</p>");
-
- List<List<String>> finishedQueries = ProfileManager.getInstance().getAllQueries();
- List<String> columnHeaders = ProfileManager.PROFILE_HEADERS;
- int queryIdIndex = 0; // the first column is 'Query ID' by default
- for (int i = 0; i < columnHeaders.size(); ++i) {
- if (columnHeaders.get(i).equals(ProfileManager.QUERY_ID)) {
- queryIdIndex = i;
- }
- }
- appendFinishedQueryTableHeader(buffer, columnHeaders, queryIdIndex);
- appendFinishedQueryTableBody(buffer, finishedQueries, columnHeaders, queryIdIndex);
- appendTableFooter(buffer);
- }
-
- private void appendFinishedQueryTableHeader(
- StringBuilder buffer,
- final List<String> columnHeaders,
- int queryIdIndex) {
- buffer.append("<table "
- + "class=\"table table-hover table-bordered table-striped table-hover\"><tr>");
- for (int i = 0; i < columnHeaders.size(); ++i) {
- if (i == queryIdIndex) {
- continue;
- }
- buffer.append("<th>" + columnHeaders.get(i) + "</th>");
- }
-
- buffer.append("<th>Profile</th>");
- buffer.append("</tr>");
- }
-
- private void appendFinishedQueryTableBody(
- StringBuilder buffer,
- List<List<String>> bodies,
- List<String> columnHeaders,
- int queryIdIndex) {
- for ( List<String> row : bodies) {
- buffer.append("<tr>");
- String queryId = row.get(queryIdIndex);
-
- for (int i = 0; i < row.size(); ++i) {
- if (i == queryIdIndex) {
- continue;
- }
- buffer.append("<td>");
- buffer.append(row.get(i));
- buffer.append("</td>");
- }
-
- // add 'Profile' column
- if (Strings.isNullOrEmpty(queryId)) {
- LOG.warn("query id is null or empty, maybe we forget to push it "
- + "into array when generate profile info.");
- buffer.append("<td>Empty Query ID</td>");
- } else {
- buffer.append("<td>");
- buffer.append("<a href=\"/query_profile?query_id="
- + queryId
- + "\">Profile</a>");
- buffer.append("</td>");
- }
-
- buffer.append("</tr>");
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java
deleted file mode 100644
index 69ac168587..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/QueryProfileAction.java
+++ /dev/null
@@ -1,68 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.common.util.ProfileManager;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-public class QueryProfileAction extends WebBaseAction {
-
- public QueryProfileAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/query_profile", new QueryProfileAction(controller));
- }
-
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- String queryId = request.getSingleParameter("query_id");
- if (Strings.isNullOrEmpty(queryId)) {
- response.appendContent("");
- response.appendContent("<p class=\"text-error\"> Must specify a query_id[]</p>");
- }
-
- String queryProfileStr = ProfileManager.getInstance().getProfile(queryId);
- if (queryProfileStr != null) {
- appendQueryProfile(response.getContent(), queryProfileStr);
- getPageFooter(response.getContent());
- writeResponse(request, response);
- } else {
- appendQueryProfile(response.getContent(), "query id " + queryId + " not found.");
- getPageFooter(response.getContent());
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- }
- }
-
- private void appendQueryProfile(StringBuilder buffer, String queryProfileStr) {
- buffer.append("<pre>");
- buffer.append(queryProfileStr);
- buffer.append("</pre>");
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java
deleted file mode 100644
index 435cacfa48..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/SessionAction.java
+++ /dev/null
@@ -1,85 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-import com.google.common.collect.Lists;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class SessionAction extends WebBaseAction {
- // we make
- private static final ArrayList<String> SESSION_TABLE_HEADER = Lists.newArrayList();
-
- static {
- SESSION_TABLE_HEADER.add("Id");
- SESSION_TABLE_HEADER.add("User");
- SESSION_TABLE_HEADER.add("Host");
- SESSION_TABLE_HEADER.add("Cluster");
- SESSION_TABLE_HEADER.add("Db");
- SESSION_TABLE_HEADER.add("Command");
- SESSION_TABLE_HEADER.add("Time");
- SESSION_TABLE_HEADER.add("State");
- SESSION_TABLE_HEADER.add("Info");
- }
-
- public SessionAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/session", new SessionAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
- appendSessionInfo(response.getContent());
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- private void appendSessionInfo(StringBuilder buffer) {
- buffer.append("<h2>Session Info</h2>");
-
- List<ConnectContext.ThreadInfo> threadInfos = ExecuteEnv.getInstance().getScheduler().listConnection("root");
- List<List<String>> rowSet = Lists.newArrayList();
- long nowMs = System.currentTimeMillis();
- for (ConnectContext.ThreadInfo info : threadInfos) {
- rowSet.add(info.toRow(nowMs));
- }
-
- buffer.append("<p>This page lists the session info, there are "
- + rowSet.size()
- + " active sessions.</p>");
-
- appendTableHeader(buffer, SESSION_TABLE_HEADER);
- appendTableBody(buffer, rowSet);
- appendTableFooter(buffer);
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java
deleted file mode 100644
index cbd540d29a..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/StaticResourceAction.java
+++ /dev/null
@@ -1,242 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.PaloFe;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import com.google.common.base.Strings;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.File;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import java.util.Locale;
-import java.util.TimeZone;
-import java.util.regex.Pattern;
-
-import javax.activation.MimetypesFileTypeMap;
-
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/**
- * A simple handler that serves incoming HTTP requests to send their respective
- * HTTP responses. It also implements {@code 'If-Modified-Since'} header to
- * take advantage of browser cache, as described in
- * <a href="http://tools.ietf.org/html/rfc2616#section-14.25">RFC 2616</a>.
- *
- * <h3>How Browser Caching Works</h3>
- *
- * Web browser caching works with HTTP headers as illustrated by the following
- * sample:
- * <ol>
- * <li>Request #1 returns the content of {@code /file1.txt}.</li>
- * <li>Contents of {@code /file1.txt} is cached by the browser.</li>
- * <li>Request #2 for {@code /file1.txt} does return the contents of the
- * file again. Rather, a 304 Not Modified is returned. This tells the
- * browser to use the contents stored in its cache.</li>
- * <li>The server knows the file has not been modified because the
- * {@code If-Modified-Since} date is the same as the file's last
- * modified date.</li>
- * </ol>
- *
- * <pre>
- * Request #1 Headers
- * ===================
- * GET /file1.txt HTTP/1.1
- *
- * Response #1 Headers
- * ===================
- * HTTP/1.1 200 OK
- * Date: Tue, 01 Mar 2011 22:44:26 GMT
- * Last-Modified: Wed, 30 Jun 2010 21:36:48 GMT
- * Expires: Tue, 01 Mar 2012 22:44:26 GMT
- * Cache-Control: private, max-age=31536000
- *
- * Request #2 Headers
- * ===================
- * GET /file1.txt HTTP/1.1
- * If-Modified-Since: Wed, 30 Jun 2010 21:36:48 GMT
- *
- * Response #2 Headers
- * ===================
- * HTTP/1.1 304 Not Modified
- * Date: Tue, 01 Mar 2011 22:44:28 GMT
- *
- * </pre>
- *
- * We use parameter named 'res' to specify the static resource path, it relative to the
- * root path of http server.
- */
-public class StaticResourceAction extends WebBaseAction {
- private static final Logger LOG = LogManager.getLogger(StaticResourceAction.class);
-
- public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
- public static final String HTTP_DATE_GMT_TIMEZONE = "GMT";
- public static final int HTTP_CACHE_SECONDS = 2592000;
-
- public MimetypesFileTypeMap mimeTypesMap;
- public String rootDir;
-
- public StaticResourceAction(ActionController controller, String rootDir) {
- super(controller);
- mimeTypesMap = new MimetypesFileTypeMap();
- mimeTypesMap.addMimeTypes("text/html html htm");
- // According to RFC 4329(http://tools.ietf.org/html/rfc4329#section-7.2), the MIME type of
- // javascript script is 'application/javascript'
- mimeTypesMap.addMimeTypes("application/javascript js");
- // mimeTypesMap.addMimeTypes("text/javascript js");
- mimeTypesMap.addMimeTypes("text/css css");
-
- this.rootDir = rootDir;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- String httpDir = PaloFe.DORIS_HOME_DIR + "/webroot";
- StaticResourceAction action = new StaticResourceAction(controller, httpDir + "/static");
- controller.registerHandler(HttpMethod.GET, "/static/js", action);
- controller.registerHandler(HttpMethod.GET, "/static/css", action);
- controller.registerHandler(HttpMethod.GET, "/static", action);
- controller.registerHandler(HttpMethod.GET, "/static/resource", action);
- controller.registerHandler(HttpMethod.GET, "/static/images", action);
- controller.registerHandler(HttpMethod.GET, "/static/Bootstrap-3.3.7/fonts/", action);
-
- StaticResourceAction action2 = new StaticResourceAction(controller, "webroot");
- controller.registerHandler(HttpMethod.GET, "/static_test", action2);
- }
-
- @Override
- public boolean needAdmin() {
- return false;
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- String resourcePath = request.getSingleParameter("res");
- if (Strings.isNullOrEmpty(resourcePath)) {
- LOG.error("Wrong request without 'res' parameter. url: {}",
- request.getRequest().uri());
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- resourcePath = sanitizePath(resourcePath);
- if (resourcePath == null) {
- LOG.error("Close this request because of risk factor in 'res' parameter.url: {}",
- request.getRequest().uri());
- writeResponse(request, response, HttpResponseStatus.FORBIDDEN);
- return;
- }
-
- String resourceAbsolutePath = rootDir + File.separator + resourcePath;
- File resFile = new File(resourceAbsolutePath);
- LOG.debug("resAbsolutePath: {}", resourceAbsolutePath);
-
- if (!resFile.exists() || resFile.isHidden() || resFile.isDirectory()) {
- LOG.error("Request with wrong path. url: {}", request.getRequest().uri());
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- }
- if (!resFile.isFile()) {
- LOG.error("Wrong request: not normal file. url: {}", request.getRequest().uri());
- writeResponse(request, response, HttpResponseStatus.FORBIDDEN);
- return;
- }
-
- // Cache validation
- String ifModifiedSince = request.getRequest()
- .headers().get(HttpHeaderNames.IF_MODIFIED_SINCE.toString());
- if (!Strings.isNullOrEmpty(ifModifiedSince)) {
- SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
- Date ifModifiedSinceDate;
- try {
- ifModifiedSinceDate = dateFormatter.parse(ifModifiedSince);
- // Only compare up to the second because the datetime format we send to the client
- // does not have milliseconds
- long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
- long fileLastModifiedSeconds = resFile.lastModified() / 1000;
- if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
- writeResponse(request, response, HttpResponseStatus.NOT_MODIFIED);
- return;
- }
- } catch (ParseException e) {
- LOG.error("Fail to analyse IF_MODIFIED_SINCE header: ", e);
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- }
-
- response.updateHeader(HttpHeaderNames.CONTENT_TYPE.toString(), getContentType(resourceAbsolutePath));
- setDateAndCacheHeaders(response, resFile);
-
- writeObjectResponse(request, response, HttpResponseStatus.OK, resFile, resFile.getName(), false);
- }
-
-
-
- // Gets the content type header for the HTTP Response
- private String getContentType(String filename) {
- return mimeTypesMap.getContentType(filename);
- }
-
- private void setDateAndCacheHeaders(BaseResponse response, File fileToCache) {
- SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
- dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE));
-
- // Date header
- Calendar time = new GregorianCalendar();
- response.updateHeader(HttpHeaderNames.DATE.toString(), dateFormatter.format(time.getTime()));
-
- // Add cache headers
- time.add(Calendar.SECOND, HTTP_CACHE_SECONDS);
- response.updateHeader(HttpHeaderNames.EXPIRES.toString(), dateFormatter.format(time.getTime()));
- response.updateHeader(HttpHeaderNames.CACHE_CONTROL.toString(), "private, max-age=" + HTTP_CACHE_SECONDS);
- response.updateHeader(HttpHeaderNames.LAST_MODIFIED.toString(),
- dateFormatter.format(new Date(fileToCache.lastModified())));
- }
-
- private static final Pattern INSECURE_URI = Pattern.compile(".*[<>&\"].*");
-
- private String sanitizePath(String path) {
- if (Strings.isNullOrEmpty(path)) {
- return null;
- }
- // Convert file separators
- String newPath = path.replace('/', File.separatorChar);
-
- // Simplistic dumb security check.
- if (newPath.contains(File.separator + '.')
- || newPath.contains('.' + File.separator)
- || newPath.charAt(0) == '.'
- || newPath.charAt(newPath.length() - 1) == '.'
- || INSECURE_URI.matcher(newPath).matches()) {
- return null;
- }
-
- return newPath;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java
deleted file mode 100644
index 3c941d9c7f..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/SystemAction.java
+++ /dev/null
@@ -1,184 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-import org.apache.doris.analysis.RedirectStatus;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.proc.ProcDirInterface;
-import org.apache.doris.common.proc.ProcNodeInterface;
-import org.apache.doris.common.proc.ProcResult;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.MasterOpExecutor;
-import org.apache.doris.qe.OriginStatement;
-import org.apache.doris.qe.ShowResultSet;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-
-import org.apache.commons.validator.routines.UrlValidator;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class SystemAction extends WebBaseAction {
- private static final Logger LOG = LogManager.getLogger(SystemAction.class);
-
- public SystemAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/system", new SystemAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- String currentPath = request.getSingleParameter("path");
- if (Strings.isNullOrEmpty(currentPath)) {
- currentPath = "/";
- }
- appendSystemInfo(response.getContent(), currentPath, currentPath);
-
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- private void appendSystemInfo(StringBuilder buffer, String procPath, String path) {
- buffer.append("<h2>System Info</h2>");
- buffer.append("<p>This page lists the system info, like /proc in Linux.</p>");
- buffer.append("<p class=\"text-info\"> Current path: " + path + "<a href=\"?path=" + getParentPath(path)
- + "\" class=\"btn btn-primary\" style=\"float: right;\">"
- + "Parent Dir</a></p><br/>");
-
- ProcNodeInterface procNode = getProcNode(procPath);
- if (procNode == null) {
- buffer.append("<p class=\"text-error\"> No such proc path[" + path + "]</p>");
- return;
- }
- boolean isDir = (procNode instanceof ProcDirInterface);
-
- List<String> columnNames = null;
- List<List<String>> rows = null;
- if (!Catalog.getCurrentCatalog().isMaster() && !Config.enable_bdbje_debug_mode) {
- // forward to master
- String showProcStmt = "SHOW PROC \"" + procPath + "\"";
- MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0),
- ConnectContext.get(), RedirectStatus.FORWARD_NO_SYNC, true);
- try {
- masterOpExecutor.execute();
- } catch (Exception e) {
- LOG.warn("Fail to forward. ", e);
- buffer.append("<p class=\"text-error\"> Failed to forward request to master</p>");
- return;
- }
-
- ShowResultSet resultSet = masterOpExecutor.getProxyResultSet();
- if (resultSet == null) {
- buffer.append("<p class=\"text-error\"> Failed to get result from master</p>");
- return;
- }
-
- columnNames = resultSet.getMetaData().getColumns().stream().map(c -> c.getName()).collect(
- Collectors.toList());
- rows = resultSet.getResultRows();
- } else {
- ProcResult result;
- try {
- result = procNode.fetchResult();
- } catch (AnalysisException e) {
- buffer.append("<p class=\"text-error\"> The result is null, "
- + "maybe haven't be implemented completely[" + e.getMessage() + "], please check.</p>");
- buffer.append("<p class=\"text-info\"> "
- + "INFO: ProcNode type is [" + procNode.getClass().getName()
- + "]</p>");
- return;
- }
-
- columnNames = result.getColumnNames();
- rows = result.getRows();
- }
-
- Preconditions.checkNotNull(columnNames);
- Preconditions.checkNotNull(rows);
-
- appendTableHeader(buffer, columnNames);
- appendSystemTableBody(buffer, rows, isDir, path);
- appendTableFooter(buffer);
- }
-
- private void appendSystemTableBody(StringBuilder buffer, List<List<String>> rows, boolean isDir, String path) {
- UrlValidator validator = new UrlValidator();
- for ( List<String> strList : rows) {
- buffer.append("<tr>");
- int columnIndex = 1;
- for (String str : strList) {
- buffer.append("<td>");
- if (isDir && columnIndex == 1) {
- String escapeStr = str.replace("%", "%25");
- buffer.append("<a href=\"?path=" + path + "/" + escapeStr + "\">");
- buffer.append(str);
- buffer.append("</a>");
- } else if (validator.isValid(str)) {
- buffer.append("<a href=\"" + str + "\">");
- buffer.append("URL");
- buffer.append("</a>");
- } else {
- buffer.append(str.replaceAll("\\n", "<br/>"));
- }
- buffer.append("</td>");
- ++columnIndex;
- }
- buffer.append("</tr>");
- }
- }
-
- // some example:
- // '/' => '/'
- // '///aaa' => '///'
- // '/aaa/bbb///' => '/aaa'
- // '/aaa/bbb/ccc' => '/aaa/bbb'
- // ATTN: the root path's parent is itself.
- private String getParentPath(String path) {
- int lastSlashIndex = path.length() - 1;
- while (lastSlashIndex > 0) {
- int tempIndex = path.lastIndexOf('/', lastSlashIndex);
- if (tempIndex > 0) {
- if (tempIndex == lastSlashIndex) {
- lastSlashIndex = tempIndex - 1;
- continue;
- } else if (tempIndex < lastSlashIndex) { // '//aaa/bbb'
- lastSlashIndex = tempIndex;
- return path.substring(0, lastSlashIndex);
- }
- }
- }
- return "/";
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java
deleted file mode 100644
index 0df3428f6f..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/VariableAction.java
+++ /dev/null
@@ -1,84 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.analysis.SetType;
-import org.apache.doris.common.Config;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.VariableMgr;
-
-import com.google.common.collect.Lists;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-
-public class VariableAction extends WebBaseAction {
-
- public VariableAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/variable", new VariableAction(controller));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- getPageHeader(request, response.getContent());
-
- appendConfigureInfo(response.getContent());
- appendVariableInfo(response.getContent());
-
- getPageFooter(response.getContent());
- writeResponse(request, response);
- }
-
- public void appendConfigureInfo(StringBuilder buffer) {
- buffer.append("<h2>Configure Info</h2>");
- buffer.append("<pre>");
- HashMap<String, String> confmap;
- try {
- confmap = Config.dump();
- List<String> keyList = Lists.newArrayList(confmap.keySet());
- Collections.sort(keyList);
- for (String key : keyList) {
- buffer.append(key + "=" + confmap.get(key) + "\n");
- }
- } catch (Exception e) {
- buffer.append("read conf exception" + e.toString());
- }
- buffer.append("</pre>");
- }
-
- private void appendVariableInfo(StringBuilder buffer) {
- buffer.append("<h2>Variable Info</h2>");
- buffer.append("<pre>");
- List<List<String>> variableInfo = VariableMgr.dump(SetType.GLOBAL, null, null);
- for (List<String> list : variableInfo) {
- buffer.append(list.get(0) + "=" + list.get(1) + "\n");
- }
- buffer.append("</pre>");
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java
deleted file mode 100644
index e710d68cfe..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/action/WebBaseAction.java
+++ /dev/null
@@ -1,374 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.action;
-
-import org.apache.doris.analysis.CompoundPredicate.Operator;
-import org.apache.doris.analysis.UserIdentity;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.proc.ProcNodeInterface;
-import org.apache.doris.common.proc.ProcService;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseAction;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.HttpAuthManager;
-import org.apache.doris.http.HttpAuthManager.SessionValue;
-import org.apache.doris.http.UnauthorizedException;
-import org.apache.doris.http.rest.RestBaseResult;
-import org.apache.doris.mysql.privilege.PaloPrivilege;
-import org.apache.doris.mysql.privilege.PrivBitSet;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.system.SystemInfoService;
-
-import com.google.common.base.Strings;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.List;
-import java.util.UUID;
-
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http.cookie.DefaultCookie;
-
-public class WebBaseAction extends BaseAction {
- private static final Logger LOG = LogManager.getLogger(WebBaseAction.class);
-
- protected static final String PALO_SESSION_ID = "PALO_SESSION_ID";
- private static final long PALO_SESSION_EXPIRED_TIME = 3600 * 24; // one day
-
- protected static final String PAGE_HEADER = "<!DOCTYPE html>"
- + "<html>"
- + "<head>"
- + " <title>Apache Doris(Incubating)</title>"
- + " <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" >"
- + " <link rel=\"shortcut icon\" href=\"/static/images?res=favicon.ico\">"
- + " <link href=\"/static/css?res=Bootstrap-3.3.7/css/bootstrap.css\" "
- + " rel=\"stylesheet\" media=\"screen\"/>"
- + " <link href=\"/static/css?res=Bootstrap-3.3.7/css/bootstrap-theme.css\" "
- + " rel=\"stylesheet\" media=\"screen\"/>"
- + " <link href=\"/static/css?res=DataTables-1.10.25/css/dataTables.bootstrap.css\" "
- + " rel=\"stylesheet\" media=\"screen\"/>"
-
- + " <script type=\"text/javascript\" src=\"/static?res=jQuery-3.3.1/jquery-3.3.1.min.js\"></script>"
- + " <script type=\"text/javascript\" src=\"/static?res=DataTables-1.10.25/js/jquery.dataTables.js\"></script>"
- + " <script type=\"text/javascript\" src=\"/static?res=DataTables-1.10.25/js/dataTables.bootstrap.js\"></script>"
-
- + " <script type=\"text/javascript\"> "
- + " $(document).ready(function() { "
- + " $('#table_id').dataTable({ "
- + " \"aaSorting\": [],"
- + " \"lengthMenu\": [[10, 25, 50, 100,-1], [10, 25, 50, 100, \"All\"]],"
- + " \"iDisplayLength\": 50,"
- + " });"
- + " }); "
- + " $(document).ready(function () {"
- + " var location = window.location.pathname;"
- + " var id = location.substring(location.lastIndexOf('/') + 1);"
- + " if (id != null || $.trim(id) != \"\") {"
- + " $(\"#nav_\" + id).addClass(\"active\");"
- + " }"
- + " });"
- + " </script> "
-
- + " <style>"
- + " body {"
- + " padding-top: 60px;"
- + " }"
- + " </style>"
- + "</head>"
- + "<body>";
- protected static final String PAGE_FOOTER = "</div></body></html>";
- protected static final String NAVIGATION_BAR_PREFIX =
- " <nav class=\"navbar navbar-default navbar-fixed-top\" role=\"navigation\">"
- + " <div class=\"container-fluid\">"
- + " <div class=\"navbar-header\">"
- + " <a class=\"navbar-brand\" href=\"/\" style=\"padding: unset;\">"
- + " <img alt=\"Doris\" style=\"height: inherit;\" src=\"/static/images?res=doris-logo.png\">"
- + " </a>"
- + " </div>"
- + " <div>"
- + " <ul class=\"nav navbar-nav\" role=\"tablist\">";
- protected static final String NAVIGATION_BAR_SUFFIX =
- " </ul>"
- + " </div>"
- + " </nav>"
- + " <div class=\"container\">";
-
- public WebBaseAction(ActionController controller) {
- super(controller);
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- if (!checkAuthWithCookie(request, response)) {
- return;
- }
-
- HttpMethod method = request.getRequest().method();
- if (method.equals(HttpMethod.GET)) {
- executeGet(request, response);
- } else if (method.equals(HttpMethod.POST)) {
- executePost(request, response);
- } else {
- response.appendContent(new RestBaseResult("HTTP method is not allowed: " + method.name()).toJson());
- writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
- }
- }
-
- // Sub Action class should override this method
- public void executeGet(BaseRequest request, BaseResponse response) {
- response.appendContent(new RestBaseResult("Not implemented").toJson());
- writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED);
- }
-
- // Sub Action class should override this method
- public void executePost(BaseRequest request, BaseResponse response) {
- response.appendContent(new RestBaseResult("Not implemented").toJson());
- writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED);
- }
-
- // We first check cookie, if not admin, we check http's authority header
- private boolean checkAuthWithCookie(BaseRequest request, BaseResponse response) {
- if (!needPassword()) {
- return true;
- }
-
- if (checkCookie(request, response)) {
- return true;
- }
-
- // cookie is invalid.
- ActionAuthorizationInfo authInfo;
- try {
- authInfo = getAuthorizationInfo(request);
- UserIdentity currentUser = checkPassword(authInfo);
- if (needAdmin()) {
- checkGlobalAuth(currentUser, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV,
- PaloPrivilege.NODE_PRIV), Operator.OR));
- }
- request.setAuthorized(true);
- SessionValue value = new SessionValue();
- value.currentUser = currentUser;
- addSession(request, response, value);
-
- ConnectContext ctx = new ConnectContext(null);
- ctx.setQualifiedUser(authInfo.fullUserName);
- ctx.setRemoteIP(authInfo.remoteIp);
- ctx.setCurrentUserIdentity(currentUser);
- ctx.setCatalog(Catalog.getCurrentCatalog());
- ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER);
- ctx.setThreadLocalInfo();
-
- return true;
- } catch (UnauthorizedException e) {
- response.appendContent("Authentication Failed. <br/> " + e.getMessage());
- writeAuthResponse(request, response);
- return false;
- }
- }
-
- private boolean checkCookie(BaseRequest request, BaseResponse response) {
- String sessionId = request.getCookieValue(PALO_SESSION_ID);
- HttpAuthManager authMgr = HttpAuthManager.getInstance();
- if (!Strings.isNullOrEmpty(sessionId)) {
- SessionValue sessionValue = authMgr.getSessionValue(sessionId);
- if (sessionValue == null) {
- return false;
- }
- if (Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(sessionValue.currentUser,
- PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV,
- PaloPrivilege.NODE_PRIV),
- Operator.OR))) {
- response.updateCookieAge(request, PALO_SESSION_ID, PALO_SESSION_EXPIRED_TIME);
- request.setAuthorized(true);
-
- ConnectContext ctx = new ConnectContext(null);
- ctx.setQualifiedUser(sessionValue.currentUser.getQualifiedUser());
- ctx.setRemoteIP(request.getHostString());
- ctx.setCurrentUserIdentity(sessionValue.currentUser);
- ctx.setCatalog(Catalog.getCurrentCatalog());
- ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER);
- ctx.setThreadLocalInfo();
- return true;
- }
- }
- return false;
- }
-
- // return true if this Action need to check password.
- // Currently, all sub actions need to check password except for MetaBaseAction.
- // if needPassword() is false, then needAdmin() should also return false
- public boolean needPassword() {
- return true;
- }
-
- // return true if this Action need Admin privilege.
- public boolean needAdmin() {
- return true;
- }
-
- protected void writeAuthResponse(BaseRequest request, BaseResponse response) {
- response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\"");
- writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED);
- }
-
- protected void writeResponse(BaseRequest request, BaseResponse response) {
- writeResponse(request, response, HttpResponseStatus.OK);
- }
-
- protected void addSession(BaseRequest request, BaseResponse response, SessionValue value) {
- String key = UUID.randomUUID().toString();
- DefaultCookie cookie = new DefaultCookie(PALO_SESSION_ID, key);
- cookie.setMaxAge(PALO_SESSION_EXPIRED_TIME);
- response.addCookie(cookie);
- HttpAuthManager.getInstance().addSessionValue(key, value);
- }
-
- protected void getPageHeader(BaseRequest request, StringBuilder sb) {
- String newPageHeaderString = PAGE_HEADER;
- newPageHeaderString = newPageHeaderString.replaceAll("<title>Apache Doris</title>",
- "<title>" + Config.cluster_name + "</title>");
-
- sb.append(newPageHeaderString);
- sb.append(NAVIGATION_BAR_PREFIX);
-
- if (request.isAuthorized()) {
- sb.append("<li id=\"nav_system\"><a href=\"/system\">")
- .append("system")
- .append("</a></li>");
- sb.append("<li id=\"nav_backend\"><a href=\"/backend\">")
- .append("backends")
- .append("</a></li>");
- sb.append("<li id=\"nav_log\"><a href=\"/log\">")
- .append("logs")
- .append("</a></li>");
- sb.append("<li id=\"nav_query\"><a href=\"/query\">")
- .append("queries")
- .append("</a></li>");
- sb.append("<li id=\"nav_session\"><a href=\"/session\">")
- .append("sessions")
- .append("</a></li>");
- sb.append("<li id=\"nav_variable\"><a href=\"/variable\">")
- .append("variables")
- .append("</a></li>");
- sb.append("<li id=\"nav_ha\"><a href=\"/ha\">")
- .append("ha")
- .append("</a></li>");
- }
- sb.append("<li id=\"nav_help\"><a href=\"/help\">")
- .append("help")
- .append("</a></li></tr>");
-
- sb.append(NAVIGATION_BAR_SUFFIX);
- }
-
- protected void getPageFooter(StringBuilder sb) {
- sb.append(PAGE_FOOTER);
- }
-
- // Note: DO NOT omit '<thead>', because it is useful in 'datatable' plugin.
- protected void appendTableHeader(StringBuilder buffer, List<String> columnHeaders) {
- buffer.append("<table id=\"table_id\" "
- + "class=\"table table-hover table-bordered table-striped table-condensed\">");
- buffer.append("<thead><tr> ");
- for (String str : columnHeaders) {
- buffer.append("<th>" + str + "</th>");
- }
- buffer.append(" </tr></thead>");
- }
-
- protected void appendTableBody(StringBuilder buffer, List<List<String>> bodies) {
- buffer.append("<tbody>");
- for ( List<String> row : bodies) {
- buffer.append("<tr>");
- for (String column : row) {
- buffer.append("<td>");
- buffer.append(column);
- buffer.append("</td>");
- }
- buffer.append("</tr>");
- }
- buffer.append("</tbody>");
- }
-
- protected void appendTableFooter(StringBuilder buffer) {
- buffer.append("</table>");
- }
-
- protected ProcNodeInterface getProcNode(String path) {
- ProcService instance = ProcService.getInstance();
- ProcNodeInterface node;
- try {
- if (Strings.isNullOrEmpty(path)) {
- node = instance.open("/");
- } else {
- node = instance.open(path);
- }
- } catch (AnalysisException e) {
- LOG.warn(e.getMessage());
- return null;
- }
- return node;
- }
-
- // Because org.apache.commons.lang.StringEscapeUtils.escapeHtml() not only escape tags in html,
- // but also escape Chinese character code, which may cause Chinese garbled in browser, so we
- // define our own simplified escape method here.
- // ATTN: we should make sure file-encoding of help files is utf-8
- protected String escapeHtmlInPreTag(String oriStr) {
- if (oriStr == null) {
- return "";
- }
-
- StringBuilder buff = new StringBuilder();
- char[] chars = oriStr.toCharArray();
- for (int i = 0; i < chars.length; ++i) {
- switch (chars[i]) {
- case '<':
- buff.append("<");
- break;
- case '>':
- buff.append("<");
- break;
- case '"':
- buff.append(""");
- break;
- case '&':
- buff.append("&");
- break;
- default:
- buff.append(chars[i]);
- break;
- }
- }
-
- return buff.toString();
- }
-
- private static final NotFoundAction NOT_FOUND_ACTION = new NotFoundAction(null);
-
- public static NotFoundAction getNotFoundAction() {
- return NOT_FOUND_ACTION;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java b/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java
deleted file mode 100644
index d904c421a3..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/common/DorisHttpPostObjectAggregator.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.common;
-
-import io.netty.channel.ChannelPipeline;
-import io.netty.handler.codec.http.HttpContent;
-import io.netty.handler.codec.http.HttpMessage;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpObject;
-import io.netty.handler.codec.http.HttpObjectAggregator;
-import io.netty.handler.codec.http.HttpRequest;
-
-/*
- * only handle post request, avoid conflicting with {@link LoadAction}
- * don't handle 100-continue header
- */
-public class DorisHttpPostObjectAggregator extends HttpObjectAggregator {
- private boolean startAggregated = false;
-
- public DorisHttpPostObjectAggregator(int maxContentLength) {
- super(maxContentLength, false);
- }
-
- @Override
- protected boolean isStartMessage(HttpObject msg) throws Exception {
- if (msg instanceof HttpMessage) {
- HttpRequest request = (HttpRequest) msg;
- if (request.method().equals(HttpMethod.POST)) {
- startAggregated = true;
- return true;
- }
- }
- return false;
- }
-
- @Override
- protected boolean isContentMessage(HttpObject msg) throws Exception {
- return msg instanceof HttpContent && startAggregated;
- }
-
- // Doris FE needn't handle 100-continue header
- @Override
- protected Object newContinueResponse(HttpMessage start, int maxContentLength, ChannelPipeline pipeline) {
- return null;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java
deleted file mode 100644
index 42b713d13c..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/ColocateMetaService.java
+++ /dev/null
@@ -1,229 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.meta;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.ColocateGroupSchema;
-import org.apache.doris.catalog.ColocateTableIndex;
-import org.apache.doris.catalog.ColocateTableIndex.GroupId;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.http.rest.RestBaseAction;
-import org.apache.doris.http.rest.RestBaseResult;
-import org.apache.doris.http.rest.RestResult;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.lang.reflect.Type;
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/*
- * the colocate meta define in {@link ColocateTableIndex}
- * The actions in ColocateMetaService is for modifying or showing colocate group info manually.
- *
- * ColocateMetaAction:
- * get all information in ColocateTableIndex, as a json string
- * eg:
- * GET /api/colocate
- * return:
- * {"colocate_meta":{"groupName2Id":{...},"group2Tables":{}, ...},"status":"OK"}
- *
- * eg:
- * POST /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as unstable)
- * DELETE /api/colocate/group_stable?db_id=123&group_id=456 (mark group[123.456] as stable)
- *
- * BucketSeqAction:
- * change the backends per bucket sequence of a group
- * eg:
- * POST /api/colocate/bucketseq?db_id=123&group_id=456
- */
-public class ColocateMetaService {
- private static final Logger LOG = LogManager.getLogger(ColocateMetaService.class);
- private static final String GROUP_ID = "group_id";
- private static final String TABLE_ID = "table_id";
- private static final String DB_ID = "db_id";
-
- private static ColocateTableIndex colocateIndex = Catalog.getCurrentColocateIndex();
-
- private static GroupId checkAndGetGroupId(BaseRequest request) throws DdlException {
- long grpId = Long.valueOf(request.getSingleParameter(GROUP_ID).trim());
- long dbId = Long.valueOf(request.getSingleParameter(DB_ID).trim());
- GroupId groupId = new GroupId(dbId, grpId);
-
- if (!colocateIndex.isGroupExist(groupId)) {
- throw new DdlException("the group " + groupId + "isn't exist");
- }
- return groupId;
- }
-
- public static class ColocateMetaBaseAction extends RestBaseAction {
- ColocateMetaBaseAction(ActionController controller) {
- super(controller);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- if (redirectToMaster(request, response)) {
- return;
- }
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
- executeInMasterWithAdmin(request, response);
- }
-
- // implement in derived classes
- protected void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
- throws DdlException {
- throw new DdlException("Not implemented");
- }
- }
-
- // get all colocate meta
- public static class ColocateMetaAction extends ColocateMetaBaseAction {
- ColocateMetaAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ColocateMetaAction action = new ColocateMetaAction(controller);
- controller.registerHandler(HttpMethod.GET, "/api/colocate", action);
- }
-
- @Override
- public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
- throws DdlException {
- response.setContentType("application/json");
- RestResult result = new RestResult();
- result.addResultEntry("colocate_meta", Catalog.getCurrentColocateIndex());
- sendResult(request, response, result);
- }
- }
-
- // mark a colocate group as stable or unstable
- public static class MarkGroupStableAction extends ColocateMetaBaseAction {
- MarkGroupStableAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- MarkGroupStableAction action = new MarkGroupStableAction(controller);
- controller.registerHandler(HttpMethod.POST, "/api/colocate/group_stable", action);
- controller.registerHandler(HttpMethod.DELETE, "/api/colocate/group_stable", action);
- }
-
- @Override
- public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
- throws DdlException {
- GroupId groupId = checkAndGetGroupId(request);
-
- HttpMethod method = request.getRequest().method();
- if (method.equals(HttpMethod.POST)) {
- colocateIndex.markGroupUnstable(groupId, "mark unstable via http api", true);
- } else if (method.equals(HttpMethod.DELETE)) {
- colocateIndex.markGroupStable(groupId, true);
- } else {
- response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson());
- writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
- }
-
- sendResult(request, response);
- }
- }
-
- // update a backendsPerBucketSeq meta for a colocate group
- public static class BucketSeqAction extends ColocateMetaBaseAction {
- private static final Logger LOG = LogManager.getLogger(BucketSeqAction.class);
-
- BucketSeqAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- BucketSeqAction action = new BucketSeqAction(controller);
- controller.registerHandler(HttpMethod.POST, "/api/colocate/bucketseq", action);
- }
-
- @Override
- public void executeInMasterWithAdmin(BaseRequest request, BaseResponse response)
- throws DdlException {
- final String clusterName = ConnectContext.get().getClusterName();
- if (Strings.isNullOrEmpty(clusterName)) {
- throw new DdlException("No cluster selected.");
- }
- GroupId groupId = checkAndGetGroupId(request);
-
- String meta = request.getContent();
- Type type = new TypeToken<List<List<Long>>>() {}.getType();
- List<List<Long>> backendsPerBucketSeq = new Gson().fromJson(meta, type);
- LOG.info("get buckets sequence: {}", backendsPerBucketSeq);
-
- ColocateGroupSchema groupSchema = Catalog.getCurrentColocateIndex().getGroupSchema(groupId);
- if (backendsPerBucketSeq.size() != groupSchema.getBucketsNum()) {
- throw new DdlException("Invalid bucket num. expected: " + groupSchema.getBucketsNum() + ", actual: "
- + backendsPerBucketSeq.size());
- }
-
- List<Long> clusterBackendIds = Catalog.getCurrentSystemInfo().getClusterBackendIds(clusterName, true);
- //check the Backend id
- for (List<Long> backendIds : backendsPerBucketSeq) {
- if (backendIds.size() != groupSchema.getReplicaAlloc().getTotalReplicaNum()) {
- throw new DdlException("Invalid backend num per bucket. expected: "
- + groupSchema.getReplicaAlloc().getTotalReplicaNum() + ", actual: " + backendIds.size());
- }
- for (Long beId : backendIds) {
- if (!clusterBackendIds.contains(beId)) {
- throw new DdlException("The backend " + beId + " does not exist or not available");
- }
- }
- }
-
- int bucketsNum = colocateIndex.getBackendsPerBucketSeq(groupId).size();
- Preconditions.checkState(backendsPerBucketSeq.size() == bucketsNum,
- backendsPerBucketSeq.size() + " vs. " + bucketsNum);
- updateBackendPerBucketSeq(groupId, backendsPerBucketSeq);
- LOG.info("the group {} backendsPerBucketSeq meta has been changed to {}", groupId, backendsPerBucketSeq);
-
- sendResult(request, response);
- }
-
- private void updateBackendPerBucketSeq(GroupId groupId, List<List<Long>> backendsPerBucketSeq)
- throws DdlException {
- throw new DdlException("Currently not support");
- /*
- colocateIndex.addBackendsPerBucketSeq(groupId, backendsPerBucketSeq);
- ColocatePersistInfo info2 = ColocatePersistInfo.createForBackendsPerBucketSeq(groupId, backendsPerBucketSeq);
- Catalog.getCurrentCatalog().getEditLog().logColocateBackendsPerBucketSeq(info2);
- */
- }
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java
deleted file mode 100644
index 6c5d9dbcf0..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/InvalidClientException.java
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.meta;
-
-import org.apache.doris.common.DdlException;
-
-public class InvalidClientException extends DdlException {
- public InvalidClientException(String msg) {
- super(msg);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java
deleted file mode 100644
index b8d8843420..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaBaseAction.java
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.meta;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.action.WebBaseAction;
-import org.apache.doris.master.MetaHelper;
-import org.apache.doris.system.Frontend;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.File;
-
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-public class MetaBaseAction extends WebBaseAction {
- private static final Logger LOG = LogManager.getLogger(MetaBaseAction.class);
- private static String CONTENT_DISPOSITION = "Content-disposition";
-
- public static final String CLUSTER_ID = "cluster_id";
- public static final String TOKEN = "token";
-
- protected File imageDir;
-
- public MetaBaseAction(ActionController controller, File imageDir) {
- super(controller);
- this.imageDir = imageDir;
- }
-
- @Override
- public boolean needAdmin() {
- return false;
- }
-
- @Override
- public boolean needPassword() {
- return false;
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- if (needCheckClientIsFe()) {
- try {
- checkFromValidFe(request, response);
- } catch (InvalidClientException e) {
- response.appendContent("invalid client host.");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- }
- super.execute(request, response);
- }
-
- protected boolean needCheckClientIsFe() {
- return true;
- }
-
- protected void writeFileResponse(BaseRequest request, BaseResponse response, File file) {
- if (file == null || !file.exists()) {
- response.appendContent("File does not exist.");
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- }
-
- // add custom header
- response.updateHeader(CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
- response.updateHeader(MetaHelper.X_IMAGE_SIZE, String.valueOf(file.length()));
-
- writeObjectResponse(request, response, HttpResponseStatus.OK, file, file.getName(), true);
- return;
- }
-
- private boolean isFromValidFe(BaseRequest request) {
- String clientHost = request.getHostString();
- Frontend fe = Catalog.getCurrentCatalog().getFeByHost(clientHost);
- if (fe == null) {
- LOG.warn("request is not from valid FE. client: {}", clientHost);
- return false;
- }
- return true;
- }
-
- private void checkFromValidFe(BaseRequest request, BaseResponse response)
- throws InvalidClientException {
- if (!isFromValidFe(request)) {
- throw new InvalidClientException("invalid client host");
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java
deleted file mode 100644
index a707900035..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/meta/MetaService.java
+++ /dev/null
@@ -1,356 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.meta;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.Config;
-import org.apache.doris.ha.FrontendNodeType;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.master.MetaHelper;
-import org.apache.doris.persist.MetaCleaner;
-import org.apache.doris.persist.Storage;
-import org.apache.doris.persist.StorageInfo;
-import org.apache.doris.system.Frontend;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-public class MetaService {
- private static final int TIMEOUT_SECOND = 10;
-
- public static class ImageAction extends MetaBaseAction {
- private static final String VERSION = "version";
-
- public ImageAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction(ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/image", new ImageAction(controller, imageDir));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- String versionStr = request.getSingleParameter(VERSION);
- if (Strings.isNullOrEmpty(versionStr)) {
- response.appendContent("Miss version parameter");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- long version = checkLongParam(versionStr);
- if (version < 0) {
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- File imageFile = Storage.getImageFile(imageDir, version);
- if (!imageFile.exists()) {
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- }
-
- writeFileResponse(request, response, imageFile);
- }
- }
-
- public static class InfoAction extends MetaBaseAction {
- private static final Logger LOG = LogManager.getLogger(InfoAction.class);
-
- public InfoAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction (ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/info", new InfoAction(controller, imageDir));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- try {
- Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath());
- StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(),
- currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq());
-
- response.setContentType("application/json");
- Gson gson = new Gson();
- response.appendContent(gson.toJson(storageInfo));
- writeResponse(request, response);
- return;
- } catch (IOException e) {
- LOG.warn("IO error.", e);
- response.appendContent("failed to get master info.");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- }
- }
-
- public static class VersionAction extends MetaBaseAction {
- public VersionAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction (ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/version", new VersionAction(controller, imageDir));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- File versionFile = new File(imageDir, Storage.VERSION_FILE);
- writeFileResponse(request, response, versionFile);
- }
- }
-
- public static class PutAction extends MetaBaseAction {
- private static final Logger LOG = LogManager.getLogger(PutAction.class);
-
- private static final String VERSION = "version";
- private static final String PORT = "port";
-
- public PutAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction (ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/put", new PutAction(controller, imageDir));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- String machine = request.getHostString();
- String portStr = request.getSingleParameter(PORT);
- // check port to avoid SSRF(Server-Side Request Forgery)
- if (Strings.isNullOrEmpty(portStr)) {
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- {
- int port = Integer.parseInt(portStr);
- if (port < 0 || port > 65535) {
- LOG.warn("port is invalid. port={}", port);
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- }
-
- String versionStr = request.getSingleParameter(VERSION);
- if (Strings.isNullOrEmpty(versionStr)) {
- response.appendContent("Miss version parameter");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- long version = checkLongParam(versionStr);
-
- // for master node, reject image put
- if (Catalog.getCurrentCatalog().isMaster()) {
- response.appendContent("this node is master, reject image put");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- LOG.error("this node is master, but receive image put from host {}, reject it", machine);
- return;
- }
-
- // do not accept image whose version is bigger than max journalId
- // if accepted, newly added log will not be replayed when restart
- long maxJournalId = Catalog.getCurrentCatalog().getMaxJournalId();
- if (version > maxJournalId) {
- response.appendContent("image version [" + version + "] is bigger than local max journal id ["
- + maxJournalId + "], reject image put");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- LOG.error("receive image whose version [{}] is bigger than local max journal id [{}], reject it",
- version, maxJournalId);
- return;
- }
-
- String url = "http://" + machine + ":" + portStr
- + "/image?version=" + versionStr;
- String filename = Storage.IMAGE + "." + versionStr;
-
- File dir = new File(Catalog.getCurrentCatalog().getImageDir());
- try {
- OutputStream out = MetaHelper.getOutputStream(filename, dir);
- MetaHelper.getRemoteFile(url, TIMEOUT_SECOND * 1000, out);
- MetaHelper.complete(filename, dir);
- writeResponse(request, response);
- } catch (FileNotFoundException e) {
- LOG.warn("file not found. file: {}", filename, e);
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- } catch (IOException e) {
- LOG.warn("failed to get remote file. url: {}", url, e);
- writeResponse(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
- return;
- }
-
- // Delete old image files
- MetaCleaner cleaner = new MetaCleaner(Config.meta_dir + "/image");
- try {
- cleaner.clean();
- } catch (IOException e) {
- LOG.error("Follower/Observer delete old image file fail.", e);
- }
- }
- }
-
- public static class JournalIdAction extends MetaBaseAction {
- public JournalIdAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction (ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/journal_id", new JournalIdAction(controller, imageDir));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- long id = Catalog.getCurrentCatalog().getReplayedJournalId();
- response.updateHeader("id", Long.toString(id));
- writeResponse(request, response);
- }
- }
-
- public static class RoleAction extends MetaBaseAction {
- private static final String HOST = "host";
- private static final String PORT = "port";
-
- public RoleAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction (ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/role", new RoleAction(controller, imageDir));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- String host = request.getSingleParameter(HOST);
- String portString = request.getSingleParameter(PORT);
-
- if (!Strings.isNullOrEmpty(host) && !Strings.isNullOrEmpty(portString)) {
- int port = Integer.parseInt(portString);
- Frontend fe = Catalog.getCurrentCatalog().checkFeExist(host, port);
- if (fe == null) {
- response.updateHeader("role", FrontendNodeType.UNKNOWN.name());
- } else {
- response.updateHeader("role", fe.getRole().name());
- response.updateHeader("name", fe.getNodeName());
- }
- writeResponse(request, response);
- } else {
- response.appendContent("Miss parameter");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- }
- }
-
- /*
- * This action is used to get the electable_nodes config and the cluster id of
- * the fe with the given ip and port. When one frontend start, it should check
- * the local electable_nodes config and local cluster id with other frontends.
- * If there is any difference, local fe will exit. This is designed to protect
- * the consistency of the cluster.
- */
- public static class CheckAction extends MetaBaseAction {
- private static final Logger LOG = LogManager.getLogger(CheckAction.class);
-
- public CheckAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction(ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/check",
- new CheckAction(controller, imageDir));
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- try {
- Storage storage = new Storage(imageDir.getAbsolutePath());
- response.updateHeader(MetaBaseAction.CLUSTER_ID, Integer.toString(storage.getClusterID()));
- response.updateHeader(MetaBaseAction.TOKEN, storage.getToken());
- } catch (IOException e) {
- LOG.error(e);
- }
- writeResponse(request, response);
- }
- }
-
- public static class DumpAction extends MetaBaseAction {
- private static final Logger LOG = LogManager.getLogger(CheckAction.class);
-
- public DumpAction(ActionController controller, File imageDir) {
- super(controller, imageDir);
- }
-
- public static void registerAction (ActionController controller, File imageDir)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/dump", new DumpAction(controller, imageDir));
- }
-
- @Override
- public boolean needAdmin() {
- return true;
- }
-
- @Override
- protected boolean needCheckClientIsFe() {
- return false;
- }
-
- @Override
- public void executeGet(BaseRequest request, BaseResponse response) {
- /*
- * Before dump, we acquired the catalog read lock and all databases' read lock and all
- * the jobs' read lock. This will guarantee the consistency of database and job queues.
- * But Backend may still inconsistent.
- */
-
- // TODO: Still need to lock ClusterInfoService to prevent add or drop Backends
- String dumpFilePath = Catalog.getCurrentCatalog().dumpImage();
- if (dumpFilePath == null) {
- response.appendContent("dump failed. " + dumpFilePath);
- }
-
- response.appendContent("dump finished. " + dumpFilePath);
- writeResponse(request, response);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java
deleted file mode 100644
index 7af15429ee..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/BootstrapFinishAction.java
+++ /dev/null
@@ -1,162 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-
-import io.netty.handler.codec.http.HttpMethod;
-import org.apache.doris.common.Version;
-
-/*
- * fe_host:fe_http_port/api/bootstrap
- * return:
- * {"status":"OK","msg":"Success","replayedJournal"=123456, "queryPort"=9000, "rpcPort"=9001}
- * {"status":"FAILED","msg":"err info..."}
- */
-public class BootstrapFinishAction extends RestBaseAction {
- private static final String CLUSTER_ID = "cluster_id";
- private static final String TOKEN = "token";
-
- public static final String REPLAYED_JOURNAL_ID = "replayedJournalId";
- public static final String QUERY_PORT = "queryPort";
- public static final String RPC_PORT = "rpcPort";
- public static final String VERSION = "version";
-
- public BootstrapFinishAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/bootstrap", new BootstrapFinishAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) throws DdlException {
- boolean isReady = Catalog.getCurrentCatalog().isReady();
-
- // to json response
- BootstrapResult result = null;
- if (isReady) {
- result = new BootstrapResult();
- String clusterIdStr = request.getSingleParameter(CLUSTER_ID);
- String token = request.getSingleParameter(TOKEN);
- if (!Strings.isNullOrEmpty(clusterIdStr) && !Strings.isNullOrEmpty(token)) {
- // cluster id or token is provided, return more info
- int clusterId = 0;
- try {
- clusterId = Integer.valueOf(clusterIdStr);
- } catch (NumberFormatException e) {
- result.status = ActionStatus.FAILED;
- result.msg = "invalid cluster id format: " + clusterIdStr;
- }
-
- if (result.status == ActionStatus.OK) {
- if (clusterId != Catalog.getCurrentCatalog().getClusterId()) {
- result.status = ActionStatus.FAILED;
- result.msg = "invalid cluster id: " + Catalog.getCurrentCatalog().getClusterId();
- }
- }
-
- if (result.status == ActionStatus.OK) {
- if (!token.equals(Catalog.getCurrentCatalog().getToken())) {
- result.status = ActionStatus.FAILED;
- result.msg = "invalid token: " + Catalog.getCurrentCatalog().getToken();
- }
- }
-
- if (result.status == ActionStatus.OK) {
- // cluster id and token are valid, return replayed journal id
- long replayedJournalId = Catalog.getCurrentCatalog().getReplayedJournalId();
- result.setMaxReplayedJournal(replayedJournalId);
- result.setQueryPort(Config.query_port);
- result.setRpcPort(Config.rpc_port);
- result.setVersion(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH);
- }
- }
- } else {
- result = new BootstrapResult("not ready");
- }
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result.toJson());
- sendResult(request, response);
- }
-
- public static class BootstrapResult extends RestBaseResult {
- private long replayedJournalId = 0;
- private int queryPort = 0;
- private int rpcPort = 0;
- private String version = "";
-
- public BootstrapResult() {
- super();
- }
-
- public BootstrapResult(String msg) {
- super(msg);
- }
-
- public void setMaxReplayedJournal(long replayedJournalId) {
- this.replayedJournalId = replayedJournalId;
- }
-
- public long getMaxReplayedJournal() {
- return replayedJournalId;
- }
-
- public void setQueryPort(int queryPort) {
- this.queryPort = queryPort;
- }
-
- public int getQueryPort() {
- return queryPort;
- }
-
- public void setRpcPort(int rpcPort) {
- this.rpcPort = rpcPort;
- }
-
- public int getRpcPort() {
- return rpcPort;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- @Override
- public String toJson() {
- Gson gson = new Gson();
- return gson.toJson(this);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java
deleted file mode 100644
index 23f12f441a..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CancelStreamLoad.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.UserException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class CancelStreamLoad extends RestBaseAction {
- public CancelStreamLoad(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller)
- throws IllegalArgException {
- CancelStreamLoad action = new CancelStreamLoad(controller);
- controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_cancel", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
-
- if (redirectToMaster(request, response)) {
- return;
- }
-
- final String clusterName = ConnectContext.get().getClusterName();
- if (Strings.isNullOrEmpty(clusterName)) {
- throw new DdlException("No cluster selected.");
- }
-
- String dbName = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(dbName)) {
- throw new DdlException("No database selected.");
- }
-
- String fullDbName = ClusterNamespace.getFullName(clusterName, dbName);
-
- String label = request.getSingleParameter(LABEL_KEY);
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected.");
- }
-
- // FIXME(cmy)
- // checkWritePriv(authInfo.fullUserName, fullDbName);
-
- Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName);
-
- try {
- Catalog.getCurrentGlobalTransactionMgr().abortTransaction(db.getId(), label, "user cancel");
- } catch (UserException e) {
- throw new DdlException(e.getMessage());
- }
-
- sendResult(request, response, new RestBaseResult());
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java
deleted file mode 100644
index 10a555c222..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/CheckDecommissionAction.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.alter.SystemHandler;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.Pair;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.system.SystemInfoService;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-/*
- * calc row count from replica to table
- * fe_host:fe_http_port/api/check_decommission?host_ports=host:port,host2:port2...
- * return:
- * {"status":"OK","msg":"Success"}
- * {"status":"FAILED","msg":"err info..."}
- */
-public class CheckDecommissionAction extends RestBaseAction {
- public static final String HOST_PORTS = "host_ports";
-
- public CheckDecommissionAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/check_decommission", new CheckDecommissionAction(controller));
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.OPERATOR);
-
- String hostPorts = request.getSingleParameter(HOST_PORTS);
- if (Strings.isNullOrEmpty(hostPorts)) {
- throw new DdlException("No host:port specified.");
- }
-
- String[] hostPortArr = hostPorts.split(",");
- if (hostPortArr.length == 0) {
- throw new DdlException("No host:port specified.");
- }
-
- List<Pair<String, Integer>> hostPortPairs = Lists.newArrayList();
- for (String hostPort : hostPortArr) {
- Pair<String, Integer> pair;
- try {
- pair = SystemInfoService.validateHostAndPort(hostPort);
- } catch (AnalysisException e) {
- throw new DdlException(e.getMessage());
- }
- hostPortPairs.add(pair);
- }
-
- SystemHandler.checkDecommission(hostPortPairs);
-
- // to json response
- RestBaseResult result = new RestBaseResult();
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result.toJson());
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java
deleted file mode 100644
index 3ac8f288be..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ConnectionAction.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import org.apache.doris.common.util.DebugUtil;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-// This class is used to get current query_id of connection_id.
-// Every connection holds at most one query at every point.
-// Some we can get query_id firstly, and get query by query_id.
-public class ConnectionAction extends RestBaseAction {
- public ConnectionAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/connection", new ConnectionAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- String connStr = request.getSingleParameter("connection_id");
- if (connStr == null) {
- response.getContent().append("not valid parameter");
- sendResult(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- int connectionId = Integer.valueOf(connStr.trim());
- ConnectContext context = ExecuteEnv.getInstance().getScheduler().getContext(connectionId);
- if (context == null || context.queryId() == null) {
- response.getContent().append("connection id " + connectionId + " not found.");
- sendResult(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- }
- String queryId = DebugUtil.printId(context.queryId());
- response.setContentType("application/json");
- response.getContent().append("{\"query_id\" : \"" + queryId + "\"}");
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java
deleted file mode 100644
index e9c23429fa..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetDdlStmtAction.java
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.util.List;
-import java.util.Map;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-/*
- * used to get a table's ddl stmt
- * eg:
- * fe_host:http_port/api/_get_ddl?db=xxx&tbl=yyy
- */
-public class GetDdlStmtAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(GetDdlStmtAction.class);
- private static final String DB_PARAM = "db";
- private static final String TABLE_PARAM = "tbl";
-
- public GetDdlStmtAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- GetDdlStmtAction action = new GetDdlStmtAction(controller);
- controller.registerHandler(HttpMethod.GET, "/api/_get_ddl", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
-
- String dbName = request.getSingleParameter(DB_PARAM);
- String tableName = request.getSingleParameter(TABLE_PARAM);
-
- if (Strings.isNullOrEmpty(dbName) || Strings.isNullOrEmpty(tableName)) {
- throw new DdlException("Missing params. Need database name and Table name");
- }
-
- Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName);
- Table table = db.getTableOrDdlException(tableName);
-
- List<String> createTableStmt = Lists.newArrayList();
- List<String> addPartitionStmt = Lists.newArrayList();
- List<String> createRollupStmt = Lists.newArrayList();
-
- table.readLock();
- try {
- Catalog.getDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt, true, false /* show password */);
- } finally {
- table.readUnlock();
- }
-
- Map<String, List<String>> results = Maps.newHashMap();
- results.put("TABLE", createTableStmt);
- results.put("PARTITION", addPartitionStmt);
- results.put("ROLLUP", createRollupStmt);
-
- // to json response
- String result = "";
- ObjectMapper mapper = new ObjectMapper();
- try {
- result = mapper.writeValueAsString(results);
- } catch (Exception e) {
- // do nothing
- }
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response);
- }
-}
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java
deleted file mode 100644
index b215702817..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLoadInfoAction.java
+++ /dev/null
@@ -1,88 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.MetaNotFoundException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.load.Load;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-// Get load information of one load job
-public class GetLoadInfoAction extends RestBaseAction {
- public GetLoadInfoAction(ActionController controller, boolean isStreamLoad) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller)
- throws IllegalArgException {
- GetLoadInfoAction action = new GetLoadInfoAction(controller, false);
- controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/_load_info", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- Load.JobInfo info = new Load.JobInfo(request.getSingleParameter(DB_KEY),
- request.getSingleParameter(LABEL_KEY),
- ConnectContext.get().getClusterName());
- if (Strings.isNullOrEmpty(info.dbName)) {
- throw new DdlException("No database selected");
- }
- if (Strings.isNullOrEmpty(info.label)) {
- throw new DdlException("No label selected");
- }
- if (Strings.isNullOrEmpty(info.clusterName)) {
- throw new DdlException("No cluster name selected");
- }
-
- if (redirectToMaster(request, response)) {
- return;
- }
-
- try {
- catalog.getLoadInstance().getJobInfo(info);
- if (info.tblNames.isEmpty()) {
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, PrivPredicate.LOAD);
- } else {
- for (String tblName : info.tblNames) {
- checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), info.dbName, tblName,
- PrivPredicate.LOAD);
- }
- }
- } catch (DdlException | MetaNotFoundException e) {
- catalog.getLoadManager().getLoadJobInfo(info);
- }
- sendResult(request, response, new Result(info));
- }
-
- private static class Result extends RestBaseResult {
- private Load.JobInfo jobInfo;
- public Result(Load.JobInfo info) {
- jobInfo = info;
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java
deleted file mode 100644
index 9e80219e64..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetLogFileAction.java
+++ /dev/null
@@ -1,131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.common.Config;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.io.File;
-import java.util.Map;
-import java.util.Set;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/*
- * get log file infos:
- * curl -I http://fe_host:http_port/api/get_log_file?type=fe.audit.log
- * return:
- * HTTP/1.1 200 OK
- * file_infos: {"fe.audit.log":24759,"fe.audit.log.20190528.1":132934}
- * content-type: text/html
- * connection: keep-alive
- *
- * get log file:
- * curl -X GET http://fe_host:http_port/api/get_log_file?type=fe.audit.log&file=fe.audit.log.20190528.1
- */
-public class GetLogFileAction extends RestBaseAction {
- private final Set<String> logFileTypes = Sets.newHashSet("fe.audit.log");
-
- public GetLogFileAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/get_log_file", new GetLogFileAction(controller));
- controller.registerHandler(HttpMethod.HEAD, "/api/get_log_file", new GetLogFileAction(controller));
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response) {
- String logType = request.getSingleParameter("type");
- String logFile = request.getSingleParameter("file");
-
- // check param empty
- if (Strings.isNullOrEmpty(logType)) {
- response.appendContent("Miss type parameter");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- // check type valid or not
- if (!logFileTypes.contains(logType)) {
- response.appendContent("log type: " + logType + " is invalid!");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- HttpMethod method = request.getRequest().method();
- if (method.equals(HttpMethod.HEAD)) {
- String fileInfos = getFileInfos(logType);
- response.updateHeader("file_infos", fileInfos);
- writeResponse(request, response, HttpResponseStatus.OK);
- return;
- } else if (method.equals(HttpMethod.GET)) {
- File log = getLogFile(logType, logFile);
- if (!log.exists() || !log.isFile()) {
- response.appendContent("Log file not exist: " + log.getName());
- writeResponse(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- }
- writeObjectResponse(request, response, HttpResponseStatus.OK, log, log.getName(), true);
- } else {
- response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson());
- writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
- }
- }
-
- private String getFileInfos(String logType) {
- Map<String, Long> fileInfos = Maps.newTreeMap();
- if (logType.equals("fe.audit.log")) {
- File logDir = new File(Config.audit_log_dir);
- File[] files = logDir.listFiles();
- for (int i = 0; i < files.length; i++) {
- if (files[i].isFile() && files[i].getName().startsWith("fe.audit.log")) {
- fileInfos.put(files[i].getName(), files[i].length());
- }
- }
- }
-
- String result = "";
- ObjectMapper mapper = new ObjectMapper();
- try {
- result = mapper.writeValueAsString(fileInfos);
- } catch (Exception e) {
- // do nothing
- }
- return result;
- }
-
- private File getLogFile(String logType, String logFile) {
- String logPath = "";
- if ("fe.audit.log".equals(logType)) {
- logPath = Config.audit_log_dir + "/" + logFile;
- }
- return new File(logPath);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java
deleted file mode 100644
index c678b8a10a..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetSmallFileAction.java
+++ /dev/null
@@ -1,92 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.util.SmallFileMgr;
-import org.apache.doris.common.util.SmallFileMgr.SmallFile;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import com.google.common.base.Strings;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-public class GetSmallFileAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(GetSmallFileAction.class);
-
- public GetSmallFileAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/get_small_file", new GetSmallFileAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- String token = request.getSingleParameter("token");
- String fileIdStr = request.getSingleParameter("file_id");
-
- // check param empty
- if (Strings.isNullOrEmpty(token) || Strings.isNullOrEmpty(fileIdStr)) {
- response.appendContent("Missing parameter");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- // check token
- if (!token.equals(Catalog.getCurrentCatalog().getToken())) {
- response.appendContent("Invalid token");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- long fileId = -1;
- try {
- fileId = Long.valueOf(fileIdStr);
- } catch (NumberFormatException e) {
- response.appendContent("Invalid file id format: " + fileIdStr);
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- SmallFileMgr fileMgr = Catalog.getCurrentCatalog().getSmallFileMgr();
- SmallFile smallFile = fileMgr.getSmallFile(fileId);
- if (smallFile == null || !smallFile.isContent) {
- response.appendContent("File not found or is not content");
- writeResponse(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
-
- HttpMethod method = request.getRequest().method();
- if (method.equals(HttpMethod.GET)) {
- writeObjectResponse(request, response, HttpResponseStatus.OK, smallFile.getContentBytes(),
- smallFile.name, true);
- } else {
- response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson());
- writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java
deleted file mode 100644
index 6be12368eb..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/GetStreamLoadState.java
+++ /dev/null
@@ -1,86 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class GetStreamLoadState extends RestBaseAction {
- public GetStreamLoadState(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller)
- throws IllegalArgException {
- GetStreamLoadState action = new GetStreamLoadState(controller);
- controller.registerHandler(HttpMethod.GET, "/api/{" + DB_KEY + "}/get_load_state", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
-
- if (redirectToMaster(request, response)) {
- return;
- }
-
- final String clusterName = ConnectContext.get().getClusterName();
- if (Strings.isNullOrEmpty(clusterName)) {
- throw new DdlException("No cluster selected.");
- }
-
- String dbName = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(dbName)) {
- throw new DdlException("No database selected.");
- }
-
- String fullDbName = ClusterNamespace.getFullName(clusterName, dbName);
-
- String label = request.getSingleParameter(LABEL_KEY);
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected.");
- }
-
- // FIXME(cmy)
- // checkReadPriv(authInfo.fullUserName, fullDbName);
-
- Database db = Catalog.getCurrentCatalog().getDbOrDdlException(fullDbName);
-
- String state = Catalog.getCurrentGlobalTransactionMgr().getLabelState(db.getId(), label).toString();
-
- sendResult(request, response, new Result(state));
- }
-
- private static class Result extends RestBaseResult {
- private String state;
- public Result(String state) {
- this.state = state;
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java
deleted file mode 100644
index 78322a68ad..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/HealthAction.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class HealthAction extends RestBaseAction {
- public HealthAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller)
- throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/health", new HealthAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- response.setContentType("application/json");
-
- RestResult result = new RestResult();
- result.addResultEntry("total_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(false).size());
- result.addResultEntry("online_backend_num", Catalog.getCurrentSystemInfo().getBackendIds(true).size());
- sendResult(request, response, result);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java
deleted file mode 100644
index f0d51065eb..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/LoadAction.java
+++ /dev/null
@@ -1,141 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-import org.apache.doris.system.Backend;
-import org.apache.doris.system.SystemInfoService;
-import org.apache.doris.thrift.TNetworkAddress;
-
-import com.google.common.base.Strings;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpMethod;
-
-public class LoadAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(LoadAction.class);
-
- public static final String SUB_LABEL_NAME_PARAM = "sub_label";
-
- private ExecuteEnv execEnv;
- private boolean isStreamLoad = false;
-
- public LoadAction(ActionController controller, ExecuteEnv execEnv) {
- this(controller, execEnv, false);
- }
-
- public LoadAction(ActionController controller, ExecuteEnv execEnv, boolean isStreamLoad) {
- super(controller);
- this.execEnv = execEnv;
- this.isStreamLoad = isStreamLoad;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ExecuteEnv execEnv = ExecuteEnv.getInstance();
- LoadAction action = new LoadAction(controller, execEnv);
- controller.registerHandler(HttpMethod.PUT,
- "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", action);
-
- controller.registerHandler(HttpMethod.PUT,
- "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load",
- new LoadAction(controller, execEnv, true));
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
-
- // A 'Load' request must have 100-continue header
- if (!request.getRequest().headers().contains(HttpHeaders.Names.EXPECT)) {
- throw new DdlException("There is no 100-continue header");
- }
-
- final String clusterName = ConnectContext.get().getClusterName();
- if (Strings.isNullOrEmpty(clusterName)) {
- throw new DdlException("No cluster selected.");
- }
-
- String dbName = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(dbName)) {
- throw new DdlException("No database selected.");
- }
-
- String tableName = request.getSingleParameter(TABLE_KEY);
- if (Strings.isNullOrEmpty(tableName)) {
- throw new DdlException("No table selected.");
- }
-
- String fullDbName = ClusterNamespace.getFullName(clusterName, dbName);
-
- String label = request.getSingleParameter(LABEL_KEY);
- if (!isStreamLoad) {
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected.");
- }
- } else {
- label = request.getRequest().headers().get(LABEL_KEY);
- }
-
- // check auth
- checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.LOAD);
-
- TNetworkAddress redirectAddr;
- if (!isStreamLoad && !Strings.isNullOrEmpty(request.getSingleParameter(SUB_LABEL_NAME_PARAM))) {
- // only multi mini load need to redirect to Master, because only Master has the info of table to
- // the Backend which the file exists.
- if (redirectToMaster(request, response)) {
- return;
- }
- redirectAddr = execEnv.getMultiLoadMgr().redirectAddr(fullDbName, label);
- } else {
- // Choose a backend sequentially.
- SystemInfoService.BeAvailablePredicate beAvailablePredicate =
- new SystemInfoService.BeAvailablePredicate(false, false, true);
- List<Long> backendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIdsByStorageMediumAndTag(
- 1, beAvailablePredicate, false, clusterName, null, null);
- if (backendIds == null) {
- throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG);
- }
-
- Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0));
- if (backend == null) {
- throw new DdlException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG);
- }
-
- redirectAddr = new TNetworkAddress(backend.getHost(), backend.getHttpPort());
- }
-
- LOG.info("redirect load action to destination={}, stream: {}, db: {}, tbl: {}, label: {}",
- redirectAddr.toString(), isStreamLoad, dbName, tableName, label);
- redirectTo(request, response, redirectAddr);
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java
deleted file mode 100644
index f874ecf9bd..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetaReplayerCheckAction.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.util.Map;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-/*
- * used to get meta replay info
- * eg:
- * fe_host:http_port/api/_meta_replay_state
- */
-public class MetaReplayerCheckAction extends RestBaseAction {
- public MetaReplayerCheckAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- MetaReplayerCheckAction action = new MetaReplayerCheckAction(controller);
- controller.registerHandler(HttpMethod.GET, "/api/_meta_replay_state", action);
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
-
- Map<String, String> resultMap = Catalog.getCurrentCatalog().getMetaReplayState().getInfo();
-
- // to json response
- String result = "";
- ObjectMapper mapper = new ObjectMapper();
- try {
- result = mapper.writeValueAsString(resultMap);
- } catch (Exception e) {
- // do nothing
- }
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response);
- }
-}
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java
deleted file mode 100644
index 34bc7854e7..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MetricsAction.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.metric.MetricRepo;
-import org.apache.doris.metric.MetricVisitor;
-import org.apache.doris.metric.JsonMetricVisitor;
-import org.apache.doris.metric.PrometheusMetricVisitor;
-import org.apache.doris.metric.SimpleCoreMetricVisitor;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-//fehost:port/metrics
-//fehost:port/metrics?type=core
-//fehost:port/metrics?type=json
-public class MetricsAction extends RestBaseAction {
-
- private static final String TYPE_PARAM = "type";
-
- public MetricsAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/metrics", new MetricsAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- String type = request.getSingleParameter(TYPE_PARAM);
- MetricVisitor visitor = null;
- if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("core")) {
- visitor = new SimpleCoreMetricVisitor("doris_fe");
- } else if (!Strings.isNullOrEmpty(type) && type.equalsIgnoreCase("json")) {
- visitor = new JsonMetricVisitor("doris_fe");
- } else {
- visitor = new PrometheusMetricVisitor("doris_fe");
- }
-
- response.setContentType("text/plain");
- response.getContent().append(MetricRepo.getMetric(visitor));
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java
deleted file mode 100644
index 6f36c5736f..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MigrationAction.java
+++ /dev/null
@@ -1,164 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.MaterializedIndex;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Partition;
-import org.apache.doris.catalog.Replica;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
-import org.apache.doris.catalog.Tablet;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.util.ListComparator;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.util.Collections;
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-/*
- * used to get table's sorted tablet info
- * eg:
- * fe_host:http_port/api/_migration?db=xxx&tbl=yyy
- */
-public class MigrationAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(MigrationAction.class);
- private static final String DB_PARAM = "db";
- private static final String TABLE_PARAM = "tbl";
-
- public MigrationAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- MigrationAction action = new MigrationAction(controller);
- controller.registerHandler(HttpMethod.GET, "/api/_migration", action);
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
-
- String dbName = request.getSingleParameter(DB_PARAM);
- String tableName = request.getSingleParameter(TABLE_PARAM);
-
- if (Strings.isNullOrEmpty(dbName)) {
- throw new DdlException("Missing params. Need database name");
- }
-
- Database db = Catalog.getCurrentCatalog().getDbOrDdlException(dbName);
-
- List<List<Comparable>> rows = Lists.newArrayList();
-
-
-
- if (!Strings.isNullOrEmpty(tableName)) {
- OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
- olapTable.readLock();
- try {
- for (Partition partition : olapTable.getPartitions()) {
- String partitionName = partition.getName();
- MaterializedIndex baseIndex = partition.getBaseIndex();
- for (Tablet tablet : baseIndex.getTablets()) {
- List<Comparable> row = Lists.newArrayList();
- row.add(tableName);
- row.add(partitionName);
- row.add(tablet.getId());
- row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId()));
- for (Replica replica : tablet.getReplicas()) {
- row.add(replica.getBackendId());
- break;
- }
- rows.add(row);
- }
- }
- } finally {
- olapTable.readUnlock();
- }
- } else {
- List<Table> tableList = db.getTables();
-
- // get all olap table
- for (Table table : tableList) {
- if (table.getType() != TableType.OLAP) {
- continue;
- }
-
- OlapTable olapTable = (OlapTable) table;
- table.readLock();
- try {
- tableName = table.getName();
- for (Partition partition : olapTable.getPartitions()) {
- String partitionName = partition.getName();
- MaterializedIndex baseIndex = partition.getBaseIndex();
- for (Tablet tablet : baseIndex.getTablets()) {
- List<Comparable> row = Lists.newArrayList();
- row.add(tableName);
- row.add(partitionName);
- row.add(tablet.getId());
- row.add(olapTable.getSchemaHashByIndexId(baseIndex.getId()));
- for (Replica replica : tablet.getReplicas()) {
- row.add(replica.getBackendId());
- break;
- }
- rows.add(row);
- }
- }
- } finally {
- table.readUnlock();
- }
- }
- }
- ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(0, 1, 2);
- Collections.sort(rows, comparator);
-
- // to json response
- String result = "";
- ObjectMapper mapper = new ObjectMapper();
- try {
- result = mapper.writeValueAsString(rows);
- } catch (Exception e) {
- // do nothing
- }
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response);
- }
-
- public static void print(String msg) {
- System.out.println(System.currentTimeMillis() + " " + msg);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java
deleted file mode 100644
index 893a47b92b..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiAbort.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class MultiAbort extends RestBaseAction {
- private ExecuteEnv execEnv;
-
- public MultiAbort(ActionController controller, ExecuteEnv execEnv) {
- super(controller);
- this.execEnv = execEnv;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ExecuteEnv executeEnv = ExecuteEnv.getInstance();
- MultiAbort action = new MultiAbort(controller, executeEnv);
- controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_abort", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- String db = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(db)) {
- throw new DdlException("No database selected");
- }
- String label = request.getSingleParameter(LABEL_KEY);
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected");
- }
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
-
- // only Master has these load info
- if (redirectToMaster(request, response)) {
- return;
- }
-
- execEnv.getMultiLoadMgr().abort(fullDbName, label);
- sendResult(request, response, RestBaseResult.getOk());
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java
deleted file mode 100644
index 956c73dc39..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiCommit.java
+++ /dev/null
@@ -1,77 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class MultiCommit extends RestBaseAction {
- private ExecuteEnv execEnv;
-
- public MultiCommit(ActionController controller, ExecuteEnv execEnv) {
- super(controller);
- this.execEnv = execEnv;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ExecuteEnv executeEnv = ExecuteEnv.getInstance();
- MultiCommit action = new MultiCommit(controller, executeEnv);
- controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_commit", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- String db = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(db)) {
- throw new DdlException("No database selected");
- }
- String label = request.getSingleParameter(LABEL_KEY);
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected");
- }
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
-
- // only Master has these load info
- if (redirectToMaster(request, response)) {
- return;
- }
- RestBaseResult result = new RestBaseResult();
- try {
- execEnv.getMultiLoadMgr().commit(fullDbName, label);
- } catch (Exception e) {
- result.msg = e.getMessage();
- result.status = ActionStatus.FAILED;
- }
- sendResult(request, response, result);
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java
deleted file mode 100644
index 5f023c7a42..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiDesc.java
+++ /dev/null
@@ -1,85 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-// List all labels of one multi-load
-public class MultiDesc extends RestBaseAction {
- private ExecuteEnv execEnv;
-
- public MultiDesc(ActionController controller, ExecuteEnv execEnv) {
- super(controller);
- this.execEnv = execEnv;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ExecuteEnv executeEnv = ExecuteEnv.getInstance();
- MultiDesc action = new MultiDesc(controller, executeEnv);
- controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_desc", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- String db = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(db)) {
- throw new DdlException("No database selected");
- }
- String label = request.getSingleParameter(LABEL_KEY);
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected");
- }
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
-
- // only Master has these load info
- if (redirectToMaster(request, response)) {
- return;
- }
-
- final List<String> labels = Lists.newArrayList();
- execEnv.getMultiLoadMgr().desc(fullDbName, label, labels);
- sendResult(request, response, new Result(labels));
- }
-
- private static class Result extends RestBaseResult {
- private List<String> labels;
-
- public Result(List<String> labels) {
- this.labels = labels;
- }
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java
deleted file mode 100644
index 407edca9c2..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiList.java
+++ /dev/null
@@ -1,82 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-// list all multi load before commit
-public class MultiList extends RestBaseAction {
-
- private ExecuteEnv execEnv;
-
- public MultiList(ActionController controller, ExecuteEnv execEnv) {
- super(controller);
- this.execEnv = execEnv;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ExecuteEnv executeEnv = ExecuteEnv.getInstance();
- MultiList action = new MultiList(controller, executeEnv);
- controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_list", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- String db = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(db)) {
- throw new DdlException("No database selected");
- }
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
-
- // only Master has these load info
- if (redirectToMaster(request, response)) {
- return;
- }
-
- final List<String> labels = Lists.newArrayList();
- execEnv.getMultiLoadMgr().list(fullDbName, labels);
- sendResult(request, response, new Result(labels));
- }
-
- private static class Result extends RestBaseResult {
- private List<String> labels;
-
- public Result(List<String> labels) {
- this.labels = labels;
- }
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java
deleted file mode 100644
index da8581dbd0..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiStart.java
+++ /dev/null
@@ -1,92 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.analysis.LoadStmt;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-
-import java.util.Map;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-// Start multi action
-public class MultiStart extends RestBaseAction {
- private ExecuteEnv execEnv;
-
- public MultiStart(ActionController controller, ExecuteEnv execEnv) {
- super(controller);
- this.execEnv = execEnv;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ExecuteEnv executeEnv = ExecuteEnv.getInstance();
- MultiStart action = new MultiStart(controller, executeEnv);
- controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_start", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- String db = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(db)) {
- throw new DdlException("No database selected");
- }
- String label = request.getSingleParameter(LABEL_KEY);
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected");
- }
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
-
- // Multi start request must redirect to master, because all following sub requests will be handled
- // on Master
- if (redirectToMaster(request, response)) {
- return;
- }
-
- Map<String, String> properties = Maps.newHashMap();
- String[] keys = {LoadStmt.TIMEOUT_PROPERTY, LoadStmt.MAX_FILTER_RATIO_PROPERTY};
- for (String key : keys) {
- String value = request.getSingleParameter(key);
- if (!Strings.isNullOrEmpty(value)) {
- properties.put(key, value);
- }
- }
- for (String key : keys) {
- String value = request.getRequest().headers().get(key);
- if (!Strings.isNullOrEmpty(value)) {
- properties.put(key, value);
- }
- }
- execEnv.getMultiLoadMgr().startMulti(fullDbName, label, properties);
- sendResult(request, response, RestBaseResult.getOk());
- }
-}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java
deleted file mode 100644
index 8e5f89ad5c..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/MultiUnload.java
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.service.ExecuteEnv;
-
-import com.google.common.base.Strings;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class MultiUnload extends RestBaseAction {
- private static final String SUB_LABEL_KEY = "sub_label";
-
- private ExecuteEnv execEnv;
-
- public MultiUnload(ActionController controller, ExecuteEnv execEnv) {
- super(controller);
- this.execEnv = execEnv;
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- ExecuteEnv executeEnv = ExecuteEnv.getInstance();
- MultiUnload action = new MultiUnload(controller, executeEnv);
- controller.registerHandler(HttpMethod.POST, "/api/{" + DB_KEY + "}/_multi_unload", action);
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- String db = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(db)) {
- throw new DdlException("No database selected");
- }
- String label = request.getSingleParameter(LABEL_KEY);
- if (Strings.isNullOrEmpty(label)) {
- throw new DdlException("No label selected");
- }
- String subLabel = request.getSingleParameter(SUB_LABEL_KEY);
- if (Strings.isNullOrEmpty(subLabel)) {
- throw new DdlException("No sub_label selected");
- }
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), db);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD);
-
- if (redirectToMaster(request, response)) {
- return;
- }
-
- execEnv.getMultiLoadMgr().unload(fullDbName, label, subLabel);
- sendResult(request, response, RestBaseResult.getOk());
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java
deleted file mode 100644
index 0bb35ed7fb..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ProfileAction.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.common.util.ProfileManager;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-// This class is a RESTFUL interface to get query profile.
-// It will be used in query monitor to collect profiles.
-// Usage:
-// wget http://fe_host:fe_http_port/api/profile?query_id=123456
-public class ProfileAction extends RestBaseAction {
-
- public ProfileAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/profile", new ProfileAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- String queryId = request.getSingleParameter("query_id");
- if (queryId == null) {
- response.getContent().append("not valid parameter");
- sendResult(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- String queryProfileStr = ProfileManager.getInstance().getProfile(queryId);
- if (queryProfileStr != null) {
- response.getContent().append(queryProfileStr);
- sendResult(request, response);
- } else {
- response.getContent().append("query id " + queryId + " not found.");
- sendResult(request, response, HttpResponseStatus.NOT_FOUND);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java
deleted file mode 100644
index c12a640e4a..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/QueryDetailAction.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import com.google.gson.Gson;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import java.util.List;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.qe.QueryDetail;
-import org.apache.doris.qe.QueryDetailQueue;
-
-public class QueryDetailAction extends RestBaseAction {
-
- public QueryDetailAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/query_detail", new QueryDetailAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- String eventTimeStr = request.getSingleParameter("event_time");
- if (eventTimeStr == null) {
- response.getContent().append("not valid parameter");
- sendResult(request, response, HttpResponseStatus.BAD_REQUEST);
- return;
- }
- long eventTime = Long.valueOf(eventTimeStr.trim());
- List<QueryDetail> queryDetails = QueryDetailQueue.getQueryDetails(eventTime);
- Gson gson = new Gson();
- String json_string = gson.toJson(queryDetails);
- response.getContent().append(json_string);
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java
deleted file mode 100644
index 8fea6046b2..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java
+++ /dev/null
@@ -1,127 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.analysis.UserIdentity;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseAction;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.UnauthorizedException;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.thrift.TNetworkAddress;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-public class RestBaseAction extends BaseAction {
- protected static final String DB_KEY = "db";
- protected static final String TABLE_KEY = "table";
- protected static final String LABEL_KEY = "label";
- private static final Logger LOG = LogManager.getLogger(RestBaseAction.class);
-
- public RestBaseAction(ActionController controller) {
- super(controller);
- }
-
- @Override
- public void handleRequest(BaseRequest request) throws Exception {
- LOG.debug("receive http request. url={}", request.getRequest().uri());
- BaseResponse response = new BaseResponse();
- try {
- execute(request, response);
- } catch (DdlException e) {
- if (e instanceof UnauthorizedException) {
- response.appendContent(e.getMessage());
- response.updateHeader(HttpHeaderNames.WWW_AUTHENTICATE.toString(), "Basic realm=\"\"");
- writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED);
- } else {
- sendResult(request, response, new RestBaseResult(e.getMessage()));
- }
- }
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) throws DdlException {
- ActionAuthorizationInfo authInfo = getAuthorizationInfo(request);
- // check password
- UserIdentity currentUser = checkPassword(authInfo);
- ConnectContext ctx = new ConnectContext(null);
- ctx.setCatalog(Catalog.getCurrentCatalog());
- ctx.setQualifiedUser(authInfo.fullUserName);
- ctx.setRemoteIP(authInfo.remoteIp);
- ctx.setCurrentUserIdentity(currentUser);
- ctx.setCluster(authInfo.cluster);
- ctx.setThreadLocalInfo();
- executeWithoutPassword(request, response);
- }
-
- // If user password should be checked, the derived class should implement this method, NOT 'execute()',
- // otherwise, override 'execute()' directly
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- throw new DdlException("Not implemented");
- }
-
- public void sendResult(BaseRequest request, BaseResponse response, RestBaseResult result) {
- response.appendContent(result.toJson());
- writeResponse(request, response, HttpResponseStatus.OK);
- }
-
- public void sendResult(BaseRequest request, BaseResponse response, HttpResponseStatus status) {
- writeResponse(request, response, status);
- }
-
- public void sendResult(BaseRequest request, BaseResponse response) {
- writeResponse(request, response, HttpResponseStatus.OK);
- }
-
- public void redirectTo(BaseRequest request, BaseResponse response, TNetworkAddress addr)
- throws DdlException {
- String urlStr = request.getRequest().uri();
- URI urlObj = null;
- URI resultUriObj = null;
- try {
- urlObj = new URI(urlStr);
- resultUriObj = new URI("http", null, addr.getHostname(),
- addr.getPort(), urlObj.getPath(), urlObj.getQuery(), null);
- } catch (URISyntaxException e) {
- LOG.warn(e.getMessage());
- throw new DdlException(e.getMessage());
- }
- response.updateHeader(HttpHeaderNames.LOCATION.toString(), resultUriObj.toString());
- writeResponse(request, response, HttpResponseStatus.TEMPORARY_REDIRECT);
- }
-
- public boolean redirectToMaster(BaseRequest request, BaseResponse response) throws DdlException {
- Catalog catalog = Catalog.getCurrentCatalog();
- if (catalog.isMaster()) {
- return false;
- }
- redirectTo(request, response, new TNetworkAddress(catalog.getMasterIp(), catalog.getMasterHttpPort()));
- return true;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java
deleted file mode 100644
index c343607148..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java
+++ /dev/null
@@ -1,115 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.MaterializedIndex;
-import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Partition;
-import org.apache.doris.catalog.Replica;
-import org.apache.doris.catalog.Tablet;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.util.Map;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-/*
- * calc row count from replica to table
- * fe_host:fe_http_port/api/rowcount?db=dbname&table=tablename
- */
-public class RowCountAction extends RestBaseAction {
-
- public RowCountAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/rowcount", new RowCountAction(controller));
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
-
- String dbName = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(dbName)) {
- throw new DdlException("No database selected.");
- }
-
- String tableName = request.getSingleParameter(TABLE_KEY);
- if (Strings.isNullOrEmpty(tableName)) {
- throw new DdlException("No table selected.");
- }
-
- Map<String, Long> indexRowCountMap = Maps.newHashMap();
- Catalog catalog = Catalog.getCurrentCatalog();
- Database db = catalog.getDbOrDdlException(dbName);
- OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
- olapTable.writeLockOrDdlException();
- try {
- for (Partition partition : olapTable.getAllPartitions()) {
- long version = partition.getVisibleVersion();
- for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
- long indexRowCount = 0L;
- for (Tablet tablet : index.getTablets()) {
- long tabletRowCount = 0L;
- for (Replica replica : tablet.getReplicas()) {
- if (replica.checkVersionCatchUp(version, false)
- && replica.getRowCount() > tabletRowCount) {
- tabletRowCount = replica.getRowCount();
- }
- }
- indexRowCount += tabletRowCount;
- } // end for tablets
- index.setRowCount(indexRowCount);
- indexRowCountMap.put(olapTable.getIndexNameById(index.getId()), indexRowCount);
- } // end for indices
- } // end for partitions
- } finally {
- olapTable.writeUnlock();
- }
-
- // to json response
- String result = "";
- ObjectMapper mapper = new ObjectMapper();
- try {
- result = mapper.writeValueAsString(indexRowCountMap);
- } catch (Exception e) {
- // do nothing
- }
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
deleted file mode 100644
index f096b10bf7..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
+++ /dev/null
@@ -1,136 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-import org.apache.doris.common.ConfigBase;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.collect.Maps;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/*
- * used to set fe config
- * eg:
- * fe_host:http_port/api/_set_config?config_key1=config_value1&config_key2=config_value2&...
- */
-public class SetConfigAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(SetConfigAction.class);
-
- private static final String PERSIST_PARAM = "persist";
- private static final String RESET_PERSIST = "reset_persist";
-
- public SetConfigAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- SetConfigAction action = new SetConfigAction(controller);
- controller.registerHandler(HttpMethod.GET, "/api/_set_config", action);
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
-
- boolean needPersist = false;
- boolean resetPersist = true;
- Map<String, List<String>> configs = request.getAllParameters();
- if (configs.containsKey(PERSIST_PARAM)) {
- List<String> val = configs.remove(PERSIST_PARAM);
- if (val.size() == 1 && val.get(0).equals("true")) {
- needPersist = true;
- }
- }
- if (configs.containsKey(RESET_PERSIST)) {
- List<String> val = configs.remove(RESET_PERSIST);
- if (val.size() == 1 && val.get(0).equals("false")) {
- resetPersist = false;
- }
- }
-
- Map<String, String> setConfigs = Maps.newHashMap();
- Map<String, String> errConfigs = Maps.newHashMap();
-
- LOG.debug("get config from url: {}, need persist: {}", configs, needPersist);
-
- for (Map.Entry<String, List<String>> config : configs.entrySet()) {
- String confKey = config.getKey();
- List<String> confValue = config.getValue();
- try {
- if (confValue != null && confValue.size() == 1) {
- ConfigBase.setMutableConfig(confKey, confValue.get(0));
- setConfigs.put(confKey, confValue.get(0));
- } else {
- throw new DdlException("conf value size != 1");
- }
- } catch (DdlException e) {
- LOG.warn("failed to set config {}:{}", confKey, confValue, e);
- errConfigs.put(confKey, String.valueOf(confValue));
- }
- }
-
- String persistMsg = "";
- if (needPersist) {
- try {
- ConfigBase.persistConfig(setConfigs, resetPersist);
- persistMsg = "ok";
- } catch (IOException e) {
- LOG.warn("failed to persist config", e);
- persistMsg = e.getMessage();
- }
- }
-
- Map<String, Object> resultMap = Maps.newHashMap();
- resultMap.put("set", setConfigs);
- resultMap.put("err", errConfigs);
- resultMap.put("persist", persistMsg);
-
- // to json response
- String result = "";
- ObjectMapper mapper = new ObjectMapper();
- try {
- result = mapper.writeValueAsString(resultMap);
- } catch (Exception e) {
- // do nothing
- }
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response);
- }
-
- public static void print(String msg) {
- System.out.println(System.currentTimeMillis() + " " + msg);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java
deleted file mode 100644
index af8036eb8b..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowDataAction.java
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-
-import io.netty.handler.codec.http.HttpMethod;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.List;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class ShowDataAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(ShowDataAction.class);
-
- public ShowDataAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction (ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/show_data", new ShowDataAction(controller));
- }
-
- public long getDataSizeOfDatabase(Database db) {
- long totalSize = 0;
- long tableSize = 0;
- // sort by table name
- List<Table> tables = db.getTables();
- for (Table table : tables) {
- if (table.getType() != TableType.OLAP) {
- continue;
- }
- table.readLock();
- try {
- tableSize = ((OlapTable)table).getDataSize();
- } finally {
- table.readUnlock();
- }
- totalSize += tableSize;
- } // end for tables
- return totalSize;
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- String dbName = request.getSingleParameter("db");
- ConcurrentHashMap<String, Database> fullNameToDb = Catalog.getCurrentCatalog().getFullNameToDb();
- long totalSize = 0;
- if (dbName != null) {
- Database db = fullNameToDb.get("default_cluster:"+dbName);
- if (db == null) {
- response.getContent().append("database " + dbName + " not found.");
- sendResult(request, response, HttpResponseStatus.NOT_FOUND);
- return;
- }
- totalSize = getDataSizeOfDatabase(db);
- } else {
- for (Database db : fullNameToDb.values()) {
- LOG.info("database name: {}", db.getFullName());
- totalSize += getDataSizeOfDatabase(db);
- }
- }
- response.getContent().append(String.valueOf(totalSize));
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java
deleted file mode 100644
index e3bbef3c40..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java
+++ /dev/null
@@ -1,203 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.MaterializedIndex;
-import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Partition;
-import org.apache.doris.catalog.Replica;
-import org.apache.doris.catalog.Replica.ReplicaState;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
-import org.apache.doris.catalog.Tablet;
-import org.apache.doris.common.Config;
-import org.apache.doris.ha.HAProtocol;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.persist.Storage;
-
-import com.google.gson.Gson;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class ShowMetaInfoAction extends RestBaseAction {
- private enum Action {
- SHOW_DB_SIZE,
- SHOW_HA,
- INVALID;
-
- public static Action getAction(String str) {
- try {
- return valueOf(str);
- } catch (Exception ex) {
- return INVALID;
- }
- }
- }
-
- private static final Logger LOG = LogManager.getLogger(ShowMetaInfoAction.class);
-
- public ShowMetaInfoAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/show_meta_info",
- new ShowMetaInfoAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- String action = request.getSingleParameter("action");
- Gson gson = new Gson();
- response.setContentType("application/json");
-
- switch (Action.getAction(action.toUpperCase())) {
- case SHOW_DB_SIZE:
- response.getContent().append(gson.toJson(getDataSize()));
- break;
- case SHOW_HA:
- response.getContent().append(gson.toJson(getHaInfo()));
- break;
- default:
- break;
- }
- sendResult(request, response);
- }
-
- public Map<String, String> getHaInfo() {
- HashMap<String, String> feInfo = new HashMap<String, String>();
- feInfo.put("role", Catalog.getCurrentCatalog().getFeType().toString());
- if (Catalog.getCurrentCatalog().isMaster()) {
- feInfo.put("current_journal_id",
- String.valueOf(Catalog.getCurrentCatalog().getEditLog().getMaxJournalId()));
- } else {
- feInfo.put("current_journal_id",
- String.valueOf(Catalog.getCurrentCatalog().getReplayedJournalId()));
- }
-
- HAProtocol haProtocol = Catalog.getCurrentCatalog().getHaProtocol();
- if (haProtocol != null) {
-
- InetSocketAddress master = null;
- try {
- master = haProtocol.getLeader();
- } catch (Exception e) {
- // this may happen when majority of FOLLOWERS are down and no MASTER right now.
- LOG.warn("failed to get leader: {}", e.getMessage());
- }
- if (master != null) {
- feInfo.put("master", master.getHostString());
- } else {
- feInfo.put("master", "unknown");
- }
-
- List<InetSocketAddress> electableNodes = haProtocol.getElectableNodes(false);
- ArrayList<String> electableNodeNames = new ArrayList<String>();
- if (!electableNodes.isEmpty()) {
- for (InetSocketAddress node : electableNodes) {
- electableNodeNames.add(node.getHostString());
- }
- feInfo.put("electable_nodes", StringUtils.join(electableNodeNames.toArray(), ","));
- }
-
- List<InetSocketAddress> observerNodes = haProtocol.getObserverNodes();
- ArrayList<String> observerNodeNames = new ArrayList<String>();
- if (observerNodes != null) {
- for (InetSocketAddress node : observerNodes) {
- observerNodeNames.add(node.getHostString());
- }
- feInfo.put("observer_nodes", StringUtils.join(observerNodeNames.toArray(), ","));
- }
- }
-
- feInfo.put("can_read", String.valueOf(Catalog.getCurrentCatalog().canRead()));
- feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady()));
- try {
- Storage storage = new Storage(Config.meta_dir + "/image");
- feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq()));
- long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
- feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime));
- } catch (IOException e) {
- LOG.warn(e.getMessage());
- }
- return feInfo;
- }
-
- public Map<String, Long> getDataSize() {
- Map<String, Long> result = new HashMap<String, Long>();
- List<String> dbNames = Catalog.getCurrentCatalog().getDbNames();
-
- for (String dbName : dbNames) {
- Database db = Catalog.getCurrentCatalog().getDbNullable(dbName);
- if (db == null) {
- continue;
- }
-
- long totalSize = 0;
- List<Table> tables = db.getTables();
- for (Table table : tables) {
- if (table.getType() != TableType.OLAP) {
- continue;
- }
-
- OlapTable olapTable = (OlapTable) table;
- long tableSize = 0;
- for (Partition partition : olapTable.getAllPartitions()) {
- long partitionSize = 0;
- for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
- long indexSize = 0;
- for (Tablet tablet : mIndex.getTablets()) {
- long maxReplicaSize = 0;
- for (Replica replica : tablet.getReplicas()) {
- if (replica.getState() == ReplicaState.NORMAL
- || replica.getState() == ReplicaState.SCHEMA_CHANGE) {
- if (replica.getDataSize() > maxReplicaSize) {
- maxReplicaSize = replica.getDataSize();
- }
- }
- } // end for replicas
- indexSize += maxReplicaSize;
- } // end for tablets
- partitionSize += indexSize;
- } // end for tables
- tableSize += partitionSize;
- } // end for partitions
- totalSize += tableSize;
- } // end for tables
- result.put(dbName, totalSize);
- } // end for dbs
- return result;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java
deleted file mode 100644
index bb53d9c7e7..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowProcAction.java
+++ /dev/null
@@ -1,136 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.analysis.RedirectStatus;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.proc.ProcNodeInterface;
-import org.apache.doris.common.proc.ProcResult;
-import org.apache.doris.common.proc.ProcService;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.MasterOpExecutor;
-import org.apache.doris.qe.OriginStatement;
-import org.apache.doris.qe.ShowResultSet;
-import org.apache.doris.system.SystemInfoService;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.List;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-// Format:
-// http://username:password@192.168.1.1:8030/api/show_proc?path=/
-public class ShowProcAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(ShowProcAction.class);
-
- public ShowProcAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/show_proc", new ShowProcAction(controller));
- }
-
- @Override
- public void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
- // check authority
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
-
- String path = request.getSingleParameter("path");
- String forward = request.getSingleParameter("forward");
- boolean isForward = false;
- if (!Strings.isNullOrEmpty(forward) && forward.equals("true")) {
- isForward = true;
- }
-
- // forward to master if necessary
- if (!Catalog.getCurrentCatalog().isMaster() && isForward) {
- String showProcStmt = "SHOW PROC \"" + path + "\"";
- ConnectContext context = new ConnectContext(null);
- context.setCatalog(Catalog.getCurrentCatalog());
- context.setCluster(SystemInfoService.DEFAULT_CLUSTER);
- context.setQualifiedUser(ConnectContext.get().getQualifiedUser());
- context.setRemoteIP(ConnectContext.get().getRemoteIP());
- MasterOpExecutor masterOpExecutor = new MasterOpExecutor(new OriginStatement(showProcStmt, 0), context,
- RedirectStatus.FORWARD_NO_SYNC, true);
- LOG.debug("need to transfer to Master. stmt: {}", context.getStmtId());
-
- try {
- masterOpExecutor.execute();
- } catch (Exception e) {
- response.appendContent("Failed to forward stmt: " + e.getMessage());
- sendResult(request, response);
- return;
- }
-
- ShowResultSet resultSet = masterOpExecutor.getProxyResultSet();
- if (resultSet == null) {
- response.appendContent("Failed to get result set");
- sendResult(request, response);
- return;
- }
-
- Gson gson = new Gson();
- response.setContentType("application/json");
- response.getContent().append(gson.toJson(resultSet.getResultRows()));
-
- } else {
- ProcNodeInterface procNode = null;
- ProcService instance = ProcService.getInstance();
- try {
- if (Strings.isNullOrEmpty(path)) {
- procNode = instance.open("/");
- } else {
- procNode = instance.open(path);
- }
- } catch (AnalysisException e) {
- LOG.warn(e.getMessage());
- response.getContent().append("[]");
- }
-
- if (procNode != null) {
- ProcResult result;
- try {
- result = procNode.fetchResult();
- List<List<String>> rows = result.getRows();
-
- Gson gson = new Gson();
- response.setContentType("application/json");
- response.getContent().append(gson.toJson(rows));
- } catch (AnalysisException e) {
- LOG.warn(e.getMessage());
- response.getContent().append("[]");
- }
- }
- }
-
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java
deleted file mode 100644
index 66ec5c4c35..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ShowRuntimeInfoAction.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import com.google.gson.Gson;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.HashMap;
-
-public class ShowRuntimeInfoAction extends RestBaseAction {
- private static final Logger LOG = LogManager.getLogger(ShowRuntimeInfoAction.class);
-
- public ShowRuntimeInfoAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET, "/api/show_runtime_info",
- new ShowRuntimeInfoAction(controller));
- }
-
- @Override
- public void execute(BaseRequest request, BaseResponse response) {
- HashMap<String, String> feInfo = new HashMap<String, String>();
-
- // Get memory info
- Runtime r = Runtime.getRuntime();
- feInfo.put("free_mem", String.valueOf(r.freeMemory()));
- feInfo.put("total_mem", String.valueOf(r.totalMemory()));
- feInfo.put("max_mem", String.valueOf(r.maxMemory()));
-
- // Get thread count
- ThreadGroup parentThread;
- for (parentThread = Thread.currentThread().getThreadGroup();
- parentThread.getParent() != null;
- parentThread = parentThread.getParent()) {
- };
- feInfo.put("thread_cnt", String.valueOf(parentThread.activeCount()));
-
- Gson gson = new Gson();
- response.setContentType("application/json");
- response.getContent().append(gson.toJson(feInfo));
-
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java
deleted file mode 100644
index bf6d271ebe..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java
+++ /dev/null
@@ -1,98 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.MaterializedIndexMeta;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.thrift.TStorageType;
-
-import com.google.common.base.Strings;
-
-import org.json.simple.JSONObject;
-
-import java.util.List;
-import java.util.Map;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-public class StorageTypeCheckAction extends RestBaseAction {
- public StorageTypeCheckAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- StorageTypeCheckAction action = new StorageTypeCheckAction(controller);
- controller.registerHandler(HttpMethod.GET, "/api/_check_storagetype", action);
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
-
- String dbName = request.getSingleParameter(DB_KEY);
- if (Strings.isNullOrEmpty(dbName)) {
- throw new DdlException("Parameter db is missing");
- }
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
- Database db = catalog.getDbOrDdlException(fullDbName);
-
- JSONObject root = new JSONObject();
- List<Table> tableList = db.getTables();
-
- for (Table tbl : tableList) {
- if (tbl.getType() != TableType.OLAP) {
- continue;
- }
-
- OlapTable olapTbl = (OlapTable) tbl;
- olapTbl.readLock();
- try {
- JSONObject indexObj = new JSONObject();
- for (Map.Entry<Long, MaterializedIndexMeta> entry : olapTbl.getIndexIdToMeta().entrySet()) {
- MaterializedIndexMeta indexMeta = entry.getValue();
- if (indexMeta.getStorageType() == TStorageType.ROW) {
- indexObj.put(olapTbl.getIndexNameById(entry.getKey()), indexMeta.getStorageType().name());
- }
- }
- root.put(tbl.getName(), indexObj);
- } finally {
- olapTbl.readUnlock();
- }
- }
-
- // to json response
- String result = root.toString();
-
- // send result
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java
deleted file mode 100644
index d252ba36db..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java
+++ /dev/null
@@ -1,306 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.analysis.InlineViewRef;
-import org.apache.doris.analysis.SelectStmt;
-import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.analysis.TableName;
-import org.apache.doris.analysis.TableRef;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.DorisHttpException;
-import org.apache.doris.common.MetaNotFoundException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.planner.PlanFragment;
-import org.apache.doris.planner.Planner;
-import org.apache.doris.planner.ScanNode;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.OriginStatement;
-import org.apache.doris.qe.StmtExecutor;
-import org.apache.doris.thrift.TDataSink;
-import org.apache.doris.thrift.TDataSinkType;
-import org.apache.doris.thrift.TMemoryScratchSink;
-import org.apache.doris.thrift.TNetworkAddress;
-import org.apache.doris.thrift.TPaloScanRange;
-import org.apache.doris.thrift.TPlanFragment;
-import org.apache.doris.thrift.TQueryOptions;
-import org.apache.doris.thrift.TQueryPlanInfo;
-import org.apache.doris.thrift.TScanRangeLocations;
-import org.apache.doris.thrift.TTabletVersionInfo;
-import org.apache.doris.thrift.TUniqueId;
-
-import com.google.common.base.Strings;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/**
- * This class responsible for parse the sql and generate the query plan fragment for a (only one) table{@see OlapTable}
- * the related tablet maybe pruned by query planer according the `where` predicate.
- */
-public class TableQueryPlanAction extends RestBaseAction {
-
- public static final Logger LOG = LogManager.getLogger(TableQueryPlanAction.class);
-
- public TableQueryPlanAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.POST,
- "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan",
- new TableQueryPlanAction(controller));
- controller.registerHandler(HttpMethod.GET,
- "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_query_plan",
- new TableQueryPlanAction(controller));
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- // just allocate 2 slot for top holder map
- Map<String, Object> resultMap = new HashMap<>(4);
- String dbName = request.getSingleParameter(DB_KEY);
- String tableName = request.getSingleParameter(TABLE_KEY);
- String postContent = request.getContent();
- try {
- // may be these common validate logic should be moved to one base class
- if (Strings.isNullOrEmpty(dbName)
- || Strings.isNullOrEmpty(tableName)) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected");
- }
- if (Strings.isNullOrEmpty(postContent)) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object");
- }
- JSONObject jsonObject = (JSONObject) JSONValue.parse(postContent);
- if (jsonObject == null) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "malformed json [ " + postContent + " ]");
- }
- String sql = (String) jsonObject.get("sql");
- if (Strings.isNullOrEmpty(sql)) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object");
- }
- LOG.info("receive SQL statement [{}] from external service [ user [{}]] for database [{}] table [{}]",
- sql, ConnectContext.get().getCurrentUserIdentity(), dbName, tableName);
-
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
- // check privilege for select, otherwise return HTTP 401
- checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT);
- Table table;
- try {
- Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName);
- table = db.getTableOrMetaException(tableName, Table.TableType.OLAP);
- } catch (MetaNotFoundException e) {
- throw new DorisHttpException(HttpResponseStatus.FORBIDDEN, e.getMessage());
- }
-
- // may be should acquire writeLock
- table.readLock();
- try {
- // parse/analysis/plan the sql and acquire tablet distributions
- handleQuery(ConnectContext.get(), fullDbName, tableName, sql, resultMap);
- } finally {
- table.readUnlock();
- }
- } catch (DorisHttpException e) {
- // status code should conforms to HTTP semantic
- resultMap.put("status", e.getCode().code());
- resultMap.put("exception", e.getMessage());
- }
- ObjectMapper mapper = new ObjectMapper();
- try {
- String result = mapper.writeValueAsString(resultMap);
- // send result with extra information
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status")))));
- } catch (Exception e) {
- // may be this never happen
- response.getContent().append(e.getMessage());
- sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
- }
- }
-
-
- /**
- * process the sql syntax and return the resolved pruned tablet
- *
- * @param context context for analyzer
- * @param sql the single table select statement
- * @param result the acquired results
- * @return
- * @throws DorisHttpException
- */
- private void handleQuery(ConnectContext context, String requestDb, String requestTable, String sql,
- Map<String, Object> result) throws DorisHttpException {
- // use SE to resolve sql
- StmtExecutor stmtExecutor = new StmtExecutor(context, new OriginStatement(sql, 0), false);
- try {
- TQueryOptions tQueryOptions = context.getSessionVariable().toThrift();
- // Conduct Planner create SingleNodePlan#createPlanFragments
- tQueryOptions.num_nodes = 1;
- // analyze sql
- stmtExecutor.analyze(tQueryOptions);
- } catch (Exception e) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage());
- }
- // the parsed logical statement
- StatementBase query = stmtExecutor.getParsedStmt();
- // only process select semantic
- if (!(query instanceof SelectStmt)) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement needed, but found [" + sql + " ]");
- }
- SelectStmt stmt = (SelectStmt) query;
- // just only process sql like `select * from table where <predicate>`, only support executing scan semantic
- if (stmt.hasAggInfo() || stmt.hasAnalyticInfo()
- || stmt.hasOrderByClause() || stmt.hasOffset() || stmt.hasLimit() || stmt.isExplain()) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "only support single table filter-prune-scan, but found [ " + sql + "]");
- }
- // process only one table by one http query
- List<TableRef> fromTables = stmt.getTableRefs();
- if (fromTables.size() != 1) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must have only one table");
- }
-
- TableRef fromTable = fromTables.get(0);
- if (fromTable instanceof InlineViewRef) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "Select statement must not embed another statement");
- }
- // check consistent http requested resource with sql referenced
- // if consistent in this way, can avoid check privilege
- TableName tableAndDb = fromTables.get(0).getName();
- if (!(tableAndDb.getDb().equals(requestDb) && tableAndDb.getTbl().equals(requestTable))) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "requested database and table must consistent with sql: request [ "
- + requestDb + "." + requestTable + "]" + "and sql [" + tableAndDb.toString() + "]");
- }
-
- // acquired Planner to get PlanNode and fragment templates
- Planner planner = stmtExecutor.planner();
- // acquire ScanNode to obtain pruned tablet
- // in this way, just retrieve only one scannode
- List<ScanNode> scanNodes = planner.getScanNodes();
- if (scanNodes.size() != 1) {
- throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one ScanNode but found [ " + scanNodes.size() + "]");
- }
- List<TScanRangeLocations> scanRangeLocations = scanNodes.get(0).getScanRangeLocations(0);
- // acquire the PlanFragment which the executable template
- List<PlanFragment> fragments = planner.getFragments();
- if (fragments.size() != 1) {
- throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Planner should plan just only one PlanFragment but found [ " + fragments.size() + "]");
- }
-
- TQueryPlanInfo tQueryPlanInfo = new TQueryPlanInfo();
-
-
- // acquire TPlanFragment
- TPlanFragment tPlanFragment = fragments.get(0).toThrift();
- // set up TMemoryScratchSink
- TDataSink tDataSink = new TDataSink();
- tDataSink.type = TDataSinkType.MEMORY_SCRATCH_SINK;
- tDataSink.memory_scratch_sink = new TMemoryScratchSink();
- tPlanFragment.output_sink = tDataSink;
-
- tQueryPlanInfo.plan_fragment = tPlanFragment;
- tQueryPlanInfo.desc_tbl = query.getAnalyzer().getDescTbl().toThrift();
- // set query_id
- UUID uuid = UUID.randomUUID();
- tQueryPlanInfo.query_id = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
-
- Map<Long, TTabletVersionInfo> tablet_info = new HashMap<>();
- // acquire resolved tablet distribution
- Map<String, Node> tabletRoutings = assemblePrunedPartitions(scanRangeLocations);
- tabletRoutings.forEach((tabletId, node) -> {
- long tablet = Long.parseLong(tabletId);
- tablet_info.put(tablet, new TTabletVersionInfo(tablet, node.version, 0l /*versionHash*/, node.schemaHash));
- });
- tQueryPlanInfo.tablet_info = tablet_info;
-
- // serialize TQueryPlanInfo and encode plan with Base64 to string in order to translate by json format
- TSerializer serializer = new TSerializer();
- String opaqued_query_plan;
- try {
- byte[] query_plan_stream = serializer.serialize(tQueryPlanInfo);
- opaqued_query_plan = Base64.getEncoder().encodeToString(query_plan_stream);
- } catch (TException e) {
- throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "TSerializer failed to serialize PlanFragment, reason [ " + e.getMessage() + " ]");
- }
- result.put("partitions", tabletRoutings);
- result.put("opaqued_query_plan", opaqued_query_plan);
- result.put("status", 200);
- }
-
- /**
- * acquire all involved (already pruned) tablet routing
- *
- * @param scanRangeLocationsList
- * @return
- */
- private Map<String, Node> assemblePrunedPartitions(List<TScanRangeLocations> scanRangeLocationsList) {
- Map<String, Node> result = new HashMap<>();
- for (TScanRangeLocations scanRangeLocations : scanRangeLocationsList) {
- // only process palo(doris) scan range
- TPaloScanRange scanRange = scanRangeLocations.scan_range.palo_scan_range;
- Node tabletRouting = new Node(Long.parseLong(scanRange.version), Integer.parseInt(scanRange.schema_hash));
- for (TNetworkAddress address : scanRange.hosts) {
- tabletRouting.addRouting(address.hostname + ":" + address.port);
- }
- result.put(String.valueOf(scanRange.tablet_id), tabletRouting);
- }
- return result;
- }
-
- // helper class for json transformation
- final class Node {
- // ["host1:port1", "host2:port2", "host3:port3"]
- public List<String> routings = new ArrayList<>();
- public long version;
- public int schemaHash;
-
- public Node(long version, int schemaHash) {
- this.version = version;
- this.schemaHash = schemaHash;
- }
-
- private void addRouting(String routing) {
- routings.add(routing);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java
deleted file mode 100644
index 1b4a8edece..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java
+++ /dev/null
@@ -1,113 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.DorisHttpException;
-import org.apache.doris.common.MetaNotFoundException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/**
- * This class is responsible for fetch the approximate row count of the specified table from cluster-meta data,
- * the approximate row maybe used for some computing system to decide use which compute-algorithm can be used
- * such as shuffle join or broadcast join.
- * <p>
- * This API is not intended to compute the exact row count of the specified table, if you need the exact row count,
- * please consider using the sql syntax `select count(*) from {table}`
- */
-public class TableRowCountAction extends RestBaseAction {
- public TableRowCountAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- controller.registerHandler(HttpMethod.GET,
- "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_count",
- new TableRowCountAction(controller));
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response)
- throws DdlException {
- // just allocate 2 slot for top holder map
- Map<String, Object> resultMap = new HashMap<>(4);
- String dbName = request.getSingleParameter(DB_KEY);
- String tableName = request.getSingleParameter(TABLE_KEY);
- try {
- if (Strings.isNullOrEmpty(dbName)
- || Strings.isNullOrEmpty(tableName)) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected");
- }
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
- // check privilege for select, otherwise return HTTP 401
- checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT);
- Table table;
- try {
- Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName);
- table = db.getTableOrMetaException(tableName, Table.TableType.OLAP);
- } catch (MetaNotFoundException e) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage());
- }
-
- table.readLock();
- try {
- OlapTable olapTable = (OlapTable) table;
- resultMap.put("status", 200);
- resultMap.put("size", olapTable.proximateRowCount());
- } finally {
- table.readUnlock();
- }
- } catch (DorisHttpException e) {
- // status code should conforms to HTTP semantic
- resultMap.put("status", e.getCode().code());
- resultMap.put("exception", e.getMessage());
- }
- ObjectMapper mapper = new ObjectMapper();
- try {
- String result = mapper.writeValueAsString(resultMap);
- // send result with extra information
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status")))));
- } catch (Exception e) {
- // may be this never happen
- response.getContent().append(e.getMessage());
- sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java
deleted file mode 100644
index 9000ae4890..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableSchemaAction.java
+++ /dev/null
@@ -1,135 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.http.rest;
-
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.PrimitiveType;
-import org.apache.doris.catalog.ScalarType;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.DorisHttpException;
-import org.apache.doris.common.MetaNotFoundException;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.BaseRequest;
-import org.apache.doris.http.BaseResponse;
-import org.apache.doris.http.IllegalArgException;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-/**
- * Get table schema for specified cluster.database.table with privilege checking
- */
-public class TableSchemaAction extends RestBaseAction {
-
- public TableSchemaAction(ActionController controller) {
- super(controller);
- }
-
- public static void registerAction(ActionController controller) throws IllegalArgException {
- // the extra `/api` path is so disgusting
- controller.registerHandler(HttpMethod.GET,
- "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_schema", new TableSchemaAction
- (controller));
- }
-
- @Override
- protected void executeWithoutPassword(BaseRequest request, BaseResponse response) throws DdlException {
- // just allocate 2 slot for top holder map
- Map<String, Object> resultMap = new HashMap<>(2);
- String dbName = request.getSingleParameter(DB_KEY);
- String tableName = request.getSingleParameter(TABLE_KEY);
- try {
- if (Strings.isNullOrEmpty(dbName)
- || Strings.isNullOrEmpty(tableName)) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "No database or table selected.");
- }
- String fullDbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(), dbName);
- // check privilege for select, otherwise return 401 HTTP status
- checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, tableName, PrivPredicate.SELECT);
- Table table;
- try {
- Database db = Catalog.getCurrentCatalog().getDbOrMetaException(fullDbName);
- table = db.getTableOrMetaException(tableName, Table.TableType.OLAP);
- } catch (MetaNotFoundException e) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage());
- }
-
- table.readLock();
- try {
- List<Column> columns = table.getBaseSchema();
- List<Map<String, String>> propList = new ArrayList(columns.size());
- for (Column column : columns) {
- Map<String, String> baseInfo = new HashMap<>(2);
- Type colType = column.getOriginType();
- PrimitiveType primitiveType = colType.getPrimitiveType();
- if (primitiveType == PrimitiveType.DECIMALV2) {
- ScalarType scalarType = (ScalarType) colType;
- baseInfo.put("precision", scalarType.getPrecision() + "");
- baseInfo.put("scale", scalarType.getScalarScale() + "");
- }
- baseInfo.put("type", primitiveType.toString());
- baseInfo.put("comment", column.getComment());
- baseInfo.put("name", column.getDisplayName());
- Optional aggregationType = Optional.ofNullable(column.getAggregationType());
- baseInfo.put("aggregation_type", aggregationType.isPresent() ? column.getAggregationType().toSql() : "");
- propList.add(baseInfo);
- }
- resultMap.put("status", 200);
- resultMap.put("properties", propList);
- } catch (Exception e) {
- // Transform the general Exception to custom DorisHttpException
- throw new DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, e.getMessage() == null ? "Null Pointer Exception" : e.getMessage());
- } finally {
- table.readUnlock();
- }
- } catch (DorisHttpException e) {
- // status code should conforms to HTTP semantic
- resultMap.put("status", e.getCode().code());
- resultMap.put("exception", e.getMessage());
- }
- ObjectMapper mapper = new ObjectMapper();
- try {
- String result = mapper.writeValueAsString(resultMap);
- // send result with extra information
- response.setContentType("application/json");
- response.getContent().append(result);
- sendResult(request, response, HttpResponseStatus.valueOf(Integer.parseInt(String.valueOf(resultMap.get("status")))));
- } catch (Exception e) {
- // may be this never happen
- response.getContent().append(e.getMessage());
- sendResult(request, response, HttpResponseStatus.INTERNAL_SERVER_ERROR);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java
similarity index 72%
rename from fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java
rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java
index 9e82202d83..b77cf08449 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/IllegalArgException.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/IllegalArgException.java
@@ -14,21 +14,20 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-
-package org.apache.doris.http;
+package org.apache.doris.httpv2;
public class IllegalArgException extends Exception {
- private static final long serialVersionUID = 3344697787301861667L;
+ private static final long serialVersionUID = 3344697787301861667L;
- public IllegalArgException() {
- super("");
- }
+ public IllegalArgException() {
+ super("");
+ }
- public IllegalArgException(String msg) {
- super(msg);
- }
+ public IllegalArgException(String msg) {
+ super(msg);
+ }
- public IllegalArgException(String msg, Throwable cause) {
- super(msg, cause);
- }
-}
+ public IllegalArgException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java
similarity index 95%
rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java
rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java
index a72b94cd94..23cd6f7fbc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/ActionStatus.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/ActionStatus.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.http.rest;
+package org.apache.doris.httpv2.entity;
// Status of RESTful action
public enum ActionStatus {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java
similarity index 97%
rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java
rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java
index 63cd07cc02..5d27d9e691 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseResult.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestBaseResult.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-
-package org.apache.doris.http.rest;
+package org.apache.doris.httpv2.entity;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
// Base restful result
public class RestBaseResult {
+
private static final RestBaseResult OK = new RestBaseResult();
public ActionStatus status;
public String msg;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java
similarity index 94%
rename from fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java
rename to fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java
index d8819848c1..d501bfdc59 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RestResult.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/entity/RestResult.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.http.rest;
+package org.apache.doris.httpv2.entity;
import java.util.Map;
@@ -23,6 +23,7 @@ import com.google.common.collect.Maps;
import com.google.gson.Gson;
public class RestResult extends RestBaseResult {
+
private Map<String, Object> resultMap;
public RestResult() {
@@ -38,7 +39,7 @@ public class RestResult extends RestBaseResult {
Gson gson = new Gson();
addResultEntry("status", status);
if (status != ActionStatus.OK) {
- addResultEntry("msg", msg);
+ addResultEntry("msg", msg);
}
return gson.toJson(resultMap);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java
index d79cd5b997..2213034cfe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java
@@ -20,7 +20,7 @@ package org.apache.doris.httpv2.rest;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
-import org.apache.doris.http.rest.RestBaseResult;
+import org.apache.doris.httpv2.entity.RestBaseResult;
import org.apache.doris.load.Load;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
index 8cacb0832a..d5b2b71e58 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
@@ -20,7 +20,7 @@ package org.apache.doris.httpv2.rest;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.DdlException;
-import org.apache.doris.http.rest.RestBaseResult;
+import org.apache.doris.httpv2.entity.RestBaseResult;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.service.ExecuteEnv;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java
index 943aa26bac..86983901d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java
@@ -19,7 +19,7 @@ package org.apache.doris.httpv2.rest;
import org.apache.doris.analysis.LoadStmt;
import org.apache.doris.common.DdlException;
-import org.apache.doris.http.rest.RestBaseResult;
+import org.apache.doris.httpv2.entity.RestBaseResult;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.service.ExecuteEnv;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java
index 1dc7e77be7..35533433d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java
@@ -20,8 +20,8 @@ package org.apache.doris.journal.bdbje;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.ThreadPoolManager;
-import org.apache.doris.http.HttpServer;
-import org.apache.doris.http.IllegalArgException;
+import org.apache.doris.httpv2.HttpServer;
+import org.apache.doris.httpv2.IllegalArgException;
import org.apache.doris.journal.JournalEntity;
import org.apache.doris.meta.MetaContext;
import org.apache.doris.qe.QeService;
@@ -86,20 +86,10 @@ public class BDBDebugger {
// Only start MySQL and HttpServer
private void startService(String dorisHomeDir) throws IllegalArgException, IOException {
// HTTP server
- if (!Config.enable_http_server_v2) {
- HttpServer httpServer = new HttpServer(
- Config.http_port,
- Config.http_max_line_length,
- Config.http_max_header_size,
- Config.http_max_chunk_size
- );
- httpServer.setup();
- httpServer.start();
- } else {
- org.apache.doris.httpv2.HttpServer httpServer2 = new org.apache.doris.httpv2.HttpServer();
- httpServer2.setPort(Config.http_port);
- httpServer2.start(dorisHomeDir);
- }
+
+ HttpServer httpServer = new HttpServer();
+ httpServer.setPort(Config.http_port);
+ httpServer.start(dorisHomeDir);
// MySQl server
QeService qeService = new QeService(Config.query_port, Config.mysql_service_nio_enabled, ExecuteEnv.getInstance().getScheduler());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java
index 66531d6ed3..afeef0db1f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/path/PathTrieTest.java
@@ -19,11 +19,6 @@ package org.apache.doris.common.path;
import static com.google.common.collect.Maps.newHashMap;
-import org.apache.doris.http.ActionController;
-import org.apache.doris.http.rest.MultiStart;
-import org.apache.doris.http.rest.TableQueryPlanAction;
-import org.apache.doris.http.rest.TableRowCountAction;
-import org.apache.doris.http.rest.TableSchemaAction;
import org.apache.doris.service.ExecuteEnv;
import com.google.common.collect.Maps;
@@ -165,18 +160,4 @@ public class PathTrieTest {
Assert.assertEquals(params.get("test"), "*");
}
- @Test
- public void testInsert(@Injectable ActionController controller,
- @Injectable ExecuteEnv execEnv) {
- PathTrie pathTrie = new PathTrie();
- pathTrie.insert("/api/{db}/_multi_start", new MultiStart(controller, execEnv));
- pathTrie.insert("/api/{db}/{table}/_count", new TableRowCountAction(controller));
- pathTrie.insert("/api/{db}/{table}/_schema", new TableSchemaAction(controller));
- pathTrie.insert("/api/{db}/{table}/_query_plan", new TableQueryPlanAction(controller));
- Map<String, String> params = Maps.newHashMap();
- pathTrie.retrieve("/api/test/_multi_start", params);
- Assert.assertEquals("test", params.get("db"));
-
-
- }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
index 8f2a1c444c..71f4adaf24 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
@@ -38,9 +38,12 @@ import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker.ThrowingRunnable;
import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.httpv2.HttpServer;
+import org.apache.doris.httpv2.IllegalArgException;
import org.apache.doris.load.Load;
import org.apache.doris.mysql.privilege.PaloAuth;
import org.apache.doris.persist.EditLog;
@@ -110,6 +113,8 @@ abstract public class DorisHttpTestCase {
protected String rootAuth = Credentials.basic("root", "");
+ public static final String DORIS_HOME_DIR = System.getenv("DORIS_HOME");
+
@Mocked
private static EditLog editLog;
@@ -291,13 +296,13 @@ abstract public class DorisHttpTestCase {
}
}
- httpServer = new HttpServer(HTTP_PORT);
- httpServer.setup();
- httpServer.start();
- // must ensure the http server started before any unit test
- while (!httpServer.isStarted()) {
- Thread.sleep(500);
- }
+ httpServer = new HttpServer();
+ httpServer.setPort(Config.http_port);
+ httpServer.setMaxHttpPostSize(Config.jetty_server_max_http_post_size);
+ httpServer.setAcceptors(Config.jetty_server_acceptors);
+ httpServer.setSelectors(Config.jetty_server_selectors);
+ httpServer.setWorkers(Config.jetty_server_workers);
+ httpServer.start(DORIS_HOME_DIR);
}
@@ -337,10 +342,6 @@ abstract public class DorisHttpTestCase {
public void tearDown() {
}
- @AfterClass
- public static void closeHttpServer() {
- httpServer.shutDown();
- }
public void doSetUp() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java
index a6ed325d2b..2cd23dc34d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/http/HttpAuthManagerTest.java
@@ -17,9 +17,12 @@
package org.apache.doris.http;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.doris.analysis.UserIdentity;
-import org.apache.doris.http.HttpAuthManager.SessionValue;
+import org.apache.doris.httpv2.HttpAuthManager.SessionValue;
+import org.apache.doris.httpv2.HttpAuthManager;
import org.junit.Assert;
import org.junit.Test;
@@ -31,15 +34,20 @@ public class HttpAuthManagerTest {
HttpAuthManager authMgr = HttpAuthManager.getInstance();
String sessionId = "test_session_id";
String username = "test-user";
- SessionValue sessionValue = new SessionValue();
+ HttpAuthManager.SessionValue sessionValue = new HttpAuthManager.SessionValue();
sessionValue.currentUser = UserIdentity.createAnalyzedUserIdentWithIp(username, "%");
authMgr.addSessionValue(sessionId, sessionValue);
Assert.assertEquals(1, authMgr.getAuthSessions().size());
- System.out.println("username in test: " + authMgr.getSessionValue(sessionId).currentUser);
- Assert.assertEquals(username, authMgr.getSessionValue(sessionId).currentUser.getQualifiedUser());
+ List<String> sessionIds = new ArrayList<>();
+ sessionIds.add (sessionId);
+ System.out.println("username in test: " + authMgr.getSessionValue(sessionIds).currentUser);
+ Assert.assertEquals(username, authMgr.getSessionValue(sessionIds).currentUser.getQualifiedUser());
+
String noExistSession = "no-exist-session-id";
- Assert.assertNull(authMgr.getSessionValue(noExistSession));
+ sessionIds.clear();
+ sessionIds.add (noExistSession);
+ Assert.assertNull(authMgr.getSessionValue(sessionIds));
Assert.assertEquals(1, authMgr.getAuthSessions().size());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 06/14: [feature](diagnose) support "ADMIN DIAGNOSE TABLET" stmt (#8839)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit cf94d9fe0f362db70dbf0deadc500bc94913799d
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Thu Apr 7 11:30:03 2022 +0800
[feature](diagnose) support "ADMIN DIAGNOSE TABLET" stmt (#8839)
`ADMIN DIAGNOSE TABLET tablet_id`
This statement makes it easier to quickly diagnose the status of a tablet.
See "ADMIN-DIAGNOSE-TABLET.md" for details
```
mysql> admin diagnose tablet 10196;
+----------------------------------+------------------------------+------------+
| Item | Info | Suggestion |
+----------------------------------+------------------------------+------------+
| TabletExist | Yes | |
| TabletId | 10196 | |
| Database | default_cluster:db1: 10192 | |
| Table | tbl1: 10194 | |
| Partition | tbl1: 10193 | |
| MaterializedIndex | tbl1: 10195 | |
| Replicas(ReplicaId -> BackendId) | {"10197":10002} | |
| ReplicasNum | OK | |
| ReplicaBackendStatus | Backend 10002 is not alive. | |
| ReplicaVersionStatus | OK | |
| ReplicaStatus | OK | |
| ReplicaCompactionStatus | OK | |
+----------------------------------+------------------------------+------------+
```
---
docs/.vuepress/sidebar/en.js | 1 +
docs/.vuepress/sidebar/zh-CN.js | 1 +
.../Administration/ADMIN-DIAGNOSE-TABLET.md | 59 ++++++++
.../Administration/ADMIN-DIAGNOSE-TABLET.md | 59 ++++++++
fe/fe-core/pom.xml | 4 +-
fe/fe-core/src/main/cup/sql_parser.cup | 8 +-
.../doris/analysis/AdminDiagnoseTabletStmt.java | 76 ++++++++++
.../java/org/apache/doris/qe/ShowExecutor.java | 11 ++
.../java/org/apache/doris/system/Diagnoser.java | 163 +++++++++++++++++++++
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../doris/clone/TabletReplicaTooSlowTest.java | 8 +
11 files changed, 388 insertions(+), 3 deletions(-)
diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js
index 896805f6c2..2361963495 100644
--- a/docs/.vuepress/sidebar/en.js
+++ b/docs/.vuepress/sidebar/en.js
@@ -588,6 +588,7 @@ module.exports = [
"ADMIN SHOW CONFIG",
"ADMIN SHOW REPLICA DISTRIBUTION",
"ADMIN SHOW REPLICA STATUS",
+ "ADMIN-DIAGNOSE-TABLET",
"ALTER CLUSTER",
"ALTER SYSTEM",
"CANCEL DECOMMISSION",
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index 9aa410753b..dcb4a87597 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -590,6 +590,7 @@ module.exports = [
"ADMIN SHOW CONFIG",
"ADMIN SHOW REPLICA DISTRIBUTION",
"ADMIN SHOW REPLICA STATUS",
+ "ADMIN-DIAGNOSE-TABLET",
"ALTER CLUSTER",
"ALTER SYSTEM",
"CANCEL DECOMMISSION",
diff --git a/docs/en/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md b/docs/en/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md
new file mode 100644
index 0000000000..e41bf98c53
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md
@@ -0,0 +1,59 @@
+---
+{
+ "title": "ADMIN DIAGNOSE TABLET",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# ADMIN DIAGNOSE TABLET
+## description
+
+ This statement is used to diagnose the specified tablet. The results will show information about the tablet and some potential problems.
+
+ grammar:
+
+ ADMIN DIAGNOSE TABLET tblet_id
+
+ illustrate:
+
+ The lines of information in the result are as follows:
+ 1. TabletExist: Whether the Tablet exists
+ 2. TabletId: Tablet ID
+ 3. Database: The DB to which the Tablet belongs and its ID
+ 4. Table: The Table to which Tablet belongs and its ID
+ 5. Partition: The Partition to which the Tablet belongs and its ID
+ 6. MaterializedIndex: The materialized view to which the Tablet belongs and its ID
+ 7. Replicas(ReplicaId -> BackendId): Tablet replicas and their BE.
+ 8. ReplicasNum: Whether the number of replicas is correct.
+ 9. ReplicaBackendStatus: Whether the BE node where the replica is located is normal.
+ 10.ReplicaVersionStatus: Whether the version number of the replica is normal.
+ 11.ReplicaStatus: Whether the replica status is normal.
+ 12.ReplicaCompactionStatus: Whether the replica Compaction status is normal.
+
+## example
+
+ 1. Diagnose tablet 10001
+
+ ADMIN DIAGNOSE TABLET 10001;
+
+## keyword
+ ADMIN,DIAGNOSE,TABLET
diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md
new file mode 100644
index 0000000000..74d6e6699a
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-statements/Administration/ADMIN-DIAGNOSE-TABLET.md
@@ -0,0 +1,59 @@
+---
+{
+ "title": "ADMIN DIAGNOSE TABLET",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# ADMIN DIAGNOSE TABLET
+## description
+
+ 该语句用于诊断指定 tablet。结果中将显示这个 tablet 的信息和一些潜在的问题。
+
+ 语法：
+
+ ADMIN DIAGNOSE TABLET tblet_id
+
+ 说明：
+
+ 结果中的各行信息如下：
+ 1. TabletExist: Tablet是否存在
+ 2. TabletId: Tablet ID
+ 3. Database: Tablet 所属 DB 和其 ID
+ 4. Table: Tablet 所属 Table 和其 ID
+ 5. Partition: Tablet 所属 Partition 和其 ID
+ 6. MaterializedIndex: Tablet 所属物化视图和其 ID
+ 7. Replicas(ReplicaId -> BackendId): Tablet 各副本和其所在 BE。
+ 8. ReplicasNum: 副本数量是否正确。
+ 9. ReplicaBackendStatus: 副本所在 BE 节点是否正常。
+ 10.ReplicaVersionStatus: 副本的版本号是否正常。
+ 11.ReplicaStatus: 副本状态是否正常。
+ 12.ReplicaCompactionStatus: 副本 Compaction 状态是否正常。
+
+## example
+
+ 1. 查看 Tablet 10001 的诊断结果
+
+ ADMIN DIAGNOSE TABLET 10001;
+
+## keyword
+ ADMIN,DIAGNOSE,TABLET
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 7a8fe89ef5..2f271b6ad9 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -688,9 +688,9 @@ under the License.
<goal>run</goal>
</goals>
<configuration>
- <protocCommand>${doris.thirdparty}/installed/bin/protoc</protocCommand>
+ <!--protocCommand>${doris.thirdparty}/installed/bin/protoc</protocCommand-->
<!-->You can use following protocArtifact instead of protocCommand, so that you don't need to install protobuf tools<-->
- <!--protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact-->
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact>
<protocVersion>${protobuf.version}</protocVersion>
<inputDirectories>
<include>${doris.home}/gensrc/proto</include>
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index f42e864734..9e288b2fc2 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -241,7 +241,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_A
KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMMIT, KW_COMMITTED, KW_COMPACT,
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT, KW_COUNT, KW_CREATE, KW_CREATION, KW_CROSS, KW_CUBE, KW_CURRENT, KW_CURRENT_USER,
KW_DATA, KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DAY, KW_DECIMAL, KW_DECOMMISSION, KW_DEFAULT, KW_DESC, KW_DESCRIBE,
- KW_DELETE, KW_UPDATE, KW_DISTINCT, KW_DISTINCTPC, KW_DISTINCTPCSA, KW_DISTRIBUTED, KW_DISTRIBUTION, KW_DYNAMIC, KW_BUCKETS, KW_DIV, KW_DOUBLE, KW_DROP, KW_DROPP, KW_DUPLICATE,
+ KW_DELETE, KW_UPDATE, KW_DIAGNOSE, KW_DISTINCT, KW_DISTINCTPC, KW_DISTINCTPCSA, KW_DISTRIBUTED, KW_DISTRIBUTION, KW_DYNAMIC, KW_BUCKETS, KW_DIV, KW_DOUBLE, KW_DROP, KW_DROPP, KW_DUPLICATE,
KW_ELSE, KW_ENABLE, KW_ENCRYPTKEY, KW_ENCRYPTKEYS, KW_END, KW_ENGINE, KW_ENGINES, KW_ENTER, KW_ERRORS, KW_EVENTS, KW_EXCEPT, KW_EXCLUDE,
KW_EXISTS, KW_EXPORT, KW_EXTENDED, KW_EXTERNAL, KW_EXTRACT,
KW_FALSE, KW_FEATURE, KW_FOLLOWER, KW_FOLLOWING, KW_FREE, KW_FROM, KW_FIELDS, KW_FILE, KW_FILTER, KW_FIRST, KW_FLOAT, KW_FOR, KW_FORCE, KW_FORMAT, KW_FRONTEND, KW_FRONTENDS, KW_FULL, KW_FUNCTION, KW_FUNCTIONS,
@@ -5283,6 +5283,10 @@ admin_stmt ::=
{:
RESULT = new AdminCleanTrashStmt(null);
:}
+ | KW_ADMIN KW_DIAGNOSE KW_TABLET INTEGER_LITERAL:tabletId
+ {:
+ RESULT = new AdminDiagnoseTabletStmt(tabletId);
+ :}
;
truncate_stmt ::=
@@ -5458,6 +5462,8 @@ keyword ::=
{: RESULT = id; :}
| KW_DECIMAL:id
{: RESULT = id; :}
+ | KW_DIAGNOSE:id
+ {: RESULT = id; :}
| KW_DISTINCTPC:id
{: RESULT = id; :}
| KW_DISTINCTPCSA:id
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminDiagnoseTabletStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminDiagnoseTabletStmt.java
new file mode 100644
index 0000000000..38dae3fbd7
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminDiagnoseTabletStmt.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.UserException;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSetMetaData;
+
+import com.google.common.collect.ImmutableList;
+
+// ADMIN DIAGNOSE TABLET tablet_id
+public class AdminDiagnoseTabletStmt extends ShowStmt {
+ public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+ .add("Item").add("Info").add("Suggestion")
+ .build();
+
+ private long tabletId;
+
+ public AdminDiagnoseTabletStmt(long tabletId) {
+ this.tabletId = tabletId;
+ }
+
+ @Override
+ public void analyze(Analyzer analyzer) throws UserException {
+ super.analyze(analyzer);
+
+ // check auth
+ if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+ }
+ }
+
+ public long getTabletId() {
+ return tabletId;
+ }
+
+ @Override
+ public String toSql() {
+ return "ADMIN DIAGNOSE TABLET " + tabletId;
+ }
+
+ @Override
+ public ShowResultSetMetaData getMetaData() {
+ ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+ for (String title : TITLE_NAMES) {
+ builder.addColumn(new Column(title, ScalarType.createVarchar(1024)));
+ }
+ return builder.build();
+ }
+
+ @Override
+ public RedirectStatus getRedirectStatus() {
+ return RedirectStatus.FORWARD_NO_SYNC;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 330b816dec..5fb6ade642 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -17,6 +17,7 @@
package org.apache.doris.qe;
+import org.apache.doris.analysis.AdminDiagnoseTabletStmt;
import org.apache.doris.analysis.AdminShowConfigStmt;
import org.apache.doris.analysis.AdminShowReplicaDistributionStmt;
import org.apache.doris.analysis.AdminShowReplicaStatusStmt;
@@ -155,6 +156,7 @@ import org.apache.doris.load.LoadJob.JobState;
import org.apache.doris.load.routineload.RoutineLoadJob;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.system.Backend;
+import org.apache.doris.system.Diagnoser;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TUnit;
import org.apache.doris.transaction.GlobalTransactionMgr;
@@ -333,6 +335,8 @@ public class ShowExecutor {
handleShowTableCreation();
} else if (stmt instanceof ShowLastInsertStmt) {
handleShowLastInsert();
+ } else if (stmt instanceof AdminDiagnoseTabletStmt) {
+ handleAdminDiagnoseTablet();
} else {
handleEmtpy();
}
@@ -2128,4 +2132,11 @@ public class ShowExecutor {
resultSet = new ShowResultSet(showMetaData, resultRowSet);
}
+ private void handleAdminDiagnoseTablet() {
+ AdminDiagnoseTabletStmt showStmt = (AdminDiagnoseTabletStmt) stmt;
+ List<List<String>> resultRowSet = Diagnoser.diagnoseTablet(showStmt.getTabletId());
+ ShowResultSetMetaData showMetaData = showStmt.getMetaData();
+ resultSet = new ShowResultSet(showMetaData, resultRowSet);
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Diagnoser.java b/fe/fe-core/src/main/java/org/apache/doris/system/Diagnoser.java
new file mode 100644
index 0000000000..24048ac251
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/Diagnoser.java
@@ -0,0 +1,163 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.system;
+
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.MaterializedIndex;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Replica;
+import org.apache.doris.catalog.Tablet;
+import org.apache.doris.catalog.TabletInvertedIndex;
+import org.apache.doris.catalog.TabletMeta;
+import org.apache.doris.common.Config;
+
+import com.google.common.collect.Lists;
+
+import org.json.simple.JSONObject;
+
+import java.util.List;
+
+// This is a util class to diagnose the Doris system
+public class Diagnoser {
+ // To diagnose a given tablet and return the info and issues about it
+ // - tablet exist:
+ // - tablet id
+ // - database
+ // - table
+ // - partition
+ // - materialized view
+ // - replica info: {"replica_id" : "backend id"}
+ // - replica num
+ // - ReplicaBackendStatus
+ // - ReplicaVersionStatus
+ // - ReplicaStatus
+ // - ReplicaCompactionStatus
+ //
+ public static List<List<String>> diagnoseTablet(long tabletId) {
+ List<List<String>> results = Lists.newArrayList();
+ TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex();
+ TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId);
+ if (tabletMeta == null) {
+ results.add(Lists.newArrayList("TabletExist", "No", ""));
+ return results;
+ }
+ results.add(Lists.newArrayList("TabletExist", "Yes", ""));
+ results.add(Lists.newArrayList("TabletId", String.valueOf(tabletId), ""));
+ // database
+ Database db = Catalog.getCurrentCatalog().getDbNullable(tabletMeta.getDbId());
+ if (db == null) {
+ results.add(Lists.newArrayList("Database", "Not exist", ""));
+ return results;
+ }
+ results.add(Lists.newArrayList("Database", db.getFullName() + ": " + db.getId(), ""));
+ // table
+ OlapTable tbl = (OlapTable) db.getTableNullable(tabletMeta.getTableId());
+ if (tbl == null) {
+ results.add(Lists.newArrayList("Table", "Not exist", ""));
+ return results;
+ }
+ results.add(Lists.newArrayList("Table", tbl.getName() + ": " + tbl.getId(), ""));
+ // partition
+ Partition partition = tbl.getPartition(tabletMeta.getPartitionId());
+ if (partition == null) {
+ results.add(Lists.newArrayList("Partition", "Not exist", ""));
+ return results;
+ }
+ results.add(Lists.newArrayList("Partition", partition.getName() + ": " + partition.getId(), ""));
+ // materialized index
+ MaterializedIndex mIndex = partition.getIndex(tabletMeta.getIndexId());
+ if (mIndex == null) {
+ results.add(Lists.newArrayList("MaterializedIndex", "Not exist", ""));
+ return results;
+ }
+ results.add(Lists.newArrayList("MaterializedIndex", tbl.getIndexNameById(mIndex.getId()) + ": " + mIndex.getId(), ""));
+ // replica info
+ Tablet tablet = mIndex.getTablet(tabletId);
+ List<Replica> replicas = tablet.getReplicas();
+ JSONObject jobj = new JSONObject();
+ for (Replica replica : replicas) {
+ jobj.put(replica.getId(), replica.getBackendId());
+ }
+ results.add(Lists.newArrayList("Replicas(ReplicaId -> BackendId)", jobj.toJSONString(), ""));
+ // replica
+ short replicaNum = tbl.getPartitionInfo().getReplicaAllocation(partition.getId()).getTotalReplicaNum();
+ if (replicas.size() != replicaNum) {
+ results.add(Lists.newArrayList("ReplicasNum", "Replica num is " + replicas.size() + ", expected: " + replicaNum, ""));
+ } else {
+ results.add(Lists.newArrayList("ReplicasNum", "OK", ""));
+ }
+
+ SystemInfoService infoService = Catalog.getCurrentSystemInfo();
+ StringBuilder backendErr = new StringBuilder();
+ StringBuilder versionErr = new StringBuilder();
+ StringBuilder statusErr = new StringBuilder();
+ StringBuilder compactionErr = new StringBuilder();
+ for (Replica replica : replicas) {
+ // backend
+ do {
+ Backend be = infoService.getBackend(replica.getBackendId());
+ if (be == null) {
+ backendErr.append("Backend " + replica.getBackendId() + " does not exist. ");
+ break;
+ }
+ if (!be.isAlive()) {
+ backendErr.append("Backend " + replica.getBackendId() + " is not alive. ");
+ break;
+ }
+ if (be.isDecommissioned()) {
+ backendErr.append("Backend " + replica.getBackendId() + " is decommission. ");
+ break;
+ }
+ if (!be.isLoadAvailable()) {
+ backendErr.append("Backend " + replica.getBackendId() + " is not load available. ");
+ break;
+ }
+ if (!be.isQueryAvailable()) {
+ backendErr.append("Backend " + replica.getBackendId() + " is not query available. ");
+ break;
+ }
+ if (be.diskExceedLimit()) {
+ backendErr.append("Backend " + replica.getBackendId() + " has no space left. ");
+ break;
+ }
+ } while (false);
+ // version
+ if (replica.getVersion() != partition.getVisibleVersion()) {
+ versionErr.append("Replica on backend " + replica.getBackendId() + "'s version (" +
+ replica.getVersion() + ") does not equal" +
+ " to partition visible version (" + partition.getVisibleVersion() + ")");
+ }
+ // status
+ if (!replica.isAlive()) {
+ statusErr.append("Replica on backend " + replica.getBackendId() + "'s state is " + replica.getState()
+ + ", and is bad: " + (replica.isBad() ? "Yes" : "No"));
+ }
+ if (replica.getVersionCount() > Config.min_version_count_indicate_replica_compaction_too_slow) {
+ compactionErr.append("Replica on backend " + replica.getBackendId() + "'s version count is too high: "
+ + replica.getVersionCount());
+ }
+ }
+ results.add(Lists.newArrayList("ReplicaBackendStatus", (backendErr.length() == 0 ? "OK" : backendErr.toString()), ""));
+ results.add(Lists.newArrayList("ReplicaVersionStatus", (versionErr.length() == 0 ? "OK" : versionErr.toString()), ""));
+ results.add(Lists.newArrayList("ReplicaStatus", (statusErr.length() == 0 ? "OK" : statusErr.toString()), ""));
+ results.add(Lists.newArrayList("ReplicaCompactionStatus", (compactionErr.length() == 0 ? "OK" : compactionErr.toString()), ""));
+ return results;
+ }
+}
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex
index 6f356569ce..1fb4a9d47a 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -162,6 +162,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("delete", new Integer(SqlParserSymbols.KW_DELETE));
keywordMap.put("desc", new Integer(SqlParserSymbols.KW_DESC));
keywordMap.put("describe", new Integer(SqlParserSymbols.KW_DESCRIBE));
+ keywordMap.put("diagnose", new Integer(SqlParserSymbols.KW_DIAGNOSE));
keywordMap.put("distinct", new Integer(SqlParserSymbols.KW_DISTINCT));
keywordMap.put("distinctpc", new Integer(SqlParserSymbols.KW_DISTINCTPC));
keywordMap.put("distinctpc", new Integer(SqlParserSymbols.KW_DISTINCTPC));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletReplicaTooSlowTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletReplicaTooSlowTest.java
index 2a48e64065..bc88fe94ff 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletReplicaTooSlowTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletReplicaTooSlowTest.java
@@ -29,6 +29,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.resource.Tag;
import org.apache.doris.system.Backend;
+import org.apache.doris.system.Diagnoser;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TStorageMedium;
@@ -132,7 +133,9 @@ public class TabletReplicaTooSlowTest {
private static void updateReplicaVersionCount() {
Table<Long, Long, Replica> replicaMetaTable = Catalog.getCurrentInvertedIndex().getReplicaMetaTable();
int versionCount = 1;
+ long tabletId = -1;
for (Table.Cell<Long, Long, Replica> cell : replicaMetaTable.cellSet()) {
+ tabletId = cell.getRowKey();
long beId = cell.getColumnKey();
Backend be = Catalog.getCurrentSystemInfo().getBackend(beId);
List<Long> pathHashes = be.getDisks().values().stream().map(DiskInfo::getPathHash).collect(Collectors.toList());
@@ -145,6 +148,10 @@ public class TabletReplicaTooSlowTest {
replica.setPathHash(pathHashes.get(0));
}
+
+ List<List<String>> result = Diagnoser.diagnoseTablet(tabletId);
+ Assert.assertEquals(12, result.size());
+ Assert.assertTrue(result.get(11).get(1).contains("version count is too high"));
}
@Test
@@ -158,6 +165,7 @@ public class TabletReplicaTooSlowTest {
" \"replication_num\" = \"3\"\n" +
")";
ExceptionChecker.expectThrowsNoException(() -> createTable(createStr));
+
int maxLoop = 300;
boolean delete = false;
while (maxLoop-- > 0) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 03/14: [improvement](hll) Optimize Hyperloglog (#8829)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 7aace71325f1f0cbea446f14f7acc176741501b8
Author: zbtzbtzbt <35...@users.noreply.github.com>
AuthorDate: Fri Apr 8 09:06:08 2022 +0800
[improvement](hll) Optimize Hyperloglog (#8829)
In meituan, pr #6625 was revert due to the oom probleam.
currently, we are trying to modify the old hyperloglog, based on pr #8555, we did some works.
via some test, we find it better than old hll, and better than apache:master hll.
Changes summary:
- use SIMD max tp speed up heavy function _merge_registers
- use phmap::flat_hash_set rather than std::set
- replace std::max
- other small changes
---
be/src/exprs/aggregate_functions.cpp | 6 +-
be/src/olap/hll.cpp | 143 ++++++++---------------------------
be/src/olap/hll.h | 118 ++++++++++-------------------
3 files changed, 75 insertions(+), 192 deletions(-)
diff --git a/be/src/exprs/aggregate_functions.cpp b/be/src/exprs/aggregate_functions.cpp
index 09d9b1b68f..9e442b3081 100644
--- a/be/src/exprs/aggregate_functions.cpp
+++ b/be/src/exprs/aggregate_functions.cpp
@@ -1108,11 +1108,9 @@ void AggregateFunctions::hll_merge(FunctionContext* ctx, const StringVal& src, S
DCHECK(!src.is_null);
DCHECK_EQ(dst->len, std::pow(2, HLL_COLUMN_PRECISION));
DCHECK_EQ(src.len, std::pow(2, HLL_COLUMN_PRECISION));
-
- auto dp = dst->ptr;
- auto sp = src.ptr;
+
for (int i = 0; i < src.len; ++i) {
- dp[i] = (dp[i] < sp[i] ? sp[i] : dp[i]);
+ dst->ptr[i] = (dst->ptr[i] < src.ptr[i] ? src.ptr[i] : dst->ptr[i]);
}
}
diff --git a/be/src/olap/hll.cpp b/be/src/olap/hll.cpp
index 12fb912b3a..a2ec6e39a0 100644
--- a/be/src/olap/hll.cpp
+++ b/be/src/olap/hll.cpp
@@ -43,15 +43,13 @@ HyperLogLog::HyperLogLog(const Slice& src) {
void HyperLogLog::_convert_explicit_to_register() {
DCHECK(_type == HLL_DATA_EXPLICIT)
<< "_type(" << _type << ") should be explicit(" << HLL_DATA_EXPLICIT << ")";
- _registers = new uint8_t[HLL_REGISTERS_COUNT]();
-
- for (uint32_t i = 0; i < _explicit_data_num; ++i) {
- _update_registers(_explicit_data[i]);
+ _registers = new uint8_t[HLL_REGISTERS_COUNT];
+ memset(_registers, 0, HLL_REGISTERS_COUNT);
+ for (auto value : _hash_set) {
+ _update_registers(value);
}
-
- delete [] _explicit_data;
- _explicit_data = nullptr;
- _explicit_data_num = 0;
+ // clear _hash_set
+ phmap::flat_hash_set<uint64_t>().swap(_hash_set);
}
// Change HLL_DATA_EXPLICIT to HLL_DATA_FULL directly, because HLL_DATA_SPARSE
@@ -59,14 +57,12 @@ void HyperLogLog::_convert_explicit_to_register() {
void HyperLogLog::update(uint64_t hash_value) {
switch (_type) {
case HLL_DATA_EMPTY:
- _explicit_data = new uint64_t[HLL_EXPLICIT_INT64_NUM_DOUBLE];
- _explicit_data[0] = hash_value;
- _explicit_data_num = 1;
+ _hash_set.insert(hash_value);
_type = HLL_DATA_EXPLICIT;
break;
case HLL_DATA_EXPLICIT:
- if (_explicit_data_num < HLL_EXPLICIT_INT64_NUM) {
- _explicit_data_insert(hash_value);
+ if (_hash_set.size() < HLL_EXPLICIT_INT64_NUM) {
+ _hash_set.insert(hash_value);
break;
}
_convert_explicit_to_register();
@@ -90,10 +86,7 @@ void HyperLogLog::merge(const HyperLogLog& other) {
_type = other._type;
switch (other._type) {
case HLL_DATA_EXPLICIT:
- _explicit_data_num = other._explicit_data_num;
- _explicit_data = new uint64_t[HLL_EXPLICIT_INT64_NUM_DOUBLE];
- memcpy(_explicit_data, other._explicit_data,
- sizeof(*_explicit_data) * _explicit_data_num);
+ _hash_set = other._hash_set;
break;
case HLL_DATA_SPARSE:
case HLL_DATA_FULL:
@@ -110,47 +103,8 @@ void HyperLogLog::merge(const HyperLogLog& other) {
case HLL_DATA_EXPLICIT: {
// Merge other's explicit values first, then check if the number is exceed
// HLL_EXPLICIT_INT64_NUM. This is OK because the max value is 2 * 160.
- if (other._explicit_data_num > HLL_EXPLICIT_INT64_NUM / 2) { //merge
- uint64_t explicit_data[HLL_EXPLICIT_INT64_NUM * 2];
- memcpy(explicit_data, _explicit_data, sizeof(*_explicit_data) * _explicit_data_num);
- uint32_t explicit_data_num = _explicit_data_num;
- _explicit_data_num = 0;
-
- // merge _explicit_data and other's _explicit_data to _explicit_data
- uint32_t i = 0, j = 0, k = 0;
- while (i < explicit_data_num || j < other._explicit_data_num) {
- if (i == explicit_data_num) {
- uint32_t n = other._explicit_data_num - j;
- memcpy(_explicit_data + k, other._explicit_data + j,
- n * sizeof(*_explicit_data));
- k += n;
- break;
- } else if (j == other._explicit_data_num) {
- uint32_t n = explicit_data_num - i;
- memcpy(_explicit_data + k, explicit_data + i, n * sizeof(*_explicit_data));
- k += n;
- break;
- } else {
- if (explicit_data[i] < other._explicit_data[j]) {
- _explicit_data[k++] = explicit_data[i++];
- } else if (explicit_data[i] > other._explicit_data[j]) {
- _explicit_data[k++] = other._explicit_data[j++];
- } else {
- _explicit_data[k++] = explicit_data[i++];
- j++;
- }
- }
- }
- _explicit_data_num = k;
- } else { //insert one by one
- int32_t n = other._explicit_data_num;
- const uint64_t* data = other._explicit_data;
- for (int32_t i = 0; i < n; ++i) {
- _explicit_data_insert(data[i]);
- }
- }
-
- if (_explicit_data_num > HLL_EXPLICIT_INT64_NUM) {
+ _hash_set.insert(other._hash_set.begin(), other._hash_set.end());
+ if (_hash_set.size() > HLL_EXPLICIT_INT64_NUM) {
_convert_explicit_to_register();
_type = HLL_DATA_FULL;
}
@@ -170,8 +124,8 @@ void HyperLogLog::merge(const HyperLogLog& other) {
case HLL_DATA_FULL: {
switch (other._type) {
case HLL_DATA_EXPLICIT:
- for (int32_t i = 0; i < other._explicit_data_num; ++i) {
- _update_registers(other._explicit_data[i]);
+ for (auto hash_value : other._hash_set) {
+ _update_registers(hash_value);
}
break;
case HLL_DATA_SPARSE:
@@ -192,7 +146,7 @@ size_t HyperLogLog::max_serialized_size() const {
default:
return 1;
case HLL_DATA_EXPLICIT:
- return 2 + _explicit_data_num * 8;
+ return 2 + _hash_set.size() * 8;
case HLL_DATA_SPARSE:
case HLL_DATA_FULL:
return 1 + HLL_REGISTERS_COUNT;
@@ -201,32 +155,24 @@ size_t HyperLogLog::max_serialized_size() const {
size_t HyperLogLog::serialize(uint8_t* dst) const {
uint8_t* ptr = dst;
-
switch (_type) {
case HLL_DATA_EMPTY:
default: {
// When the _type is unknown, which may not happen, we encode it as
// Empty HyperLogLog object.
*ptr++ = HLL_DATA_EMPTY;
-
break;
}
case HLL_DATA_EXPLICIT: {
- DCHECK(_explicit_data_num < HLL_EXPLICIT_INT64_NUM)
- << "Number of explicit elements(" << _explicit_data_num
+ DCHECK(_hash_set.size() <= HLL_EXPLICIT_INT64_NUM)
+ << "Number of explicit elements(" << _hash_set.size()
<< ") should be less or equal than " << HLL_EXPLICIT_INT64_NUM;
*ptr++ = _type;
- *ptr++ = (uint8_t)_explicit_data_num;
-
-#if __BYTE_ORDER == __LITTLE_ENDIAN
- memcpy(ptr, _explicit_data, _explicit_data_num * sizeof(*_explicit_data));
- ptr += _explicit_data_num * sizeof(*_explicit_data);
-#else
- for (int32_t i = 0; i < _explicit_data_num; ++i) {
- *(uint64_t*)ptr = (uint64_t)gbswap_64(_explicit_data[i]);
+ *ptr++ = (uint8_t)_hash_set.size();
+ for (auto hash_value : _hash_set) {
+ encode_fixed64_le(ptr, hash_value);
ptr += 8;
}
-#endif
break;
}
case HLL_DATA_SPARSE:
@@ -249,39 +195,15 @@ size_t HyperLogLog::serialize(uint8_t* dst) const {
encode_fixed32_le(ptr, num_non_zero_registers);
ptr += 4;
- for (uint32_t i = 0; i < HLL_REGISTERS_COUNT;) {
- if (*(uint32_t*)(&_registers[i]) == 0) {
- i += 4;
+ for (uint32_t i = 0; i < HLL_REGISTERS_COUNT; ++i) {
+ if (_registers[i] == 0) {
continue;
}
-
- if (UNLIKELY(_registers[i])) {
- encode_fixed16_le(ptr, i);
- ptr += 2; // 2 bytes: register index
- *ptr++ = _registers[i]; // 1 byte: register value
- }
- ++i;
-
- if (UNLIKELY(_registers[i])) {
- encode_fixed16_le(ptr, i);
- ptr += 2; // 2 bytes: register index
- *ptr++ = _registers[i]; // 1 byte: register value
- }
- ++i;
-
- if (UNLIKELY(_registers[i])) {
- encode_fixed16_le(ptr, i);
- ptr += 2; // 2 bytes: register index
- *ptr++ = _registers[i]; // 1 byte: register value
- }
- ++i;
-
- if (UNLIKELY(_registers[i])) {
- encode_fixed16_le(ptr, i);
- ptr += 2; // 2 bytes: register index
- *ptr++ = _registers[i]; // 1 byte: register value
- }
- ++i;
+ // 2 bytes: register index
+ // 1 byte: register value
+ encode_fixed16_le(ptr, i);
+ ptr += 2;
+ *ptr++ = _registers[i];
}
}
break;
@@ -355,24 +277,23 @@ bool HyperLogLog::deserialize(const Slice& slice) {
// 2: number of explicit values
// make sure that num_explicit is positive
uint8_t num_explicits = *ptr++;
- _explicit_data = new uint64_t[HLL_EXPLICIT_INT64_NUM_DOUBLE];
// 3+: 8 bytes hash value
for (int i = 0; i < num_explicits; ++i) {
- _explicit_data_insert(decode_fixed64_le(ptr));
+ _hash_set.insert(decode_fixed64_le(ptr));
ptr += 8;
}
break;
}
case HLL_DATA_SPARSE: {
- _registers = new uint8_t[HLL_REGISTERS_COUNT]();
+ _registers = new uint8_t[HLL_REGISTERS_COUNT];
+ memset(_registers, 0, HLL_REGISTERS_COUNT);
// 2-5(4 byte): number of registers
uint32_t num_registers = decode_fixed32_le(ptr);
- uint16_t register_idx = 0;
ptr += 4;
for (uint32_t i = 0; i < num_registers; ++i) {
// 2 bytes: register index
// 1 byte: register value
- register_idx = decode_fixed16_le(ptr);
+ uint16_t register_idx = decode_fixed16_le(ptr);
ptr += 2;
_registers[register_idx] = *ptr++;
}
@@ -397,7 +318,7 @@ int64_t HyperLogLog::estimate_cardinality() const {
return 0;
}
if (_type == HLL_DATA_EXPLICIT) {
- return _explicit_data_num;
+ return _hash_set.size();
}
const int num_streams = HLL_REGISTERS_COUNT;
diff --git a/be/src/olap/hll.h b/be/src/olap/hll.h
index 1ae56e2254..dc157f886e 100644
--- a/be/src/olap/hll.h
+++ b/be/src/olap/hll.h
@@ -24,6 +24,11 @@
#include <map>
#include <set>
#include <string>
+#include <parallel_hashmap/phmap.h>
+
+#ifdef __x86_64__
+#include <immintrin.h>
+#endif
#include "gutil/macros.h"
@@ -34,7 +39,6 @@ struct Slice;
const static int HLL_COLUMN_PRECISION = 14;
const static int HLL_ZERO_COUNT_BITS = (64 - HLL_COLUMN_PRECISION);
const static int HLL_EXPLICIT_INT64_NUM = 160;
-const static int HLL_EXPLICIT_INT64_NUM_DOUBLE = HLL_EXPLICIT_INT64_NUM * 2;
const static int HLL_SPARSE_THRESHOLD = 4096;
const static int HLL_REGISTERS_COUNT = 16 * 1024;
// maximum size in byte of serialized HLL: type(1) + registers (2^14)
@@ -83,10 +87,9 @@ class HyperLogLog {
public:
HyperLogLog() = default;
explicit HyperLogLog(uint64_t hash_value) : _type(HLL_DATA_EXPLICIT) {
- _explicit_data = new uint64_t[HLL_EXPLICIT_INT64_NUM_DOUBLE];
- _explicit_data[0] = hash_value;
- _explicit_data_num = 1;
+ _hash_set.emplace(hash_value);
}
+ explicit HyperLogLog(const Slice& src);
HyperLogLog(const HyperLogLog& other) {
this->_type = other._type;
@@ -94,10 +97,7 @@ public:
case HLL_DATA_EMPTY:
break;
case HLL_DATA_EXPLICIT: {
- this->_explicit_data_num = other._explicit_data_num;
- _explicit_data = new uint64_t[HLL_EXPLICIT_INT64_NUM_DOUBLE];
- memcpy(_explicit_data, other._explicit_data,
- sizeof(*_explicit_data) * _explicit_data_num);
+ this->_hash_set = other._hash_set;
break;
}
case HLL_DATA_SPARSE:
@@ -105,10 +105,10 @@ public:
_registers = new uint8_t[HLL_REGISTERS_COUNT];
memcpy(_registers, other._registers, HLL_REGISTERS_COUNT);
break;
+ }
default:
break;
}
- }
}
HyperLogLog(HyperLogLog&& other) {
@@ -117,10 +117,7 @@ public:
case HLL_DATA_EMPTY:
break;
case HLL_DATA_EXPLICIT: {
- this->_explicit_data_num = other._explicit_data_num;
- this->_explicit_data = other._explicit_data;
- other._explicit_data_num = 0;
- other._explicit_data = nullptr;
+ this->_hash_set = std::move(other._hash_set);
other._type = HLL_DATA_EMPTY;
break;
}
@@ -130,33 +127,25 @@ public:
other._registers = nullptr;
other._type = HLL_DATA_EMPTY;
break;
+ }
default:
break;
}
- }
}
HyperLogLog& operator=(HyperLogLog&& other) {
if (this != &other) {
- if (_registers) {
+ if (_registers != nullptr) {
delete[] _registers;
_registers = nullptr;
}
- if (_explicit_data) {
- delete[] _explicit_data;
- _explicit_data = nullptr;
- }
- _explicit_data_num = 0;
this->_type = other._type;
switch (other._type) {
case HLL_DATA_EMPTY:
break;
case HLL_DATA_EXPLICIT: {
- this->_explicit_data_num = other._explicit_data_num;
- this->_explicit_data = other._explicit_data;
- other._explicit_data_num = 0;
- other._explicit_data = nullptr;
+ this->_hash_set = std::move(other._hash_set);
other._type = HLL_DATA_EMPTY;
break;
}
@@ -166,35 +155,27 @@ public:
other._registers = nullptr;
other._type = HLL_DATA_EMPTY;
break;
+ }
default:
break;
}
- }
}
return *this;
}
HyperLogLog& operator=(const HyperLogLog& other) {
if (this != &other) {
- if (_registers) {
+ if (_registers != nullptr) {
delete[] _registers;
_registers = nullptr;
}
- if (_explicit_data) {
- delete[] _explicit_data;
- _explicit_data = nullptr;
- }
- _explicit_data_num = 0;
this->_type = other._type;
switch (other._type) {
case HLL_DATA_EMPTY:
break;
case HLL_DATA_EXPLICIT: {
- this->_explicit_data_num = other._explicit_data_num;
- _explicit_data = new uint64_t[HLL_EXPLICIT_INT64_NUM_DOUBLE];
- memcpy(_explicit_data, other._explicit_data,
- sizeof(*_explicit_data) * _explicit_data_num);
+ this->_hash_set = other._hash_set;
break;
}
case HLL_DATA_SPARSE:
@@ -202,25 +183,20 @@ public:
_registers = new uint8_t[HLL_REGISTERS_COUNT];
memcpy(_registers, other._registers, HLL_REGISTERS_COUNT);
break;
+ }
default:
break;
}
- }
}
return *this;
}
- explicit HyperLogLog(const Slice& src);
-
~HyperLogLog() { clear(); }
-
void clear() {
_type = HLL_DATA_EMPTY;
+ _hash_set.clear();
delete[] _registers;
_registers = nullptr;
- delete[] _explicit_data;
- _explicit_data = nullptr;
- _explicit_data_num = 0;
}
typedef uint8_t SetTypeValueType;
@@ -239,8 +215,10 @@ public:
size_t memory_consumed() const {
size_t size = sizeof(*this);
- if (_explicit_data) size += HLL_EXPLICIT_INT64_NUM_DOUBLE;
- if (_registers) size += HLL_REGISTERS_COUNT;
+ if (_type == HLL_DATA_EXPLICIT)
+ size += _hash_set.size() * sizeof(uint64_t);
+ else if (_type == HLL_DATA_SPARSE || _type == HLL_DATA_FULL)
+ size += HLL_REGISTERS_COUNT;
return size;
}
@@ -277,7 +255,7 @@ public:
case HLL_DATA_SPARSE:
case HLL_DATA_FULL: {
std::string str {"hash set size: "};
- str.append(std::to_string((size_t)_explicit_data_num));
+ str.append(std::to_string(_hash_set.size()));
str.append("\ncardinality:\t");
str.append(std::to_string(estimate_cardinality()));
str.append("\ntype:\t");
@@ -291,9 +269,7 @@ public:
private:
HllDataType _type = HLL_DATA_EMPTY;
-
- uint32_t _explicit_data_num = 0;
- uint64_t* _explicit_data = nullptr;
+ phmap::flat_hash_set<uint64_t> _hash_set;
// This field is much space consuming(HLL_REGISTERS_COUNT), we create
// it only when it is really needed.
@@ -312,40 +288,28 @@ private:
// make sure max first_one_bit is HLL_ZERO_COUNT_BITS + 1
hash_value |= ((uint64_t)1 << HLL_ZERO_COUNT_BITS);
uint8_t first_one_bit = __builtin_ctzl(hash_value) + 1;
- _registers[idx] = _registers[idx] > first_one_bit ? _registers[idx] : first_one_bit;
+ _registers[idx] = (_registers[idx] < first_one_bit ? first_one_bit : _registers[idx]);
}
// absorb other registers into this registers
- void _merge_registers(const uint8_t* other) {
- for (int i = 0; i < HLL_REGISTERS_COUNT; ++i) {
- _registers[i] = _registers[i] < other[i] ? other[i] : _registers[i];
+ void _merge_registers(const uint8_t* other_registers) {
+#ifdef __AVX2__
+ int loop = HLL_REGISTERS_COUNT / 32; // 32 = 256/8
+ uint8_t* dst = _registers;
+ const uint8_t* src = other_registers;
+ for (int i = 0; i < loop; i++) {
+ __m256i xa = _mm256_loadu_si256((const __m256i*)dst);
+ __m256i xb = _mm256_loadu_si256((const __m256i*)src);
+ _mm256_storeu_si256((__m256i*)dst, _mm256_max_epu8(xa, xb));
+ src += 32;
+ dst += 32;
}
- }
-
- bool _explicit_data_insert(uint64_t data) {
- //find insert pos
- int32_t i = (int32_t)_explicit_data_num - 1;
- while (i >= 0) {
- if (_explicit_data[i] == data) {
- return false;
- } else if (_explicit_data[i] < data) {
- break;
- } else {
- --i;
- }
- }
-
- ++i; //now, i is the insert position
-
- size_t n = (_explicit_data_num - i) * sizeof(*_explicit_data);
- if (n) {
- memmove(_explicit_data + i + 1, _explicit_data + i, n);
+#else
+ for (int i = 0; i < HLL_REGISTERS_COUNT; ++i) {
+ _registers[i] =
+ (_registers[i] < other_registers[i] ? other_registers[i] : _registers[i]);
}
-
- //insert data
- _explicit_data[i] = data;
- _explicit_data_num++;
- return true;
+#endif
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 09/14: [improvement](restore) set table property 'dynamic_partition.enable' to false after restore (#8852)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 8c4c4b237fad39560883f762ee0d7d0d64a62905
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Thu Apr 7 11:34:01 2022 +0800
[improvement](restore) set table property 'dynamic_partition.enable' to false after restore (#8852)
when restore table with dynamic partition properties, 'dynamic_partition.enable' is set to the backup time value.
but Doris could not turn on dynamic partition automatically when restore.
So we cloud see table never do dynamic partition with dynamic_partition.enable is set to 'true'.
---
.../java/org/apache/doris/backup/RestoreJob.java | 3 ++
.../java/org/apache/doris/catalog/OlapTable.java | 10 ++++++
.../org/apache/doris/catalog/TableProperty.java | 12 +++++++
.../org/apache/doris/catalog/OlapTableTest.java | 42 ++++++++++++++++++++++
4 files changed, 67 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index f6f183017f..30878a29ca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -641,6 +641,9 @@ public class RestoreJob extends AbstractJob {
return;
}
+ // Reset properties to correct values.
+ remoteOlapTbl.resetPropertiesForRestore();
+
// DO NOT set remote table's new name here, cause we will still need the origin name later
// remoteOlapTbl.setName(jobInfo.getAliasByOriginNameIfSet(tblInfo.name));
remoteOlapTbl.setState(allowLoad ? OlapTableState.RESTORE_WITH_LOAD : OlapTableState.RESTORE);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 13590e5f9c..d23a4609a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -420,6 +420,16 @@ public class OlapTable extends Table {
}
}
+ /**
+ * Reset properties to correct values.
+ */
+ public void resetPropertiesForRestore() {
+ // disable dynamic partition
+ if (tableProperty != null) {
+ tableProperty.resetPropertiesForRestore();
+ }
+ }
+
public Status resetIdsForRestore(Catalog catalog, Database db, ReplicaAllocation restoreReplicaAlloc) {
// table id
id = catalog.getNextId();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 427b09af0a..7c0d6f277e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -102,6 +102,18 @@ public class TableProperty implements Writable {
return this;
}
+ /**
+ * Reset properties to correct values.
+ * @return this for chained
+ */
+ public TableProperty resetPropertiesForRestore() {
+ if (properties.containsKey(DynamicPartitionProperty.ENABLE)) {
+ properties.put(DynamicPartitionProperty.ENABLE, "false");
+ executeBuildDynamicProperty();
+ }
+ return this;
+ }
+
public TableProperty buildDynamicProperty() throws DdlException {
if (properties.containsKey(DynamicPartitionProperty.ENABLE)
&& Boolean.valueOf(properties.get(DynamicPartitionProperty.ENABLE))
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index 76dea1d59a..cc9d7f47f5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog;
+import com.google.common.collect.Maps;
import mockit.Mock;
import mockit.MockUp;
@@ -28,12 +29,14 @@ import org.apache.doris.common.util.UnitTestUtil;
import com.google.common.collect.Lists;
+import org.junit.Assert;
import org.junit.Test;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
public class OlapTableTest {
@@ -73,4 +76,43 @@ public class OlapTableTest {
}
+ @Test
+ public void testResetPropertiesForRestore() {
+ // restore with other key
+ String otherKey = "other_key";
+ String otherValue = "other_value";
+
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put(otherKey, otherValue);
+ TableProperty tableProperty = new TableProperty(properties);
+
+ OlapTable olapTable = new OlapTable();
+ olapTable.setTableProperty(tableProperty);
+
+ olapTable.resetPropertiesForRestore();
+ Assert.assertEquals(tableProperty.getProperties(), olapTable.getTableProperty().getProperties());
+ Assert.assertFalse(tableProperty.getDynamicPartitionProperty().isExist());
+
+ // restore with dynamic partition keys
+ properties = Maps.newHashMap();
+ properties.put(DynamicPartitionProperty.ENABLE, "true");
+ properties.put(DynamicPartitionProperty.TIME_UNIT, "HOUR");
+ properties.put(DynamicPartitionProperty.TIME_ZONE, "Asia/Shanghai");
+ properties.put(DynamicPartitionProperty.START, "-2147483648");
+ properties.put(DynamicPartitionProperty.END, "3");
+ properties.put(DynamicPartitionProperty.PREFIX, "dynamic");
+ properties.put(DynamicPartitionProperty.BUCKETS, "10");
+ properties.put(DynamicPartitionProperty.REPLICATION_NUM, "3");
+ properties.put(DynamicPartitionProperty.CREATE_HISTORY_PARTITION, "false");
+
+ tableProperty = new TableProperty(properties);
+ olapTable.setTableProperty(tableProperty);
+ olapTable.resetPropertiesForRestore();
+
+ Map<String, String> expectedProperties = Maps.newHashMap(properties);
+ expectedProperties.put(DynamicPartitionProperty.ENABLE, "false");
+ Assert.assertEquals(expectedProperties, olapTable.getTableProperty().getProperties());
+ Assert.assertTrue(olapTable.getTableProperty().getDynamicPartitionProperty().isExist());
+ Assert.assertFalse(olapTable.getTableProperty().getDynamicPartitionProperty().getEnable());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 05/14: [fix](ut) fix fe run CreateTableAsSelectStmtTest ,UserPropertyTest, ProjectPlannerFunctionTest and AggregateTest failed (#8838)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 1eebb658e18ea3707bf7ba09f0e668166cad2cea
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Wed Apr 6 15:23:49 2022 +0800
[fix](ut) fix fe run CreateTableAsSelectStmtTest ,UserPropertyTest, ProjectPlannerFunctionTest and AggregateTest failed (#8838)
---
.../apache/doris/analysis/FunctionCallExpr.java | 12 +++----
.../java/org/apache/doris/catalog/Catalog.java | 3 +-
.../main/java/org/apache/doris/catalog/Column.java | 5 +--
.../org/apache/doris/analysis/AggregateTest.java | 4 +--
.../analysis/CreateTableAsSelectStmtTest.java | 25 +++++++------
.../org/apache/doris/catalog/UserPropertyTest.java | 41 ++++++++++++++++++++++
.../doris/planner/ProjectPlannerFunctionTest.java | 2 +-
.../doris/planner/TableFunctionPlanTest.java | 8 ++---
8 files changed, 68 insertions(+), 32 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index e64a2d121c..a023da2e6c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -236,17 +236,17 @@ public class FunctionCallExpr extends Expr {
&& fnParams.isStar() == o.fnParams.isStar();
}
- private String paramsToSql(FunctionParams params) {
+ private String paramsToSql() {
StringBuilder sb = new StringBuilder();
sb.append("(");
- if (params.isStar()) {
+ if (fnParams.isStar()) {
sb.append("*");
}
- if (params.isDistinct()) {
+ if (fnParams.isDistinct()) {
sb.append("DISTINCT ");
}
- int len = params.exprs().size();
+ int len = children.size();
List<String> result = Lists.newArrayList();
if (fnName.getFunction().equalsIgnoreCase("json_array") ||
fnName.getFunction().equalsIgnoreCase("json_object")) {
@@ -265,7 +265,7 @@ public class FunctionCallExpr extends Expr {
fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) {
result.add("\'***\'");
} else {
- result.add(params.exprs().get(i).toSql());
+ result.add(children.get(i).toSql());
}
}
sb.append(Joiner.on(", ").join(result)).append(")");
@@ -282,7 +282,7 @@ public class FunctionCallExpr extends Expr {
}
StringBuilder sb = new StringBuilder();
sb.append(((FunctionCallExpr) expr).fnName);
- sb.append(paramsToSql(fnParams));
+ sb.append(paramsToSql());
if (fnName.getFunction().equalsIgnoreCase("json_quote") ||
fnName.getFunction().equalsIgnoreCase("json_array") ||
fnName.getFunction().equalsIgnoreCase("json_object")) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 57a066680f..beaa10d415 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -3116,8 +3116,7 @@ public class Catalog {
}
TypeDef typeDef;
Expr resultExpr = resultExprs.get(i);
- // varchar/char transfer to string
- if (resultExpr.getType().isStringType()) {
+ if (resultExpr.getType().isStringType() && resultExpr.getType().getLength() < 0) {
typeDef = new TypeDef(Type.STRING);
} else {
typeDef = new TypeDef(resultExpr.getType());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 522d6c0bdc..d5b45a83e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -230,10 +230,7 @@ public class Column implements Writable {
public PrimitiveType getDataType() { return type.getPrimitiveType(); }
public Type getType() {
- if (type.isArrayType() || type.isMapType() || type.isStructType()) {
- return type;
- }
- return ScalarType.createType(type.getPrimitiveType());
+ return type;
}
public void setType(Type type) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java
index 35225dca12..f239a303a8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java
@@ -130,7 +130,7 @@ public class AggregateTest {
try {
UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
} catch (AnalysisException e) {
- Assert.assertTrue(e.getMessage().contains("The window param of window_funnel function must be integer"));
+ Assert.assertTrue(e.getMessage().contains("The window params of window_funnel function must be integer"));
break;
} catch (Exception e) {
Assert.fail("must be AnalysisException.");
@@ -144,7 +144,7 @@ public class AggregateTest {
try {
UtFrameUtils.parseAndAnalyzeStmt(query, ctx);
} catch (AnalysisException e) {
- Assert.assertTrue(e.getMessage().contains("The mode param of window_funnel function must be string"));
+ Assert.assertTrue(e.getMessage().contains("The mode params of window_funnel function must be integer"));
break;
} catch (Exception e) {
Assert.fail("must be AnalysisException.");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index 7804f6c94a..bf2154c5bd 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -113,14 +113,14 @@ public class CreateTableAsSelectStmtTest {
}
@Test
- public void testErrorType() throws Exception {
+ public void testErrorType() {
String selectFromDecimal = "create table `test`.`select_decimal_table` PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`decimal_table`";
ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Unsupported type",
() -> UtFrameUtils.parseAndAnalyzeStmt(selectFromDecimal, connectContext));
}
@Test
- public void testErrorColumn() throws Exception {
+ public void testErrorColumn() {
String selectFromColumn = "create table `test`.`select_column_table`(test_error) PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`varchar_table`";
ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Number of columns don't equal number of SELECT statement's select list",
() -> UtFrameUtils.parseAndAnalyzeStmt(selectFromColumn, connectContext));
@@ -132,8 +132,8 @@ public class CreateTableAsSelectStmtTest {
createTableAsSelect(selectFromDecimal);
ShowResultSet showResultSet = showCreateTable("select_varchar");
Assert.assertEquals("CREATE TABLE `select_varchar` (\n" +
- " `userId` text NULL COMMENT \"\",\n" +
- " `username` text NULL COMMENT \"\"\n" +
+ " `userId` varchar(255) NULL COMMENT \"\",\n" +
+ " `username` varchar(255) NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`userId`)\n" +
"COMMENT \"OLAP\"\n" +
@@ -150,7 +150,6 @@ public class CreateTableAsSelectStmtTest {
String selectFromFunction1 = "create table `test`.`select_function_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) from `test`.`varchar_table`";
createTableAsSelect(selectFromFunction1);
ShowResultSet showResultSet1 = showCreateTable("select_function_1");
- System.out.println(showResultSet1.getResultRows().get(0).get(1));
Assert.assertEquals("CREATE TABLE `select_function_1` (\n" +
" `_col0` bigint(20) NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
@@ -203,8 +202,8 @@ public class CreateTableAsSelectStmtTest {
createTableAsSelect(selectAlias2);
ShowResultSet showResultSet2 = showCreateTable("select_alias_2");
Assert.assertEquals("CREATE TABLE `select_alias_2` (\n" +
- " `alias_name` text NULL COMMENT \"\",\n" +
- " `username` text NULL COMMENT \"\"\n" +
+ " `alias_name` varchar(255) NULL COMMENT \"\",\n" +
+ " `username` varchar(255) NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`alias_name`)\n" +
"COMMENT \"OLAP\"\n" +
@@ -223,8 +222,8 @@ public class CreateTableAsSelectStmtTest {
createTableAsSelect(selectFromJoin);
ShowResultSet showResultSet = showCreateTable("select_join");
Assert.assertEquals("CREATE TABLE `select_join` (\n" +
- " `userId` text NULL COMMENT \"\",\n" +
- " `username` text NULL COMMENT \"\",\n" +
+ " `userId` varchar(255) NULL COMMENT \"\",\n" +
+ " `username` varchar(255) NULL COMMENT \"\",\n" +
" `status` int(11) NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`userId`)\n" +
@@ -244,8 +243,8 @@ public class CreateTableAsSelectStmtTest {
createTableAsSelect(selectFromName);
ShowResultSet showResultSet = showCreateTable("select_name");
Assert.assertEquals("CREATE TABLE `select_name` (\n" +
- " `user` text NULL COMMENT \"\",\n" +
- " `testname` text NULL COMMENT \"\",\n" +
+ " `user` varchar(255) NULL COMMENT \"\",\n" +
+ " `testname` varchar(255) NULL COMMENT \"\",\n" +
" `userstatus` int(11) NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`user`)\n" +
@@ -265,7 +264,7 @@ public class CreateTableAsSelectStmtTest {
createTableAsSelect(selectFromName);
ShowResultSet showResultSet = showCreateTable("select_union");
Assert.assertEquals("CREATE TABLE `select_union` (\n" +
- " `userId` text NULL COMMENT \"\"\n" +
+ " `userId` varchar(255) NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`userId`)\n" +
"COMMENT \"OLAP\"\n" +
@@ -284,7 +283,7 @@ public class CreateTableAsSelectStmtTest {
createTableAsSelect(selectFromName);
ShowResultSet showResultSet = showCreateTable("select_cte");
Assert.assertEquals("CREATE TABLE `select_cte` (\n" +
- " `userId` text NULL COMMENT \"\"\n" +
+ " `userId` varchar(255) NULL COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`userId`)\n" +
"COMMENT \"OLAP\"\n" +
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java
index 0702e2157c..f82c378929 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java
@@ -17,6 +17,9 @@
package org.apache.doris.catalog;
+import mockit.Expectations;
+import mockit.Mocked;
+import org.apache.doris.blockrule.SqlBlockRuleMgr;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
@@ -26,7 +29,9 @@ import org.apache.doris.mysql.privilege.UserProperty;
import com.google.common.collect.Lists;
+import org.apache.doris.thrift.TStorageMedium;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import java.io.ByteArrayInputStream;
@@ -38,6 +43,42 @@ import java.util.List;
public class UserPropertyTest {
private FakeCatalog fakeCatalog;
+ @Mocked
+ private Catalog catalog;
+ @Mocked
+ private SqlBlockRuleMgr sqlBlockRuleMgr;
+
+ @Before
+ public void setUp() {
+ new Expectations(catalog) {
+ {
+ catalog.getSqlBlockRuleMgr();
+ minTimes = 0;
+ result = sqlBlockRuleMgr;
+
+ sqlBlockRuleMgr.existRule("rule1");
+ minTimes = 0;
+ result = true;
+
+ sqlBlockRuleMgr.existRule("rule2");
+ minTimes = 0;
+ result = true;
+
+ sqlBlockRuleMgr.existRule("test1");
+ minTimes = 0;
+ result = true;
+
+ sqlBlockRuleMgr.existRule("test2");
+ minTimes = 0;
+ result = true;
+
+ sqlBlockRuleMgr.existRule("test3");
+ minTimes = 0;
+ result = true;
+ }
+ };
+ }
+
@Test
public void testNormal() throws IOException, DdlException {
// mock catalog
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ProjectPlannerFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ProjectPlannerFunctionTest.java
index 58c87a6289..0159edba6c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/ProjectPlannerFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ProjectPlannerFunctionTest.java
@@ -46,7 +46,7 @@ public class ProjectPlannerFunctionTest {
connectContext = UtFrameUtils.createDefaultCtx();
// enable hash project
- Deencapsulation.setField(connectContext.getSessionVariable(), "enableHashProject", true);
+ Deencapsulation.setField(connectContext.getSessionVariable(), "enableProjection", true);
// create database
String createDbStmtStr = "create database test;";
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
index c127d2fd9c..f203c8dbf2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
@@ -199,7 +199,7 @@ public class TableFunctionPlanTest {
String sql = "explain select k1 from db1.tbl1 where explode_split(k2, \",\");";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
Assert.assertTrue(
- explainString.contains("No matching function with signature: explode_split(varchar(-1), varchar(-1))."));
+ explainString.contains("No matching function with signature: explode_split(varchar(1), varchar(-1))."));
}
// test projection
@@ -340,8 +340,8 @@ public class TableFunctionPlanTest {
Assert.assertTrue(explainString.contains("table function: explode_split(concat(`a`.`k2`, ',', `a`.`k3`), ',')"));
Assert.assertTrue(explainString.contains("lateral view tuple id: 1"));
Assert.assertTrue(explainString.contains("output slot id: 3"));
- Assert.assertTrue(explainString.contains("SlotDescriptor{id=0, col=k2, type=VARCHAR(*)}"));
- Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=k3, type=VARCHAR(*)}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=0, col=k2, type=VARCHAR(1)}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=k3, type=VARCHAR(1)}"));
}
// lateral view of subquery
@@ -398,7 +398,7 @@ public class TableFunctionPlanTest {
+ "materialized=true"
));
Assert.assertTrue(formatString.contains(
- "SlotDescriptor{id=1,col=k2,type=VARCHAR(*)}\n"
+ "SlotDescriptor{id=1,col=k2,type=VARCHAR(1)}\n"
+ "parent=0\n"
+ "materialized=true"
));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 12/14: [feature](vectorized) support vexplode_bitmap (#8890)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 32ccf29b8609335d343c29d4cb7c486fe6b23d3d
Author: Pxl <95...@qq.com>
AuthorDate: Fri Apr 8 09:20:26 2022 +0800
[feature](vectorized) support vexplode_bitmap (#8890)
---
be/src/exprs/table_function/explode_bitmap.h | 5 +-
.../table_function/table_function_factory.cpp | 13 ++--
be/src/vec/CMakeLists.txt | 1 +
.../vec/exprs/table_function/vexplode_bitmap.cpp | 81 ++++++++++++++++++++++
.../table_function/vexplode_bitmap.h} | 27 +++++---
be/src/vec/functions/function_fake.cpp | 1 +
be/src/vec/functions/function_fake.h | 8 +++
7 files changed, 120 insertions(+), 16 deletions(-)
diff --git a/be/src/exprs/table_function/explode_bitmap.h b/be/src/exprs/table_function/explode_bitmap.h
index 2abb1603e0..c75559d023 100644
--- a/be/src/exprs/table_function/explode_bitmap.h
+++ b/be/src/exprs/table_function/explode_bitmap.h
@@ -33,13 +33,12 @@ public:
virtual Status forward(bool* eos) override;
-private:
+protected:
void _reset_iterator();
-private:
// Read from tuple row.
// if _cur_bitmap_owned is true, need to delete it when deconstruction
- BitmapValue* _cur_bitmap = nullptr;
+ const BitmapValue* _cur_bitmap = nullptr;
bool _cur_bitmap_owned = false;
// iterator of _cur_bitmap
BitmapValueIterator* _cur_iter = nullptr;
diff --git a/be/src/exprs/table_function/table_function_factory.cpp b/be/src/exprs/table_function/table_function_factory.cpp
index c347d245eb..7dcce8c9c5 100644
--- a/be/src/exprs/table_function/table_function_factory.cpp
+++ b/be/src/exprs/table_function/table_function_factory.cpp
@@ -22,9 +22,10 @@
#include "exprs/table_function/explode_json_array.h"
#include "exprs/table_function/explode_split.h"
#include "exprs/table_function/table_function.h"
+#include "vec/exprs/table_function/vexplode_bitmap.h"
+#include "vec/exprs/table_function/vexplode_json_array.h"
#include "vec/exprs/table_function/vexplode_numbers.h"
#include "vec/exprs/table_function/vexplode_split.h"
-#include "vec/exprs/table_function/vexplode_json_array.h"
namespace doris {
@@ -36,13 +37,15 @@ struct TableFunctionCreator {
template <>
struct TableFunctionCreator<ExplodeJsonArrayTableFunction> {
ExplodeJsonArrayType type;
- TableFunction* operator()() { return new ExplodeJsonArrayTableFunction(type); }
+ TableFunction* operator()() const { return new ExplodeJsonArrayTableFunction(type); }
};
template <>
struct TableFunctionCreator<vectorized::VExplodeJsonArrayTableFunction> {
ExplodeJsonArrayType type;
- TableFunction* operator()() { return new vectorized::VExplodeJsonArrayTableFunction(type); }
+ TableFunction* operator()() const {
+ return new vectorized::VExplodeJsonArrayTableFunction(type);
+ }
};
inline auto ExplodeJsonArrayIntCreator =
@@ -76,7 +79,9 @@ const std::unordered_map<std::pair<std::string, bool>, std::function<TableFuncti
TableFunctionCreator<vectorized::VExplodeNumbersTableFunction>()},
{{"explode_json_array_int", true}, VExplodeJsonArrayIntCreator},
{{"explode_json_array_double", true}, VExplodeJsonArrayDoubleCreator},
- {{"explode_json_array_string", true}, VExplodeJsonArrayStringCreator}};
+ {{"explode_json_array_string", true}, VExplodeJsonArrayStringCreator},
+ {{"explode_bitmap", true},
+ TableFunctionCreator<vectorized::VExplodeBitmapTableFunction>()}};
Status TableFunctionFactory::get_fn(const std::string& fn_name, bool is_vectorized,
ObjectPool* pool, TableFunction** fn) {
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index f0781f83db..243fd1869e 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -108,6 +108,7 @@ set(VEC_FILES
exprs/vinfo_func.cpp
exprs/table_function/vexplode_split.cpp
exprs/table_function/vexplode_numbers.cpp
+ exprs/table_function/vexplode_bitmap.cpp
exprs/table_function/vexplode_json_array.cpp
functions/math.cpp
functions/function_bitmap.cpp
diff --git a/be/src/vec/exprs/table_function/vexplode_bitmap.cpp b/be/src/vec/exprs/table_function/vexplode_bitmap.cpp
new file mode 100644
index 0000000000..0dec849980
--- /dev/null
+++ b/be/src/vec/exprs/table_function/vexplode_bitmap.cpp
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exprs/table_function/vexplode_bitmap.h"
+
+#include "util/bitmap_value.h"
+#include "vec/exprs/vexpr.h"
+
+namespace doris::vectorized {
+
+VExplodeBitmapTableFunction::VExplodeBitmapTableFunction() {
+ _fn_name = "vexplode_bitmap";
+}
+
+Status VExplodeBitmapTableFunction::process_init(vectorized::Block* block) {
+ CHECK(_vexpr_context->root()->children().size() == 1)
+ << "VExplodeNumbersTableFunction must be have 1 children but have "
+ << _vexpr_context->root()->children().size();
+
+ int value_column_idx = -1;
+ _vexpr_context->root()->children()[0]->execute(_vexpr_context, block, &value_column_idx);
+ _value_column = block->get_by_position(value_column_idx).column;
+
+ return Status::OK();
+}
+
+Status VExplodeBitmapTableFunction::process_row(size_t row_idx) {
+ _eos = false;
+ _is_current_empty = false;
+ _cur_size = 0;
+ _cur_offset = 0;
+
+ StringRef value = _value_column->get_data_at(row_idx);
+
+ if (value.data == nullptr) {
+ _is_current_empty = true;
+ } else {
+ _cur_bitmap = reinterpret_cast<const BitmapValue*>(value.data);
+ _cur_bitmap_owned = false;
+
+ _cur_size = _cur_bitmap->cardinality();
+ if (_cur_size == 0) {
+ _is_current_empty = true;
+ } else {
+ _reset_iterator();
+ }
+ }
+
+ _is_current_empty = (_cur_size == 0);
+ return Status::OK();
+}
+
+Status VExplodeBitmapTableFunction::process_close() {
+ _value_column = nullptr;
+ return Status::OK();
+}
+
+Status VExplodeBitmapTableFunction::get_value_length(int64_t* length) {
+ if (_is_current_empty) {
+ *length = -1;
+ } else {
+ *length = sizeof(uint64_t);
+ }
+ return Status::OK();
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_fake.cpp b/be/src/vec/exprs/table_function/vexplode_bitmap.h
similarity index 60%
copy from be/src/vec/functions/function_fake.cpp
copy to be/src/vec/exprs/table_function/vexplode_bitmap.h
index eb6e875980..5e46ab20c8 100644
--- a/be/src/vec/functions/function_fake.cpp
+++ b/be/src/vec/exprs/table_function/vexplode_bitmap.h
@@ -15,17 +15,26 @@
// specific language governing permissions and limitations
// under the License.
-#include "vec/functions/function_fake.h"
+#pragma once
+
+#include "exprs/table_function/explode_bitmap.h"
+#include "exprs/table_function/table_function.h"
+#include "util/bitmap_value.h"
+#include "vec/columns/column.h"
namespace doris::vectorized {
-void register_function_fake(SimpleFunctionFactory& factory) {
- factory.register_function<FunctionFake<FunctionEsqueryImpl>>();
- factory.register_function<FunctionFake<FunctionExplodeSplitImpl>>();
- factory.register_function<FunctionFake<FunctionExplodeNumbersImpl>>();
- factory.register_function<FunctionFake<FunctionExplodeJsonArrayDoubleImpl>>();
- factory.register_function<FunctionFake<FunctionExplodeJsonArrayIntImpl>>();
- factory.register_function<FunctionFake<FunctionExplodeJsonArrayStringImpl>>();
-}
+class VExplodeBitmapTableFunction : public ExplodeBitmapTableFunction {
+public:
+ VExplodeBitmapTableFunction();
+
+ Status process_init(vectorized::Block* block) override;
+ Status process_row(size_t row_idx) override;
+ Status process_close() override;
+ Status get_value_length(int64_t* length) override;
+
+private:
+ ColumnPtr _value_column;
+};
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_fake.cpp b/be/src/vec/functions/function_fake.cpp
index eb6e875980..9deacb2afa 100644
--- a/be/src/vec/functions/function_fake.cpp
+++ b/be/src/vec/functions/function_fake.cpp
@@ -26,6 +26,7 @@ void register_function_fake(SimpleFunctionFactory& factory) {
factory.register_function<FunctionFake<FunctionExplodeJsonArrayDoubleImpl>>();
factory.register_function<FunctionFake<FunctionExplodeJsonArrayIntImpl>>();
factory.register_function<FunctionFake<FunctionExplodeJsonArrayStringImpl>>();
+ factory.register_function<FunctionFake<FunctionExplodeBitmapImpl>>();
}
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_fake.h b/be/src/vec/functions/function_fake.h
index 6e70eb8343..c7e536d6ea 100644
--- a/be/src/vec/functions/function_fake.h
+++ b/be/src/vec/functions/function_fake.h
@@ -67,6 +67,14 @@ struct FunctionExplodeJsonArrayDoubleImpl {
return std::make_shared<DataTypeFloat64>();
}
};
+
+struct FunctionExplodeBitmapImpl {
+ static constexpr auto name = "explode_bitmap";
+ static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
+ return std::make_shared<DataTypeInt64>();
+ }
+};
+
//FunctionFake is use for some function call expr only work at prepare/open phase, do not support execute().
template <typename Impl>
class FunctionFake : public IFunction {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 04/14: [fix](storage) Fix core bug of convert to predicate column (#8833)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 28892f28075d20177ad55ca203d23b5f9ab240b5
Author: ZenoYang <co...@qq.com>
AuthorDate: Thu Apr 7 11:29:26 2022 +0800
[fix](storage) Fix core bug of convert to predicate column (#8833)
recurrent:
When `enable_low_cardinality_optimize = true`, for the TPCH dataset, using the following SQL query will Core
```sql
select count(*) from lineitem where l_comment = 'ously even exc';
```
This SQL will trigger the execution of `ColumnDictionary::convert_to_predicate_column_if_dictionary`, and `res->reserve(_codes.size())` is problematic because the current `_codes.size()` is smaller than its reserve value, so inserting a value into `PredicateColumn` will Core.
---
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 2 +-
be/src/vec/columns/column.h | 2 +-
be/src/vec/columns/column_dictionary.h | 14 +++++++++-----
be/src/vec/columns/column_nullable.h | 6 ++----
4 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
index 28a6e62b36..95a9fd5c7b 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
@@ -240,7 +240,7 @@ void BinaryDictPageDecoder::set_dict_decoder(PageDecoder* dict_decoder, StringRe
Status BinaryDictPageDecoder::next_batch(size_t* n, vectorized::MutableColumnPtr &dst) {
if (_encoding_type == PLAIN_ENCODING) {
- dst = (*(std::move(dst->convert_to_predicate_column_if_dictionary()))).assume_mutable();
+ dst = dst->convert_to_predicate_column_if_dictionary();
return _data_page_decoder->next_batch(n, dst);
}
// dictionary encoding
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 7e717bb984..610babd91a 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -66,7 +66,7 @@ public:
/// If column isn't ColumnDictionary, return itself.
/// If column is ColumnDictionary, transforms is to predicate column.
- virtual Ptr convert_to_predicate_column_if_dictionary() { return get_ptr(); }
+ virtual MutablePtr convert_to_predicate_column_if_dictionary() { return get_ptr(); }
/// If column is ColumnDictionary, and is a range comparison predicate, convert dict encoding
virtual void convert_dict_codes_if_necessary() {}
diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h
index 499a04abdf..76f9516c9c 100644
--- a/be/src/vec/columns/column_dictionary.h
+++ b/be/src/vec/columns/column_dictionary.h
@@ -120,7 +120,10 @@ public:
LOG(FATAL) << "get_permutation not supported in ColumnDictionary";
}
- void reserve(size_t n) override { _codes.reserve(n); }
+ void reserve(size_t n) override {
+ _reserve_size = n;
+ _codes.reserve(n);
+ }
const char* get_family_name() const override { return "ColumnDictionary"; }
@@ -259,15 +262,15 @@ public:
bool is_dict_code_converted() const { return _dict_code_converted; }
- ColumnPtr convert_to_predicate_column_if_dictionary() override {
+ MutableColumnPtr convert_to_predicate_column_if_dictionary() override {
auto res = vectorized::PredicateColumnType<StringValue>::create();
- size_t size = _codes.size();
- res->reserve(size);
- for (size_t i = 0; i < size; ++i) {
+ res->reserve(_reserve_size);
+ for (size_t i = 0; i < _codes.size(); ++i) {
auto& code = reinterpret_cast<T&>(_codes[i]);
auto value = _dict.get_value(code);
res->insert_data(value.ptr, value.len);
}
+ clear();
_dict.clear();
return res;
}
@@ -365,6 +368,7 @@ public:
};
private:
+ size_t _reserve_size;
bool _dict_inited = false;
bool _dict_sorted = false;
bool _dict_code_converted = false;
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
index aa0df2dc1c..482b9a5bfe 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -267,10 +267,8 @@ public:
LOG(FATAL) << "should not call the method in column nullable";
}
- ColumnPtr convert_to_predicate_column_if_dictionary() override {
- IColumn* nested_ptr = get_nested_column_ptr().get();
- nested_ptr = (*(std::move(nested_ptr->convert_to_predicate_column_if_dictionary()
- ))).assume_mutable();
+ MutableColumnPtr convert_to_predicate_column_if_dictionary() override {
+ nested_column = get_nested_column().convert_to_predicate_column_if_dictionary();
return get_ptr();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 08/14: [Fix](Lateral View) The Error expr type when exploding a function result of inline view (#8851)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 5b6f9329c8bfe1447301905008a8874cc30c46f2
Author: EmmyMiao87 <52...@qq.com>
AuthorDate: Fri Apr 8 09:08:55 2022 +0800
[Fix](Lateral View) The Error expr type when exploding a function result of inline view (#8851)
Fixed #8850
The column in inline view maybe a function instead of slotRef.
So when this column is used as the input of explode function,
it can't be converted to slotRef.
The correct way is to treat it as an Expr and extract the required slotRef for materialization.
For example:
```
with d as (select k1+k1 as k1_plus from table)
select k1_plus from d explode_split(k1_plus, ",")
```
FnExp: SlorRef<k1_plus>
SubstituteFnExpr: functionCallExpr<k1+k1>
originSlotRefList: SlotRef<k1>
---
.../main/java/org/apache/doris/analysis/LateralViewRef.java | 12 ++++++------
.../java/org/apache/doris/planner/TableFunctionPlanTest.java | 11 +++++++++++
2 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
index bee7032c03..3af8b5481c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
@@ -26,7 +26,6 @@ import org.apache.doris.common.UserException;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import java.util.ArrayList;
import java.util.List;
/**
@@ -42,7 +41,7 @@ public class LateralViewRef extends TableRef {
// after analyzed
private FunctionCallExpr fnExpr;
- private ArrayList<Expr> originSlotRefList = Lists.newArrayList();
+ private List<SlotRef> originSlotRefList = Lists.newArrayList();
private InlineView view;
private SlotRef explodeSlotRef;
@@ -98,7 +97,6 @@ public class LateralViewRef extends TableRef {
for (Expr expr : fnExpr.getChildren()) {
checkScalarFunction(expr);
}
- fnExpr.collect(SlotRef.class, originSlotRefList);
}
@Override
@@ -116,11 +114,13 @@ public class LateralViewRef extends TableRef {
}
public void materializeRequiredSlots(ExprSubstitutionMap baseTblSmap, Analyzer analyzer) throws AnalysisException {
+ Expr substituteFnExpr = fnExpr;
if (relatedTableRef instanceof InlineViewRef) {
- originSlotRefList = Expr.trySubstituteList(originSlotRefList, baseTblSmap, analyzer, false);
+ substituteFnExpr = fnExpr.trySubstitute(baseTblSmap, analyzer, false);
}
- for (Expr originSlotRef : originSlotRefList) {
- ((SlotRef) originSlotRef).getDesc().setIsMaterialized(true);
+ substituteFnExpr.collect(SlotRef.class, originSlotRefList);
+ for (SlotRef originSlotRef : originSlotRefList) {
+ originSlotRef.getDesc().setIsMaterialized(true);
}
explodeSlotRef.getDesc().setIsMaterialized(true);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
index f203c8dbf2..58e4c9de86 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
@@ -527,4 +527,15 @@ public class TableFunctionPlanTest {
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
Assert.assertTrue(!explainString.contains("Unknown column 'e1' in 'table list'"));
}
+
+
+ // The 'k1' column in 'd' view should be materialized
+ // Fix #8850
+ @Test
+ public void testLateralViewWithInlineViewBug() throws Exception {
+ String sql = "with d as (select k1+k1 as k1 from db1.table_for_view ) "
+ + "select k1 from d lateral view explode_split(k1,',') tmp as e1;";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
+ Assert.assertTrue(!explainString.contains("Unexpected exception: org.apache.doris.analysis.FunctionCallExpr cannot be cast to org.apache.doris.analysis.SlotRef"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 11/14: [fix] Disable cast operation of object type (#8882)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit b86f290a7c4c94fcce1dd470b8167a019d763990
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Fri Apr 8 09:13:56 2022 +0800
[fix] Disable cast operation of object type (#8882)
Disable cast between string and object type(bitmap, hll, quantile_state)
---
.../java/org/apache/doris/analysis/CastExpr.java | 28 +++++++++++--------
.../apache/doris/analysis/FunctionCallExpr.java | 4 +--
.../org/apache/doris/catalog/PrimitiveType.java | 8 ------
.../java/org/apache/doris/catalog/ScalarType.java | 12 ++++++---
.../main/java/org/apache/doris/catalog/Type.java | 4 +++
.../org/apache/doris/planner/QueryPlanTest.java | 31 +++++++++++++---------
.../doris/planner/TableFunctionPlanTest.java | 4 +--
7 files changed, 52 insertions(+), 39 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 387cf1a77f..3bc9a19831 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -134,23 +134,29 @@ public class CastExpr extends Expr {
return targetTypeDef;
}
+ private static boolean disableRegisterCastingFunction(Type fromType, Type toType) {
+ // Disable casting from boolean to decimal or datetime or date
+ if (fromType.isBoolean() &&
+ (toType.equals(Type.DECIMALV2) ||
+ toType.equals(Type.DATETIME) || toType.equals(Type.DATE))) {
+ return true;
+ }
+
+ // Disable casting operation of hll/bitmap/quantile_state
+ if (fromType.isObjectStored() || toType.isObjectStored()) {
+ return true;
+ }
+ // Disable no-op casting
+ return fromType.equals(toType);
+ }
+
public static void initBuiltins(FunctionSet functionSet) {
for (Type fromType : Type.getSupportedTypes()) {
if (fromType.isNull()) {
continue;
}
for (Type toType : Type.getSupportedTypes()) {
- if (toType.isNull()) {
- continue;
- }
- // Disable casting from boolean to decimal or datetime or date
- if (fromType.isBoolean() &&
- (toType.equals(Type.DECIMALV2) ||
- toType.equals(Type.DATETIME) || toType.equals(Type.DATE))) {
- continue;
- }
- // Disable no-op casts
- if (fromType.equals(toType)) {
+ if (toType.isNull() || disableRegisterCastingFunction(fromType, toType)) {
continue;
}
String beClass = toType.isDecimalV2() || fromType.isDecimalV2() ? "DecimalV2Operators" : "CastFunctions";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index a023da2e6c..bc30b9d5f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -89,8 +89,6 @@ public class FunctionCallExpr extends Expr {
private boolean isRewrote = false;
- public static final String UNKNOWN_TABLE_FUNCTION_MSG = "This table function not supported now";
-
public void setIsAnalyticFnCall(boolean v) {
isAnalyticFnCall = v;
}
@@ -808,7 +806,7 @@ public class FunctionCallExpr extends Expr {
fn = getTableFunction(fnName.getFunction(), childTypes,
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
if (fn == null) {
- throw new AnalysisException(UNKNOWN_TABLE_FUNCTION_MSG);
+ throw new AnalysisException(getFunctionNotFoundError(argTypes));
}
} else {
// now first find function in built-in functions
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index 7def685b5a..d5f65908a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -247,8 +247,6 @@ public enum PrimitiveType {
builder.put(VARCHAR, DECIMALV2);
builder.put(VARCHAR, VARCHAR);
builder.put(VARCHAR, STRING);
- builder.put(VARCHAR, HLL);
- builder.put(VARCHAR, BITMAP);
// Varchar
builder.put(STRING, BOOLEAN);
@@ -264,8 +262,6 @@ public enum PrimitiveType {
builder.put(STRING, DECIMALV2);
builder.put(STRING, VARCHAR);
builder.put(STRING, STRING);
- builder.put(STRING, HLL);
- builder.put(STRING, BITMAP);
// DecimalV2
builder.put(DECIMALV2, BOOLEAN);
@@ -282,13 +278,9 @@ public enum PrimitiveType {
// HLL
builder.put(HLL, HLL);
- builder.put(HLL, VARCHAR);
- builder.put(HLL, STRING);
// BITMAP
builder.put(BITMAP, BITMAP);
- builder.put(BITMAP, VARCHAR);
- builder.put(BITMAP, STRING);
//TIME
builder.put(TIME, TIME);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java
index 714e067699..6809a99eea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -541,9 +541,6 @@ public class ScalarType extends Type {
if (type == PrimitiveType.VARCHAR && scalarType.isStringType()) {
return true;
}
- if (type == PrimitiveType.HLL && scalarType.isStringType()) {
- return true;
- }
if (isDecimalV2() && scalarType.isWildcardDecimal()) {
Preconditions.checkState(!isWildcardDecimal());
return true;
@@ -677,6 +674,15 @@ public class ScalarType extends Type {
return INVALID;
}
+ boolean t1IsBitMap = t1.type == PrimitiveType.BITMAP;
+ boolean t2IsBitMap = t2.type == PrimitiveType.BITMAP;
+ if (t1IsBitMap || t2IsBitMap) {
+ if (t1IsBitMap && t2IsBitMap) {
+ return BITMAP;
+ }
+ return INVALID;
+ }
+
// for cast all type
if (t1.type == PrimitiveType.ALL || t2.type == PrimitiveType.ALL) {
return Type.ALL;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
index efee561e00..438d2d1828 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
@@ -209,6 +209,10 @@ public abstract class Type {
return isScalarType(PrimitiveType.BITMAP);
}
+ public boolean isObjectStored() {
+ return isHllType() || isBitmapType();
+ }
+
public boolean isScalarType() {
return this instanceof ScalarType;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index bfaec003ad..20a5c7c704 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -476,18 +476,6 @@ public class QueryPlanTest {
Assert.assertTrue(explainString.contains("OUTPUT EXPRS:`id` | `id2`"));
Assert.assertTrue(explainString.contains("0:OlapScanNode"));
- queryStr = "explain insert into test.bitmap_table select id, to_bitmap(id2) from test.bitmap_table_2;";
- explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
- Assert.assertTrue(explainString.contains("OLAP TABLE SINK"));
- Assert.assertTrue(explainString.contains("OUTPUT EXPRS:`id` | to_bitmap(CAST(`id2` AS CHARACTER))"));
- Assert.assertTrue(explainString.contains("0:OlapScanNode"));
-
- queryStr = "explain insert into test.bitmap_table select id, bitmap_hash(id2) from test.bitmap_table_2;";
- explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
- Assert.assertTrue(explainString.contains("OLAP TABLE SINK"));
- Assert.assertTrue(explainString.contains("OUTPUT EXPRS:`id` | bitmap_hash(CAST(`id2` AS CHARACTER))"));
- Assert.assertTrue(explainString.contains("0:OlapScanNode"));
-
queryStr = "explain insert into test.bitmap_table select id, id from test.bitmap_table_2;";
String errorMsg = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
Assert.assertTrue(errorMsg.contains("bitmap column require the function return type is BITMAP"));
@@ -615,6 +603,25 @@ public class QueryPlanTest {
sql = "SHOW VARIABLES LIKE 'lower_case_%'; SHOW VARIABLES LIKE 'sql_mode'";
List<StatementBase> stmts = UtFrameUtils.parseAndAnalyzeStmts(sql, connectContext);
Assert.assertEquals(2, stmts.size());
+
+ // disable cast hll/bitmap to string
+ testHLLQueryPlan(
+ "select cast(id2 as varchar) from test.hll_table;",
+ "Invalid type cast of `id2` from HLL to VARCHAR(*)"
+ );
+ testBitmapQueryPlan(
+ "select cast(id2 as varchar) from test.bitmap_table;",
+ "Invalid type cast of `id2` from BITMAP to VARCHAR(*)"
+ );
+ // disable implicit cast hll/bitmap to string
+ testHLLQueryPlan(
+ "select length(id2) from test.hll_table;",
+ "No matching function with signature: length(hll)"
+ );
+ testBitmapQueryPlan(
+ "select length(id2) from test.bitmap_table;",
+ "No matching function with signature: length(bitmap)"
+ );
}
@Test
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
index 58e4c9de86..15d84f7f39 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
@@ -184,11 +184,11 @@ public class TableFunctionPlanTest {
public void errorParam() throws Exception {
String sql = "explain select k1, e1 from db1.tbl1 lateral view explode_split(k2) tmp as e1;";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
- Assert.assertTrue(explainString.contains(FunctionCallExpr.UNKNOWN_TABLE_FUNCTION_MSG));
+ Assert.assertTrue(explainString.contains("No matching function with signature: explode_split(varchar(1))"));
sql = "explain select k1, e1 from db1.tbl1 lateral view explode_split(k1) tmp as e1;";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
- Assert.assertTrue(explainString.contains(FunctionCallExpr.UNKNOWN_TABLE_FUNCTION_MSG));
+ Assert.assertTrue(explainString.contains("No matching function with signature: explode_split(int(11))"));
}
/* Case2 table function in where stmt
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 10/14: [fix] access parallel_flat_hash_map via thread safely methods (#8854)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 930b458901dce143c99e69d7f645888e6c06bb16
Author: dataroaring <98...@users.noreply.github.com>
AuthorDate: Thu Apr 7 11:35:59 2022 +0800
[fix] access parallel_flat_hash_map via thread safely methods (#8854)
Iterator of parallel_flat_hash_map is not thread safely, so
we should use if_contains instead.
---
be/src/util/brpc_client_cache.h | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/be/src/util/brpc_client_cache.h b/be/src/util/brpc_client_cache.h
index d0d8b73689..b15a3c6aeb 100644
--- a/be/src/util/brpc_client_cache.h
+++ b/be/src/util/brpc_client_cache.h
@@ -30,9 +30,10 @@
#include "util/doris_metrics.h"
template <typename T>
-using SubMap = phmap::parallel_flat_hash_map<
+using StubMap = phmap::parallel_flat_hash_map<
std::string, std::shared_ptr<T>, std::hash<std::string>, std::equal_to<std::string>,
std::allocator<std::pair<const std::string, std::shared_ptr<T>>>, 8, std::mutex>;
+
namespace doris {
template <class T>
@@ -63,10 +64,12 @@ public:
}
inline std::shared_ptr<T> get_client(const std::string& host_port) {
- auto stub_ptr = _stub_map.find(host_port);
- if (LIKELY(stub_ptr != _stub_map.end())) {
- return stub_ptr->second;
+ std::shared_ptr<T> stub_ptr;
+ auto get_value = [&stub_ptr](typename StubMap<T>::mapped_type& v) { stub_ptr = v; };
+ if(LIKELY(_stub_map.if_contains(host_port, get_value))) {
+ return stub_ptr;
}
+
// new one stub and insert into map
brpc::ChannelOptions options;
if constexpr (std::is_same_v<T, PFunctionService_Stub>) {
@@ -85,7 +88,9 @@ public:
}
auto stub = std::make_shared<T>(channel.release(),
google::protobuf::Service::STUB_OWNS_CHANNEL);
- _stub_map[host_port] = stub;
+ _stub_map.try_emplace_l(host_port,
+ [&stub](typename StubMap<T>::mapped_type& v) { stub = v; },
+ stub);
return stub;
}
@@ -149,7 +154,7 @@ public:
}
private:
- SubMap<T> _stub_map;
+ StubMap<T> _stub_map;
};
using InternalServiceClientCache = BrpcClientCache<PBackendService_Stub>;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 02/14: min and max window function bug fix (#8822)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit b77baafcaca0912bdac5bef72c248f5ed9fb7b83
Author: camby <10...@qq.com>
AuthorDate: Thu Apr 7 08:36:33 2022 +0800
min and max window function bug fix (#8822)
[Fix bug] min and max window function bug fix #8822
---
be/src/exprs/agg_fn_evaluator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/exprs/agg_fn_evaluator.cpp b/be/src/exprs/agg_fn_evaluator.cpp
index 2b0679cf4c..ecd0b6e666 100644
--- a/be/src/exprs/agg_fn_evaluator.cpp
+++ b/be/src/exprs/agg_fn_evaluator.cpp
@@ -345,7 +345,7 @@ inline void AggFnEvaluator::set_any_val(const void* slot, const TypeDescriptor&
inline void AggFnEvaluator::set_output_slot(const AnyVal* src, const SlotDescriptor* dst_slot_desc,
Tuple* dst) {
- if (src->is_null) {
+ if (src->is_null && dst_slot_desc->is_nullable()) {
dst->set_null(dst_slot_desc->null_indicator_offset());
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org