You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/07/31 11:49:00 UTC
[doris] branch branch-2.0 updated (6fe74e080c -> e5359ba03c)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 6fe74e080c [feature](datetime) Support timezone when insert datetime value (#21898)
new 997d071fc3 [FIX](complex-type)fix complex type nested col_const (#22375)
new bf6ca76d29 [feature](property) Add table property "is_being_synced" (#22314)
new 729b9e979c [fix](load) load core dump print load id (#22388)
new e5359ba03c [fix](show-stmt) fix show create table missing storage_medium info (#21757)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/signal_handler.h | 10 +++
be/src/runtime/thread_context.cpp | 4 +-
be/src/service/internal_service.cpp | 4 ++
.../vec/data_types/serde/data_type_array_serde.cpp | 4 +-
.../vec/data_types/serde/data_type_map_serde.cpp | 12 ++--
.../data_types/serde/data_type_nullable_serde.cpp | 1 +
.../data_types/serde/data_type_struct_serde.cpp | 4 +-
docs/en/docs/advanced/autobucket.md | 4 +-
docs/en/docs/advanced/cold_hot_separation.md | 4 +-
docs/en/docs/advanced/is-being-synced.md | 65 ++++++++++++++++++
.../docs/advanced/partition/dynamic-partition.md | 4 +-
.../join-optimization/colocation-join.md | 4 +-
.../Create/CREATE-TABLE.md | 10 +++
docs/zh-CN/docs/advanced/autobucket.md | 5 +-
docs/zh-CN/docs/advanced/cold_hot_separation.md | 4 +-
docs/zh-CN/docs/advanced/is-being-synced.md | 65 ++++++++++++++++++
.../docs/advanced/partition/dynamic-partition.md | 2 +
.../join-optimization/colocation-join.md | 4 +-
.../Create/CREATE-TABLE.md | 10 +++
.../main/java/org/apache/doris/alter/Alter.java | 6 +-
.../org/apache/doris/alter/AlterOperations.java | 8 +--
.../analysis/ModifyTablePropertiesClause.java | 16 ++---
.../org/apache/doris/analysis/RestoreStmt.java | 23 ++++++-
.../org/apache/doris/backup/BackupHandler.java | 4 +-
.../java/org/apache/doris/backup/RestoreJob.java | 20 ++++--
.../org/apache/doris/binlog/CreateTableRecord.java | 4 +-
.../org/apache/doris/catalog/DataProperty.java | 16 +++++
.../org/apache/doris/catalog/DistributionInfo.java | 6 +-
.../main/java/org/apache/doris/catalog/Env.java | 26 ++++++-
.../apache/doris/catalog/HashDistributionInfo.java | 4 +-
.../java/org/apache/doris/catalog/OlapTable.java | 65 ++++++++++++++----
.../doris/catalog/RandomDistributionInfo.java | 4 +-
.../org/apache/doris/catalog/TableProperty.java | 29 +++++---
.../doris/clone/DynamicPartitionScheduler.java | 1 +
.../doris/common/util/DynamicPartitionUtil.java | 1 +
.../apache/doris/common/util/PropertyAnalyzer.java | 14 +++-
.../apache/doris/datasource/InternalCatalog.java | 19 ++++-
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +-
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
.../analysis/CreateTableAsSelectStmtTest.java | 80 ++++++++++++++++------
.../org/apache/doris/backup/RestoreJobTest.java | 2 +-
.../org/apache/doris/catalog/CreateTableTest.java | 23 +++++++
.../org/apache/doris/catalog/OlapTableTest.java | 4 +-
.../stream_load/test_map_load_and_function.out | 3 +
.../test_array_functions_by_literal.out | 3 +
.../array_functions/test_array_with_scale_type.out | 28 ++++----
.../test_struct_functions_by_literal.out | 3 +
.../stream_load/test_map_load_and_function.groovy | 2 +
.../test_array_functions_by_literal.groovy | 4 ++
.../test_struct_functions_by_literal.groovy | 3 +
50 files changed, 527 insertions(+), 118 deletions(-)
create mode 100644 docs/en/docs/advanced/is-being-synced.md
create mode 100644 docs/zh-CN/docs/advanced/is-being-synced.md
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/04: [FIX](complex-type)fix complex type nested col_const (#22375)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 997d071fc3adebc1d9564f21573f350605d8597a
Author: amory <wa...@selectdb.com>
AuthorDate: Mon Jul 31 14:53:18 2023 +0800
[FIX](complex-type)fix complex type nested col_const (#22375)
for array/map/struct in mysql_writer unpack_if_const only unpack self column not nested , so col_const should not used in nested column.
---
.../vec/data_types/serde/data_type_array_serde.cpp | 4 ++--
.../vec/data_types/serde/data_type_map_serde.cpp | 12 +++++-----
.../data_types/serde/data_type_nullable_serde.cpp | 1 +
.../data_types/serde/data_type_struct_serde.cpp | 4 ++--
.../stream_load/test_map_load_and_function.out | 3 +++
.../test_array_functions_by_literal.out | 3 +++
.../array_functions/test_array_with_scale_type.out | 28 +++++++++++-----------
.../test_struct_functions_by_literal.out | 3 +++
.../stream_load/test_map_load_and_function.groovy | 2 ++
.../test_array_functions_by_literal.groovy | 4 ++++
.../test_struct_functions_by_literal.groovy | 3 +++
11 files changed, 43 insertions(+), 24 deletions(-)
diff --git a/be/src/vec/data_types/serde/data_type_array_serde.cpp b/be/src/vec/data_types/serde/data_type_array_serde.cpp
index 596209c9aa..93cc45414d 100644
--- a/be/src/vec/data_types/serde/data_type_array_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_array_serde.cpp
@@ -117,12 +117,12 @@ Status DataTypeArraySerDe::_write_column_to_mysql(const IColumn& column,
if (0 != result.push_string("\"", 1)) {
return Status::InternalError("pack mysql buffer failed.");
}
- RETURN_IF_ERROR(nested_serde->write_column_to_mysql(data, result, j, col_const));
+ RETURN_IF_ERROR(nested_serde->write_column_to_mysql(data, result, j, false));
if (0 != result.push_string("\"", 1)) {
return Status::InternalError("pack mysql buffer failed.");
}
} else {
- RETURN_IF_ERROR(nested_serde->write_column_to_mysql(data, result, j, col_const));
+ RETURN_IF_ERROR(nested_serde->write_column_to_mysql(data, result, j, false));
}
}
}
diff --git a/be/src/vec/data_types/serde/data_type_map_serde.cpp b/be/src/vec/data_types/serde/data_type_map_serde.cpp
index fcf67a8f53..5b7bb09147 100644
--- a/be/src/vec/data_types/serde/data_type_map_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_map_serde.cpp
@@ -146,13 +146,13 @@ Status DataTypeMapSerDe::_write_column_to_mysql(const IColumn& column,
return Status::InternalError("pack mysql buffer failed.");
}
RETURN_IF_ERROR(
- key_serde->write_column_to_mysql(nested_keys_column, result, j, col_const));
+ key_serde->write_column_to_mysql(nested_keys_column, result, j, false));
if (0 != result.push_string("\"", 1)) {
return Status::InternalError("pack mysql buffer failed.");
}
} else {
RETURN_IF_ERROR(
- key_serde->write_column_to_mysql(nested_keys_column, result, j, col_const));
+ key_serde->write_column_to_mysql(nested_keys_column, result, j, false));
}
}
if (0 != result.push_string(":", 1)) {
@@ -167,14 +167,14 @@ Status DataTypeMapSerDe::_write_column_to_mysql(const IColumn& column,
if (0 != result.push_string("\"", 1)) {
return Status::InternalError("pack mysql buffer failed.");
}
- RETURN_IF_ERROR(value_serde->write_column_to_mysql(nested_values_column, result, j,
- col_const));
+ RETURN_IF_ERROR(
+ value_serde->write_column_to_mysql(nested_values_column, result, j, false));
if (0 != result.push_string("\"", 1)) {
return Status::InternalError("pack mysql buffer failed.");
}
} else {
- RETURN_IF_ERROR(value_serde->write_column_to_mysql(nested_values_column, result, j,
- col_const));
+ RETURN_IF_ERROR(
+ value_serde->write_column_to_mysql(nested_values_column, result, j, false));
}
}
}
diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp
index 654890f501..d96a705def 100644
--- a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp
@@ -129,6 +129,7 @@ Status DataTypeNullableSerDe::_write_column_to_mysql(const IColumn& column,
int row_idx, bool col_const) const {
auto& col = static_cast<const ColumnNullable&>(column);
auto& nested_col = col.get_nested_column();
+ col_const = col_const || is_column_const(nested_col);
const auto col_index = index_check_const(row_idx, col_const);
if (col.has_null() && col.is_null_at(col_index)) {
if (UNLIKELY(0 != result.push_null())) {
diff --git a/be/src/vec/data_types/serde/data_type_struct_serde.cpp b/be/src/vec/data_types/serde/data_type_struct_serde.cpp
index ed6e9c1b7b..4dd8086fb9 100644
--- a/be/src/vec/data_types/serde/data_type_struct_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_struct_serde.cpp
@@ -106,13 +106,13 @@ Status DataTypeStructSerDe::_write_column_to_mysql(const IColumn& column,
return Status::InternalError("pack mysql buffer failed.");
}
RETURN_IF_ERROR(elemSerDeSPtrs[j]->write_column_to_mysql(col.get_column(j), result,
- col_index, col_const));
+ col_index, false));
if (0 != result.push_string("\"", 1)) {
return Status::InternalError("pack mysql buffer failed.");
}
} else {
RETURN_IF_ERROR(elemSerDeSPtrs[j]->write_column_to_mysql(col.get_column(j), result,
- col_index, col_const));
+ col_index, false));
}
}
begin = false;
diff --git a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
index c493af4441..71cd0b7528 100644
--- a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
+++ b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
@@ -63,6 +63,9 @@
-- !select_map3 --
{}
+-- !select_map4 --
+{1000:"k11", 2000:"k22"}
+
-- !select_element1 --
1000
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out
index 6d2e316090..63103fa437 100644
--- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out
@@ -1,4 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+[1, 1, 2, 2, 2, 2]
+
-- !sql --
true
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out
index e2b4a2a2af..b887cdf11b 100644
--- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out
@@ -58,8 +58,8 @@
[23.679]
-- !select --
-[24.990, 24.990]
-[24.990, 24.990]
+[24.990, 25.990]
+[24.990, 25.990]
-- !select --
[24.990, 25.990]
@@ -69,8 +69,8 @@
[34.679]
-- !select --
-[24.990, 24.990]
-[24.990, 24.990]
+[24.990, 25.990]
+[24.990, 25.990]
-- !select --
[24.990, 25.990]
@@ -80,8 +80,8 @@
[2022-12-02 22:23:24.999]
-- !select --
-[2022-12-02 22:23:24.999, 2022-12-02 22:23:24.999]
-[2022-12-02 22:23:24.999, 2022-12-02 22:23:24.999]
+[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]
+[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]
-- !select --
[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]
@@ -111,8 +111,8 @@
\N
-- !select --
-[2022-12-02 22:23:24.999, 2022-12-02 22:23:24.999]
-[2022-12-02 22:23:24.999, 2022-12-02 22:23:24.999]
+[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]
+[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]
-- !select --
[2022-12-01 22:23:24.999, 2022-12-01 23:23:24.999]
@@ -131,8 +131,8 @@
[{23.679, 23.679, 2022-12-02 22:23:24.999, 23.679}, {34.679, 34.679, 2022-12-02 23:23:24.999, 34.679}]
-- !select --
-[{2022-12-02 22:23:24.999}, {2022-12-02 22:23:24.999}]
-[{2022-12-02 22:23:24.999}, {2022-12-02 22:23:24.999}]
+[{2022-12-02 22:23:24.999}, {2022-12-02 22:23:23.997}]
+[{2022-12-02 22:23:24.999}, {2022-12-02 22:23:23.997}]
-- !select --
[{2022-12-01 22:23:24.999}, {2022-12-01 23:23:24.999}]
@@ -143,8 +143,8 @@
[{2022-12-02 22:23:24.999, 2022-12-02 22:23:24.999}, {2022-12-02 23:23:24.999, 2022-12-02 22:23:23.997}]
-- !select --
-[2022-12-02 22:23:23.997, 2022-12-02 22:23:23.997]
-[2022-12-02 22:23:23.997, 2022-12-02 22:23:23.997]
+[2022-12-02 22:23:23.997, 2022-12-02 22:23:24.999]
+[2022-12-02 22:23:23.997, 2022-12-02 22:23:24.999]
-- !select --
[2023-03-08 23:23:23.997, 2022-12-01 22:23:24.999, 2022-12-01 23:23:24.999]
@@ -163,8 +163,8 @@
23.679 [23.679, 34.679] [23.679, 23.679, 34.679]
-- !select --
-[2022-12-02 22:23:24.999, 2022-12-02 22:23:24.999]
-[2022-12-02 22:23:24.999, 2022-12-02 22:23:24.999]
+[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]
+[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]
-- !select --
[2022-12-01 22:23:24.999, 2022-12-01 23:23:24.999, 2023-03-08 23:23:23.997]
diff --git a/regression-test/data/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.out b/regression-test/data/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.out
index 17e391b283..d6e14a7cc4 100644
--- a/regression-test/data/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.out
+++ b/regression-test/data/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.out
@@ -1,4 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+{"a", 1, "doris", "aaaaa", 1.32}
+
-- !sql --
{1, 2, 3}
diff --git a/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy
index 5ef0208bb2..c3e6ab8a20 100644
--- a/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy
@@ -79,6 +79,8 @@ suite("test_map_load_and_function", "p0") {
qt_select_map1 "SELECT map('k11', 1000, 'k22', 2000)"
qt_select_map2 "SELECT map(1000, 'k11', 2000, 'k22')"
qt_select_map3 "SELECT map()"
+ // map nested
+ qt_select_map4 "select m from (SELECT map(1000, 'k11', 2000, 'k22') as m) t"
// map element_at
qt_select_element1 "SELECT map('k11', 1000, 'k22', 2000)['k11']"
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy
index c29bf53103..d969312431 100644
--- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy
@@ -16,6 +16,10 @@
// under the License.
suite("test_array_functions_by_literal") {
+ // array_nested function
+ qt_sql "select a from (select array(1, 1, 2, 2, 2, 2) as a) t"
+
+
// array_contains function
qt_sql "select array_contains([1,2,3], 1)"
qt_sql "select array_contains([1,2,3], 4)"
diff --git a/regression-test/suites/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.groovy b/regression-test/suites/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.groovy
index a7ae109fd3..d394b3cbd4 100644
--- a/regression-test/suites/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.groovy
+++ b/regression-test/suites/query_p0/sql_functions/struct_functions/test_struct_functions_by_literal.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("test_struct_functions_by_literal") {
+ // struct-nested
+ qt_sql "select s from (select struct('a', 1, 'doris', 'aaaaa', 1.32) as s) t"
+
// struct constructor
qt_sql "select struct(1, 2, 3)"
qt_sql "select struct(1, 1000, 10000000000)"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/04: [fix](load) load core dump print load id (#22388)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 729b9e979cc0095c273fe1bbfc71558da2bba790
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Mon Jul 31 18:29:38 2023 +0800
[fix](load) load core dump print load id (#22388)
save the load id to the thread context,
expect all task ids to be saved in thread context, compaction/schema change/etc.
---
be/src/common/signal_handler.h | 10 ++++++++++
be/src/runtime/thread_context.cpp | 4 ++--
be/src/service/internal_service.cpp | 4 ++++
3 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/be/src/common/signal_handler.h b/be/src/common/signal_handler.h
index dbc7f4db57..1c721a691a 100644
--- a/be/src/common/signal_handler.h
+++ b/be/src/common/signal_handler.h
@@ -422,6 +422,16 @@ void FailureSignalHandler(int signal_number, siginfo_t* signal_info, void* ucont
} // namespace
+inline void set_signal_task_id(PUniqueId tid) {
+ query_id_hi = tid.hi();
+ query_id_lo = tid.lo();
+}
+
+inline void set_signal_task_id(TUniqueId tid) {
+ query_id_hi = tid.hi;
+ query_id_lo = tid.lo;
+}
+
inline void InstallFailureSignalHandler() {
// Build the sigaction struct.
struct sigaction sig_action;
diff --git a/be/src/runtime/thread_context.cpp b/be/src/runtime/thread_context.cpp
index c2dd13770c..a092385f41 100644
--- a/be/src/runtime/thread_context.cpp
+++ b/be/src/runtime/thread_context.cpp
@@ -33,13 +33,13 @@ ThreadContextPtr::ThreadContextPtr() {
AttachTask::AttachTask(const std::shared_ptr<MemTrackerLimiter>& mem_tracker,
const TUniqueId& task_id, const TUniqueId& fragment_instance_id) {
SwitchBthreadLocal::switch_to_bthread_local();
+ signal::set_signal_task_id(task_id);
thread_context()->attach_task(task_id, fragment_instance_id, mem_tracker);
}
AttachTask::AttachTask(RuntimeState* runtime_state) {
SwitchBthreadLocal::switch_to_bthread_local();
- doris::signal::query_id_hi = runtime_state->query_id().hi;
- doris::signal::query_id_lo = runtime_state->query_id().lo;
+ signal::set_signal_task_id(runtime_state->query_id());
thread_context()->attach_task(runtime_state->query_id(), runtime_state->fragment_instance_id(),
runtime_state->query_mem_tracker());
}
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 2d0b20ce8f..3a22919ea2 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -51,6 +51,7 @@
#include "common/config.h"
#include "common/exception.h"
#include "common/logging.h"
+#include "common/signal_handler.h"
#include "common/status.h"
#include "gutil/integral_types.h"
#include "http/http_client.h"
@@ -233,6 +234,7 @@ void PInternalServiceImpl::tablet_writer_open(google::protobuf::RpcController* c
bool ret = _light_work_pool.try_offer([this, request, response, done]() {
VLOG_RPC << "tablet writer open, id=" << request->id()
<< ", index_id=" << request->index_id() << ", txn_id=" << request->txn_id();
+ signal::set_signal_task_id(request->id());
brpc::ClosureGuard closure_guard(done);
auto st = _exec_env->load_channel_mgr()->open(*request);
if (!st.ok()) {
@@ -373,6 +375,7 @@ void PInternalServiceImpl::_tablet_writer_add_block(google::protobuf::RpcControl
int64_t execution_time_ns = 0;
{
SCOPED_RAW_TIMER(&execution_time_ns);
+ signal::set_signal_task_id(request->id());
auto st = _exec_env->load_channel_mgr()->add_batch(*request, response);
if (!st.ok()) {
LOG(WARNING) << "tablet writer add block failed, message=" << st
@@ -400,6 +403,7 @@ void PInternalServiceImpl::tablet_writer_cancel(google::protobuf::RpcController*
bool ret = _light_work_pool.try_offer([this, request, done]() {
VLOG_RPC << "tablet writer cancel, id=" << request->id()
<< ", index_id=" << request->index_id() << ", sender_id=" << request->sender_id();
+ signal::set_signal_task_id(request->id());
brpc::ClosureGuard closure_guard(done);
auto st = _exec_env->load_channel_mgr()->cancel(*request);
if (!st.ok()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/04: [fix](show-stmt) fix show create table missing storage_medium info (#21757)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e5359ba03ce03cd4e122e4304833364397d7e862
Author: zhengyu <fr...@gmail.com>
AuthorDate: Mon Jul 31 19:26:21 2023 +0800
[fix](show-stmt) fix show create table missing storage_medium info (#21757)
---
.../org/apache/doris/catalog/DataProperty.java | 16 ++++++
.../main/java/org/apache/doris/catalog/Env.java | 4 ++
.../java/org/apache/doris/catalog/OlapTable.java | 22 ++++++++
.../apache/doris/datasource/InternalCatalog.java | 5 +-
.../analysis/CreateTableAsSelectStmtTest.java | 60 ++++++++++++++--------
.../org/apache/doris/catalog/CreateTableTest.java | 23 +++++++++
6 files changed, 109 insertions(+), 21 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
index 7028e5e449..23b4508c94 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
@@ -55,6 +55,22 @@ public class DataProperty implements Writable, GsonPostProcessable {
// for persist
}
+ public String getStorageMediumString() {
+ int val = storageMedium.getValue();
+ switch (val) {
+ case 0:
+ return "hdd";
+ case 1:
+ return "ssd";
+ case 2:
+ return "s3";
+ case 3:
+ return "remote_cache";
+ default:
+ return "unknown";
+ }
+ }
+
public DataProperty(TStorageMedium medium) {
this.storageMedium = medium;
this.cooldownTimeMs = MAX_COOLDOWN_TIME_MS;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index f1bb519290..dc5598b710 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3155,6 +3155,10 @@ public class Env {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION).append("\" = \"");
sb.append(olapTable.enableSingleReplicaCompaction()).append("\"");
+ // storage medium
+ sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM).append("\" = \"");
+ sb.append(olapTable.getStorageMedium()).append("\"");
+
// enable duplicate without keys by default
if (olapTable.isDuplicateWithoutKey()) {
sb.append(",\n\"")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index ff5e3a8570..977e7f9e45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -151,6 +151,8 @@ public class OlapTable extends Table {
private TableProperty tableProperty;
+ private String storageMedium;
+
public OlapTable() {
// for persist
super(TableType.OLAP);
@@ -1299,6 +1301,13 @@ public class OlapTable extends Table {
}
tempPartitions.write(out);
+
+ if (storageMedium == null || storageMedium.length() == 0) {
+ out.writeBoolean(false);
+ } else {
+ out.writeBoolean(true);
+ Text.writeString(out, storageMedium);
+ }
}
@Override
@@ -1395,6 +1404,10 @@ public class OlapTable extends Table {
}
tempPartitions.unsetPartitionInfo();
+ if (in.readBoolean()) {
+ storageMedium = Text.readString(in);
+ }
+
// In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV.
// After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns.
// So, here we need to rebuild the fullSchema to ensure the correctness of the properties.
@@ -1816,6 +1829,15 @@ public class OlapTable extends Table {
return false;
}
+
+ public void setStorageMedium(String medium) {
+ storageMedium = medium;
+ }
+
+ public String getStorageMedium() {
+ return storageMedium;
+ }
+
public void setStoreRowColumn(boolean storeRowColumn) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 6c1079688d..28bac7a540 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2148,6 +2148,7 @@ public class InternalCatalog implements CatalogIf<Database> {
try {
dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
+ olapTable.setStorageMedium(dataProperty.getStorageMediumString());
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
@@ -2313,10 +2314,12 @@ public class InternalCatalog implements CatalogIf<Database> {
} else if (partitionInfo.getType() == PartitionType.RANGE
|| partitionInfo.getType() == PartitionType.LIST) {
try {
+ DataProperty dataProperty = null;
// just for remove entries in stmt.getProperties(),
// and then check if there still has unknown properties
- PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
+ dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
+ olapTable.setStorageMedium(dataProperty.getStorageMediumString());
if (partitionInfo.getType() == PartitionType.RANGE) {
DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties, db);
} else if (partitionInfo.getType() == PartitionType.LIST) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index 2a5ff740d5..348c8571f6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -96,7 +96,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1));
String selectFromDecimal1 =
@@ -117,7 +118,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
} else {
@@ -134,7 +136,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
}
@@ -169,7 +172,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -193,7 +197,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet1.getResultRows().get(0).get(1));
@@ -219,7 +224,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet2.getResultRows().get(0).get(1));
}
@@ -242,7 +248,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet1.getResultRows().get(0).get(1));
String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") "
+ "as select userId as alias_name, username from `test`.`varchar_table`";
@@ -261,7 +268,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet2.getResultRows().get(0).get(1));
}
@@ -287,7 +295,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") "
@@ -310,7 +319,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet1.getResultRows().get(0).get(1));
}
@@ -337,7 +347,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -361,7 +372,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet.getResultRows().get(0).get(1));
}
@@ -384,7 +396,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")"
@@ -403,7 +416,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");", showResultSet1.getResultRows().get(0).get(1));
}
@@ -429,7 +443,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -454,7 +469,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -478,7 +494,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -503,7 +520,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
showResultSet.getResultRows().get(0).get(1));
}
@@ -552,7 +570,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
createTableStmts.get(0));
} else {
@@ -569,7 +588,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
- + "\"enable_single_replica_compaction\" = \"false\"\n"
+ + "\"enable_single_replica_compaction\" = \"false\",\n"
+ + "\"storage_medium\" = \"hdd\"\n"
+ ");",
createTableStmts.get(0));
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 7d45752426..4e52ac8e51 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -146,6 +146,16 @@ public class CreateTableTest {
.expectThrowsNoException(() -> createTable("create table test.tb7(key1 int, key2 varchar(10)) \n"
+ "distributed by hash(key1) buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
+ ConfigBase.setMutableConfig("disable_storage_medium_check", "true");
+ ExceptionChecker
+ .expectThrowsNoException(() -> createTable("create table test.tb7_1(key1 int, key2 varchar(10))\n"
+ + "PARTITION BY RANGE(`key1`) (\n"
+ + " PARTITION `p1` VALUES LESS THAN (\"10\"),\n"
+ + " PARTITION `p2` VALUES LESS THAN (\"20\"),\n"
+ + " PARTITION `p3` VALUES LESS THAN (\"30\"))\n"
+ + "distributed by hash(key1)\n"
+ + "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
+
ExceptionChecker
.expectThrowsNoException(() -> createTable("create table test.compression1(key1 int, key2 varchar(10)) \n"
+ "distributed by hash(key1) buckets 1 \n"
@@ -301,6 +311,19 @@ public class CreateTableTest {
() -> createTable("create table test.tb7(key1 int, key2 varchar(10)) distributed by hash(key1) \n"
+ "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
+ ConfigBase.setMutableConfig("disable_storage_medium_check", "false");
+ ExceptionChecker
+ .expectThrowsWithMsg(DdlException.class, "Failed to find enough backend, please check the replication num,replication tag and storage medium.\n"
+ + "Create failed replications:\n"
+ + "replication tag: {\"location\" : \"default\"}, replication num: 1, storage medium: SSD",
+ () -> createTable("create table test.tb7_1(key1 int, key2 varchar(10))\n"
+ + "PARTITION BY RANGE(`key1`) (\n"
+ + " PARTITION `p1` VALUES LESS THAN (\"10\"),\n"
+ + " PARTITION `p2` VALUES LESS THAN (\"20\"),\n"
+ + " PARTITION `p3` VALUES LESS THAN (\"30\"))\n"
+ + "distributed by hash(key1)\n"
+ + "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');"));
+
ExceptionChecker
.expectThrowsWithMsg(DdlException.class, "sequence column only support UNIQUE_KEYS",
() -> createTable("create table test.atbl8\n" + "(k1 varchar(40), k2 int, v1 int sum)\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/04: [feature](property) Add table property "is_being_synced" (#22314)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bf6ca76d29d9cc709c390ec96d7e304b8edce1c0
Author: DeadlineFen <11...@users.noreply.github.com>
AuthorDate: Mon Jul 31 18:14:13 2023 +0800
[feature](property) Add table property "is_being_synced" (#22314)
---
docs/en/docs/advanced/autobucket.md | 4 +-
docs/en/docs/advanced/cold_hot_separation.md | 4 +-
docs/en/docs/advanced/is-being-synced.md | 65 ++++++++++++++++++++++
.../docs/advanced/partition/dynamic-partition.md | 4 +-
.../join-optimization/colocation-join.md | 4 +-
.../Create/CREATE-TABLE.md | 10 ++++
docs/zh-CN/docs/advanced/autobucket.md | 5 +-
docs/zh-CN/docs/advanced/cold_hot_separation.md | 4 +-
docs/zh-CN/docs/advanced/is-being-synced.md | 65 ++++++++++++++++++++++
.../docs/advanced/partition/dynamic-partition.md | 2 +
.../join-optimization/colocation-join.md | 4 +-
.../Create/CREATE-TABLE.md | 10 ++++
.../main/java/org/apache/doris/alter/Alter.java | 6 +-
.../org/apache/doris/alter/AlterOperations.java | 8 +--
.../analysis/ModifyTablePropertiesClause.java | 16 +++---
.../org/apache/doris/analysis/RestoreStmt.java | 23 +++++++-
.../org/apache/doris/backup/BackupHandler.java | 4 +-
.../java/org/apache/doris/backup/RestoreJob.java | 20 +++++--
.../org/apache/doris/binlog/CreateTableRecord.java | 4 +-
.../org/apache/doris/catalog/DistributionInfo.java | 6 +-
.../main/java/org/apache/doris/catalog/Env.java | 22 +++++++-
.../apache/doris/catalog/HashDistributionInfo.java | 4 +-
.../java/org/apache/doris/catalog/OlapTable.java | 43 ++++++++++----
.../doris/catalog/RandomDistributionInfo.java | 4 +-
.../org/apache/doris/catalog/TableProperty.java | 29 +++++++---
.../doris/clone/DynamicPartitionScheduler.java | 1 +
.../doris/common/util/DynamicPartitionUtil.java | 1 +
.../apache/doris/common/util/PropertyAnalyzer.java | 14 ++++-
.../apache/doris/datasource/InternalCatalog.java | 14 ++++-
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +-
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
.../analysis/CreateTableAsSelectStmtTest.java | 20 +++++++
.../org/apache/doris/backup/RestoreJobTest.java | 2 +-
.../org/apache/doris/catalog/OlapTableTest.java | 4 +-
34 files changed, 359 insertions(+), 71 deletions(-)
diff --git a/docs/en/docs/advanced/autobucket.md b/docs/en/docs/advanced/autobucket.md
index d17e8d41ef..bc05e3e72f 100644
--- a/docs/en/docs/advanced/autobucket.md
+++ b/docs/en/docs/advanced/autobucket.md
@@ -32,7 +32,9 @@ DISTRIBUTED BY ... BUCKETS auto
</version>
-Users often set inappropriate buckets, leading to various problems. For now, it only works for olap tables
+Users often set inappropriate buckets, leading to various problems. For now, it only works for olap tables
+
+Node: This feature will be disabled when synchronized by CCR. If this table is copied by CCR, that is, PROPERTIES contains `is_being_synced = true`, it will be displayed as enabled in show create table, but will not actually take effect. When `is_being_synced` is set to `false`, these features will resume working, but the `is_being_synced` property is for CCR peripheral modules only and should not be manually set during CCR synchronization.
# Implementation
diff --git a/docs/en/docs/advanced/cold_hot_separation.md b/docs/en/docs/advanced/cold_hot_separation.md
index 5d7ce3063f..df441ded63 100644
--- a/docs/en/docs/advanced/cold_hot_separation.md
+++ b/docs/en/docs/advanced/cold_hot_separation.md
@@ -54,7 +54,9 @@ The storage policy is the entry to use the cold and hot separation function. Use
<version since="dev"></version> When creating an S3 RESOURCE, the S3 remote link verification will be performed to ensure that the RESOURCE is created correctly.
-In addition, fe configuration needs to be added: `enable_storage_policy=true`
+In addition, fe configuration needs to be added: `enable_storage_policy=true`
+
+Note: This property will not be synchronized by CCR. If this table is copied by CCR, that is, PROPERTIES contains `is_being_synced = true`, this property will be erased in this table.
For example:
diff --git a/docs/en/docs/advanced/is-being-synced.md b/docs/en/docs/advanced/is-being-synced.md
new file mode 100644
index 0000000000..028e740ec9
--- /dev/null
+++ b/docs/en/docs/advanced/is-being-synced.md
@@ -0,0 +1,65 @@
+---
+{
+ "title": "property is_being_synced",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Background
+
+<version since="2.0">
+
+"is_being_synced" = "true"
+
+</version>
+
+CCR will create replica tables (referred to as target tables, located in the dest cluster) of the tables (referred to as source tables, located in the source cluster) in the synchronization scope of the source cluster in the dest cluster when establishing synchronization, but some functions and properties need to be invalidated or erased when creating replica tables to ensure the correctness of the synchronization process.
+
+such as：
+- The source table contains information that may not have been synchronized to the dest cluster, such as `storage_policy`, etc., which may cause the target table creation to fail or behave abnormally.
+- The source table may contain some dynamic functions, such as dynamic partitioning, etc., which may cause the behavior of the target table to be out of syncer control and cause partition inconsistency.
+
+The properties that need to be erased due to invalidation when being replicated are:
+- `storage_policy`
+- `colocate_with`
+
+The functions that need to be invalidated when being synchronized are:
+- auto bucket
+- dynamic partition
+
+# Implementation
+When creating the target table, this properties will be added or deleted by syncer control. In CCR, there are two ways to create a target table:
+1. When synchronizing tables, syncer uses backup/restore method to perform full replication of the source table to obtain the target table.
+2. When synchronizing databases, for existing tables, syncer also uses backup/restore method to obtain the target table, for incremental tables, syncer will create the target table through binlog with CreateTableRecord.
+
+In summary, there are two entry points for inserting `is_being_synced` attribute: the restore process in full synchronization and the getDdlStmt in incremental synchronization.
+
+In the restore process of full synchronization, syncer will initiate the restore of the snapshot in the original cluster through rpc. In this process, it will add `is_being_synced` attribute to RestoreStmt and take effect in the final restoreJob, executing the relevant logic of `isBeingSynced`.
+In the getDdlStmt of incremental synchronization, add the parameter `boolean getDdlForSync` to the getDdlStmt method to distinguish whether it is an operation to convert to the target table ddl under control, and execute the relevant logic of `isBeingSynced` when creating the target table.
+
+There is no need to say more about the erasure of invalid properties. The invalidation of the above functions needs to be explained:
+1. auto bucket
+ Auto bucket takes effect when creating a table, calculating the current appropriate number of buckets, which may cause the number of buckets in the source table and the destination table to be inconsistent. Therefore, when synchronizing, you need to obtain the number of buckets in the source table, and also need to obtain the information whether the source table is an automatic bucketing table in order to restore the function after ending the synchronization. The current approach is [...]
+2. dynamic partition
+ Dynamic partition is achieved by adding `olapTable.isBeingSynced()` to the judgment of whether to execute add/drop partition, so that the target table will not periodically execute add/drop partition operation during the synchronization process.
+# Note
+When no exception occurs, the `is_being_synced` attribute should be completely controlled by syncer to turn on or off, and users should not modify the attribute by themselves.
\ No newline at end of file
diff --git a/docs/en/docs/advanced/partition/dynamic-partition.md b/docs/en/docs/advanced/partition/dynamic-partition.md
index 08c3e66cbf..0511b93af4 100644
--- a/docs/en/docs/advanced/partition/dynamic-partition.md
+++ b/docs/en/docs/advanced/partition/dynamic-partition.md
@@ -30,7 +30,9 @@ Dynamic partition is a new feature introduced in Doris version 0.12. It's design
At present, the functions of dynamically adding partitions and dynamically deleting partitions are realized.
-Dynamic partitioning is only supported for Range partitions.
+Dynamic partitioning is only supported for Range partitions.
+
+Node: This feature will be disabled when synchronized by CCR. If this table is copied by CCR, that is, PROPERTIES contains `is_being_synced = true`, it will be displayed as enabled in show create table, but will not actually take effect. When `is_being_synced` is set to `false`, these features will resume working, but the `is_being_synced` property is for CCR peripheral modules only and should not be manually set during CCR synchronization.
## Noun Interpretation
diff --git a/docs/en/docs/query-acceleration/join-optimization/colocation-join.md b/docs/en/docs/query-acceleration/join-optimization/colocation-join.md
index 15357c2913..a9b6c2ab79 100644
--- a/docs/en/docs/query-acceleration/join-optimization/colocation-join.md
+++ b/docs/en/docs/query-acceleration/join-optimization/colocation-join.md
@@ -30,7 +30,9 @@ Colocation Join is a new feature introduced in Doris 0.9. The purpose of this pa
The original design, implementation and effect can be referred to [ISSUE 245](https://github.com/apache/incubator-doris/issues/245).
-The Colocation Join function has undergone a revision, and its design and use are slightly different from the original design. This document mainly introduces Colocation Join's principle, implementation, usage and precautions.
+The Colocation Join function has undergone a revision, and its design and use are slightly different from the original design. This document mainly introduces Colocation Join's principle, implementation, usage and precautions.
+
+Note: This property will not be synchronized by CCR. If this table is copied by CCR, that is, PROPERTIES contains `is_being_synced = true`, this property will be erased in this table.
## Noun Interpretation
diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
index a2f8c4731b..ab9e3a6f0f 100644
--- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
+++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
@@ -309,6 +309,16 @@ Set table properties. The following attributes are currently supported:
Set the copy distribution according to Tag. This attribute can completely cover the function of the `replication_num` attribute.
+* `is_being_synced`
+
+ Used to identify whether this table is copied by CCR and is being synchronized by syncer. The default is `false`.
+
+ If set to `true`:
+ `colocate_with`, `storage_policy` properties will be erased
+ `dynamic partition`, `auto bucket` features will be disabled, that is, they will be displayed as enabled in `show create table`, but will not actually take effect. When `is_being_synced` is set to `false`, these features will resume working.
+
+ This property is for CCR peripheral modules only and should not be manually set during CCR synchronization.
+
* `storage_medium/storage_cooldown_time`
Data storage medium. `storage_medium` is used to declare the initial storage medium of the table data, and `storage_cooldown_time` is used to set the expiration time. Example:
diff --git a/docs/zh-CN/docs/advanced/autobucket.md b/docs/zh-CN/docs/advanced/autobucket.md
index 880f5214f5..43a1124db7 100644
--- a/docs/zh-CN/docs/advanced/autobucket.md
+++ b/docs/zh-CN/docs/advanced/autobucket.md
@@ -32,7 +32,10 @@ DISTRIBUTED BY ... BUCKETS auto
</version>
-用户经常设置不合适的bucket，导致各种问题，这里提供一种方式，来自动设置分桶数。暂时而言只对olap表生效
+用户经常设置不合适的bucket，导致各种问题，这里提供一种方式，来自动设置分桶数。暂时而言只对olap表生效
+
+注意：这个功能在被CCR同步时将会失效。如果这个表是被CCR复制而来的，即PROPERTIES中包含`is_being_synced = true`时，在`show create table`中会显示开启状态，但不会实际生效。当`is_being_synced`被设置为 `false` 时，这些功能将会恢复生效，但`is_being_synced`属性仅供CCR外围模块使用，在CCR同步的过程中不要手动设置。
+
# 实现
diff --git a/docs/zh-CN/docs/advanced/cold_hot_separation.md b/docs/zh-CN/docs/advanced/cold_hot_separation.md
index e952b42f23..3d157c8e6c 100644
--- a/docs/zh-CN/docs/advanced/cold_hot_separation.md
+++ b/docs/zh-CN/docs/advanced/cold_hot_separation.md
@@ -53,7 +53,9 @@ under the License.
<version since="dev"></version> 创建S3 RESOURCE的时候，会进行S3远端的链接校验，以保证RESOURCE创建的正确。
-此外，需要新增fe配置：`enable_storage_policy=true`
+此外，需要新增fe配置：`enable_storage_policy=true`
+
+注意：这个属性不会被CCR同步，如果这个表是被CCR复制而来的，即PROPERTIES中包含`is_being_synced = true`时，这个属性将会在这个表中被擦除。
例如：
diff --git a/docs/zh-CN/docs/advanced/is-being-synced.md b/docs/zh-CN/docs/advanced/is-being-synced.md
new file mode 100644
index 0000000000..1d98a3b4e9
--- /dev/null
+++ b/docs/zh-CN/docs/advanced/is-being-synced.md
@@ -0,0 +1,65 @@
+---
+{
+ "title": "is_being_synced属性",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# 背景
+
+<version since="2.0">
+
+"is_being_synced" = "true"
+
+</version>
+
+CCR功能在建立同步时，会在目标集群中创建源集群同步范围中表（后称源表，位于源集群）的副本表（后称目标表，位于目标集群），但是在创建副本表时需要失效或者擦除一些功能和属性以保证同步过程中的正确性。
+
+如：
+- 源表中包含了可能没有被同步到目标集群的信息，如`storage_policy`等，可能会导致目标表创建失败或者行为异常。
+- 源表中可能包含一些动态功能，如动态分区等，可能导致目标表的行为不受syncer控制导致partition不一致。
+
+在被复制时因失效而需要擦除的属性有：
+- `storage_policy`
+- `colocate_with`
+
+在被同步时需要失效的功能有：
+- 自动分桶
+- 动态分区
+
+# 实现
+在创建目标表时，这条属性将会由syncer控制添加或者删除，在CCR功能中，创建一个目标表有两个途径：
+1. 在表同步时，syncer通过backup/restore的方式对源表进行全量复制来得到目标表。
+2. 在库同步时，对于存量表而言，syncer同样通过backup/restore的方式来得到目标表，对于增量表而言，syncer会通过携带有CreateTableRecord的binlog来创建目标表。
+
+综上，对于插入`is_being_synced`属性有两个切入点：全量同步中的restore过程和增量同步时的getDdlStmt。
+
+在全量同步的restore过程中，syncer会通过rpc发起对原集群中snapshot的restore，在这个过程中为会为RestoreStmt添加`is_being_synced`属性，并在最终的restoreJob中生效，执行`isBeingSynced`的相关逻辑。
+在增量同步时的getDdlStmt中，为getDdlStmt方法添加参数`boolean getDdlForSync`，以区分是否为受控转化为目标表ddl的操作，并在创建目标表时执行`isBeingSynced`的相关逻辑。
+
+对于失效属性的擦除无需多言，对于上述功能的失效需要进行说明：
+1. 自动分桶
+ 自动分桶会在创建表时生效，计算当前合适的bucket数量，这就可能导致源表和目的表的bucket数目不一致。因此在同步时需要获得源表的bucket数目，并且也要获得源表是否为自动分桶表的信息以便结束同步后恢复功能。当前的做法是在获取distribution信息时默认autobucket为false，在恢复表时通过检查`_auto_bucket`属性来判断源表是否为自动分桶表，如是则将目标表的autobucket字段设置为true，以此来达到跳过计算bucket数量，直接应用源表bucket数量的目的。
+2. 动态分区
+ 动态分区则是通过将`olapTable.isBeingSynced()`添加到是否执行add/drop partition的判断中来实现的，这样目标表在被同步的过程中就不会周期性的执行add/drop partition操作。
+# 注意
+在未出现异常时，`is_being_synced`属性应该完全由syncer控制开启或关闭，用户不要自行修改该属性。
\ No newline at end of file
diff --git a/docs/zh-CN/docs/advanced/partition/dynamic-partition.md b/docs/zh-CN/docs/advanced/partition/dynamic-partition.md
index 38ea866070..6aca4aafc3 100644
--- a/docs/zh-CN/docs/advanced/partition/dynamic-partition.md
+++ b/docs/zh-CN/docs/advanced/partition/dynamic-partition.md
@@ -32,6 +32,8 @@ under the License.
动态分区只支持 Range 分区。
+注意：这个功能在被CCR同步时将会失效。如果这个表是被CCR复制而来的，即PROPERTIES中包含`is_being_synced = true`时，在`show create table`中会显示开启状态，但不会实际生效。当`is_being_synced`被设置为 `false` 时，这些功能将会恢复生效，但`is_being_synced`属性仅供CCR外围模块使用，在CCR同步的过程中不要手动设置。
+
## 原理
在某些使用场景下，用户会将表按照天进行分区划分，每天定时执行例行任务，这时需要使用方手动管理分区，否则可能由于使用方没有创建分区导致数据导入失败，这给使用方带来了额外的维护成本。
diff --git a/docs/zh-CN/docs/query-acceleration/join-optimization/colocation-join.md b/docs/zh-CN/docs/query-acceleration/join-optimization/colocation-join.md
index e357683dc8..95661219cd 100644
--- a/docs/zh-CN/docs/query-acceleration/join-optimization/colocation-join.md
+++ b/docs/zh-CN/docs/query-acceleration/join-optimization/colocation-join.md
@@ -30,7 +30,9 @@ Colocation Join 是在 Doris 0.9 版本中引入的新功能。旨在为某些 J
最初的设计、实现和效果可以参阅 [ISSUE 245](https://github.com/apache/incubator-doris/issues/245)。
-Colocation Join 功能经过一次改版，设计和使用方式和最初设计稍有不同。本文档主要介绍 Colocation Join 的原理、实现、使用方式和注意事项。
+Colocation Join 功能经过一次改版，设计和使用方式和最初设计稍有不同。本文档主要介绍 Colocation Join 的原理、实现、使用方式和注意事项。
+
+注意：这个属性不会被CCR同步，如果这个表是被CCR复制而来的，即PROPERTIES中包含`is_being_synced = true`时，这个属性将会在这个表中被擦除。
## 名词解释
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
index 01266687c0..2fc5d23009 100644
--- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
@@ -293,6 +293,16 @@ UNIQUE KEY(k1, k2)
根据 Tag 设置副本分布情况。该属性可以完全覆盖 `replication_num` 属性的功能。
+* `is_being_synced`
+
+ 用于标识此表是否是被CCR复制而来并且正在被syncer同步，默认为 `false`。
+
+ 如果设置为 `true`：
+ `colocate_with`，`storage_policy`属性将被擦除
+ `dynamic partition`，`auto bucket`功能将会失效，即在`show create table`中显示开启状态，但不会实际生效。当`is_being_synced`被设置为 `false` 时，这些功能将会恢复生效。
+
+ 这个属性仅供CCR外围模块使用，在CCR同步的过程中不要手动设置。
+
* `storage_medium/storage_cooldown_time`
数据存储介质。`storage_medium` 用于声明表数据的初始存储介质，而 `storage_cooldown_time` 用于设定到期时间。示例：
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index a82209a866..53eb9052e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -208,8 +208,8 @@ public class Alter {
olapTable.setStoragePolicy(currentStoragePolicy);
needProcessOutsideTableLock = true;
- } else if (currentAlterOps.checkCcrEnable(alterClauses)) {
- olapTable.setCcrEnable(currentAlterOps.isCcrEnable(alterClauses));
+ } else if (currentAlterOps.checkIsBeingSynced(alterClauses)) {
+ olapTable.setIsBeingSynced(currentAlterOps.isBeingSynced(alterClauses));
needProcessOutsideTableLock = true;
} else if (currentAlterOps.checkBinlogConfigChange(alterClauses)) {
if (!Config.enable_feature_binlog) {
@@ -514,7 +514,7 @@ public class Alter {
// currently, only in memory and storage policy property could reach here
Preconditions.checkState(properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY)
|| properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)
- || properties.containsKey(PropertyAnalyzer.PROPERTIES_CCR_ENABLE));
+ || properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED));
((SchemaChangeHandler) schemaChangeHandler).updateTableProperties(db, tableName, properties);
} else {
throw new DdlException("Invalid alter operation: " + alterClause.getOpType());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java
index 632b60be41..22104522ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java
@@ -78,10 +78,10 @@ public class AlterOperations {
).map(c -> ((ModifyTablePropertiesClause) c).getStoragePolicy()).findFirst().orElse("");
}
- public boolean checkCcrEnable(List<AlterClause> alterClauses) {
+ public boolean checkIsBeingSynced(List<AlterClause> alterClauses) {
return alterClauses.stream().filter(clause ->
clause instanceof ModifyTablePropertiesClause
- ).anyMatch(clause -> clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_CCR_ENABLE));
+ ).anyMatch(clause -> clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED));
}
public boolean checkBinlogConfigChange(List<AlterClause> alterClauses) {
@@ -93,10 +93,10 @@ public class AlterOperations {
|| clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_MAX_HISTORY_NUMS));
}
- public boolean isCcrEnable(List<AlterClause> alterClauses) {
+ public boolean isBeingSynced(List<AlterClause> alterClauses) {
return alterClauses.stream().filter(clause ->
clause instanceof ModifyTablePropertiesClause
- ).map(c -> ((ModifyTablePropertiesClause) c).isCcrEnable()).findFirst().orElse(false);
+ ).map(c -> ((ModifyTablePropertiesClause) c).isBeingSynced()).findFirst().orElse(false);
}
// MODIFY_TABLE_PROPERTY is also processed by SchemaChangeHandler
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index 43c5211e07..93ece524f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -44,14 +44,14 @@ public class ModifyTablePropertiesClause extends AlterTableClause {
private String storagePolicy;
- private boolean ccrEnable = false;
+ private boolean isBeingSynced = false;
- public void setCcrEnable(boolean ccrEnable) {
- this.ccrEnable = ccrEnable;
+ public void setIsBeingSynced(boolean isBeingSynced) {
+ this.isBeingSynced = isBeingSynced;
}
- public boolean isCcrEnable() {
- return ccrEnable;
+ public boolean isBeingSynced() {
+ return isBeingSynced;
}
public ModifyTablePropertiesClause(Map<String, String> properties) {
@@ -133,10 +133,10 @@ public class ModifyTablePropertiesClause extends AlterTableClause {
throw new AnalysisException("Can not change enable_duplicate_without_keys_by_default");
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE)) {
// do nothing, will be alter in SchemaChangeHandler.updateTableProperties
- } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_CCR_ENABLE)) {
+ } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED)) {
this.needTableStable = false;
- setCcrEnable(
- Boolean.parseBoolean(properties.getOrDefault(PropertyAnalyzer.PROPERTIES_CCR_ENABLE, "false")));
+ setIsBeingSynced(Boolean.parseBoolean(properties.getOrDefault(
+ PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED, "false")));
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_ENABLE)
|| properties.containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_TTL_SECONDS)
|| properties.containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_MAX_BYTES)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
index 895ad48ce9..3b5f46a37c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
@@ -39,6 +39,7 @@ public class RestoreStmt extends AbstractBackupStmt {
private static final String PROP_META_VERSION = "meta_version";
private static final String PROP_RESERVE_REPLICA = "reserve_replica";
private static final String PROP_RESERVE_DYNAMIC_PARTITION_ENABLE = "reserve_dynamic_partition_enable";
+ private static final String PROP_IS_BEING_SYNCED = PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED;
private boolean allowLoad = false;
private ReplicaAllocation replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
@@ -47,6 +48,7 @@ public class RestoreStmt extends AbstractBackupStmt {
private boolean reserveReplica = false;
private boolean reserveDynamicPartitionEnable = false;
private boolean isLocal = false;
+ private boolean isBeingSynced = false;
private byte[] meta = null;
private byte[] jobInfo = null;
@@ -98,8 +100,12 @@ public class RestoreStmt extends AbstractBackupStmt {
return jobInfo;
}
- public void disableDynamicPartition() {
- setProperty(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE, "false");
+ public void setIsBeingSynced() {
+ setProperty(PROP_IS_BEING_SYNCED, "true");
+ }
+
+ public boolean isBeingSynced() {
+ return isBeingSynced;
}
@Override
@@ -195,6 +201,19 @@ public class RestoreStmt extends AbstractBackupStmt {
copiedProperties.remove(PROP_META_VERSION);
}
+ // is being synced
+ if (copiedProperties.containsKey(PROP_IS_BEING_SYNCED)) {
+ if (copiedProperties.get(PROP_IS_BEING_SYNCED).equalsIgnoreCase("true")) {
+ isBeingSynced = true;
+ } else if (copiedProperties.get(PROP_IS_BEING_SYNCED).equalsIgnoreCase("false")) {
+ isBeingSynced = false;
+ } else {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Invalid is being synced value: " + copiedProperties.get(PROP_IS_BEING_SYNCED));
+ }
+ copiedProperties.remove(PROP_IS_BEING_SYNCED);
+ }
+
if (!copiedProperties.isEmpty()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"Unknown restore job properties: " + copiedProperties.keySet());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 6619c457b9..5279175fa0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -450,7 +450,7 @@ public class BackupHandler extends MasterDaemon implements Writable {
restoreJob = new RestoreJob(stmt.getLabel(), backupTimestamp,
db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(),
stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(),
- stmt.reserveDynamicPartitionEnable(),
+ stmt.reserveDynamicPartitionEnable(), stmt.isBeingSynced(),
env, Repository.KEEP_ON_LOCAL_REPO_ID, backupMeta);
} catch (IOException e) {
throw new DdlException(e.getMessage());
@@ -459,7 +459,7 @@ public class BackupHandler extends MasterDaemon implements Writable {
restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(),
db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(),
stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveDynamicPartitionEnable(),
- env, repository.getId());
+ stmt.isBeingSynced(), env, repository.getId());
}
env.getEditLog().logRestoreJob(restoreJob);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 6bd606a378..a5256dac64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -60,6 +60,7 @@ import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.DynamicPartitionUtil;
+import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.resource.Tag;
import org.apache.doris.task.AgentBatchTask;
@@ -104,6 +105,7 @@ import java.util.stream.Collectors;
public class RestoreJob extends AbstractJob {
private static final String PROP_RESERVE_REPLICA = "reserve_replica";
private static final String PROP_RESERVE_DYNAMIC_PARTITION_ENABLE = "reserve_dynamic_partition_enable";
+ private static final String PROP_IS_BEING_SYNCED = PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED;
private static final Logger LOG = LogManager.getLogger(RestoreJob.class);
@@ -165,6 +167,9 @@ public class RestoreJob extends AbstractJob {
// NOTICE: because we do not persist it, this info may be lost if Frontend restart,
// and if you don't want to losing it, backup your data again by using latest Doris version.
private int metaVersion = -1;
+
+ private boolean isBeingSynced = false;
+
// restore properties
private Map<String, String> properties = Maps.newHashMap();
@@ -174,7 +179,7 @@ public class RestoreJob extends AbstractJob {
public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, boolean allowLoad,
ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica,
- boolean reserveDynamicPartitionEnable, Env env, long repoId) {
+ boolean reserveDynamicPartitionEnable, boolean isBeingSynced, Env env, long repoId) {
super(JobType.RESTORE, label, dbId, dbName, timeoutMs, env, repoId);
this.backupTimestamp = backupTs;
this.jobInfo = jobInfo;
@@ -184,15 +189,17 @@ public class RestoreJob extends AbstractJob {
this.metaVersion = metaVersion;
this.reserveReplica = reserveReplica;
this.reserveDynamicPartitionEnable = reserveDynamicPartitionEnable;
+ this.isBeingSynced = isBeingSynced;
properties.put(PROP_RESERVE_REPLICA, String.valueOf(reserveReplica));
properties.put(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE, String.valueOf(reserveDynamicPartitionEnable));
+ properties.put(PROP_IS_BEING_SYNCED, String.valueOf(isBeingSynced));
}
public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, boolean allowLoad,
ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica,
- boolean reserveDynamicPartitionEnable, Env env, long repoId, BackupMeta backupMeta) {
+ boolean reserveDynamicPartitionEnable, boolean isBeingSynced, Env env, long repoId, BackupMeta backupMeta) {
this(label, backupTs, dbId, dbName, jobInfo, allowLoad, replicaAlloc, timeoutMs, metaVersion, reserveReplica,
- reserveDynamicPartitionEnable, env, repoId);
+ reserveDynamicPartitionEnable, isBeingSynced, env, repoId);
this.backupMeta = backupMeta;
}
@@ -212,6 +219,10 @@ public class RestoreJob extends AbstractJob {
return metaVersion;
}
+ public boolean isBeingSynced() {
+ return isBeingSynced;
+ }
+
public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) {
if (checkTaskStatus(task, task.getJobId(), request)) {
return false;
@@ -688,7 +699,7 @@ public class RestoreJob extends AbstractJob {
// Reset properties to correct values.
remoteOlapTbl.resetPropertiesForRestore(reserveDynamicPartitionEnable, reserveReplica,
- replicaAlloc);
+ replicaAlloc, isBeingSynced);
// DO NOT set remote table's new name here, cause we will still need the origin name later
// remoteOlapTbl.setName(jobInfo.getAliasByOriginNameIfSet(tblInfo.name));
@@ -2135,6 +2146,7 @@ public class RestoreJob extends AbstractJob {
}
reserveReplica = Boolean.parseBoolean(properties.get(PROP_RESERVE_REPLICA));
reserveDynamicPartitionEnable = Boolean.parseBoolean(properties.get(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE));
+ isBeingSynced = Boolean.parseBoolean(properties.get(PROP_IS_BEING_SYNCED));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java
index 50557195b4..145b45396f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java
@@ -63,8 +63,8 @@ public class CreateTableRecord {
table.readLock();
try {
- Env.getDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt, false, false /* show password */,
- -1L);
+ Env.getSyncedDdlStmt(table, createTableStmt, addPartitionStmt, createRollupStmt,
+ false, false /* show password */, -1L);
} finally {
table.readUnlock();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
index fdb3513f7d..a104ff8b57 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
@@ -95,10 +95,14 @@ public abstract class DistributionInfo implements Writable {
type = DistributionInfoType.valueOf(Text.readString(in));
}
- public String toSql() {
+ public String toSql(boolean forSync) {
return "";
}
+ public String toSql() {
+ return toSql(false);
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 9f2797fe78..f1bb519290 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -2832,7 +2832,14 @@ public class Env {
List<String> createRollupStmt, boolean separatePartition, boolean hidePassword,
long specificVersion) {
getDdlStmt(null, null, table, createTableStmt, addPartitionStmt, createRollupStmt, separatePartition,
- hidePassword, false, specificVersion, false);
+ hidePassword, false, specificVersion, false, false);
+ }
+
+ public static void getSyncedDdlStmt(TableIf table, List<String> createTableStmt, List<String> addPartitionStmt,
+ List<String> createRollupStmt, boolean separatePartition, boolean hidePassword,
+ long specificVersion) {
+ getDdlStmt(null, null, table, createTableStmt, addPartitionStmt, createRollupStmt, separatePartition,
+ hidePassword, false, specificVersion, false, true);
}
/**
@@ -2844,7 +2851,7 @@ public class Env {
List<String> addPartitionStmt, List<String> createRollupStmt,
boolean separatePartition,
boolean hidePassword, boolean getDdlForLike, long specificVersion,
- boolean getBriefDdl) {
+ boolean getBriefDdl, boolean getDdlForSync) {
StringBuilder sb = new StringBuilder();
// 1. create table
@@ -3037,6 +3044,15 @@ public class Env {
sb.append(partition.getVisibleVersion()).append("\"");
}
+ // mark this table is being synced
+ sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED).append("\" = \"");
+ sb.append(String.valueOf(olapTable.isBeingSynced() || getDdlForSync)).append("\"");
+ // mark this table if it is a auto bucket table
+ if (getDdlForSync && olapTable.isAutoBucket()) {
+ sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_AUTO_BUCKET).append("\" = \"");
+ sb.append("true").append("\"");
+ }
+
// colocateTable
String colocateTable = olapTable.getColocateGroup();
if (colocateTable != null) {
@@ -4491,7 +4507,7 @@ public class Env {
}
tableProperty.buildInMemory()
.buildStoragePolicy()
- .buildCcrEnable();
+ .buildIsBeingSynced();
// need to update partition info meta
for (Partition partition : table.getPartitions()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
index 5f30bc2098..b4dc9f85a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
@@ -138,7 +138,7 @@ public class HashDistributionInfo extends DistributionInfo {
}
@Override
- public String toSql() {
+ public String toSql(boolean forSync) {
StringBuilder builder = new StringBuilder();
builder.append("DISTRIBUTED BY HASH(");
@@ -149,7 +149,7 @@ public class HashDistributionInfo extends DistributionInfo {
String colList = Joiner.on(", ").join(colNames);
builder.append(colList);
- if (autoBucket) {
+ if (autoBucket && !forSync) {
builder.append(") BUCKETS AUTO");
} else {
builder.append(") BUCKETS ").append(bucketNum);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 0a4ad6e52a..ff5e3a8570 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -208,6 +208,15 @@ public class OlapTable extends Table {
getOrCreatTableProperty().setBinlogConfig(binlogConfig);
}
+ public void setIsBeingSynced(boolean isBeingSynced) {
+ getOrCreatTableProperty().modifyTableProperties(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED,
+ String.valueOf(isBeingSynced));
+ }
+
+ public boolean isBeingSynced() {
+ return getOrCreatTableProperty().isBeingSynced();
+ }
+
public void setTableProperty(TableProperty tableProperty) {
this.tableProperty = tableProperty;
}
@@ -497,10 +506,18 @@ public class OlapTable extends Table {
* Reset properties to correct values.
*/
public void resetPropertiesForRestore(boolean reserveDynamicPartitionEnable, boolean reserveReplica,
- ReplicaAllocation replicaAlloc) {
+ ReplicaAllocation replicaAlloc, boolean isBeingSynced) {
if (tableProperty != null) {
tableProperty.resetPropertiesForRestore(reserveDynamicPartitionEnable, reserveReplica, replicaAlloc);
}
+ if (isBeingSynced) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+ tableProperty.setIsBeingSynced();
+ tableProperty.removeInvalidProperties();
+ if (isAutoBucket()) {
+ markAutoBucket();
+ }
+ }
// remove colocate property.
setColocateGroup(null);
}
@@ -751,6 +768,10 @@ public class OlapTable extends Table {
return defaultDistributionInfo;
}
+ public void markAutoBucket() {
+ defaultDistributionInfo.markAutoBucket();
+ }
+
public Set<String> getDistributionColumnNames() {
Set<String> distributionColumnNames = Sets.newHashSet();
if (defaultDistributionInfo instanceof RandomDistributionInfo) {
@@ -1675,6 +1696,15 @@ public class OlapTable extends Table {
return hasChanged;
}
+ public void ignoreInvaildPropertiesWhenSynced(Map<String, String> properties) {
+ // ignore colocate table
+ PropertyAnalyzer.analyzeColocate(properties);
+ // ignore storage policy
+ if (!PropertyAnalyzer.analyzeStoragePolicy(properties).isEmpty()) {
+ properties.remove(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY);
+ }
+ }
+
public void setReplicationAllocation(ReplicaAllocation replicaAlloc) {
getOrCreatTableProperty().setReplicaAlloc(replicaAlloc);
}
@@ -1756,17 +1786,6 @@ public class OlapTable extends Table {
return "";
}
- public void setCcrEnable(boolean ccrEnable) throws UserException {
- // TODO(Drogon): Config.enable_ccr
- TableProperty tableProperty = getOrCreatTableProperty();
- tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_CCR_ENABLE, Boolean.toString(ccrEnable));
- tableProperty.buildCcrEnable();
- }
-
- public boolean isCcrEnable() {
- return tableProperty != null && tableProperty.isCcrEnable();
- }
-
public void setDisableAutoCompaction(boolean disableAutoCompaction) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java
index 3c63546ffc..1352f8af3f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java
@@ -53,9 +53,9 @@ public class RandomDistributionInfo extends DistributionInfo {
}
@Override
- public String toSql() {
+ public String toSql(boolean forSync) {
StringBuilder builder = new StringBuilder();
- if (autoBucket) {
+ if (autoBucket && !forSync) {
builder.append("DISTRIBUTED BY RANDOM BUCKETS AUTO");
} else {
builder.append("DISTRIBUTED BY RANDOM BUCKETS ").append(bucketNum);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 550d190604..bced77425b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -60,7 +60,7 @@ public class TableProperty implements Writable {
private boolean isInMemory = false;
private String storagePolicy = "";
- private Boolean ccrEnable = null;
+ private Boolean isBeingSynced = null;
private BinlogConfig binlogConfig;
private boolean isDynamicSchema = false;
@@ -112,7 +112,7 @@ public class TableProperty implements Writable {
case OperationType.OP_MODIFY_IN_MEMORY:
buildInMemory();
buildStoragePolicy();
- buildCcrEnable();
+ buildIsBeingSynced();
break;
default:
break;
@@ -223,16 +223,27 @@ public class TableProperty implements Writable {
return storagePolicy;
}
- public TableProperty buildCcrEnable() {
- ccrEnable = Boolean.parseBoolean(properties.getOrDefault(PropertyAnalyzer.PROPERTIES_CCR_ENABLE, "false"));
+ public TableProperty buildIsBeingSynced() {
+ isBeingSynced = Boolean.parseBoolean(properties.getOrDefault(
+ PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED, "false"));
return this;
}
- public boolean isCcrEnable() {
- if (ccrEnable == null) {
- buildCcrEnable();
+ public void setIsBeingSynced() {
+ properties.put(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED, "true");
+ isBeingSynced = true;
+ }
+
+ public boolean isBeingSynced() {
+ if (isBeingSynced == null) {
+ buildIsBeingSynced();
}
- return ccrEnable;
+ return isBeingSynced;
+ }
+
+ public void removeInvalidProperties() {
+ properties.remove(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY);
+ properties.remove(PropertyAnalyzer.PROPERTIES_COLOCATE_WITH);
}
public TableProperty buildBinlogConfig() {
@@ -429,7 +440,7 @@ public class TableProperty implements Writable {
.buildDataSortInfo()
.buildCompressionType()
.buildStoragePolicy()
- .buildCcrEnable()
+ .buildIsBeingSynced()
.buildBinlogConfig()
.buildEnableLightSchemaChange()
.buildStoreRowColumn()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index eac10bb2d0..a368e33a77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -501,6 +501,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
olapTable = (OlapTable) db.getTableNullable(tableId);
// Only OlapTable has DynamicPartitionProperty
if (olapTable == null
+ || olapTable.isBeingSynced()
|| !olapTable.dynamicPartitionExists()
|| !olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()) {
iterator.remove();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index 54c0594482..274e1a3e15 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -670,6 +670,7 @@ public class DynamicPartitionUtil {
public static void checkAlterAllowed(OlapTable olapTable) throws DdlException {
TableProperty tableProperty = olapTable.getTableProperty();
if (tableProperty != null && tableProperty.getDynamicPartitionProperty() != null
+ && !tableProperty.isBeingSynced()
&& tableProperty.getDynamicPartitionProperty().isExist()
&& tableProperty.getDynamicPartitionProperty().getEnable()) {
throw new DdlException("Cannot add/drop partition on a Dynamic Partition Table, "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index dd137077db..b80d511256 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -129,7 +129,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_MUTABLE = "mutable";
- public static final String PROPERTIES_CCR_ENABLE = "ccr_enable";
+ public static final String PROPERTIES_IS_BEING_SYNCED = "is_being_synced";
// binlog.enable, binlog.ttl_seconds, binlog.max_bytes, binlog.max_history_nums
public static final String PROPERTIES_BINLOG_PREFIX = "binlog.";
@@ -458,7 +458,7 @@ public class PropertyAnalyzer {
}
// analyze the colocation properties of table
- public static String analyzeColocate(Map<String, String> properties) throws AnalysisException {
+ public static String analyzeColocate(Map<String, String> properties) {
String colocateGroup = null;
if (properties != null && properties.containsKey(PROPERTIES_COLOCATE_WITH)) {
colocateGroup = properties.get(PROPERTIES_COLOCATE_WITH);
@@ -667,7 +667,7 @@ public class PropertyAnalyzer {
return estimatePartitionSize;
}
- public static String analyzeStoragePolicy(Map<String, String> properties) throws AnalysisException {
+ public static String analyzeStoragePolicy(Map<String, String> properties) {
String storagePolicy = "";
if (properties != null && properties.containsKey(PROPERTIES_STORAGE_POLICY)) {
storagePolicy = properties.get(PROPERTIES_STORAGE_POLICY);
@@ -815,6 +815,14 @@ public class PropertyAnalyzer {
return binlogConfigMap;
}
+ public static boolean analyzeIsBeingSynced(Map<String, String> properties, boolean defaultValue) {
+ if (properties != null && properties.containsKey(PROPERTIES_IS_BEING_SYNCED)) {
+ String value = properties.remove(PROPERTIES_IS_BEING_SYNCED);
+ return Boolean.valueOf(value);
+ }
+ return defaultValue;
+ }
+
// There are 2 kinds of replication property:
// 1. "replication_num" = "3"
// 2. "replication_allocation" = "tag.location.zone1: 2, tag.location.zone2: 1"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index a4ada8e761..6c1079688d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1147,7 +1147,7 @@ public class InternalCatalog implements CatalogIf<Database> {
}
Env.getDdlStmt(stmt, stmt.getDbName(), table, createTableStmt, null, null, false, false, true, -1L,
- false);
+ false, false);
if (createTableStmt.isEmpty()) {
ErrorReport.reportDdlException(ErrorCode.ERROR_CREATE_TABLE_LIKE_EMPTY, "CREATE");
}
@@ -2071,6 +2071,17 @@ public class InternalCatalog implements CatalogIf<Database> {
}
olapTable.setIsInMemory(false);
+ boolean isBeingSynced = PropertyAnalyzer.analyzeIsBeingSynced(properties, false);
+ olapTable.setIsBeingSynced(isBeingSynced);
+ if (isBeingSynced) {
+ // erase colocate table, storage policy
+ olapTable.ignoreInvaildPropertiesWhenSynced(properties);
+ // remark auto bucket
+ if (isAutoBucket) {
+ olapTable.markAutoBucket();
+ }
+ }
+
boolean storeRowColumn = false;
try {
storeRowColumn = PropertyAnalyzer.analyzeStoreRowColumn(properties);
@@ -2312,7 +2323,6 @@ public class InternalCatalog implements CatalogIf<Database> {
if (DynamicPartitionUtil.checkDynamicPartitionPropertiesExist(properties)) {
throw new DdlException(
"Only support dynamic partition properties on range partition table");
-
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 59bfbada21..322dcdf500 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -964,7 +964,7 @@ public class ShowExecutor {
}
List<String> createTableStmt = Lists.newArrayList();
Env.getDdlStmt(null, null, table, createTableStmt, null, null, false,
- true /* hide password */, false, -1L, showStmt.isNeedBriefDdl());
+ true /* hide password */, false, -1L, showStmt.isNeedBriefDdl(), false);
if (createTableStmt.isEmpty()) {
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
return;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 93b4d7f687..8b25aac5f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -2569,7 +2569,7 @@ public class FrontendServiceImpl implements FrontendService.Iface {
Map<String, String> properties = request.getProperties();
RestoreStmt restoreStmt = new RestoreStmt(label, repoName, null, properties, request.getMeta(),
request.getJobInfo());
- restoreStmt.disableDynamicPartition();
+ restoreStmt.setIsBeingSynced();
LOG.trace("restore snapshot info, restoreStmt: {}", restoreStmt);
try {
ConnectContext ctx = ConnectContext.get();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index e3e25b90ab..2a5ff740d5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -92,6 +92,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -112,6 +113,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -128,6 +130,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -162,6 +165,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -185,6 +189,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -210,6 +215,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -232,6 +238,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -250,6 +257,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -275,6 +283,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -297,6 +306,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId1`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -323,6 +333,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -346,6 +357,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -368,6 +380,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -386,6 +399,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -411,6 +425,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -435,6 +450,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -458,6 +474,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`username`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -482,6 +499,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -530,6 +548,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
@@ -546,6 +565,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ + "\"is_being_synced\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"light_schema_change\" = \"true\",\n"
+ "\"disable_auto_compaction\" = \"false\",\n"
diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index dcc76fdfcd..64e92b35c8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -243,7 +243,7 @@ public class RestoreJobTest {
db.dropTable(expectedRestoreTbl.getName());
job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), jobInfo, false,
- new ReplicaAllocation((short) 3), 100000, -1, false, false, env, repo.getId());
+ new ReplicaAllocation((short) 3), 100000, -1, false, false, false, env, repo.getId());
List<Table> tbls = Lists.newArrayList();
List<Resource> resources = Lists.newArrayList();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index 340161495d..aa85178c08 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -92,7 +92,7 @@ public class OlapTableTest {
Assert.assertTrue(olapTable.getDefaultReplicaAllocation() == ReplicaAllocation.DEFAULT_ALLOCATION);
ReplicaAllocation replicaAlloc = new ReplicaAllocation((short) 4);
- olapTable.resetPropertiesForRestore(false, false, replicaAlloc);
+ olapTable.resetPropertiesForRestore(false, false, replicaAlloc, false);
Assert.assertEquals(tableProperty.getProperties(), olapTable.getTableProperty().getProperties());
Assert.assertFalse(tableProperty.getDynamicPartitionProperty().isExist());
Assert.assertFalse(olapTable.isColocateTable());
@@ -112,7 +112,7 @@ public class OlapTableTest {
tableProperty = new TableProperty(properties);
olapTable.setTableProperty(tableProperty);
- olapTable.resetPropertiesForRestore(false, false, ReplicaAllocation.DEFAULT_ALLOCATION);
+ olapTable.resetPropertiesForRestore(false, false, ReplicaAllocation.DEFAULT_ALLOCATION, false);
Map<String, String> expectedProperties = Maps.newHashMap(properties);
expectedProperties.put(DynamicPartitionProperty.ENABLE, "false");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org