You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2026/01/16 05:58:29 UTC
(doris) branch routineload_flexible_update updated: fix: Handle conflicting partial_columns and unique_key_update_mode settings
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch routineload_flexible_update
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/routineload_flexible_update by this push:
new a64cdd99f80 fix: Handle conflicting partial_columns and unique_key_update_mode settings
a64cdd99f80 is described below
commit a64cdd99f808a4aabb5df378ee9adc49c3227afa
Author: Yongqiang YANG <da...@gmail.com>
AuthorDate: Thu Jan 15 21:58:14 2026 -0800
fix: Handle conflicting partial_columns and unique_key_update_mode settings
Address review comments:
- Reset uniqueKeyUpdateMode to UPSERT when partial_columns is set to false
but mode was UPDATE_FIXED_COLUMNS (in modifyCommonJobProperties)
- Add backward compatibility handling in gsonPostProcess for inconsistent
metadata states
- Add validation to reject partial_columns=false with UPDATE_FIXED_COLUMNS
or UPDATE_FLEXIBLE_COLUMNS modes in CreateRoutineLoadInfo
---
.../doris/load/routineload/RoutineLoadJob.java | 12 ++++++++++-
.../plans/commands/info/CreateRoutineLoadInfo.java | 25 ++++++++++++++++------
2 files changed, 29 insertions(+), 8 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
index 585c3167154..b098fbba662 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
@@ -1957,12 +1957,19 @@ public abstract class RoutineLoadJob
// Process remaining properties
jobProperties.forEach((k, v) -> {
if (k.equals(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) {
+ boolean partialColumns = Boolean.parseBoolean(v);
// Backward compatibility: if unique_key_update_mode is not set, use partial_columns
if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) {
- isPartialUpdate = Boolean.parseBoolean(v);
+ isPartialUpdate = partialColumns;
if (isPartialUpdate) {
uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS;
}
+ } else if (!partialColumns
+ && uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) {
+ // Backward compatibility: if partial_columns is false but unique_key_update_mode
+ // was UPDATE_FIXED_COLUMNS, reset to UPSERT for consistency
+ isPartialUpdate = false;
+ uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT;
}
} else if (k.equals(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) {
if ("ERROR".equalsIgnoreCase(v)) {
@@ -2058,6 +2065,9 @@ public abstract class RoutineLoadJob
jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_COLUMNS));
if (this.isPartialUpdate && uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) {
this.uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS;
+ } else if (!this.isPartialUpdate && uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS) {
+ // Reset to UPSERT when partial_columns is disabled
+ this.uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT;
}
this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate));
this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
index 9394257013c..ae2781c5abd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
@@ -630,13 +630,24 @@ public class CreateRoutineLoadInfo {
if (jobProperties.containsKey(UNIQUE_KEY_UPDATE_MODE)) {
String modeStr = jobProperties.get(UNIQUE_KEY_UPDATE_MODE);
TUniqueKeyUpdateMode mode = parseAndValidateUniqueKeyUpdateMode(modeStr);
- // Check for conflicting settings: partial_columns=true with unique_key_update_mode=UPSERT
- if (jobProperties.containsKey(PARTIAL_COLUMNS)
- && jobProperties.get(PARTIAL_COLUMNS).equalsIgnoreCase("true")
- && mode == TUniqueKeyUpdateMode.UPSERT) {
- throw new AnalysisException("Cannot set both 'partial_columns=true' and "
- + "'unique_key_update_mode=UPSERT'. "
- + "Use unique_key_update_mode=UPDATE_FIXED_COLUMNS instead.");
+ // Check for conflicting settings between partial_columns and unique_key_update_mode
+ if (jobProperties.containsKey(PARTIAL_COLUMNS)) {
+ String partialColumnsVal = jobProperties.get(PARTIAL_COLUMNS);
+ // partial_columns=true is not allowed with UPSERT
+ if (partialColumnsVal.equalsIgnoreCase("true")
+ && mode == TUniqueKeyUpdateMode.UPSERT) {
+ throw new AnalysisException("Cannot set both 'partial_columns=true' and "
+ + "'unique_key_update_mode=UPSERT'. "
+ + "Use unique_key_update_mode=UPDATE_FIXED_COLUMNS instead.");
+ }
+ // partial_columns=false is not allowed with UPDATE_FIXED_COLUMNS or UPDATE_FLEXIBLE_COLUMNS
+ if (partialColumnsVal.equalsIgnoreCase("false")
+ && (mode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS
+ || mode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS)) {
+ throw new AnalysisException("Cannot set 'partial_columns=false' when "
+ + "'unique_key_update_mode' is 'UPDATE_FIXED_COLUMNS' or 'UPDATE_FLEXIBLE_COLUMNS'. "
+ + "Use unique_key_update_mode=UPSERT instead, or enable partial columns.");
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org