You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ju...@apache.org on 2026/07/08 04:10:37 UTC
(iotdb) 01/01: Limit delete predicate DNF expansion
This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch fix-delete-dnf-limit
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0dd9b6d60d324e12c1b402b32af7011c62d4101c
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Wed Jul 8 12:10:01 2026 +0800
Limit delete predicate DNF expansion
---
.../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++
.../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++
.../db/queryengine/plan/analyze/AnalyzeUtils.java | 40 +++++++++++++++++-----
.../queryengine/plan/analyze/AnalyzeUtilsTest.java | 36 +++++++++++++++++++
4 files changed, 76 insertions(+), 8 deletions(-)
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 7978fce5113..4ea569ad799 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -335,6 +335,10 @@ public final class DataNodeQueryMessages {
public static final String TOO_MANY_DEVICES_MATCHED_BY_ATTRIBUTE_FILTERS_IN_DELETION =
"Too many devices (%d) matched by attribute filters in deletion, limit is %d. "
+ "Please remove all attribute filters (%s) or add more attribute filters.";
+ public static final String
+ EXCEPTION_DELETION_PREDICATE_EXPANDS_TO_TOO_MANY_DISJUNCTIVE_TERMS_ARG_LIMIT_IS_ARG_PLEASE_SIMPLIFY_THE_OR_CONDITIONS_BA138588 =
+ "Deletion predicate expands to too many disjunctive terms (%d), limit is %d. "
+ + "Please simplify the OR conditions.";
public static final String ONLY_TIME_FILTERS_ARE_SUPPORTED_IN_LAST_QUERY =
"Only time filters are supported in LAST query";
public static final String VIEWS_CANNOT_BE_USED_IN_GROUP_BY_TAGS =
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 34a7fa084bf..adcfc5b252a 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -319,6 +319,10 @@ public final class DataNodeQueryMessages {
"列 '%s' 不存在或不是标签列";
public static final String TOO_MANY_DEVICES_MATCHED_BY_ATTRIBUTE_FILTERS_IN_DELETION =
"删除语句中的属性过滤条件匹配了过多设备（%d），限制为 %d。请移除所有属性过滤条件（%s），或进一步增加属性过滤条件。";
+ public static final String
+ EXCEPTION_DELETION_PREDICATE_EXPANDS_TO_TOO_MANY_DISJUNCTIVE_TERMS_ARG_LIMIT_IS_ARG_PLEASE_SIMPLIFY_THE_OR_CONDITIONS_BA138588 =
+ "删除谓词会展开为过多析取项（%d），限制为 %d。"
+ + "请简化 OR 条件。";
public static final String ONLY_TIME_FILTERS_ARE_SUPPORTED_IN_LAST_QUERY =
"LAST 查询中仅支持时间过滤器";
public static final String VIEWS_CANNOT_BE_USED_IN_GROUP_BY_TAGS =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
index 81849ff9dfb..65fbc06b214 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
@@ -96,6 +96,7 @@ import static org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigT
public class AnalyzeUtils {
private static final int ATTRIBUTE_FILTER_DELETE_DEVICE_IN_LIMIT = 1000;
+ private static final int DELETE_DNF_EXPRESSION_LIMIT = 1000;
private static final CommonQueryAstVisitor<PredicateParseResult, PredicateParseContext>
DELETION_PREDICATE_PARSE_VISITOR =
new CommonQueryAstVisitor<>() {
@@ -455,14 +456,18 @@ public class AnalyzeUtils {
}
/**
- * Convert to a disjunctive normal forms.
+ * Convert to disjunctive normal forms.
*
* <p>For example: ( A | B ) & ( C | D ) => ( A & C ) | ( A & D ) | ( B & C ) | ( B & D)
*
- * <p>Returns the original expression if the expression is null or if the distribution will expand
- * the expression by too much.
+ * <p>Throws SemanticException if the distribution will expand the expression by too much.
*/
public static List<Expression> toDisjunctiveNormalForms(Expression expression) {
+ return toDisjunctiveNormalForms(expression, DELETE_DNF_EXPRESSION_LIMIT);
+ }
+
+ private static List<Expression> toDisjunctiveNormalForms(
+ Expression expression, int expressionLimit) {
if (!(expression instanceof LogicalExpression)) {
return Collections.singletonList(expression);
}
@@ -472,20 +477,24 @@ public class AnalyzeUtils {
// ( A | B ) & ( C | D ) => ( A & C ) | ( A & D ) | ( B & C ) | ( B & D)
List<Expression> results = null;
for (Expression term : logicalExpression.getTerms()) {
+ List<Expression> termResults = toDisjunctiveNormalForms(term, expressionLimit);
if (results == null) {
- results = toDisjunctiveNormalForms(term);
+ results = termResults;
} else {
results =
crossProductOfDisjunctiveNormalForms(
- results, toDisjunctiveNormalForms(term), Operator.AND);
+ results, termResults, Operator.AND, expressionLimit);
}
+ ensureDnfExpressionLimit(results.size(), expressionLimit);
}
return results;
} else if (logicalExpression.getOperator() == Operator.OR) {
// ( A | B ) | ( C | D ) => A | B | C | D
List<Expression> results = new ArrayList<>();
for (Expression term : logicalExpression.getTerms()) {
- results.addAll(toDisjunctiveNormalForms(term));
+ List<Expression> termResults = toDisjunctiveNormalForms(term, expressionLimit);
+ ensureDnfExpressionLimit((long) results.size() + termResults.size(), expressionLimit);
+ results.addAll(termResults);
}
return results;
} else {
@@ -495,8 +504,12 @@ public class AnalyzeUtils {
}
private static List<Expression> crossProductOfDisjunctiveNormalForms(
- List<Expression> leftList, List<Expression> rightList, Operator operator) {
- List<Expression> results = new ArrayList<>();
+ List<Expression> leftList,
+ List<Expression> rightList,
+ Operator operator,
+ int expressionLimit) {
+ ensureDnfExpressionLimit((long) leftList.size() * rightList.size(), expressionLimit);
+ List<Expression> results = new ArrayList<>(leftList.size() * rightList.size());
for (Expression leftExp : leftList) {
for (Expression rightExp : rightList) {
List<Expression> terms = new ArrayList<>();
@@ -518,6 +531,17 @@ public class AnalyzeUtils {
return results;
}
+ private static void ensureDnfExpressionLimit(final long expressionSize, final int limit) {
+ if (expressionSize > limit) {
+ throw new SemanticException(
+ String.format(
+ DataNodeQueryMessages
+ .EXCEPTION_DELETION_PREDICATE_EXPANDS_TO_TOO_MANY_DISJUNCTIVE_TERMS_ARG_LIMIT_IS_ARG_PLEASE_SIMPLIFY_THE_OR_CONDITIONS_BA138588,
+ expressionSize,
+ limit));
+ }
+ }
+
private static TableDeletionEntry parsePredicate(
Expression expression,
TsTable table,
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java
index 915006a8286..8ecf2b49787 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java
@@ -23,11 +23,14 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier;
+import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QualifiedName;
+import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Table;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema;
@@ -47,6 +50,7 @@ import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
@@ -54,6 +58,7 @@ import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
public class AnalyzeUtilsTest {
@@ -77,6 +82,24 @@ public class AnalyzeUtilsTest {
assertEquals(100, entries.get(0).getEndTime());
}
+ @Test
+ public void testDeletePredicateDnfExpansionWithinLimit() {
+ List<Expression> result =
+ AnalyzeUtils.toDisjunctiveNormalForms(buildConjunctiveOrExpression(9));
+
+ assertEquals(512, result.size());
+ }
+
+ @Test
+ public void testDeletePredicateDnfExpansionRejectsTooManyTerms() {
+ SemanticException exception =
+ assertThrows(
+ SemanticException.class,
+ () -> AnalyzeUtils.toDisjunctiveNormalForms(buildConjunctiveOrExpression(10)));
+
+ assertTrue(exception.getMessage().contains("too many disjunctive terms (1024)"));
+ }
+
@Test
public void testFetchDeleteReplicaSetsOnlyQueriesTargetDatabaseRegions() throws Exception {
final Delete delete = new Delete(new Table(QualifiedName.of("table1")));
@@ -118,6 +141,19 @@ public class AnalyzeUtilsTest {
assertEquals(40, requests.get(1).getEndTime());
}
+ private static Expression buildConjunctiveOrExpression(final int termCount) {
+ final List<Expression> terms = new ArrayList<>();
+ for (int i = 0; i < termCount; i++) {
+ terms.add(LogicalExpression.or(equalTag("tag" + i, "left"), equalTag("tag" + i, "right")));
+ }
+ return new LogicalExpression(LogicalExpression.Operator.AND, terms);
+ }
+
+ private static Expression equalTag(final String column, final String value) {
+ return new ComparisonExpression(
+ ComparisonExpression.Operator.EQUAL, new Identifier(column), new StringLiteral(value));
+ }
+
private static TGetRegionGroupsByTimeResp successRegionGroupsResp(
final Set<TRegionReplicaSet> replicaSets) {
final TGetRegionGroupsByTimeResp resp =