You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/10/11 09:17:39 UTC
[iotdb] branch fix-regex created (now 1bf41662bb)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a change to branch fix-regex
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 1bf41662bb fix dos attack
This branch includes the following new commits:
new 1bf41662bb fix dos attack
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: fix dos attack
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch fix-regex
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1bf41662bbd9543afdbe34de676889a666bc12e7
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Tue Oct 11 17:17:22 2022 +0800
fix dos attack
---
.../resources/conf/iotdb-engine.properties | 4 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 ++++
.../iotdb/tsfile/common/conf/TSFileConfig.java | 10 +++++
.../iotdb/tsfile/read/filter/operator/Regexp.java | 48 +++++++++++++++++++++-
5 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 8991b9933a..f8267b9c38 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -741,6 +741,10 @@ timestamp_precision=ms
# Datatype: long
# slow_query_threshold=5000
+# max pattern access time in regex filter
+# Datatype: int
+# pattern_matching_threshold=1000000
+
####################
### MQTT Broker Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index d92bd997e9..f4517811da 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -792,6 +792,8 @@ public class IoTDBConfig {
/** time cost(ms) threshold for slow query. Unit: millisecond */
private long slowQueryThreshold = 5000;
+ private int patternMatchingThreshold = 1000000;
+
/**
* whether enable the rpc service. This parameter has no a corresponding field in the
* iotdb-engine.properties
@@ -2796,4 +2798,12 @@ public class IoTDBConfig {
public void setChunkMetadataMemorySizeProportion(double chunkMetadataMemorySizeProportion) {
this.chunkMetadataMemorySizeProportion = chunkMetadataMemorySizeProportion;
}
+
+ public int getPatternMatchingThreshold() {
+ return patternMatchingThreshold;
+ }
+
+ public void setPatternMatchingThreshold(int patternMatchingThreshold) {
+ this.patternMatchingThreshold = patternMatchingThreshold;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7516081328..fadf77ccf0 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -928,6 +928,13 @@ public class IoTDBDescriptor {
.setDfsClientFailoverProxyProvider(
properties.getProperty(
"dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider()));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setPatternMatchingThreshold(
+ Integer.parseInt(
+ properties.getProperty(
+ "pattern_matching_threshold",
+ String.valueOf(conf.getPatternMatchingThreshold()))));
TSFileDescriptor.getInstance()
.getConfig()
.setUseKerberos(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index df5df28446..0fd1bdcaa6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -146,6 +146,8 @@ public class TSFileConfig implements Serializable {
private double bloomFilterErrorRate = 0.05;
/** The amount of data iterate each time */
private int batchSize = 1000;
+
+ private int patternMatchingThreshold = 1000000;
/** customizedProperties, this should be empty by default. */
private Properties customizedProperties = new Properties();
@@ -422,4 +424,12 @@ public class TSFileConfig implements Serializable {
public void setCustomizedProperties(Properties customizedProperties) {
this.customizedProperties = customizedProperties;
}
+
+ public int getPatternMatchingThreshold() {
+ return patternMatchingThreshold;
+ }
+
+ public void setPatternMatchingThreshold(int patternMatchingThreshold) {
+ this.patternMatchingThreshold = patternMatchingThreshold;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java
index 646b51ba57..31e7260469 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.read.filter.operator;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId;
@@ -66,7 +67,7 @@ public class Regexp<T extends Comparable<T>> implements Filter {
if (filterType != FilterType.VALUE_FILTER) {
return false;
}
- return pattern.matcher(value.toString()).find();
+ return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find();
}
@Override
@@ -124,4 +125,49 @@ public class Regexp<T extends Comparable<T>> implements Filter {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.REGEXP;
}
+
+ private static class AccessCount {
+ private int count;
+ private final int accessThreshold =
+ TSFileDescriptor.getInstance().getConfig().getPatternMatchingThreshold();
+
+ public void check() throws IllegalStateException {
+ if (this.count++ > accessThreshold) {
+ throw new IllegalStateException("Pattern access threshold exceeded");
+ }
+ }
+ }
+
+ private static class MatcherInput implements CharSequence {
+
+ private final CharSequence value;
+
+ private final AccessCount access;
+
+ public MatcherInput(CharSequence value, AccessCount access) {
+ this.value = value;
+ this.access = access;
+ }
+
+ @Override
+ public char charAt(int index) {
+ this.access.check();
+ return this.value.charAt(index);
+ }
+
+ @Override
+ public CharSequence subSequence(int start, int end) {
+ return new MatcherInput(this.value.subSequence(start, end), this.access);
+ }
+
+ @Override
+ public int length() {
+ return this.value.length();
+ }
+
+ @Override
+ public String toString() {
+ return this.value.toString();
+ }
+ }
}