You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by wa...@apache.org on 2021/07/14 01:19:42 UTC
[iotdb] branch rel/0.12 updated: [IOTDB-1485] Replace
tsfile_size_threshold by unseq_tsfile_size/seq_tsfile_size (#3558)
This is an automated email from the ASF dual-hosted git repository.
wangchao316 pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 4e78faf [IOTDB-1485] Replace tsfile_size_threshold by unseq_tsfile_size/seq_tsfile_size (#3558)
4e78faf is described below
commit 4e78fafe3b458085df15fafe7009ab3589ba4d11
Author: yanhong wang <67...@users.noreply.github.com>
AuthorDate: Wed Jul 14 09:19:16 2021 +0800
[IOTDB-1485] Replace tsfile_size_threshold by unseq_tsfile_size/seq_tsfile_size (#3558)
[IOTDB-1485] Replace tsfile_size_threshold by unseq_tsfile_size/seq_tsfile_size (#3558)
---
.../resources/conf/iotdb-engine.properties | 8 +++--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 23 ++++++++++----
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 36 ++++++++++++++++------
.../db/engine/storagegroup/TsFileProcessor.java | 8 ++++-
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +--
.../apache/iotdb/db/utils/EnvironmentUtils.java | 6 ++--
.../apache/iotdb/spark/db/EnvironmentUtils.java | 6 ++--
7 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index e646807..18aab0f 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -183,9 +183,13 @@ timestamp_precision=ms
# If it sets a value smaller than 0, use the default value 16777216
# wal_buffer_size=16777216
-# When a TsFile's file size (in byte) exceeds this, the TsFile is forced closed.
+# When a unSequence TsFile's file size (in byte) exceeds this, the TsFile is forced closed.
# It may cause memTable size smaller if it is a large value
-# tsfile_size_threshold=1
+# unseq_tsfile_size=1
+
+# When a sequence TsFile's file size (in byte) exceeds this, the TsFile is forced closed.
+# It may cause memTable size smaller if it is a large value
+# seq_tsfile_size=1
# Size of log buffer in each metadata operation plan(in byte).
# If the size of a metadata operation plan is larger than this parameter, then it will be rejected by MManager
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 462826b..999b240 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -265,8 +265,11 @@ public class IoTDBConfig {
/** index directory. */
private String indexRootFolder = "data" + File.separator + "index";
- /** When a TsFile's file size (in byte) exceed this, the TsFile is forced closed. */
- private long tsFileSizeThreshold = 1L;
+ /** When a unSequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */
+ private long unSeqTsFileSize = 1L;
+
+ /** When a sequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */
+ private long seqTsFileSize = 1L;
/** When a memTable's size (in byte) exceeds this, the memtable is flushed to disk. */
private long memtableSizeThreshold = 1024 * 1024 * 1024L;
@@ -1048,12 +1051,20 @@ public class IoTDBConfig {
this.concurrentQueryThread = concurrentQueryThread;
}
- public long getTsFileSizeThreshold() {
- return tsFileSizeThreshold;
+ public long getSeqTsFileSize() {
+ return seqTsFileSize;
+ }
+
+ public void setSeqTsFileSize(long seqTsFileSize) {
+ this.seqTsFileSize = seqTsFileSize;
+ }
+
+ public long getUnSeqTsFileSize() {
+ return unSeqTsFileSize;
}
- public void setTsFileSizeThreshold(long tsFileSizeThreshold) {
- this.tsFileSizeThreshold = tsFileSizeThreshold;
+ public void setUnSeqTsFileSize(long unSeqTsFileSize) {
+ this.unSeqTsFileSize = unSeqTsFileSize;
}
public boolean isEnableStatMonitor() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 6738921..d4f59d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -250,14 +250,22 @@ public class IoTDBDescriptor {
"enable_mem_control", Boolean.toString(conf.isEnableMemControl())))));
logger.info("IoTDB enable memory control: {}", conf.isEnableMemControl());
- long tsfileSizeThreshold =
+ long seqTsFileSize =
Long.parseLong(
properties
- .getProperty(
- "tsfile_size_threshold", Long.toString(conf.getTsFileSizeThreshold()))
+ .getProperty("seq_tsfile_size", Long.toString(conf.getSeqTsFileSize()))
+ .trim());
+ if (seqTsFileSize >= 0) {
+ conf.setSeqTsFileSize(seqTsFileSize);
+ }
+
+ long unSeqTsFileSize =
+ Long.parseLong(
+ properties
+ .getProperty("unseq_tsfile_size", Long.toString(conf.getUnSeqTsFileSize()))
.trim());
- if (tsfileSizeThreshold >= 0) {
- conf.setTsFileSizeThreshold(tsfileSizeThreshold);
+ if (unSeqTsFileSize >= 0) {
+ conf.setUnSeqTsFileSize(unSeqTsFileSize);
}
long memTableSizeThreshold =
@@ -984,14 +992,22 @@ public class IoTDBDescriptor {
// update WAL conf
loadWALProps(properties);
- long tsfileSizeThreshold =
+ long seqTsFileSize =
Long.parseLong(
properties
- .getProperty(
- "tsfile_size_threshold", Long.toString(conf.getTsFileSizeThreshold()))
+ .getProperty("seq_tsfile_size", Long.toString(conf.getSeqTsFileSize()))
+ .trim());
+ if (seqTsFileSize >= 0) {
+ conf.setSeqTsFileSize(seqTsFileSize);
+ }
+
+ long unSeqTsFileSize =
+ Long.parseLong(
+ properties
+ .getProperty("unseq_tsfile_size", Long.toString(conf.getUnSeqTsFileSize()))
.trim());
- if (tsfileSizeThreshold >= 0) {
- conf.setTsFileSizeThreshold(tsfileSizeThreshold);
+ if (unSeqTsFileSize >= 0) {
+ conf.setUnSeqTsFileSize(unSeqTsFileSize);
}
long memTableSizeThreshold =
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 9642271..0402f4b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -468,7 +468,13 @@ public class TsFileProcessor {
public boolean shouldClose() {
long fileSize = tsFileResource.getTsFileSize();
- long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getTsFileSizeThreshold();
+ long fileSizeThreshold;
+ if (sequence) {
+ fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getSeqTsFileSize();
+ } else {
+ fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getUnSeqTsFileSize();
+ }
+
if (fileSize >= fileSizeThreshold) {
logger.info(
"{} fileSize {} >= fileSizeThreshold {}",
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 65f6ea5..d29dfc3 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -168,8 +168,9 @@ public class IoTDB implements IoTDBMBean {
long end = System.currentTimeMillis() - time;
logger.info("spend {}ms to recover schema.", end);
logger.info(
- "After initializing, tsFile threshold is {}, memtableSize is {}",
- IoTDBDescriptor.getInstance().getConfig().getTsFileSizeThreshold(),
+ "After initializing, sequence tsFile threshold is {}, unsequence tsFile threshold is {}, memtableSize is {}",
+ IoTDBDescriptor.getInstance().getConfig().getSeqTsFileSize(),
+ IoTDBDescriptor.getInstance().getConfig().getUnSeqTsFileSize(),
IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold());
}
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 7245c19..a0c62a4 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -71,7 +71,8 @@ public class EnvironmentUtils {
public static long TEST_QUERY_JOB_ID = 1;
public static QueryContext TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
- private static final long oldTsFileThreshold = config.getTsFileSizeThreshold();
+ private static final long oldSeqTsFileSize = config.getSeqTsFileSize();
+ private static final long oldUnSeqTsFileSize = config.getUnSeqTsFileSize();
private static final long oldGroupSizeInByte = config.getMemtableSizeThreshold();
@@ -148,7 +149,8 @@ public class EnvironmentUtils {
// delete all directory
cleanAllDir();
- config.setTsFileSizeThreshold(oldTsFileThreshold);
+ config.setSeqTsFileSize(oldSeqTsFileSize);
+ config.setUnSeqTsFileSize(oldUnSeqTsFileSize);
config.setMemtableSizeThreshold(oldGroupSizeInByte);
}
diff --git a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
index f46c5c4..1dc3ddb 100644
--- a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
+++ b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
@@ -96,7 +96,8 @@ public class EnvironmentUtils {
.assignQueryId(true, 1024, 0);
public static QueryContext TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
- private static long oldTsFileThreshold = config.getTsFileSizeThreshold();
+ private static long oldSeqTsFileSize = config.getSeqTsFileSize();
+ private static long oldUnSeqTsFileSize = config.getUnSeqTsFileSize();
private static long oldGroupSizeInByte = config.getMemtableSizeThreshold();
@@ -129,7 +130,8 @@ public class EnvironmentUtils {
// delete all directory
cleanAllDir();
- config.setTsFileSizeThreshold(oldTsFileThreshold);
+ config.setSeqTsFileSize(oldSeqTsFileSize);
+ config.setUnSeqTsFileSize(oldUnSeqTsFileSize);
config.setMemtableSizeThreshold(oldGroupSizeInByte);
}