You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/09/29 09:11:11 UTC
[incubator-iotdb] branch dev_preallocate_data_file created (now
6e54b42)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch dev_preallocate_data_file
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 6e54b42 add preallocate data file
This branch includes the following new commits:
new 6e54b42 add preallocate data file
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: add preallocate data file
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_preallocate_data_file
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 6e54b42325b844c948865ce37044460761d03e3e
Author: jt <jt...@163.com>
AuthorDate: Sun Sep 29 17:00:11 2019 +0800
add preallocate data file
---
.../assembly/resources/conf/iotdb-engine.properties | 4 ++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 ++++++++++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +++
.../iotdb/db/engine/storagegroup/TsFileProcessor.java | 19 +++++++++++++++++++
4 files changed, 40 insertions(+)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index eacc84b..11d5ff4 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -165,6 +165,10 @@ concurrent_flush_thread=0
# (i.e., whether use ChunkBufferPool), value true, false
chunk_buffer_pool_enable=false
+# When enabled, write empty bytes into a new TsFile to make the disk distribution of it
+# more local.
+preallocate_data_file=false;
+
####################
### Merge Configurations
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index b99ec35..38e94a9 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -346,6 +346,12 @@ public class IoTDBConfig {
*/
private String hdfsPort = "9000";
+ /**
+ * When enabled, write empty bytes into a new TsFile to make the disk distribution of it
+ * more local.
+ */
+ private boolean preAllocateDataFile = false;
+
public IoTDBConfig() {
// empty constructor
}
@@ -951,4 +957,12 @@ public class IoTDBConfig {
public void setHdfsPort(String hdfsPort) {
this.hdfsPort = hdfsPort;
}
+
+ public boolean isPreAllocateDataFile() {
+ return preAllocateDataFile;
+ }
+
+ public void setPreAllocateDataFile(boolean preAllocateDataFile) {
+ this.preAllocateDataFile = preAllocateDataFile;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 700f634..54e4975 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -279,6 +279,9 @@ public class IoTDBDescriptor {
TSFileDescriptor.getInstance().getConfig().setHdfsIp(properties.getProperty("hdfs_ip"));
TSFileDescriptor.getInstance().getConfig().setHdfsPort(properties.getProperty("hdfs_port"));
+ conf.setPreAllocateDataFile(Boolean.parseBoolean(properties.getProperty("preallocate_data_file",
+ String.valueOf(conf.isPreAllocateDataFile()))));
+
} catch (IOException e) {
logger.warn("Cannot load config file because, use default configuration", e);
} catch (Exception e) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 902f14f..dc6af4f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -20,6 +20,10 @@ package org.apache.iotdb.db.engine.storagegroup;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.OpenOption;
+import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -127,6 +131,21 @@ public class TsFileProcessor {
this.updateLatestFlushTimeCallback = updateLatestFlushTimeCallback;
this.sequence = sequence;
logger.info("create a new tsfile processor {}", tsfile.getAbsolutePath());
+
+ if (IoTDBDescriptor.getInstance().getConfig().isPreAllocateDataFile()) {
+ preAllocateTsFile(tsfile);
+ }
+ }
+
+ private void preAllocateTsFile(File tsFile) throws IOException {
+ try (FileChannel fileChannel = FileChannel.open(tsFile.toPath(), StandardOpenOption.CREATE_NEW)) {
+ long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getTsFileSizeThreshold();
+ // usually, jvm cannot allocate size of just Integer.maxValue, so we use a smaller value
+ // instead
+ int preAllocateSize =
+ (fileSizeThreshold > Integer.MAX_VALUE / 2) ? Integer.MAX_VALUE / 2 : (int) fileSizeThreshold;
+ fileChannel.write(ByteBuffer.allocate(preAllocateSize));
+ }
}
/**