You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/04/29 08:11:36 UTC
[iotdb] 01/01: [IOTDB-1330]fix the load tsfile bug when the cross
multi partition's tsfile only have one page (#3080)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch jira1330_12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e8c0c68f788c86d416a18c07d9a71eb57f43519e
Author: HouliangQi <ne...@163.com>
AuthorDate: Thu Apr 29 16:03:57 2021 +0800
[IOTDB-1330]fix the load tsfile bug when the cross multi partition's tsfile only have one page (#3080)
---
.../apache/iotdb/db/tools/TsFileRewriteTool.java | 15 +++---
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 16 ++++++
...oTDBLoadExternalTsFileWithTimePartitionIT.java} | 44 ++++++++++++-----
.../iotdb/db/utils/TsFileRewriteToolTest.java | 57 ++++++++++++++++++++++
4 files changed, 111 insertions(+), 21 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java
index 1cfd792..842fd00 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java
@@ -270,17 +270,16 @@ public class TsFileRewriteTool implements AutoCloseable {
}
/**
- * Due to TsFile version-3 changed the serialize way of integer in TEXT data and INT32 data with
- * PLAIN encoding, and also add a sum statistic for BOOLEAN data, these types of data need to
- * decode to points and rewrite in new TsFile.
+ * If the page have no statistics or crosses multi partitions, will return true, otherwise return
+ * false.
*/
protected boolean checkIfNeedToDecode(
TSDataType dataType, TSEncoding encoding, PageHeader pageHeader) {
- return dataType == TSDataType.BOOLEAN
- || dataType == TSDataType.TEXT
- || (dataType == TSDataType.INT32 && encoding == TSEncoding.PLAIN)
- || StorageEngine.getTimePartition(pageHeader.getStartTime())
- != StorageEngine.getTimePartition(pageHeader.getEndTime());
+ if (pageHeader.getStatistics() == null) {
+ return true;
+ }
+ return StorageEngine.getTimePartition(pageHeader.getStartTime())
+ != StorageEngine.getTimePartition(pageHeader.getEndTime());
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
index 8549858..a876044 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.tools.upgrade;
+import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -237,6 +238,21 @@ public class TsFileOnlineUpgradeTool extends TsFileRewriteTool {
return name[0] + "-0" + TsFileConstant.TSFILE_SUFFIX;
}
+ /**
+ * Due to TsFile version-3 changed the serialize way of integer in TEXT data and INT32 data with
+ * PLAIN encoding, and also add a sum statistic for BOOLEAN data, these types of data need to
+ * decode to points and rewrite in new TsFile.
+ */
+ @Override
+ protected boolean checkIfNeedToDecode(
+ TSDataType dataType, TSEncoding encoding, PageHeader pageHeader) {
+ return dataType == TSDataType.BOOLEAN
+ || dataType == TSDataType.TEXT
+ || (dataType == TSDataType.INT32 && encoding == TSEncoding.PLAIN)
+ || StorageEngine.getTimePartition(pageHeader.getStartTime())
+ != StorageEngine.getTimePartition(pageHeader.getEndTime());
+ }
+
@Override
protected void decodeAndWritePageInToFiles(
MeasurementSchema schema,
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartitionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsFileWithTimePartitionIT.java
similarity index 82%
rename from server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartitionIT.java
rename to server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsFileWithTimePartitionIT.java
index 5de4c9e..d991123 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartitionIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsFileWithTimePartitionIT.java
@@ -52,7 +52,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
-public class IoTDBLoadExternalTsfileWithTimePartitionIT {
+public class IoTDBLoadExternalTsFileWithTimePartitionIT {
String DOT = ".";
String tempDir = "temp";
@@ -61,12 +61,16 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT {
String[] devices = new String[] {"d1", "d2"};
String[] measurements = new String[] {"s1", "s2"};
- // generate several tsfiles, with timestamp from startTime(inclusive) to endTime(exclusive)
+ // generate several tsFiles, with timestamp from startTime(inclusive) to endTime(exclusive)
long startTime = 0;
long endTime = 1000_000;
long timePartition = 100; // unit s
+ long recordTimeGap = 1000;
+
+ int originalTsFileNum = 0;
+
boolean originalIsEnablePartition;
long originalPartitionInterval;
@@ -134,7 +138,7 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT {
}
/** create multiple tsfiles, each correspond to a time partition. */
- private void prepareData() throws IOException {
+ private void prepareData() {
File dir = new File(tempDir);
if (dir.exists()) {
FileUtils.deleteDirectory(dir);
@@ -144,8 +148,8 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT {
File f;
TsFileWriter tsFileWriter = null;
int counter = 0;
- for (long timestamp = startTime; timestamp < endTime; timestamp += 1000) {
- if (timestamp % (timePartition * 1000) == 0) {
+ for (long timestamp = startTime; timestamp < endTime; timestamp += recordTimeGap) {
+ if (timestamp % (timePartition * recordTimeGap) == 0) {
if (tsFileWriter != null) {
tsFileWriter.flushAllChunkGroups();
tsFileWriter.close();
@@ -160,13 +164,14 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT {
}
tsFileWriter.flushAllChunkGroups();
tsFileWriter.close();
+ originalTsFileNum = counter + 1;
} catch (Throwable e) {
e.printStackTrace();
}
}
@Test
- public void loadTsfileWithTimePartition() {
+ public void loadTsFileWithTimePartition() {
try (Connection connection =
DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
@@ -174,16 +179,29 @@ public class IoTDBLoadExternalTsfileWithTimePartitionIT {
statement.execute(String.format("load \"%s\"", new File(tempDir).getAbsolutePath()));
String dataDir = config.getDataDirs()[0];
- File f = new File(dataDir, new PartialPath("sequence") + File.separator + "root.ln");
- System.out.println(Arrays.toString(f.list()));
- Assert.assertEquals((endTime - startTime) / (timePartition * 1000), f.list().length);
+ // sequence/logical_sg/virtual_sg/time_partitions
+ File f =
+ new File(
+ dataDir,
+ new PartialPath("sequence") + File.separator + "root.ln" + File.separator + "0");
+ Assert.assertEquals(
+ (endTime - startTime) / (timePartition), f.list().length * originalTsFileNum);
+
+ int totalPartitionsNum = (int) ((endTime - startTime) / (timePartition) / originalTsFileNum);
+ int[] splitTsFilePartitions = new int[totalPartitionsNum];
+ for (int i = 0; i < splitTsFilePartitions.length; i++) {
+ splitTsFilePartitions[i] = Integer.parseInt(f.list()[i]);
+ }
+ Arrays.sort(splitTsFilePartitions);
- for (int i = 0; i < (endTime - startTime) / (timePartition * 1000); i++) {
- Assert.assertEquals("" + i, f.list()[i]);
+ for (int i = 0; i < (endTime - startTime) / (timePartition) / originalTsFileNum; i++) {
+ Assert.assertEquals((i * originalTsFileNum), splitTsFilePartitions[i]);
}
+
// each time partition folder should contain 2 files, the tsfile and the resource file
- for (int i = 0; i < (endTime - startTime) / (timePartition * 1000); i++) {
- Assert.assertEquals(2, new File(f.getAbsolutePath(), "" + i).list().length);
+ for (int i = 0; i < (endTime - startTime) / (timePartition) / originalTsFileNum; i++) {
+ Assert.assertEquals(
+ 2, new File(f.getAbsolutePath(), "" + i * originalTsFileNum).list().length);
}
} catch (SQLException | IllegalPathException throwables) {
throwables.printStackTrace();
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java b/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java
index 4cba60f..58126ef 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java
@@ -181,6 +181,22 @@ public class TsFileRewriteToolTest {
}
}
+ @Test
+ public void loadFileWithOnlyOnePageTest() {
+ HashMap<String, List<String>> deviceSensorsMap = new HashMap<>();
+ List<String> sensors = new ArrayList<>();
+ sensors.add(SENSOR1);
+ deviceSensorsMap.put(DEVICE1, sensors);
+ createOneTsFileWithOnlyOnePage(deviceSensorsMap);
+ // try load the tsfile
+ String sql = "load \"" + path + "\"" + " true";
+ try {
+ queryExecutor.processNonQuery(processor.parseSQLToPhysicalPlan(sql));
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
private void splitFileAndQueryCheck(HashMap<String, List<String>> deviceSensorsMap) {
File tsFile = new File(path);
TsFileResource tsFileResource = new TsFileResource(tsFile);
@@ -203,6 +219,47 @@ public class TsFileRewriteToolTest {
}
}
+ private void createOneTsFileWithOnlyOnePage(HashMap<String, List<String>> deviceSensorsMap) {
+ try {
+ File f = FSFactoryProducer.getFSFactory().getFile(path);
+ TsFileWriter tsFileWriter = new TsFileWriter(f);
+ // add measurements into file schema
+ try {
+ for (Map.Entry<String, List<String>> entry : deviceSensorsMap.entrySet()) {
+ String device = entry.getKey();
+ for (String sensor : entry.getValue()) {
+ tsFileWriter.registerTimeseries(
+ new Path(device, sensor),
+ new MeasurementSchema(sensor, TSDataType.INT64, TSEncoding.RLE));
+ }
+ }
+ } catch (WriteProcessException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ int count = 0;
+ for (long timestamp = 1; ; timestamp += newPartitionInterval) {
+ if (count == 2) {
+ break;
+ }
+ count++;
+ for (Map.Entry<String, List<String>> entry : deviceSensorsMap.entrySet()) {
+ String device = entry.getKey();
+ TSRecord tsRecord = new TSRecord(timestamp, device);
+ for (String sensor : entry.getValue()) {
+ DataPoint dataPoint = new LongDataPoint(sensor, timestamp + VALUE_OFFSET);
+ tsRecord.addTuple(dataPoint);
+ }
+ tsFileWriter.write(tsRecord);
+ }
+ tsFileWriter.flushAllChunkGroups();
+ }
+ tsFileWriter.close();
+ } catch (Throwable e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
private void createOneTsFile(HashMap<String, List<String>> deviceSensorsMap) {
try {
File f = FSFactoryProducer.getFSFactory().getFile(path);