You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/11/29 09:15:14 UTC
[incubator-iotdb] 03/03: add unit test
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch add_load_tsfile_feature
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 41bf8ea15fb610e62f33011b30f26615ca42ffcf
Author: lta <li...@163.com>
AuthorDate: Fri Nov 29 17:14:32 2019 +0800
add unit test
---
.../org/apache/iotdb/db/engine/StorageEngine.java | 11 +-
.../engine/storagegroup/StorageGroupProcessor.java | 6 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 5 +-
.../integration/IoTDBLoadExternalTsfileTest.java | 155 +++++++++++++++++++--
4 files changed, 159 insertions(+), 18 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index e8c4c34..7bb0260 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -435,9 +435,14 @@ public class StorageEngine implements IService {
}
public void loadNewTsFile(TsFileResource newTsFileResource)
- throws TsFileProcessorException, StorageEngineException {
- getProcessor(newTsFileResource.getFile().getParentFile().getName())
- .loadNewTsFile(newTsFileResource);
+ throws TsFileProcessorException, StorageEngineException, StorageGroupException {
+ Map<String, Long> startTimeMap = newTsFileResource.getStartTimeMap();
+ if (startTimeMap == null || startTimeMap.isEmpty()) {
+ throw new StorageEngineException("Can not get the corresponding storage group.");
+ }
+ String device = startTimeMap.keySet().iterator().next();
+ String storageGroupName = MManager.getInstance().getStorageGroupNameByPath(device);
+ getProcessor(storageGroupName).loadNewTsFile(newTsFileResource);
}
public boolean deleteTsfile(File deletedTsfile) throws StorageEngineException {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 70ee1e7..ead863e 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1387,8 +1387,7 @@ public class StorageGroupProcessor {
case LOAD_UNSEQUENCE:
targetFile =
new File(DirectoryManager.getInstance().getNextFolderForUnSequenceFile(),
- syncedTsFile.getParentFile().getName() + File.separatorChar + syncedTsFile
- .getName());
+ storageGroupName + File.separatorChar + syncedTsFile.getName());
tsFileResource.setFile(targetFile);
unSequenceFileList.add(index, tsFileResource);
logger
@@ -1399,8 +1398,7 @@ public class StorageGroupProcessor {
case LOAD_SEQUENCE:
targetFile =
new File(DirectoryManager.getInstance().getNextFolderForSequenceFile(),
- syncedTsFile.getParentFile().getName() + File.separatorChar + syncedTsFile
- .getName());
+ storageGroupName + File.separatorChar + syncedTsFile.getName());
tsFileResource.setFile(targetFile);
sequenceFileList.add(index, tsFileResource);
logger
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
index 146e27f..be4de18 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.qp.executor;
import static org.apache.iotdb.db.conf.IoTDBConstant.PRIVILEGE;
import static org.apache.iotdb.db.conf.IoTDBConstant.ROLE;
import static org.apache.iotdb.db.conf.IoTDBConstant.USER;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
import java.io.File;
import java.io.IOException;
@@ -179,7 +180,7 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor {
for (File file : files) {
if (file.isDirectory()) {
recursionFileDir(file);
- } else {
+ } else if (file.getName().endsWith(TSFILE_SUFFIX)) {
loadFile(file);
}
}
@@ -196,7 +197,7 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor {
file.getAbsolutePath()));
}
StorageEngine.getInstance().loadNewTsFile(tsFileResource);
- } catch (IOException | TsFileProcessorException | StorageEngineException e) {
+ } catch (IOException | TsFileProcessorException | StorageEngineException | StorageGroupException e) {
throw new QueryProcessException(
String.format("Cannot load file %s because %s", file.getAbsolutePath(), e.getMessage()));
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileTest.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileTest.java
index 6e7a88b..21b13a5 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileTest.java
@@ -1,9 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.iotdb.db.integration;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -12,6 +35,7 @@ import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -25,13 +49,31 @@ public class IoTDBLoadExternalTsfileTest {
"CREATE TIMESERIES root.vehicle.d0.s1 WITH DATATYPE=TEXT, ENCODING=PLAIN",
"CREATE TIMESERIES root.vehicle.d1.s2 WITH DATATYPE=FLOAT, ENCODING=RLE",
"CREATE TIMESERIES root.vehicle.d1.s3 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.test.d0.s0 WITH DATATYPE=INT32, ENCODING=RLE",
+ "CREATE TIMESERIES root.test.d0.s1 WITH DATATYPE=TEXT, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.test.d1.g0.s0 WITH DATATYPE=INT32, ENCODING=RLE",
"insert into root.vehicle.d0(timestamp,s0) values(10,100)",
"insert into root.vehicle.d0(timestamp,s0,s1) values(12,101,'102')",
"insert into root.vehicle.d0(timestamp,s1) values(19,'103')",
"insert into root.vehicle.d1(timestamp,s2) values(11,104.0)",
"insert into root.vehicle.d1(timestamp,s2,s3) values(15,105.0,true)",
"insert into root.vehicle.d1(timestamp,s3) values(17,false)",
- "insert into root.vehicle.d0(timestamp,s0) values(20,1000)"
+ "insert into root.vehicle.d0(timestamp,s0) values(20,1000)",
+ "insert into root.test.d0(timestamp,s0) values(10,106)",
+ "insert into root.test.d0(timestamp,s0,s1) values(14,107,'108')",
+ "insert into root.test.d0(timestamp,s1) values(16,'109')",
+ "insert into root.test.d1.g0(timestamp,s0) values(1,110)",
+ "insert into root.test.d0(timestamp,s0) values(30,1006)",
+ "insert into root.test.d0(timestamp,s0,s1) values(34,1007,'1008')",
+ "insert into root.test.d0(timestamp,s1) values(36,'1090')",
+ "insert into root.test.d1.g0(timestamp,s0) values(10,1100)",
+ "flush",
+ "insert into root.test.d0(timestamp,s0) values(150,126)",
+ "insert into root.test.d0(timestamp,s0,s1) values(80,127,'128')",
+ "insert into root.test.d0(timestamp,s1) values(200,'129')",
+ "insert into root.test.d1.g0(timestamp,s0) values(140,430)",
+ "insert into root.test.d0(timestamp,s0) values(150,426)",
+ "flush"
};
private static final String TIMESTAMP_STR = "Time";
@@ -56,18 +98,114 @@ public class IoTDBLoadExternalTsfileTest {
}
@Test
- public void LoadNewTsfileTest() throws SQLException {
+ public void moveTsfileTest() throws SQLException {
try (Connection connection = DriverManager.
getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
- statement.execute("flush");
- List<TsFileResource> resources = StorageEngine.getInstance().getProcessor("root.vehicle")
- .getSequenceFileList();
- for(TsFileResource resource:resources){
- System.out.println(resource.getFile().getAbsolutePath());
+
+ // move root.vehicle
+ List<TsFileResource> resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.vehicle")
+ .getSequenceFileList());
+ assertEquals(1, resources.size());
+ File tmpDir = new File(resources.get(0).getFile().getParentFile().getParentFile(), "tmp");
+ if (!tmpDir.exists()) {
+ tmpDir.mkdirs();
+ }
+ for (TsFileResource resource : resources) {
+ statement.execute(String.format("move %s %s", resource.getFile().getPath(), tmpDir));
+ }
+ assertEquals(0, StorageEngine.getInstance().getProcessor("root.vehicle")
+ .getSequenceFileList().size());
+ assertNotNull(tmpDir.listFiles());
+ assertEquals(1, tmpDir.listFiles().length >> 1);
+
+ // move root.test
+ resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.test")
+ .getSequenceFileList());
+ assertEquals(2, resources.size());
+ for (TsFileResource resource : resources) {
+ statement.execute(String.format("move %s %s", resource.getFile().getPath(), tmpDir));
}
+ assertEquals(0, StorageEngine.getInstance().getProcessor("root.test")
+ .getSequenceFileList().size());
+ assertNotNull(tmpDir.listFiles());
+ assertEquals(3, tmpDir.listFiles().length >> 1);
} catch (StorageEngineException e) {
- e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void loadTsfileTest() throws SQLException {
+ try (Connection connection = DriverManager.
+ getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ // move root.vehicle
+ List<TsFileResource> resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.vehicle")
+ .getSequenceFileList());
+ File tmpDir = new File(resources.get(0).getFile().getParentFile().getParentFile(), "tmp");
+ if (!tmpDir.exists()) {
+ tmpDir.mkdirs();
+ }
+ for (TsFileResource resource : resources) {
+ statement.execute(String.format("move %s %s", resource.getFile().getPath(), tmpDir));
+ }
+
+ // move root.test
+ resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.test")
+ .getSequenceFileList());
+ for (TsFileResource resource : resources) {
+ statement.execute(String.format("move %s %s", resource.getFile().getPath(), tmpDir));
+ }
+
+ // load all tsfile in tmp dir
+ statement.execute(String.format("load %s", tmpDir.getAbsolutePath()));
+ resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.vehicle")
+ .getSequenceFileList());
+ assertEquals(1, resources.size());
+ resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.test")
+ .getSequenceFileList());
+ assertEquals(2, resources.size());
+ assertNotNull(tmpDir.listFiles());
+ assertEquals(0, tmpDir.listFiles().length >> 1);
+ } catch (StorageEngineException e) {
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void removeTsfileTest() throws SQLException {
+ try (Connection connection = DriverManager.
+ getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ List<TsFileResource> resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.vehicle")
+ .getSequenceFileList());
+ assertEquals(1, resources.size());
+ for (TsFileResource resource : resources) {
+ statement.execute(String.format("remove %s", resource.getFile().getPath()));
+ }
+ assertEquals(0, StorageEngine.getInstance().getProcessor("root.vehicle")
+ .getSequenceFileList().size());
+
+ resources = new ArrayList<>(
+ StorageEngine.getInstance().getProcessor("root.test")
+ .getSequenceFileList());
+ assertEquals(2, resources.size());
+ for (TsFileResource resource : resources) {
+ statement.execute(String.format("remove %s", resource.getFile().getPath()));
+ }
+ assertEquals(0, StorageEngine.getInstance().getProcessor("root.test")
+ .getSequenceFileList().size());
+ } catch (StorageEngineException e) {
+ Assert.fail();
}
}
@@ -77,7 +215,6 @@ public class IoTDBLoadExternalTsfileTest {
"root");
Statement statement = connection.createStatement()) {
-
for (String sql : sqls) {
statement.execute(sql);
}