You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/09/02 09:50:13 UTC
[incubator-iotdb] 03/03: add snapshot unit test
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch reimpl_sync
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 2060feaa0517de2a2a3d535613c4bcabbcdf0694
Author: lta <li...@163.com>
AuthorDate: Mon Sep 2 17:49:52 2019 +0800
add snapshot unit test
---
.../sender/recover/ISyncSenderLogAnalyzer.java | 2 +-
.../sync/sender/recover/SyncSenderLogAnalyzer.java | 9 +-
.../sync/sender/transfer/DataTransferManager.java | 6 +-
.../java/org/apache/iotdb/db/utils/SyncUtils.java | 2 +-
.../sender/recover/SyncSenderLogAnalyzerTest.java | 18 +++
.../sender/transfer/DataTransferManagerTest.java | 135 +++++++++++++++++++++
6 files changed, 165 insertions(+), 7 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java
index 3d7a356..652839b 100644
--- a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/ISyncSenderLogAnalyzer.java
@@ -34,6 +34,6 @@ public interface ISyncSenderLogAnalyzer {
void loadLogger(Set<String> deletedFiles, Set<String> newFiles);
- void clearLogger(Set<String> currentLocalFiles);
+ void updateLastLocalFile(Set<String> currentLocalFiles);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java
index 6eaec50..5f97f05 100644
--- a/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzer.java
@@ -58,7 +58,7 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer {
loadLogger(deletedFiles, newFiles);
lastLocalFiles.removeAll(deletedFiles);
lastLocalFiles.addAll(newFiles);
- clearLogger(lastLocalFiles);
+ updateLastLocalFile(lastLocalFiles);
}
FileUtils.deleteDirectory(new File(senderPath, Constans.DATA_SNAPSHOT_NAME));
syncLogFile.delete();
@@ -67,6 +67,7 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer {
@Override
public void loadLastLocalFiles(Set<String> lastLocalFiles) {
if (!lastLocalFile.exists()) {
+ LOGGER.info("last local file {} doesn't exist.", syncLogFile.getAbsolutePath());
return;
}
try (BufferedReader br = new BufferedReader(new FileReader(lastLocalFile))) {
@@ -83,6 +84,10 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer {
@Override
public void loadLogger(Set<String> deletedFiles, Set<String> newFiles) {
+ if (!syncLogFile.exists()) {
+ LOGGER.info("log file {} doesn't exist.", syncLogFile.getAbsolutePath());
+ return;
+ }
try (BufferedReader br = new BufferedReader(new FileReader(syncLogFile))) {
String line;
int mode = 0;
@@ -107,7 +112,7 @@ public class SyncSenderLogAnalyzer implements ISyncSenderLogAnalyzer {
}
@Override
- public void clearLogger(Set<String> currentLocalFiles) {
+ public void updateLastLocalFile(Set<String> currentLocalFiles) {
try (BufferedWriter bw = new BufferedWriter(new FileWriter(currentLocalFile))) {
for (String line : currentLocalFiles) {
bw.write(line);
diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java
index 6331d2c..aecf59e 100644
--- a/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManager.java
@@ -524,18 +524,18 @@ public class DataTransferManager implements IDataTransferManager {
*
* @param file new tsfile to be synced
*/
- private File makeFileSnapshot(File file) throws IOException {
+ File makeFileSnapshot(File file) throws IOException {
File snapshotFile = SyncUtils.getSnapshotFile(file);
if (!snapshotFile.getParentFile().exists()) {
snapshotFile.getParentFile().mkdirs();
}
Path link = FileSystems.getDefault().getPath(snapshotFile.getAbsolutePath());
- Path target = FileSystems.getDefault().getPath(snapshotFile.getAbsolutePath());
+ Path target = FileSystems.getDefault().getPath(file.getAbsolutePath());
Files.createLink(link, target);
link = FileSystems.getDefault()
.getPath(snapshotFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX);
target = FileSystems.getDefault()
- .getPath(snapshotFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX);
+ .getPath(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX);
Files.createLink(link, target);
return snapshotFile;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java
index 13247cb..599edac 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SyncUtils.java
@@ -38,7 +38,7 @@ public class SyncUtils {
* sender.
*/
public static File getSnapshotFile(File file) {
- String relativeFilePath = file.getParent() + File.separator + file.getName();
+ String relativeFilePath = file.getParentFile().getName() + File.separator + file.getName();
String snapshotDir = SyncSenderDescriptor.getInstance().getConfig().getSnapshotPath();
if (!new File(snapshotDir).exists()) {
new File(snapshotDir).mkdirs();
diff --git a/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java b/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java
index c30eb8f..71c9629 100644
--- a/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/sync/sender/recover/SyncSenderLogAnalyzerTest.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.iotdb.db.sync.sender.recover;
import java.io.File;
diff --git a/server/src/test/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManagerTest.java b/server/src/test/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManagerTest.java
new file mode 100644
index 0000000..ef49dd5
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/sync/sender/transfer/DataTransferManagerTest.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.sync.sender.transfer;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.exception.StartupException;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.sync.sender.conf.SyncSenderConfig;
+import org.apache.iotdb.db.sync.sender.conf.SyncSenderDescriptor;
+import org.apache.iotdb.db.sync.sender.recover.SyncSenderLogAnalyzer;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.db.utils.FilePathUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DataTransferManagerTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(DataTransferManagerTest.class);
+ private DataTransferManager manager = DataTransferManager.getInstance();
+ private SyncSenderConfig config = SyncSenderDescriptor.getInstance().getConfig();
+ private String dataDir;
+ private SyncSenderLogAnalyzer senderLogAnalyzer;
+
+ @Before
+ public void setUp()
+ throws IOException, InterruptedException, StartupException, DiskSpaceInsufficientException {
+ EnvironmentUtils.envSetUp();
+ dataDir = new File(DirectoryManager.getInstance().getNextFolderForSequenceFile())
+ .getParentFile().getAbsolutePath();
+ config.update(dataDir);
+ senderLogAnalyzer = new SyncSenderLogAnalyzer(config.getSenderFolderPath());
+ }
+
+ @After
+ public void tearDown() throws InterruptedException, IOException, StorageEngineException {
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void makeFileSnapshot() throws IOException {
+ Map<String, Set<File>> allFileList = new HashMap<>();
+
+ Random r = new Random(0);
+ for (int i = 0; i < 3; i++) {
+ for (int j = 0; j < 5; j++) {
+ if (!allFileList.containsKey(String.valueOf(i))) {
+ allFileList.put(String.valueOf(i), new HashSet<>());
+ }
+ String rand = String.valueOf(r.nextInt(10000));
+ String fileName = FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME
+ + File.separator + i
+ + File.separator + rand;
+ File file = new File(fileName);
+ allFileList.get(String.valueOf(i)).add(file);
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdirs();
+ }
+ if (!file.exists() && !file.createNewFile()) {
+ logger.error("Can not create new file {}", file.getPath());
+ }
+ if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists()
+ && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) {
+ logger.error("Can not create new file {}", file.getPath());
+ }
+ }
+ }
+
+ Map<String, Set<String>> dataFileMap = new HashMap<>();
+ File sequenceFile = new File(dataDir, IoTDBConstant.SEQUENCE_FLODER_NAME);
+ for(File sgFile: sequenceFile.listFiles()){
+ dataFileMap.putIfAbsent(sgFile.getName(), new HashSet<>());
+ for (File tsfile : sgFile.listFiles()) {
+ if (!tsfile.getName().endsWith(TsFileResource.RESOURCE_SUFFIX)) {
+ manager.makeFileSnapshot(tsfile);
+ }
+ dataFileMap.get(sgFile.getName()).add(tsfile.getName());
+ }
+ }
+
+ assert new File(config.getSenderFolderPath()).exists();
+ assert new File(config.getSnapshotPath()).exists();
+
+ Map<String, Set<String>> snapFileMap = new HashMap<>();
+ for(File sgFile: new File(config.getSnapshotPath()).listFiles()){
+ snapFileMap.putIfAbsent(sgFile.getName(), new HashSet<>());
+ for(File snapshotTsfile: sgFile.listFiles()){
+ snapFileMap.get(sgFile.getName()).add(snapshotTsfile.getName());
+ }
+ }
+
+ assert dataFileMap.size() == snapFileMap.size();
+ for(Entry<String, Set<String>> entry: dataFileMap.entrySet()){
+ String sg = entry.getKey();
+ Set<String> tsfiles = entry.getValue();
+ assert snapFileMap.containsKey(sg);
+ assert snapFileMap.get(sg).size() == tsfiles.size();
+ assert snapFileMap.get(sg).containsAll(tsfiles);
+ }
+
+ assert !new File(config.getLastFileInfo()).exists();
+ senderLogAnalyzer.recover();
+ assert !new File(config.getSnapshotPath()).exists();
+ assert new File(config.getLastFileInfo()).exists();
+ }
+}
\ No newline at end of file