You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2019/06/20 08:15:31 UTC
[incubator-iotdb] 01/01: add ut for DirectoriesStrategy
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch mult_dir_add_ut
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 59939248a3e452c21131437638f97ff58b4883e3
Author: mdf369 <95...@qq.com>
AuthorDate: Thu Jun 20 16:15:10 2019 +0800
add ut for DirectoriesStrategy
---
iotdb/pom.xml | 18 +++
.../directories/strategy/DirectoryStrategy.java | 24 ++--
.../strategy/MaxDiskUsableSpaceFirstStrategy.java | 8 +-
.../MinFolderOccupiedSpaceFirstStrategy.java | 35 ++---
.../directories/strategy/SequenceStrategy.java | 10 +-
.../java/org/apache/iotdb/db/utils/FileUtils.java | 21 +++
.../strategy/DirectoryStrategyTest.java | 157 +++++++++++++++++++++
7 files changed, 228 insertions(+), 45 deletions(-)
diff --git a/iotdb/pom.xml b/iotdb/pom.xml
index 7ff37e9..bc230e6 100644
--- a/iotdb/pom.xml
+++ b/iotdb/pom.xml
@@ -70,6 +70,24 @@
<artifactId>commons-lang3</artifactId>
<version>${common.lang3.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-core</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
index dd397b8..1239e0a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
@@ -18,9 +18,9 @@
*/
package org.apache.iotdb.db.conf.directories.strategy;
-import java.io.File;
import java.util.List;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,6 +46,18 @@ public abstract class DirectoryStrategy {
*/
public void init(List<String> folders) throws DiskSpaceInsufficientException {
this.folders = folders;
+
+ boolean hasSpace = false;
+ for (String folder : folders) {
+ if (FileUtils.hasSpace(folder)) {
+ hasSpace = true;
+ break;
+ }
+ }
+ if (!hasSpace) {
+ throw new DiskSpaceInsufficientException(
+ String.format("All disks of folders %s are full, can't init.", folders));
+ }
}
/**
@@ -74,14 +86,4 @@ public abstract class DirectoryStrategy {
public void setFolderForTest(String path) {
folders.set(0, path);
}
-
- protected long getUsableSpace(String dir) {
- long space = new File(dir).getFreeSpace();
- LOGGER.trace("Folder {} has {} available bytes.", dir, space);
- return space;
- }
-
- protected boolean hasSpace(String dir) {
- return getUsableSpace(dir) > 0;
- }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java
index 8f3ea7d..98b0e75 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.conf.directories.strategy;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
public class MaxDiskUsableSpaceFirstStrategy extends DirectoryStrategy {
@@ -35,7 +36,12 @@ public class MaxDiskUsableSpaceFirstStrategy extends DirectoryStrategy {
long maxSpace = 0;
for (int i = 0; i < folders.size(); i++) {
- long space = getUsableSpace(folders.get(i));
+ String folder = folders.get(i);
+ if (!FileUtils.hasSpace(folder)) {
+ continue;
+ }
+
+ long space = FileUtils.getUsableSpace(folder);
if (space > maxSpace) {
maxSpace = space;
maxIndex = i;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java
index 81f0f73..87c3814 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java
@@ -19,17 +19,11 @@
package org.apache.iotdb.db.conf.directories.strategy;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.stream.Stream;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
- // directory space is measured by MB
- private static final long DATA_SIZE_SHIFT = 1024L * 1024;
-
@Override
public int nextFolderIndex() throws DiskSpaceInsufficientException {
return getMinOccupiedSpaceFolder();
@@ -37,15 +31,20 @@ public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
private int getMinOccupiedSpaceFolder() throws DiskSpaceInsufficientException {
int minIndex = -1;
- long minSpace = Integer.MAX_VALUE;
+ long minSpace = Long.MAX_VALUE;
for (int i = 0; i < folders.size(); i++) {
String folder = folders.get(i);
- if (!hasSpace(folder)) {
+ if (!FileUtils.hasSpace(folder)) {
continue;
}
- long space = getOccupiedSpace(folder);
+ long space = 0;
+ try {
+ space = FileUtils.getOccupiedSpace(folder);
+ } catch (IOException e) {
+ LOGGER.error("Cannot calculate occupied space for path {}.", folder, e);
+ }
if (space < minSpace) {
minSpace = space;
minIndex = i;
@@ -58,20 +57,4 @@ public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
return minIndex;
}
-
- private long getOccupiedSpace(String path) {
- Path folder = Paths.get(path);
- long size = Long.MAX_VALUE;
- try {
- try (Stream<Path> stream = Files.walk(folder)) {
- size = stream.filter(p -> p.toFile().isFile())
- .mapToLong(p -> p.toFile().length())
- .sum();
- }
- } catch (IOException e) {
- LOGGER.error("Cannot calculate occupied space for seriesPath {}.", path, e);
- }
-
- return size / DATA_SIZE_SHIFT;
- }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java
index b9c9ff5..7b05ab2 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.conf.directories.strategy;
import java.util.List;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
public class SequenceStrategy extends DirectoryStrategy {
@@ -31,16 +32,11 @@ public class SequenceStrategy extends DirectoryStrategy {
currentIndex = -1;
for (int i = 0; i < folders.size(); i++) {
- if (hasSpace(folders.get(i))) {
+ if (FileUtils.hasSpace(folders.get(i))) {
currentIndex = i;
break;
}
}
-
- if (currentIndex == -1) {
- throw new DiskSpaceInsufficientException(
- String.format("All disks of folders %s are full, can't init.", folders));
- }
}
@Override
@@ -53,7 +49,7 @@ public class SequenceStrategy extends DirectoryStrategy {
private int tryGetNextIndex(int start) throws DiskSpaceInsufficientException {
int index = (start + 1) % folders.size();
- while (!hasSpace(folders.get(index))) {
+ while (!FileUtils.hasSpace(folders.get(index))) {
index = (index + 1) % folders.size();
if (index == start) {
throw new DiskSpaceInsufficientException(folders);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
index c276f01..f51fe93 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.db.utils;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
/**
* FileUtils is just used to calculate file attributes like file size, including some measurement
@@ -84,4 +88,21 @@ public class FileUtils {
public enum Unit {
B, KB, MB, GB, TB, PB, EB
}
+
+ public static long getUsableSpace(String dir) {
+ long space = new File(dir).getFreeSpace();
+ return space;
+ }
+
+ public static boolean hasSpace(String dir) {
+ return getUsableSpace(dir) > 0;
+ }
+
+ public static long getOccupiedSpace(String path) throws IOException {
+ Path folder = Paths.get(path);
+ long size = Files.walk(folder).filter(p -> p.toFile().isFile())
+ .mapToLong(p -> p.toFile().length()).sum();
+
+ return size;
+ }
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java b/iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java
new file mode 100644
index 0000000..676e2f3
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java
@@ -0,0 +1,157 @@
+package org.apache.iotdb.db.conf.directories.strategy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(FileUtils.class)
+public class DirectoryStrategyTest {
+
+ List<String> dataDirList;
+ Set<Integer> fullDirIndexSet;
+
+ @Before
+ public void setUp() throws DiskSpaceInsufficientException, IOException {
+ dataDirList = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ dataDirList.add("data" + i);
+ }
+
+ fullDirIndexSet = new HashSet<>();
+ fullDirIndexSet.add(1);
+ fullDirIndexSet.add(3);
+
+ PowerMockito.mockStatic(FileUtils.class);
+ for (int i = 0; i < dataDirList.size(); i++) {
+ boolean res = !fullDirIndexSet.contains(i);
+ PowerMockito.when(FileUtils.hasSpace(dataDirList.get(i))).thenReturn(res);
+ PowerMockito.when(FileUtils.getUsableSpace(dataDirList.get(i))).thenReturn(res ? (long) (i + 1) : 0L);
+ PowerMockito.when(FileUtils.getOccupiedSpace(dataDirList.get(i))).thenReturn(res ? (long) (i + 1) : Long.MAX_VALUE);
+ }
+ }
+
+ @After
+ public void tearDown() {
+
+ }
+
+ @Test
+ public void testSequenceStrategy() throws DiskSpaceInsufficientException {
+ SequenceStrategy sequenceStrategy = new SequenceStrategy();
+ sequenceStrategy.init(dataDirList);
+
+ // loop two times of data dir size to fully loop
+ int index = 0;
+ for (int i = 0; i < dataDirList.size() * 2; i++, index++) {
+ index = index % dataDirList.size();
+ while (fullDirIndexSet.contains(index)) {
+ index = (index + 1) % dataDirList.size();
+ }
+ assertEquals(index, sequenceStrategy.nextFolderIndex());
+ }
+ }
+
+ @Test
+ public void testMaxDiskUsableSpaceFirstStrategy() throws DiskSpaceInsufficientException {
+ MaxDiskUsableSpaceFirstStrategy maxDiskUsableSpaceFirstStrategy = new MaxDiskUsableSpaceFirstStrategy();
+ maxDiskUsableSpaceFirstStrategy.init(dataDirList);
+
+ int maxIndex = getIndexOfMaxSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(maxIndex, maxDiskUsableSpaceFirstStrategy.nextFolderIndex());
+ }
+
+ PowerMockito.when(FileUtils.getUsableSpace(dataDirList.get(maxIndex))).thenReturn(0L);
+ maxIndex = getIndexOfMaxSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(maxIndex, maxDiskUsableSpaceFirstStrategy.nextFolderIndex());
+ }
+ }
+
+ private int getIndexOfMaxSpace() {
+ int index = -1;
+ long maxSpace = -1;
+ for (int i = 0; i < dataDirList.size(); i++) {
+ long space = FileUtils.getUsableSpace(dataDirList.get(i));
+ if (maxSpace < space) {
+ index = i;
+ maxSpace = space;
+ }
+ }
+ return index;
+ }
+
+ @Test
+ public void testMinFolderOccupiedSpaceFirstStrategy()
+ throws DiskSpaceInsufficientException, IOException {
+ MinFolderOccupiedSpaceFirstStrategy minFolderOccupiedSpaceFirstStrategy = new MinFolderOccupiedSpaceFirstStrategy();
+ minFolderOccupiedSpaceFirstStrategy.init(dataDirList);
+
+ int minIndex = getIndexOfMinOccupiedSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(minIndex, minFolderOccupiedSpaceFirstStrategy.nextFolderIndex());
+ }
+
+ PowerMockito.when(FileUtils.getOccupiedSpace(dataDirList.get(minIndex))).thenReturn(Long.MAX_VALUE);
+ minIndex = getIndexOfMinOccupiedSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(minIndex, minFolderOccupiedSpaceFirstStrategy.nextFolderIndex());
+ }
+ }
+
+ private int getIndexOfMinOccupiedSpace() throws IOException {
+ int index = -1;
+ long minOccupied = Long.MAX_VALUE;
+ for (int i = 0; i < dataDirList.size(); i++) {
+ long space = FileUtils.getOccupiedSpace(dataDirList.get(i));
+ if (minOccupied > space) {
+ index = i;
+ minOccupied = space;
+ }
+ }
+ return index;
+ }
+
+ @Test
+ public void testAllDiskFull() {
+ for (int i = 0; i < dataDirList.size(); i++) {
+ PowerMockito.when(FileUtils.hasSpace(dataDirList.get(i))).thenReturn(false);
+ }
+
+ SequenceStrategy sequenceStrategy = new SequenceStrategy();
+ try {
+ sequenceStrategy.init(dataDirList);
+ fail();
+ } catch (DiskSpaceInsufficientException e) {
+ }
+
+ MaxDiskUsableSpaceFirstStrategy maxDiskUsableSpaceFirstStrategy = new MaxDiskUsableSpaceFirstStrategy();
+ try {
+ maxDiskUsableSpaceFirstStrategy.init(dataDirList);
+ fail();
+ } catch (DiskSpaceInsufficientException e) {
+ }
+
+ MinFolderOccupiedSpaceFirstStrategy minFolderOccupiedSpaceFirstStrategy = new MinFolderOccupiedSpaceFirstStrategy();
+ try {
+ minFolderOccupiedSpaceFirstStrategy.init(dataDirList);
+ fail();
+ } catch (DiskSpaceInsufficientException e) {
+ }
+ }
+}
\ No newline at end of file