You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2019/06/20 08:15:30 UTC
[incubator-iotdb] branch mult_dir_add_ut created (now 5993924)
This is an automated email from the ASF dual-hosted git repository.
east pushed a change to branch mult_dir_add_ut
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 5993924 add ut for DirectoriesStrategy
This branch includes the following new commits:
new 5993924 add ut for DirectoriesStrategy
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: add ut for DirectoriesStrategy
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch mult_dir_add_ut
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 59939248a3e452c21131437638f97ff58b4883e3
Author: mdf369 <95...@qq.com>
AuthorDate: Thu Jun 20 16:15:10 2019 +0800
add ut for DirectoriesStrategy
---
iotdb/pom.xml | 18 +++
.../directories/strategy/DirectoryStrategy.java | 24 ++--
.../strategy/MaxDiskUsableSpaceFirstStrategy.java | 8 +-
.../MinFolderOccupiedSpaceFirstStrategy.java | 35 ++---
.../directories/strategy/SequenceStrategy.java | 10 +-
.../java/org/apache/iotdb/db/utils/FileUtils.java | 21 +++
.../strategy/DirectoryStrategyTest.java | 157 +++++++++++++++++++++
7 files changed, 228 insertions(+), 45 deletions(-)
diff --git a/iotdb/pom.xml b/iotdb/pom.xml
index 7ff37e9..bc230e6 100644
--- a/iotdb/pom.xml
+++ b/iotdb/pom.xml
@@ -70,6 +70,24 @@
<artifactId>commons-lang3</artifactId>
<version>${common.lang3.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-core</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
index dd397b8..1239e0a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
@@ -18,9 +18,9 @@
*/
package org.apache.iotdb.db.conf.directories.strategy;
-import java.io.File;
import java.util.List;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,6 +46,18 @@ public abstract class DirectoryStrategy {
*/
public void init(List<String> folders) throws DiskSpaceInsufficientException {
this.folders = folders;
+
+ boolean hasSpace = false;
+ for (String folder : folders) {
+ if (FileUtils.hasSpace(folder)) {
+ hasSpace = true;
+ break;
+ }
+ }
+ if (!hasSpace) {
+ throw new DiskSpaceInsufficientException(
+ String.format("All disks of folders %s are full, can't init.", folders));
+ }
}
/**
@@ -74,14 +86,4 @@ public abstract class DirectoryStrategy {
public void setFolderForTest(String path) {
folders.set(0, path);
}
-
- protected long getUsableSpace(String dir) {
- long space = new File(dir).getFreeSpace();
- LOGGER.trace("Folder {} has {} available bytes.", dir, space);
- return space;
- }
-
- protected boolean hasSpace(String dir) {
- return getUsableSpace(dir) > 0;
- }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java
index 8f3ea7d..98b0e75 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MaxDiskUsableSpaceFirstStrategy.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.conf.directories.strategy;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
public class MaxDiskUsableSpaceFirstStrategy extends DirectoryStrategy {
@@ -35,7 +36,12 @@ public class MaxDiskUsableSpaceFirstStrategy extends DirectoryStrategy {
long maxSpace = 0;
for (int i = 0; i < folders.size(); i++) {
- long space = getUsableSpace(folders.get(i));
+ String folder = folders.get(i);
+ if (!FileUtils.hasSpace(folder)) {
+ continue;
+ }
+
+ long space = FileUtils.getUsableSpace(folder);
if (space > maxSpace) {
maxSpace = space;
maxIndex = i;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java
index 81f0f73..87c3814 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/MinFolderOccupiedSpaceFirstStrategy.java
@@ -19,17 +19,11 @@
package org.apache.iotdb.db.conf.directories.strategy;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.stream.Stream;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
- // directory space is measured by MB
- private static final long DATA_SIZE_SHIFT = 1024L * 1024;
-
@Override
public int nextFolderIndex() throws DiskSpaceInsufficientException {
return getMinOccupiedSpaceFolder();
@@ -37,15 +31,20 @@ public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
private int getMinOccupiedSpaceFolder() throws DiskSpaceInsufficientException {
int minIndex = -1;
- long minSpace = Integer.MAX_VALUE;
+ long minSpace = Long.MAX_VALUE;
for (int i = 0; i < folders.size(); i++) {
String folder = folders.get(i);
- if (!hasSpace(folder)) {
+ if (!FileUtils.hasSpace(folder)) {
continue;
}
- long space = getOccupiedSpace(folder);
+ long space = 0;
+ try {
+ space = FileUtils.getOccupiedSpace(folder);
+ } catch (IOException e) {
+ LOGGER.error("Cannot calculate occupied space for path {}.", folder, e);
+ }
if (space < minSpace) {
minSpace = space;
minIndex = i;
@@ -58,20 +57,4 @@ public class MinFolderOccupiedSpaceFirstStrategy extends DirectoryStrategy {
return minIndex;
}
-
- private long getOccupiedSpace(String path) {
- Path folder = Paths.get(path);
- long size = Long.MAX_VALUE;
- try {
- try (Stream<Path> stream = Files.walk(folder)) {
- size = stream.filter(p -> p.toFile().isFile())
- .mapToLong(p -> p.toFile().length())
- .sum();
- }
- } catch (IOException e) {
- LOGGER.error("Cannot calculate occupied space for seriesPath {}.", path, e);
- }
-
- return size / DATA_SIZE_SHIFT;
- }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java
index b9c9ff5..7b05ab2 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/SequenceStrategy.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.conf.directories.strategy;
import java.util.List;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
public class SequenceStrategy extends DirectoryStrategy {
@@ -31,16 +32,11 @@ public class SequenceStrategy extends DirectoryStrategy {
currentIndex = -1;
for (int i = 0; i < folders.size(); i++) {
- if (hasSpace(folders.get(i))) {
+ if (FileUtils.hasSpace(folders.get(i))) {
currentIndex = i;
break;
}
}
-
- if (currentIndex == -1) {
- throw new DiskSpaceInsufficientException(
- String.format("All disks of folders %s are full, can't init.", folders));
- }
}
@Override
@@ -53,7 +49,7 @@ public class SequenceStrategy extends DirectoryStrategy {
private int tryGetNextIndex(int start) throws DiskSpaceInsufficientException {
int index = (start + 1) % folders.size();
- while (!hasSpace(folders.get(index))) {
+ while (!FileUtils.hasSpace(folders.get(index))) {
index = (index + 1) % folders.size();
if (index == start) {
throw new DiskSpaceInsufficientException(folders);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
index c276f01..f51fe93 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/FileUtils.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.db.utils;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
/**
* FileUtils is just used to calculate file attributes like file size, including some measurement
@@ -84,4 +88,21 @@ public class FileUtils {
public enum Unit {
B, KB, MB, GB, TB, PB, EB
}
+
+ public static long getUsableSpace(String dir) {
+ long space = new File(dir).getFreeSpace();
+ return space;
+ }
+
+ public static boolean hasSpace(String dir) {
+ return getUsableSpace(dir) > 0;
+ }
+
+ public static long getOccupiedSpace(String path) throws IOException {
+ Path folder = Paths.get(path);
+ long size = Files.walk(folder).filter(p -> p.toFile().isFile())
+ .mapToLong(p -> p.toFile().length()).sum();
+
+ return size;
+ }
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java b/iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java
new file mode 100644
index 0000000..676e2f3
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategyTest.java
@@ -0,0 +1,157 @@
+package org.apache.iotdb.db.conf.directories.strategy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.utils.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(FileUtils.class)
+public class DirectoryStrategyTest {
+
+ List<String> dataDirList;
+ Set<Integer> fullDirIndexSet;
+
+ @Before
+ public void setUp() throws DiskSpaceInsufficientException, IOException {
+ dataDirList = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ dataDirList.add("data" + i);
+ }
+
+ fullDirIndexSet = new HashSet<>();
+ fullDirIndexSet.add(1);
+ fullDirIndexSet.add(3);
+
+ PowerMockito.mockStatic(FileUtils.class);
+ for (int i = 0; i < dataDirList.size(); i++) {
+ boolean res = !fullDirIndexSet.contains(i);
+ PowerMockito.when(FileUtils.hasSpace(dataDirList.get(i))).thenReturn(res);
+ PowerMockito.when(FileUtils.getUsableSpace(dataDirList.get(i))).thenReturn(res ? (long) (i + 1) : 0L);
+ PowerMockito.when(FileUtils.getOccupiedSpace(dataDirList.get(i))).thenReturn(res ? (long) (i + 1) : Long.MAX_VALUE);
+ }
+ }
+
+ @After
+ public void tearDown() {
+
+ }
+
+ @Test
+ public void testSequenceStrategy() throws DiskSpaceInsufficientException {
+ SequenceStrategy sequenceStrategy = new SequenceStrategy();
+ sequenceStrategy.init(dataDirList);
+
+ // loop two times of data dir size to fully loop
+ int index = 0;
+ for (int i = 0; i < dataDirList.size() * 2; i++, index++) {
+ index = index % dataDirList.size();
+ while (fullDirIndexSet.contains(index)) {
+ index = (index + 1) % dataDirList.size();
+ }
+ assertEquals(index, sequenceStrategy.nextFolderIndex());
+ }
+ }
+
+ @Test
+ public void testMaxDiskUsableSpaceFirstStrategy() throws DiskSpaceInsufficientException {
+ MaxDiskUsableSpaceFirstStrategy maxDiskUsableSpaceFirstStrategy = new MaxDiskUsableSpaceFirstStrategy();
+ maxDiskUsableSpaceFirstStrategy.init(dataDirList);
+
+ int maxIndex = getIndexOfMaxSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(maxIndex, maxDiskUsableSpaceFirstStrategy.nextFolderIndex());
+ }
+
+ PowerMockito.when(FileUtils.getUsableSpace(dataDirList.get(maxIndex))).thenReturn(0L);
+ maxIndex = getIndexOfMaxSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(maxIndex, maxDiskUsableSpaceFirstStrategy.nextFolderIndex());
+ }
+ }
+
+ private int getIndexOfMaxSpace() {
+ int index = -1;
+ long maxSpace = -1;
+ for (int i = 0; i < dataDirList.size(); i++) {
+ long space = FileUtils.getUsableSpace(dataDirList.get(i));
+ if (maxSpace < space) {
+ index = i;
+ maxSpace = space;
+ }
+ }
+ return index;
+ }
+
+ @Test
+ public void testMinFolderOccupiedSpaceFirstStrategy()
+ throws DiskSpaceInsufficientException, IOException {
+ MinFolderOccupiedSpaceFirstStrategy minFolderOccupiedSpaceFirstStrategy = new MinFolderOccupiedSpaceFirstStrategy();
+ minFolderOccupiedSpaceFirstStrategy.init(dataDirList);
+
+ int minIndex = getIndexOfMinOccupiedSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(minIndex, minFolderOccupiedSpaceFirstStrategy.nextFolderIndex());
+ }
+
+ PowerMockito.when(FileUtils.getOccupiedSpace(dataDirList.get(minIndex))).thenReturn(Long.MAX_VALUE);
+ minIndex = getIndexOfMinOccupiedSpace();
+ for (int i = 0; i < dataDirList.size(); i++) {
+ assertEquals(minIndex, minFolderOccupiedSpaceFirstStrategy.nextFolderIndex());
+ }
+ }
+
+ private int getIndexOfMinOccupiedSpace() throws IOException {
+ int index = -1;
+ long minOccupied = Long.MAX_VALUE;
+ for (int i = 0; i < dataDirList.size(); i++) {
+ long space = FileUtils.getOccupiedSpace(dataDirList.get(i));
+ if (minOccupied > space) {
+ index = i;
+ minOccupied = space;
+ }
+ }
+ return index;
+ }
+
+ @Test
+ public void testAllDiskFull() {
+ for (int i = 0; i < dataDirList.size(); i++) {
+ PowerMockito.when(FileUtils.hasSpace(dataDirList.get(i))).thenReturn(false);
+ }
+
+ SequenceStrategy sequenceStrategy = new SequenceStrategy();
+ try {
+ sequenceStrategy.init(dataDirList);
+ fail();
+ } catch (DiskSpaceInsufficientException e) {
+ }
+
+ MaxDiskUsableSpaceFirstStrategy maxDiskUsableSpaceFirstStrategy = new MaxDiskUsableSpaceFirstStrategy();
+ try {
+ maxDiskUsableSpaceFirstStrategy.init(dataDirList);
+ fail();
+ } catch (DiskSpaceInsufficientException e) {
+ }
+
+ MinFolderOccupiedSpaceFirstStrategy minFolderOccupiedSpaceFirstStrategy = new MinFolderOccupiedSpaceFirstStrategy();
+ try {
+ minFolderOccupiedSpaceFirstStrategy.init(dataDirList);
+ fail();
+ } catch (DiskSpaceInsufficientException e) {
+ }
+ }
+}
\ No newline at end of file