You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/02/21 08:52:08 UTC
[incubator-iotdb] branch master updated: [IOTDB-7] Fix
OpenFileNumUtilTest (#53)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1f71db1 [IOTDB-7] Fix OpenFileNumUtilTest (#53)
1f71db1 is described below
commit 1f71db1fe614e7d76bcce3e2b5a86f1596c0f73b
Author: Rui Liu <24...@qq.com>
AuthorDate: Thu Feb 21 16:52:04 2019 +0800
[IOTDB-7] Fix OpenFileNumUtilTest (#53)
* check lsof command output for unsupported situation
* add warn log about unsupported OS for OpenFileNumUtil getting Pid
---
.../org/apache/iotdb/db/utils/OpenFileNumUtil.java | 121 +++++++++++----------
.../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 108 +++++++++---------
2 files changed, 121 insertions(+), 108 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
index 7101694..cf0c24b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
@@ -16,14 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.utils;
-import com.sun.management.UnixOperatingSystemMXBean;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.lang.management.ManagementFactory;
-import java.lang.management.OperatingSystemMXBean;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
@@ -34,9 +32,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
// Notice : statistics in this class may not be accurate because of limited user authority.
-// TODO: now only TOTAL_OPEN_FILE_NUM is general avaiable on multiple Linux-like OSs.
public class OpenFileNumUtil {
- private static final Logger log = LoggerFactory.getLogger(OpenFileNumUtil.class);
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(OpenFileNumUtil.class);
private static final int PID_ERROR_CODE = -1;
private static final int UNSUPPORTED_OS_ERROR_CODE = -2;
private static final int UNKNOWN_STATISTICS_ERROR_CODE = -3;
@@ -45,22 +43,19 @@ public class OpenFileNumUtil {
private static final String MAC_OS_NAME = "mac";
private static final String SEARCH_PID_LINUX = "ps -aux | grep -i %s | grep -v grep";
private static final String SEARCH_PID_MAC = "ps aux | grep -i %s | grep -v grep";
- //TODO lsof command is only avaiable on some OSs.
+ //command 'lsof -p' is available on most Linux distro except CentOS.
private static final String SEARCH_OPEN_DATA_FILE_BY_PID = "lsof -p %d";
private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private static Directories directories = Directories.getInstance();
- private final String[] cmds = {"/bin/bash", "-c", ""};
+ private static final String[] COMMAND_TEMPLATE = {"/bin/bash", "-c", ""};
+ private static boolean isOutputValid = false;
private int pid;
- private String processName;
-
- private OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
/**
* constructor, process key word is defined by IOTDB_PROCESS_KEY_WORD.
*/
private OpenFileNumUtil() {
- processName = IOTDB_PROCESS_KEY_WORD;
pid = getPid();
}
@@ -98,34 +93,42 @@ public class OpenFileNumUtil {
*
* @return pid
*/
- private int getPid() {
+ private static int getPid() {
int iotdbPid = -1;
Process pro1;
Runtime r = Runtime.getRuntime();
- String osName = System.getProperty("os.name").toLowerCase();
- try {
- String command;
- if (osName.startsWith(LINUX_OS_NAME)) {
- command = String.format(SEARCH_PID_LINUX, processName);
- } else {
- command = String.format(SEARCH_PID_MAC, processName);
- }
- cmds[2] = command;
- pro1 = r.exec(cmds);
- BufferedReader in1 = new BufferedReader(new InputStreamReader(pro1.getInputStream()));
- String line;
- while ((line = in1.readLine()) != null) {
- line = line.trim();
- String[] temp = line.split("\\s+");
- if (temp.length > 1 && isNumeric(temp[1])) {
- iotdbPid = Integer.parseInt(temp[1]);
- break;
+ // System.getProperty("os.name") can detect which type of OS is using now.
+ // this code can detect Windows, Mac, Unix and Solaris.
+ String os = System.getProperty("os.name");
+ String osName = os.toLowerCase();
+ if (osName.startsWith(LINUX_OS_NAME) || osName.startsWith(MAC_OS_NAME)) {
+ try {
+ String command;
+ String processName = IOTDB_PROCESS_KEY_WORD;
+ if (osName.startsWith(LINUX_OS_NAME)) {
+ command = String.format(SEARCH_PID_LINUX, processName);
+ } else {
+ command = String.format(SEARCH_PID_MAC, processName);
}
+ COMMAND_TEMPLATE[2] = command;
+ pro1 = r.exec(COMMAND_TEMPLATE);
+ BufferedReader in1 = new BufferedReader(new InputStreamReader(pro1.getInputStream()));
+ String line;
+ while ((line = in1.readLine()) != null) {
+ line = line.trim();
+ String[] temp = line.split("\\s+");
+ if (temp.length > 1 && isNumeric(temp[1])) {
+ iotdbPid = Integer.parseInt(temp[1]);
+ break;
+ }
+ }
+ in1.close();
+ pro1.destroy();
+ } catch (IOException e) {
+ LOGGER.error("Cannot get pid of IoTDB process because {}", e.getMessage());
}
- in1.close();
- pro1.destroy();
- } catch (IOException e) {
- log.error("Cannot get pid of IoTDB process. ", e);
+ } else {
+ LOGGER.warn("Unsupported OS {} for OpenFileNumUtil getting Pid.", os);
}
return iotdbPid;
}
@@ -141,33 +144,35 @@ public class OpenFileNumUtil {
/**
* return statistic Map, whose key belongs to enum OpenFileNumStatistics: TOTAL_OPEN_FILE_NUM is
- * the current total open file number of IoTDB service process DATA_OPEN_FILE_NUM is the current
- * open file number under data directory DELTA_OPEN_FILE_NUM is the current open file number of
- * tsfile OVERFLOW_OPEN_FILE_NUM is the current open file number of overflow file
- * WAL_OPEN_FILE_NUM is the current open file number of WAL file METADATA_OPEN_FILE_NUM is the
- * current open file number of metadata DIGEST_OPEN_FILE_NUM is the current open file number of
- * fileNodeDir SOCKET_OPEN_FILE_NUM is the current open socket connection of IoTDB service
+ * the current total open file number of IoTDB service process; DATA_OPEN_FILE_NUM is the current
+ * open file number under data directory; DELTA_OPEN_FILE_NUM is the current open file number of
+ * TsFile; OVERFLOW_OPEN_FILE_NUM is the current open file number of overflow file;
+ * WAL_OPEN_FILE_NUM is the current open file number of WAL file; METADATA_OPEN_FILE_NUM is the
+ * current open file number of metadata; DIGEST_OPEN_FILE_NUM is the current open file number of
+ * fileNodeDir; SOCKET_OPEN_FILE_NUM is the current open socket connection of IoTDB service
* process.
*
* @param pid : IoTDB service pid
* @return list : statistics list
*/
- private EnumMap<OpenFileNumStatistics, Integer> getOpenFile(int pid) {
+ private static EnumMap<OpenFileNumStatistics, Integer> getOpenFile(int pid) {
EnumMap<OpenFileNumStatistics, Integer> resultMap = new EnumMap<>(OpenFileNumStatistics.class);
// initialize resultMap
for (OpenFileNumStatistics openFileNumStatistics : OpenFileNumStatistics.values()) {
resultMap.put(openFileNumStatistics, 0);
}
Process pro;
+ int lineCount = 0;
Runtime r = Runtime.getRuntime();
try {
String command = String.format(SEARCH_OPEN_DATA_FILE_BY_PID, pid);
- cmds[2] = command;
- pro = r.exec(cmds);
+ COMMAND_TEMPLATE[2] = command;
+ pro = r.exec(COMMAND_TEMPLATE);
BufferedReader in = new BufferedReader(new InputStreamReader(pro.getInputStream()));
String line;
int oldValue;
while ((line = in.readLine()) != null) {
+ lineCount++;
String[] temp = line.split("\\s+");
if (line.contains(Integer.toString(pid)) && temp.length > 8) {
oldValue = resultMap.get(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
@@ -188,22 +193,22 @@ public class OpenFileNumUtil {
}
}
}
+ if (lineCount < OpenFileNumStatistics.values().length) {
+ isOutputValid = false;
+ for (OpenFileNumStatistics statistics : OpenFileNumStatistics.values()) {
+ resultMap.put(statistics, UNSUPPORTED_OS_ERROR_CODE);
+ }
+ } else {
+ isOutputValid = true;
+ }
in.close();
pro.destroy();
} catch (Exception e) {
- log.error("Cannot get open file number of IoTDB process.", e);
+ LOGGER.error("Cannot get open file number of IoTDB process because {}", e.getMessage());
}
return resultMap;
}
- private long getCurrentOpenedFiles() {
- if(os instanceof UnixOperatingSystemMXBean){
- return ((UnixOperatingSystemMXBean) os).getOpenFileDescriptorCount();
- }else {
- return UNSUPPORTED_OS_ERROR_CODE;
- }
- }
-
/**
* Check if runtime OS is supported then return the result list. If pid is abnormal then all
* statistics returns -1, if OS is not supported then all statistics returns -2
@@ -240,13 +245,14 @@ public class OpenFileNumUtil {
* @return open file number
*/
public int get(OpenFileNumStatistics statistics) {
- if(statistics.equals(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM)){
- return (int) getCurrentOpenedFiles();
- }
EnumMap<OpenFileNumStatistics, Integer> statisticsMap = getStatisticMap();
return statisticsMap.getOrDefault(statistics, UNKNOWN_STATISTICS_ERROR_CODE);
}
+ boolean isCommandValid() {
+ return isOutputValid;
+ }
+
public enum OpenFileNumStatistics {
TOTAL_OPEN_FILE_NUM(null),
DATA_OPEN_FILE_NUM(Collections.singletonList(config.dataDir)),
@@ -271,7 +277,10 @@ public class OpenFileNumUtil {
}
private static class OpenFileNumUtilHolder {
- private OpenFileNumUtilHolder(){}
+
+ private OpenFileNumUtilHolder() {
+ }
+
private static final OpenFileNumUtil INSTANCE = new OpenFileNumUtil();
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
index efab193..b2fbec3 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
@@ -16,16 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.utils;
import static org.junit.Assert.assertEquals;
-import com.sun.management.UnixOperatingSystemMXBean;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean;
import java.nio.file.Files;
import java.util.ArrayList;
@@ -43,6 +42,7 @@ public class OpenFileNumUtilTest {
private static final String TEST_FILE_PREFIX = "testFileForOpenFileNumUtil";
private static final String MAC_OS_NAME = "mac";
private static final String LINUX_OS_NAME = "linux";
+ private static final int UNSUPPORTED_OS_ERROR_CODE = -2;
private OpenFileNumUtil openFileNumUtil = OpenFileNumUtil.getInstance();
private ArrayList<File> fileList = new ArrayList<>();
private ArrayList<FileWriter> fileWriterList = new ArrayList<>();
@@ -57,9 +57,7 @@ public class OpenFileNumUtilTest {
private static int getProcessID() {
RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
- int pid = Integer.parseInt(runtimeMXBean.getName().split("@")[0]);
- System.out.println("Test process ID of OpenFileNumUtilTest is " + pid);
- return pid;
+ return Integer.parseInt(runtimeMXBean.getName().split("@")[0]);
}
@Before
@@ -72,10 +70,8 @@ public class OpenFileNumUtilTest {
testDataDirRoot = new File(currDir);
currDir = currDir + File.separator + dataFilePath;
File testDataDir = new File(currDir);
- if(!testDataDir.isDirectory()){
- if (!testDataDir.mkdirs()) {
- LOGGER.error("Create test file dir {} failed.", testDataDir.getPath());
- }
+ if (!testDataDir.isDirectory() && !testDataDir.mkdirs()) {
+ LOGGER.error("Create test file dir {} failed.", testDataDir.getPath());
}
testFileName = TEST_FILE_PREFIX + testProcessID;
}
@@ -111,34 +107,37 @@ public class OpenFileNumUtilTest {
}
@Test
- public void testTotalOpenFileNumWhenCreateFile() {
+ public void testDataOpenFileNumWhenCreateFile() {
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
- // get total open file number statistics of original state
- totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ // get total open file number under /data/data before create new files
+ totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
for (int i = 0; i < testFileNum; i++) {
fileList.add(new File(currDir + testFileName + i));
}
// create testFileNum File, then get total open file number statistics
- totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// create test file shall not affect total open file number statistics
- assertEquals(0, totalOpenFileNumChange);
+ // if 'lsof' command is valid
+ if (openFileNumUtil.isCommandValid()) {
+ assertEquals(0, totalOpenFileNumChange);
+ } else {
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+ }
} else {
- assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+ openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
}
}
@Test
- public void testTotalOpenFileNumWhenCreateFileWriter() {
+ public void testDataOpenFileNumWhenCreateFileWriter() {
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
for (int i = 0; i < testFileNum; i++) {
- fileList.add(new File(currDir + File.separator + testFileName + i));
+ fileList.add(new File(currDir + testFileName + i));
}
- totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
for (File file : fileList) {
if (file.exists()) {
try {
@@ -149,32 +148,34 @@ public class OpenFileNumUtilTest {
} else {
try {
boolean flag = file.createNewFile();
- if(flag) {
+ if (flag) {
LOGGER.debug("Create a file {} successfully", file);
fileWriterList.add(new FileWriter(file));
- }else{
- LOGGER.error(
- "create test file {} failed when execute testTotalOpenFileNumWhenCreateFileWriter().",
- file.getPath());
+ } else {
+ LOGGER.error("create test file {} failed.", file.getPath());
}
} catch (IOException e) {
LOGGER.error(e.getMessage());
}
}
}
- totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// create FileWriter shall cause total open file number increase by testFileNum
- assertEquals(testFileNum, totalOpenFileNumChange);
+ if (openFileNumUtil.isCommandValid()) {
+ assertEquals(testFileNum, totalOpenFileNumChange);
+ } else {
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+ }
} else {
- assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+ openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
}
}
@Test
- public void testTotalOpenFileNumWhenFileWriterWriting() {
+ public void testDataOpenFileNumWhenFileWriterWriting() {
LOGGER.debug("testDataOpenFileNumWhenFileWriterWriting...");
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
for (int i = 0; i < testFileNum; i++) {
@@ -202,8 +203,7 @@ public class OpenFileNumUtilTest {
}
}
}
- totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
for (FileWriter fw : fileWriterList) {
try {
fw.write("this is a test file for open file number counting.");
@@ -211,19 +211,23 @@ public class OpenFileNumUtilTest {
LOGGER.error(e.getMessage());
}
}
- totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// writing test file shall not affect total open file number statistics
- assertEquals(0, totalOpenFileNumChange);
+ if (openFileNumUtil.isCommandValid()) {
+ assertEquals(0, totalOpenFileNumChange);
+ } else {
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+ }
} else {
- assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+ openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
}
}
@Test
- public void testTotalOpenFileNumWhenFileWriterClose() {
+ public void testDataOpenFileNumWhenFileWriterClose() {
LOGGER.debug("testDataOpenFileNumWhenFileWriterClose...");
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
for (int i = 0; i < testFileNum; i++) {
@@ -239,9 +243,7 @@ public class OpenFileNumUtilTest {
} else {
try {
if (!file.createNewFile()) {
- LOGGER.error(
- "create test file {} failed when execute testTotalOpenFileNumWhenFileWriterClose().",
- file.getPath());
+ LOGGER.error("create test file {} failed.", file.getPath());
}
} catch (IOException e) {
LOGGER.error(e.getMessage());
@@ -260,8 +262,7 @@ public class OpenFileNumUtilTest {
LOGGER.error(e.getMessage());
}
}
- totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
for (FileWriter fw : fileWriterList) {
try {
fw.close();
@@ -269,15 +270,18 @@ public class OpenFileNumUtilTest {
LOGGER.error(e.getMessage());
}
}
- totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+ totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// close FileWriter shall cause total open file number decrease by testFileNum
- assertEquals(-testFileNum, totalOpenFileNumChange);
+ if (openFileNumUtil.isCommandValid()) {
+ assertEquals(-testFileNum, totalOpenFileNumChange);
+ } else {
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+ }
} else {
- assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+ assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+ openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
}
}
-
}