You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/01/22 14:28:26 UTC
[incubator-iotdb] branch fix_OpenFileNumUtilTest_failure updated:
using UnixOperatingSystemMXBean to replace lsof command for counting the
opened files totally
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch fix_OpenFileNumUtilTest_failure
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/fix_OpenFileNumUtilTest_failure by this push:
new 505eb7b using UnixOperatingSystemMXBean to replace lsof command for counting the opened files totally
505eb7b is described below
commit 505eb7bdca7b5785664a666e4883784ce24481f2
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Tue Jan 22 22:28:15 2019 +0800
using UnixOperatingSystemMXBean to replace lsof command for counting the opened files totally
---
.../org/apache/iotdb/db/utils/OpenFileNumUtil.java | 22 ++++++++++++++
.../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 35 ++++++++++++----------
2 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
index cfce9ce..007363b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
@@ -18,9 +18,16 @@
*/
package org.apache.iotdb.db.utils;
+import com.sun.management.UnixOperatingSystemMXBean;
import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileDescriptor;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
@@ -31,6 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
// Notice : statistics in this class may not be accurate because of limited user authority.
+// TODO: now only TOTAL_OPEN_FILE_NUM is general avaiable on multiple Linux-like OSs.
public class OpenFileNumUtil {
private static final Logger log = LoggerFactory.getLogger(OpenFileNumUtil.class);
private static final int PID_ERROR_CODE = -1;
@@ -41,6 +49,7 @@ public class OpenFileNumUtil {
private static final String MAC_OS_NAME = "mac";
private static final String SEARCH_PID_LINUX = "ps -aux | grep -i %s | grep -v grep";
private static final String SEARCH_PID_MAC = "ps aux | grep -i %s | grep -v grep";
+ //TODO lsof command is only avaiable on some OSs.
private static final String SEARCH_OPEN_DATA_FILE_BY_PID = "lsof -p %d";
private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -49,6 +58,8 @@ public class OpenFileNumUtil {
private int pid;
private String processName;
+ OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
+
/**
* constructor, process key word is defined by IOTDB_PROCESS_KEY_WORD.
*/
@@ -192,6 +203,14 @@ public class OpenFileNumUtil {
return resultMap;
}
+ private long getCurrentOpenedFiles() {
+ if(os instanceof UnixOperatingSystemMXBean){
+ return ((UnixOperatingSystemMXBean) os).getOpenFileDescriptorCount();
+ }else {
+ return UNSUPPORTED_OS_ERROR_CODE;
+ }
+ }
+
/**
* Check if runtime OS is supported then return the result list. If pid is abnormal then all
* statistics returns -1, if OS is not supported then all statistics returns -2
@@ -228,6 +247,9 @@ public class OpenFileNumUtil {
* @return open file number
*/
public int get(OpenFileNumStatistics statistics) {
+ if(statistics.equals(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM)){
+ return (int) getCurrentOpenedFiles();
+ }
EnumMap<OpenFileNumStatistics, Integer> statisticsMap = getStatisticMap();
return statisticsMap.getOrDefault(statistics, UNKNOWN_STATISTICS_ERROR_CODE);
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
index 01b94b1..efab193 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
@@ -20,14 +20,17 @@ package org.apache.iotdb.db.utils;
import static org.junit.Assert.assertEquals;
+import com.sun.management.UnixOperatingSystemMXBean;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean;
import java.nio.file.Files;
import java.util.ArrayList;
import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.utils.OpenFileNumUtil.OpenFileNumStatistics;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -108,34 +111,34 @@ public class OpenFileNumUtilTest {
}
@Test
- public void testDataOpenFileNumWhenCreateFile() {
+ public void testTotalOpenFileNumWhenCreateFile() {
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
// get total open file number statistics of original state
totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
for (int i = 0; i < testFileNum; i++) {
fileList.add(new File(currDir + testFileName + i));
}
// create testFileNum File, then get total open file number statistics
totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// create test file shall not affect total open file number statistics
assertEquals(0, totalOpenFileNumChange);
} else {
assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
+ openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
}
}
@Test
- public void testDataOpenFileNumWhenCreateFileWriter() {
+ public void testTotalOpenFileNumWhenCreateFileWriter() {
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
for (int i = 0; i < testFileNum; i++) {
fileList.add(new File(currDir + File.separator + testFileName + i));
}
totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
for (File file : fileList) {
if (file.exists()) {
try {
@@ -160,18 +163,18 @@ public class OpenFileNumUtilTest {
}
}
totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// create FileWriter shall cause total open file number increase by testFileNum
assertEquals(testFileNum, totalOpenFileNumChange);
} else {
assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
+ openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
}
}
@Test
- public void testDataOpenFileNumWhenFileWriterWriting() {
+ public void testTotalOpenFileNumWhenFileWriterWriting() {
LOGGER.debug("testDataOpenFileNumWhenFileWriterWriting...");
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
for (int i = 0; i < testFileNum; i++) {
@@ -200,7 +203,7 @@ public class OpenFileNumUtilTest {
}
}
totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
for (FileWriter fw : fileWriterList) {
try {
fw.write("this is a test file for open file number counting.");
@@ -209,18 +212,18 @@ public class OpenFileNumUtilTest {
}
}
totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// writing test file shall not affect total open file number statistics
assertEquals(0, totalOpenFileNumChange);
} else {
assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
+ openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
}
}
@Test
- public void testDataOpenFileNumWhenFileWriterClose() {
+ public void testTotalOpenFileNumWhenFileWriterClose() {
LOGGER.debug("testDataOpenFileNumWhenFileWriterClose...");
if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
for (int i = 0; i < testFileNum; i++) {
@@ -258,7 +261,7 @@ public class OpenFileNumUtilTest {
}
}
totalOpenFileNumBefore = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
for (FileWriter fw : fileWriterList) {
try {
fw.close();
@@ -267,13 +270,13 @@ public class OpenFileNumUtilTest {
}
}
totalOpenFileNumAfter = openFileNumUtil
- .get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
+ .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
// close FileWriter shall cause total open file number decrease by testFileNum
assertEquals(-testFileNum, totalOpenFileNumChange);
} else {
assertEquals(-2,
- openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
+ openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
}
}