You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/02/21 08:52:08 UTC

[incubator-iotdb] branch master updated: [IOTDB-7] Fix OpenFileNumUtilTest (#53)

This is an automated email from the ASF dual-hosted git repository.

hxd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 1f71db1  [IOTDB-7] Fix OpenFileNumUtilTest (#53)
1f71db1 is described below

commit 1f71db1fe614e7d76bcce3e2b5a86f1596c0f73b
Author: Rui Liu <24...@qq.com>
AuthorDate: Thu Feb 21 16:52:04 2019 +0800

    [IOTDB-7] Fix OpenFileNumUtilTest (#53)
    
    * check lsof command output for unsupported situation
    * add warn log about unsupported OS for OpenFileNumUtil getting Pid
---
 .../org/apache/iotdb/db/utils/OpenFileNumUtil.java | 121 +++++++++++----------
 .../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 108 +++++++++---------
 2 files changed, 121 insertions(+), 108 deletions(-)

diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
index 7101694..cf0c24b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/OpenFileNumUtil.java
@@ -16,14 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.iotdb.db.utils;
 
-import com.sun.management.UnixOperatingSystemMXBean;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.lang.management.ManagementFactory;
-import java.lang.management.OperatingSystemMXBean;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.List;
@@ -34,9 +32,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 // Notice : statistics in this class may not be accurate because of limited user authority.
-// TODO: now only TOTAL_OPEN_FILE_NUM is general avaiable on multiple Linux-like OSs.
 public class OpenFileNumUtil {
-  private static final Logger log = LoggerFactory.getLogger(OpenFileNumUtil.class);
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(OpenFileNumUtil.class);
   private static final int PID_ERROR_CODE = -1;
   private static final int UNSUPPORTED_OS_ERROR_CODE = -2;
   private static final int UNKNOWN_STATISTICS_ERROR_CODE = -3;
@@ -45,22 +43,19 @@ public class OpenFileNumUtil {
   private static final String MAC_OS_NAME = "mac";
   private static final String SEARCH_PID_LINUX = "ps -aux | grep -i %s | grep -v grep";
   private static final String SEARCH_PID_MAC = "ps aux | grep -i %s | grep -v grep";
-  //TODO lsof command is only avaiable on some OSs.
+  //command 'lsof -p' is available on most Linux distro except CentOS.
   private static final String SEARCH_OPEN_DATA_FILE_BY_PID = "lsof -p %d";
 
   private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
   private static Directories directories = Directories.getInstance();
-  private final String[] cmds = {"/bin/bash", "-c", ""};
+  private static final String[] COMMAND_TEMPLATE = {"/bin/bash", "-c", ""};
+  private static boolean isOutputValid = false;
   private int pid;
-  private String processName;
-
-  private OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
 
   /**
    * constructor, process key word is defined by IOTDB_PROCESS_KEY_WORD.
    */
   private OpenFileNumUtil() {
-    processName = IOTDB_PROCESS_KEY_WORD;
     pid = getPid();
   }
 
@@ -98,34 +93,42 @@ public class OpenFileNumUtil {
    *
    * @return pid
    */
-  private int getPid() {
+  private static int getPid() {
     int iotdbPid = -1;
     Process pro1;
     Runtime r = Runtime.getRuntime();
-    String osName = System.getProperty("os.name").toLowerCase();
-    try {
-      String command;
-      if (osName.startsWith(LINUX_OS_NAME)) {
-        command = String.format(SEARCH_PID_LINUX, processName);
-      } else {
-        command = String.format(SEARCH_PID_MAC, processName);
-      }
-      cmds[2] = command;
-      pro1 = r.exec(cmds);
-      BufferedReader in1 = new BufferedReader(new InputStreamReader(pro1.getInputStream()));
-      String line;
-      while ((line = in1.readLine()) != null) {
-        line = line.trim();
-        String[] temp = line.split("\\s+");
-        if (temp.length > 1 && isNumeric(temp[1])) {
-          iotdbPid = Integer.parseInt(temp[1]);
-          break;
+    // System.getProperty("os.name") can detect which type of OS is using now.
+    // this code can detect Windows, Mac, Unix and Solaris.
+    String os = System.getProperty("os.name");
+    String osName = os.toLowerCase();
+    if (osName.startsWith(LINUX_OS_NAME) || osName.startsWith(MAC_OS_NAME)) {
+      try {
+        String command;
+        String processName = IOTDB_PROCESS_KEY_WORD;
+        if (osName.startsWith(LINUX_OS_NAME)) {
+          command = String.format(SEARCH_PID_LINUX, processName);
+        } else {
+          command = String.format(SEARCH_PID_MAC, processName);
         }
+        COMMAND_TEMPLATE[2] = command;
+        pro1 = r.exec(COMMAND_TEMPLATE);
+        BufferedReader in1 = new BufferedReader(new InputStreamReader(pro1.getInputStream()));
+        String line;
+        while ((line = in1.readLine()) != null) {
+          line = line.trim();
+          String[] temp = line.split("\\s+");
+          if (temp.length > 1 && isNumeric(temp[1])) {
+            iotdbPid = Integer.parseInt(temp[1]);
+            break;
+          }
+        }
+        in1.close();
+        pro1.destroy();
+      } catch (IOException e) {
+        LOGGER.error("Cannot get pid of IoTDB process because {}", e.getMessage());
       }
-      in1.close();
-      pro1.destroy();
-    } catch (IOException e) {
-      log.error("Cannot get pid of IoTDB process. ", e);
+    } else {
+      LOGGER.warn("Unsupported OS {} for OpenFileNumUtil getting Pid.", os);
     }
     return iotdbPid;
   }
@@ -141,33 +144,35 @@ public class OpenFileNumUtil {
 
   /**
    * return statistic Map, whose key belongs to enum OpenFileNumStatistics: TOTAL_OPEN_FILE_NUM is
-   * the current total open file number of IoTDB service process DATA_OPEN_FILE_NUM is the current
-   * open file number under data directory DELTA_OPEN_FILE_NUM is the current open file number of
-   * tsfile OVERFLOW_OPEN_FILE_NUM is the current open file number of overflow file
-   * WAL_OPEN_FILE_NUM is the current open file number of WAL file METADATA_OPEN_FILE_NUM is the
-   * current open file number of metadata DIGEST_OPEN_FILE_NUM is the current open file number of
-   * fileNodeDir SOCKET_OPEN_FILE_NUM is the current open socket connection of IoTDB service
+   * the current total open file number of IoTDB service process; DATA_OPEN_FILE_NUM is the current
+   * open file number under data directory; DELTA_OPEN_FILE_NUM is the current open file number of
+   * TsFile; OVERFLOW_OPEN_FILE_NUM is the current open file number of overflow file;
+   * WAL_OPEN_FILE_NUM is the current open file number of WAL file; METADATA_OPEN_FILE_NUM is the
+   * current open file number of metadata; DIGEST_OPEN_FILE_NUM is the current open file number of
+   * fileNodeDir; SOCKET_OPEN_FILE_NUM is the current open socket connection of IoTDB service
    * process.
    *
    * @param pid : IoTDB service pid
    * @return list : statistics list
    */
-  private EnumMap<OpenFileNumStatistics, Integer> getOpenFile(int pid) {
+  private static EnumMap<OpenFileNumStatistics, Integer> getOpenFile(int pid) {
     EnumMap<OpenFileNumStatistics, Integer> resultMap = new EnumMap<>(OpenFileNumStatistics.class);
     // initialize resultMap
     for (OpenFileNumStatistics openFileNumStatistics : OpenFileNumStatistics.values()) {
       resultMap.put(openFileNumStatistics, 0);
     }
     Process pro;
+    int lineCount = 0;
     Runtime r = Runtime.getRuntime();
     try {
       String command = String.format(SEARCH_OPEN_DATA_FILE_BY_PID, pid);
-      cmds[2] = command;
-      pro = r.exec(cmds);
+      COMMAND_TEMPLATE[2] = command;
+      pro = r.exec(COMMAND_TEMPLATE);
       BufferedReader in = new BufferedReader(new InputStreamReader(pro.getInputStream()));
       String line;
       int oldValue;
       while ((line = in.readLine()) != null) {
+        lineCount++;
         String[] temp = line.split("\\s+");
         if (line.contains(Integer.toString(pid)) && temp.length > 8) {
           oldValue = resultMap.get(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
@@ -188,22 +193,22 @@ public class OpenFileNumUtil {
           }
         }
       }
+      if (lineCount < OpenFileNumStatistics.values().length) {
+        isOutputValid = false;
+        for (OpenFileNumStatistics statistics : OpenFileNumStatistics.values()) {
+          resultMap.put(statistics, UNSUPPORTED_OS_ERROR_CODE);
+        }
+      } else {
+        isOutputValid = true;
+      }
       in.close();
       pro.destroy();
     } catch (Exception e) {
-      log.error("Cannot get open file number of IoTDB process.", e);
+      LOGGER.error("Cannot get open file number of IoTDB process because {}", e.getMessage());
     }
     return resultMap;
   }
 
-  private long getCurrentOpenedFiles() {
-    if(os instanceof UnixOperatingSystemMXBean){
-      return ((UnixOperatingSystemMXBean) os).getOpenFileDescriptorCount();
-    }else {
-      return UNSUPPORTED_OS_ERROR_CODE;
-    }
-  }
-
   /**
    * Check if runtime OS is supported then return the result list. If pid is abnormal then all
    * statistics returns -1, if OS is not supported then all statistics returns -2
@@ -240,13 +245,14 @@ public class OpenFileNumUtil {
    * @return open file number
    */
   public int get(OpenFileNumStatistics statistics) {
-    if(statistics.equals(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM)){
-      return (int) getCurrentOpenedFiles();
-    }
     EnumMap<OpenFileNumStatistics, Integer> statisticsMap = getStatisticMap();
     return statisticsMap.getOrDefault(statistics, UNKNOWN_STATISTICS_ERROR_CODE);
   }
 
+  boolean isCommandValid() {
+    return isOutputValid;
+  }
+
   public enum OpenFileNumStatistics {
     TOTAL_OPEN_FILE_NUM(null),
     DATA_OPEN_FILE_NUM(Collections.singletonList(config.dataDir)),
@@ -271,7 +277,10 @@ public class OpenFileNumUtil {
   }
 
   private static class OpenFileNumUtilHolder {
-    private OpenFileNumUtilHolder(){}
+
+    private OpenFileNumUtilHolder() {
+    }
+
     private static final OpenFileNumUtil INSTANCE = new OpenFileNumUtil();
   }
 
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
index efab193..b2fbec3 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/OpenFileNumUtilTest.java
@@ -16,16 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.iotdb.db.utils;
 
 import static org.junit.Assert.assertEquals;
 
-import com.sun.management.UnixOperatingSystemMXBean;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.lang.management.OperatingSystemMXBean;
 import java.lang.management.RuntimeMXBean;
 import java.nio.file.Files;
 import java.util.ArrayList;
@@ -43,6 +42,7 @@ public class OpenFileNumUtilTest {
   private static final String TEST_FILE_PREFIX = "testFileForOpenFileNumUtil";
   private static final String MAC_OS_NAME = "mac";
   private static final String LINUX_OS_NAME = "linux";
+  private static final int UNSUPPORTED_OS_ERROR_CODE = -2;
   private OpenFileNumUtil openFileNumUtil = OpenFileNumUtil.getInstance();
   private ArrayList<File> fileList = new ArrayList<>();
   private ArrayList<FileWriter> fileWriterList = new ArrayList<>();
@@ -57,9 +57,7 @@ public class OpenFileNumUtilTest {
 
   private static int getProcessID() {
     RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
-    int pid = Integer.parseInt(runtimeMXBean.getName().split("@")[0]);
-    System.out.println("Test process ID of OpenFileNumUtilTest is " + pid);
-    return pid;
+    return Integer.parseInt(runtimeMXBean.getName().split("@")[0]);
   }
 
   @Before
@@ -72,10 +70,8 @@ public class OpenFileNumUtilTest {
     testDataDirRoot = new File(currDir);
     currDir = currDir + File.separator + dataFilePath;
     File testDataDir = new File(currDir);
-    if(!testDataDir.isDirectory()){
-      if (!testDataDir.mkdirs()) {
-        LOGGER.error("Create test file dir {} failed.", testDataDir.getPath());
-      }
+    if (!testDataDir.isDirectory() && !testDataDir.mkdirs()) {
+      LOGGER.error("Create test file dir {} failed.", testDataDir.getPath());
     }
     testFileName = TEST_FILE_PREFIX + testProcessID;
   }
@@ -111,34 +107,37 @@ public class OpenFileNumUtilTest {
   }
 
   @Test
-  public void testTotalOpenFileNumWhenCreateFile() {
+  public void testDataOpenFileNumWhenCreateFile() {
     if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
-      // get total open file number statistics of original state
-      totalOpenFileNumBefore = openFileNumUtil
-          .get(OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      // get total open file number under /data/data before create new files
+      totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       for (int i = 0; i < testFileNum; i++) {
         fileList.add(new File(currDir + testFileName + i));
       }
       // create testFileNum File, then get total open file number statistics
-      totalOpenFileNumAfter = openFileNumUtil
-          .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
       // create test file shall not affect total open file number statistics
-      assertEquals(0, totalOpenFileNumChange);
+      // if 'lsof' command is valid
+      if (openFileNumUtil.isCommandValid()) {
+        assertEquals(0, totalOpenFileNumChange);
+      } else {
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+      }
     } else {
-      assertEquals(-2,
-          openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+      assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+          openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
     }
   }
 
   @Test
-  public void testTotalOpenFileNumWhenCreateFileWriter() {
+  public void testDataOpenFileNumWhenCreateFileWriter() {
     if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
       for (int i = 0; i < testFileNum; i++) {
-        fileList.add(new File(currDir + File.separator + testFileName + i));
+        fileList.add(new File(currDir + testFileName + i));
       }
-      totalOpenFileNumBefore = openFileNumUtil
-          .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       for (File file : fileList) {
         if (file.exists()) {
           try {
@@ -149,32 +148,34 @@ public class OpenFileNumUtilTest {
         } else {
           try {
             boolean flag = file.createNewFile();
-            if(flag) {
+            if (flag) {
               LOGGER.debug("Create a file {} successfully", file);
               fileWriterList.add(new FileWriter(file));
-            }else{
-              LOGGER.error(
-                  "create test file {} failed when execute testTotalOpenFileNumWhenCreateFileWriter().",
-                  file.getPath());
+            } else {
+              LOGGER.error("create test file {} failed.", file.getPath());
             }
           } catch (IOException e) {
             LOGGER.error(e.getMessage());
           }
         }
       }
-      totalOpenFileNumAfter = openFileNumUtil
-          .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
       // create FileWriter shall cause total open file number increase by testFileNum
-      assertEquals(testFileNum, totalOpenFileNumChange);
+      if (openFileNumUtil.isCommandValid()) {
+        assertEquals(testFileNum, totalOpenFileNumChange);
+      } else {
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+      }
     } else {
-      assertEquals(-2,
-          openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+      assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+          openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
     }
   }
 
   @Test
-  public void testTotalOpenFileNumWhenFileWriterWriting() {
+  public void testDataOpenFileNumWhenFileWriterWriting() {
     LOGGER.debug("testDataOpenFileNumWhenFileWriterWriting...");
     if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
       for (int i = 0; i < testFileNum; i++) {
@@ -202,8 +203,7 @@ public class OpenFileNumUtilTest {
           }
         }
       }
-      totalOpenFileNumBefore = openFileNumUtil
-          .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       for (FileWriter fw : fileWriterList) {
         try {
           fw.write("this is a test file for open file number counting.");
@@ -211,19 +211,23 @@ public class OpenFileNumUtilTest {
           LOGGER.error(e.getMessage());
         }
       }
-      totalOpenFileNumAfter = openFileNumUtil
-          .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
       // writing test file shall not affect total open file number statistics
-      assertEquals(0, totalOpenFileNumChange);
+      if (openFileNumUtil.isCommandValid()) {
+        assertEquals(0, totalOpenFileNumChange);
+      } else {
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+      }
     } else {
-      assertEquals(-2,
-          openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+      assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+          openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
     }
   }
 
   @Test
-  public void testTotalOpenFileNumWhenFileWriterClose() {
+  public void testDataOpenFileNumWhenFileWriterClose() {
     LOGGER.debug("testDataOpenFileNumWhenFileWriterClose...");
     if (os.startsWith(MAC_OS_NAME) || os.startsWith(LINUX_OS_NAME)) {
       for (int i = 0; i < testFileNum; i++) {
@@ -239,9 +243,7 @@ public class OpenFileNumUtilTest {
         } else {
           try {
             if (!file.createNewFile()) {
-              LOGGER.error(
-                  "create test file {} failed when execute testTotalOpenFileNumWhenFileWriterClose().",
-                  file.getPath());
+              LOGGER.error("create test file {} failed.", file.getPath());
             }
           } catch (IOException e) {
             LOGGER.error(e.getMessage());
@@ -260,8 +262,7 @@ public class OpenFileNumUtilTest {
           LOGGER.error(e.getMessage());
         }
       }
-      totalOpenFileNumBefore = openFileNumUtil
-          .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      totalOpenFileNumBefore = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       for (FileWriter fw : fileWriterList) {
         try {
           fw.close();
@@ -269,15 +270,18 @@ public class OpenFileNumUtilTest {
           LOGGER.error(e.getMessage());
         }
       }
-      totalOpenFileNumAfter = openFileNumUtil
-          .get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM);
+      totalOpenFileNumAfter = openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM);
       totalOpenFileNumChange = totalOpenFileNumAfter - totalOpenFileNumBefore;
       // close FileWriter shall cause total open file number decrease by testFileNum
-      assertEquals(-testFileNum, totalOpenFileNumChange);
+      if (openFileNumUtil.isCommandValid()) {
+        assertEquals(-testFileNum, totalOpenFileNumChange);
+      } else {
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumBefore);
+        assertEquals(UNSUPPORTED_OS_ERROR_CODE, totalOpenFileNumAfter);
+      }
     } else {
-      assertEquals(-2,
-          openFileNumUtil.get(OpenFileNumUtil.OpenFileNumStatistics.TOTAL_OPEN_FILE_NUM));
+      assertEquals(UNSUPPORTED_OS_ERROR_CODE,
+          openFileNumUtil.get(OpenFileNumStatistics.DATA_OPEN_FILE_NUM));
     }
   }
-
 }