svn commit: r1687883 - in /lucene/dev/trunk/lucene: CHANGES.txt core/src/java/org/apache/lucene/util/IOUtils.java core/src/test/org/apache/lucene/util/TestIOUtils.java tools/junit4/tests.policy

[us...@apache.org]

Author: uschindler
Date: Sat Jun 27 03:35:00 2015
New Revision: 1687883

URL: http://svn.apache.org/r1687883
Log:
LUCENE-6614: Improve partition detection in IOUtils#spins() so it works with NVMe drives

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
    lucene/dev/trunk/lucene/tools/junit4/tests.policy

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1687883&r1=1687882&r2=1687883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Sat Jun 27 03:35:00 2015
@@ -179,6 +179,9 @@ Bug fixes
 * LUCENE-6608: Fix potential resource leak in BigramDictionary.
   (Rishabh Patel via Uwe Schindler)
 
+* LUCENE-6614: Improve partition detection in IOUtils#spins() so it
+  works with NVMe drives.  (Uwe Schindler, Mike McCandless)
+
 Changes in Runtime Behavior
 
 * LUCENE-6501: The subreader structure in ParallelCompositeReader

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1687883&r1=1687882&r2=1687883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Sat Jun 27 03:35:00 2015
@@ -28,6 +28,7 @@ import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CodingErrorAction;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
 import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
@@ -493,23 +494,28 @@ public final class IOUtils {
     // /devices/XXX -> sda0
     devName = path.getRoot().resolve(devName).toRealPath().getFileName().toString();
   
-    // now read:
-    Path sysinfo = path.getRoot().resolve("sys/block");
-    Path devinfo = sysinfo.resolve(devName);
-    
-    // tear away partition numbers until we find it.
-    while (!Files.exists(devinfo)) {
-      if (!devName.isEmpty() && Character.isDigit(devName.charAt(devName.length()-1))) {
-        devName = devName.substring(0, devName.length()-1);
-      } else {
-        break; // give up
+    // now try to find the longest matching device folder in /sys/block
+    // (that starts with our dev name):
+    Path sysinfo = path.getRoot().resolve("sys").resolve("block");
+    Path devsysinfo = null;
+    int matchlen = 0;
+    try (DirectoryStream<Path> stream = Files.newDirectoryStream(sysinfo)) {
+      for (Path device : stream) {
+        String name = device.getFileName().toString();
+        if (name.length() > matchlen && devName.startsWith(name)) {
+          devsysinfo = device;
+          matchlen = name.length();
+        }
       }
-      devinfo = sysinfo.resolve(devName);
+    }
+    
+    if (devsysinfo == null) {
+      return true; // give up
     }
     
     // read first byte from rotational, it's a 1 if it spins.
-    Path info = devinfo.resolve("queue/rotational");
-    try (InputStream stream = Files.newInputStream(info)) {
+    Path rotational = devsysinfo.resolve("queue").resolve("rotational");
+    try (InputStream stream = Files.newInputStream(rotational)) {
       return stream.read() == '1'; 
     }
   }

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java?rev=1687883&r1=1687882&r2=1687883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java Sat Jun 27 03:35:00 2015
@@ -32,6 +32,8 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
 
 import org.apache.lucene.mockfile.FilterFileSystem;
 import org.apache.lucene.mockfile.FilterFileSystemProvider;
@@ -353,6 +355,34 @@ public class TestIOUtils extends LuceneT
     assertFalse(IOUtils.spinsLinux(mockPath));
   }
   
+  public void testNVME() throws Exception {
+    assumeFalse("windows is not supported", Constants.WINDOWS);
+    Path dir = createTempDir();
+    dir = FilterPath.unwrap(dir).toRealPath();
+    
+    // fake ssd
+    FileStore root = new MockFileStore(dir.toString() + " (/dev/nvme0n1p1)", "btrfs", "/dev/nvme0n1p1");
+    // make a fake /dev/nvme0n1p1 for it
+    Path devdir = dir.resolve("dev");
+    Files.createDirectories(devdir);
+    Files.createFile(devdir.resolve("nvme0n1p1"));
+    // make a fake /sys/block/nvme0n1/queue/rotational file for it
+    Path sysdir = dir.resolve("sys").resolve("block").resolve("nvme0n1").resolve("queue");
+    Files.createDirectories(sysdir);
+    try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
+      o.write("0\n".getBytes(StandardCharsets.US_ASCII));
+    }
+    // As test for the longest path match, add some other devices (that have no queue/rotational), too:
+    Files.createFile(dir.resolve("sys").resolve("block").resolve("nvme0"));
+    Files.createFile(dir.resolve("sys").resolve("block").resolve("dummy"));
+    Files.createFile(dir.resolve("sys").resolve("block").resolve("nvm"));
+    Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
+    FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+    
+    Path mockPath = mockLinux.getPath(dir.toString());
+    assertFalse(IOUtils.spinsLinux(mockPath));
+  }
+  
   public void testRotatingPlatters() throws Exception {
     assumeFalse("windows is not supported", Constants.WINDOWS);
     Path dir = createTempDir();
@@ -393,6 +423,41 @@ public class TestIOUtils extends LuceneT
     Files.createDirectories(sysdir);
     try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
       o.write("0\n".getBytes(StandardCharsets.US_ASCII));
+    }
+    Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
+    FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+    
+    Path mockPath = mockLinux.getPath(dir.toString());
+    assertFalse(IOUtils.spinsLinux(mockPath));
+  }
+  
+  public void testSymlinkSSD() throws Exception {
+    assumeFalse("windows is not supported", Constants.WINDOWS);
+    Path dir = createTempDir();
+    dir = FilterPath.unwrap(dir).toRealPath();
+    
+    // fake SSD with a symlink mount (Ubuntu-like):
+    Random rnd = random();
+    String partitionUUID = new UUID(rnd.nextLong(), rnd.nextLong()).toString();
+    FileStore root = new MockFileStore(dir.toString() + " (/dev/disk/by-uuid/"+partitionUUID+")", "btrfs", "/dev/disk/by-uuid/"+partitionUUID);
+    // make a fake /dev/sda1 for it
+    Path devdir = dir.resolve("dev");
+    Files.createDirectories(devdir);
+    Path deviceFile = devdir.resolve("sda1");
+    Files.createFile(deviceFile);
+    // create a symlink to the above device file
+    Path symlinkdir = devdir.resolve("disk").resolve("by-uuid");
+    Files.createDirectories(symlinkdir);
+    try {
+      Files.createSymbolicLink(symlinkdir.resolve(partitionUUID), deviceFile);
+    } catch (UnsupportedOperationException | IOException e) {
+      assumeNoException("test requires filesystem that supports symbolic links", e);
+    }
+    // make a fake /sys/block/sda/queue/rotational file for it
+    Path sysdir = dir.resolve("sys").resolve("block").resolve("sda").resolve("queue");
+    Files.createDirectories(sysdir);
+    try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
+      o.write("0\n".getBytes(StandardCharsets.US_ASCII));
     }
     Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
     FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);

Modified: lucene/dev/trunk/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/tools/junit4/tests.policy?rev=1687883&r1=1687882&r2=1687883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/trunk/lucene/tools/junit4/tests.policy Sat Jun 27 03:35:00 2015
@@ -38,6 +38,9 @@ grant {
   permission java.io.FilePermission "${junit4.childvm.cwd}${/}jacoco.db", "write";
   permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
   permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
+  
+  // needed by SSD detection tests in TestIOUtils (creates symlinks)
+  permission java.nio.file.LinkPermission "symbolic";
 
   // needed by gson serialization of junit4 runner: TODO clean that up
   permission java.lang.RuntimePermission "accessDeclaredMembers";