svn commit: r1586407 - in /lucene/dev/trunk/lucene: CHANGES.txt core/src/java/org/apache/lucene/store/FSDirectory.java core/src/java/org/apache/lucene/util/IOUtils.java

[us...@apache.org]

Author: uschindler
Date: Thu Apr 10 18:46:33 2014
New Revision: 1586407

URL: http://svn.apache.org/r1586407
Log:
LUCENE-5588: Lucene now calls fsync() on the index directory, ensuring that all file metadata is persisted on disk in case of power failure.

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1586407&r1=1586406&r2=1586407&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Apr 10 18:46:33 2014
@@ -166,6 +166,12 @@ New Features
 * LUCENE-5583: Added DataInput.skipBytes. ChecksumIndexInput can now seek, but
   only forward. (Adrien Grand, Mike McCandless, Simon Willnauer, Uwe Schindler)
 
+* LUCENE-5588: Lucene now calls fsync() on the index directory, ensuring
+  that all file metadata is persisted on disk in case of power failure.
+  This does not work on all file systems and operating systems, but Linux
+  and MacOSX are known to work. On Windows, fsyncing a directory is not
+  possible with Java APIs.  (Mike McCandless, Uwe Schindler)
+
 API Changes
 
 * LUCENE-5454: Add RandomAccessOrds, an optional extension of SortedSetDocValues

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1586407&r1=1586406&r2=1586407&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Thu Apr 10 18:46:33 2014
@@ -291,7 +291,9 @@ public abstract class FSDirectory extend
 
     for (String name : toSync)
       fsync(name);
-
+    
+    IOUtils.fsync(directory, true);
+    
     staleFiles.removeAll(toSync);
   }
 
@@ -395,6 +397,6 @@ public abstract class FSDirectory extend
   }
 
   protected void fsync(String name) throws IOException {
-    IOUtils.fsync(new File(directory, name));
+    IOUtils.fsync(new File(directory, name), false);
   }
 }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1586407&r1=1586406&r2=1586407&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Thu Apr 10 18:46:33 2014
@@ -372,29 +372,47 @@ public final class IOUtils {
   /**
    * Ensure that any writes to the given file is written to the storage device that contains it.
    * @param fileToSync the file to fsync
+   * @param isDir if true, the given file is a directory (we open for read and ignore IOExceptions,
+   *  because not all file systems and operating systems allow to fsync on a directory)
    */
-  public static void fsync(File fileToSync) throws IOException {
+  public static void fsync(File fileToSync, boolean isDir) throws IOException {
     IOException exc = null;
-    for (int retry = 0; retry < 5; retry++) {
-      try {
-        try (FileChannel file = FileChannel.open(fileToSync.toPath(), StandardOpenOption.WRITE)) {
-          file.force(true); // TODO: we probably dont care about metadata, but this is what we did before...
-          return;
-        }
-      } catch (IOException ioe) {
-        if (exc == null) {
-          exc = ioe;
-        }
+    
+    // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect.
+    // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
+    try (final FileChannel file = FileChannel.open(fileToSync.toPath(), isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) {
+      for (int retry = 0; retry < 5; retry++) {
         try {
-          // Pause 5 msec
-          Thread.sleep(5);
-        } catch (InterruptedException ie) {
-          ThreadInterruptedException ex = new ThreadInterruptedException(ie);
-          ex.addSuppressed(exc);
-          throw ex;
+          file.force(true);
+          return;
+        } catch (IOException ioe) {
+          if (exc == null) {
+            exc = ioe;
+          }
+          try {
+            // Pause 5 msec
+            Thread.sleep(5L);
+          } catch (InterruptedException ie) {
+            ThreadInterruptedException ex = new ThreadInterruptedException(ie);
+            ex.addSuppressed(exc);
+            throw ex;
+          }
         }
       }
+    } catch (IOException ioe) {
+      if (exc == null) {
+        exc = ioe;
+      }
+    }
+    
+    if (isDir) {
+      assert (Constants.LINUX || Constants.MAC_OS_X) == false :
+        "On Linux and MacOSX fsyncing a directory should not throw IOException, "+
+        "we just don't want to rely on that in production (undocumented). Got: " + exc;
+      // Ignore exception if it is a directory
+      return;
     }
+    
     // Throw original exception
     throw exc;
   }