You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2007/11/10 18:51:03 UTC
svn commit: r593799 - in /lucene/java/trunk: ./
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/
src/java/org/apache/lucene/store/ src/test/org/apache/lucene/index/
src/test/org/apache/lucene/store/
Author: mikemccand
Date: Sat Nov 10 09:51:00 2007
New Revision: 593799
URL: http://svn.apache.org/viewvc?rev=593799&view=rev
Log:
LUCENE-1044: add doSync option to FSDirectory.getDirectory, defaulting to true, to sync() each file descriptor before close()
Modified:
lucene/java/trunk/CHANGES.txt
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html
lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java
lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java
Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Sat Nov 10 09:51:00 2007
@@ -168,8 +168,15 @@
obtained or released, throwing an exception if an illegal lock
obtain occurred. (Patrick Kimber via Mike McCandless)
- 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs.
- Added support into SortField for sorting on doubles and longs as well. (Grant Ingersoll)
+ 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs.
+ Added support into SortField for sorting on doubles and longs as well. (Grant Ingersoll)
+
+ 7. LUCENE-1044: Added optional doSync boolean to
+ FSDirectory.getDirectory(...). If true (the default) then we will
+ always sync() a file before closing it, which improves the
+ likelihood that the index will remain consistent when the OS or
+ machine crashes, or power to the machine is cut. (Venkat Rangan
+ via Mike McCandless)
Optimizations
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java Sat Nov 10 09:51:00 2007
@@ -126,7 +126,8 @@
FileUtils.fullyDelete(indexDir);
}
indexDir.mkdirs();
- directory = FSDirectory.getDirectory(indexDir);
+ final boolean doSync = config.get("fsdirectory.dosync", true);
+ directory = FSDirectory.getDirectory(indexDir, null, doSync);
} else {
directory = new RAMDirectory();
}
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html Sat Nov 10 09:51:00 2007
@@ -479,6 +479,12 @@
</li>
<li>
+ <font color="#FF0066">fsdirectory.dosync</font> - true/false
+ If "true", the FSDirectory will call sync() before close() on each
+ file.
+ </li>
+
+ <li>
<b>Index work parameters</b>:
Multi int/boolean values would be iterated with calls to NewRound.
There would be also added as columns in the reports, first string in the
@@ -543,6 +549,7 @@
</li><li>merge.factor
</li><li>max.buffered
</li><li>directory
+ </li><li>fsdirectory.dosync
</li><li>ram.flush.mb
</li><li>autocommit
</li></ul>
@@ -676,4 +683,4 @@
</DIV>
<DIV> </DIV>
</BODY>
-</HTML>
\ No newline at end of file
+</HTML>
Modified: lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java Sat Nov 10 09:51:00 2007
@@ -62,6 +62,14 @@
private static boolean disableLocks = false;
+ private static boolean DEFAULT_DO_SYNC = true;
+
+ // True if we should call sync() before closing a file.
+ // This improves chances that index will still be
+ // consistent if the machine or OS abruptly crashes. See
+ // LUCENE-1044.
+ private boolean doSync = DEFAULT_DO_SYNC;
+
// TODO: should this move up to the Directory base class? Also: should we
// make a per-instance (in addition to the static "default") version?
@@ -136,17 +144,34 @@
* @return the FSDirectory for the named file. */
public static FSDirectory getDirectory(String path)
throws IOException {
- return getDirectory(new File(path), null);
+ return getDirectory(new File(path), null, DEFAULT_DO_SYNC);
}
/** Returns the directory instance for the named location.
* @param path the path to the directory.
* @param lockFactory instance of {@link LockFactory} providing the
- * locking implementation.
+ * locking implementation. If null, the default
+ * {@link SimpleFSLockFactory} is used.
* @return the FSDirectory for the named file. */
public static FSDirectory getDirectory(String path, LockFactory lockFactory)
throws IOException {
- return getDirectory(new File(path), lockFactory);
+ return getDirectory(new File(path), lockFactory, DEFAULT_DO_SYNC);
+ }
+
+ /** Returns the directory instance for the named location.
+ * @param path the path to the directory.
+ * @param lockFactory instance of {@link LockFactory} providing the
+ * locking implementation. If null, the default
+ * {@link SimpleFSLockFactory} is used.
+ * @param doSync if true (the default), sync() is called
+ * on all file descriptors before close(). This
+ * improves the likelihood that the index will
+ * remain consistent when the OS or machine crashes
+ * or the power cord is pulled.
+ * @return the FSDirectory for the named file. */
+ public static FSDirectory getDirectory(String path, LockFactory lockFactory, boolean doSync)
+ throws IOException {
+ return getDirectory(new File(path), lockFactory, doSync);
}
/** Returns the directory instance for the named location.
@@ -154,15 +179,32 @@
* @return the FSDirectory for the named file. */
public static FSDirectory getDirectory(File file)
throws IOException {
- return getDirectory(file, null);
+ return getDirectory(file, null, DEFAULT_DO_SYNC);
}
/** Returns the directory instance for the named location.
* @param file the path to the directory.
* @param lockFactory instance of {@link LockFactory} providing the
- * locking implementation.
+ * locking implementation. If null, the default
+ * {@link SimpleFSLockFactory} is used.
* @return the FSDirectory for the named file. */
public static FSDirectory getDirectory(File file, LockFactory lockFactory)
+ throws IOException {
+ return getDirectory(file, lockFactory, DEFAULT_DO_SYNC);
+ }
+
+ /** Returns the directory instance for the named location.
+ * @param file the path to the directory.
+ * @param lockFactory instance of {@link LockFactory} providing the
+ * locking implementation. If null, the default
+ * {@link SimpleFSLockFactory} is used.
+ * @param doSync if true (the default), sync() is called
+ * on all file descriptors before close(). This
+ * improves the likelihood that the index will
+ * remain consistent when the OS or machine crashes
+ * or the power cord is pulled.
+ * @return the FSDirectory for the named file. */
+ public static FSDirectory getDirectory(File file, LockFactory lockFactory, boolean doSync)
throws IOException
{
file = new File(file.getCanonicalPath());
@@ -183,7 +225,7 @@
} catch (Exception e) {
throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e);
}
- dir.init(file, lockFactory);
+ dir.init(file, lockFactory, doSync);
DIRECTORIES.put(file, dir);
} else {
// Catch the case where a Directory is pulled from the cache, but has a
@@ -254,7 +296,7 @@
protected FSDirectory() {}; // permit subclassing
- private void init(File path, LockFactory lockFactory) throws IOException {
+ private void init(File path, LockFactory lockFactory, boolean doSync) throws IOException {
// Set up lockFactory with cascaded defaults: if an instance was passed in,
// use that; else if locks are disabled, use NoLockFactory; else if the
@@ -262,6 +304,7 @@
// instantiate that; else, use SimpleFSLockFactory:
directory = path;
+ this.doSync = doSync;
boolean doClearLockID = false;
@@ -432,7 +475,7 @@
if (file.exists() && !file.delete()) // delete existing, if any
throw new IOException("Cannot overwrite: " + file);
- return new FSIndexOutput(file);
+ return new FSIndexOutput(file, doSync);
}
// Inherit javadoc
@@ -588,10 +631,13 @@
// remember if the file is open, so that we don't try to close it
// more than once
private boolean isOpen;
-
- public FSIndexOutput(File path) throws IOException {
+ private boolean doSync;
+
+ public FSIndexOutput(File path, boolean doSync) throws IOException {
file = new RandomAccessFile(path, "rw");
+
isOpen = true;
+ this.doSync = doSync;
}
/** output methods: */
@@ -601,9 +647,14 @@
public void close() throws IOException {
// only close the file if it has not been closed yet
if (isOpen) {
- super.close();
- file.close();
- isOpen = false;
+ try {
+ super.close();
+ if (doSync)
+ file.getFD().sync();
+ } finally {
+ file.close();
+ isOpen = false;
+ }
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java Sat Nov 10 09:51:00 2007
@@ -177,7 +177,7 @@
// Second in an FSDirectory:
String tempDir = System.getProperty("java.io.tmpdir");
File dirPath = new File(tempDir, "lucene.test.atomic");
- directory = FSDirectory.getDirectory(dirPath);
+ directory = FSDirectory.getDirectory(dirPath, null, false);
runTest(directory);
directory.close();
_TestUtil.rmDir(dirPath);
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sat Nov 10 09:51:00 2007
@@ -152,7 +152,7 @@
dirName = fullDir(dirName);
- Directory dir = FSDirectory.getDirectory(dirName);
+ Directory dir = FSDirectory.getDirectory(dirName, null, false);
IndexSearcher searcher = new IndexSearcher(dir);
Hits hits = searcher.search(new TermQuery(new Term("content", "aaa")));
@@ -172,7 +172,7 @@
dirName = fullDir(dirName);
- Directory dir = FSDirectory.getDirectory(dirName);
+ Directory dir = FSDirectory.getDirectory(dirName, null, false);
// open writer
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
@@ -232,7 +232,7 @@
dirName = fullDir(dirName);
- Directory dir = FSDirectory.getDirectory(dirName);
+ Directory dir = FSDirectory.getDirectory(dirName, null, false);
// make sure searching sees right # hits
IndexSearcher searcher = new IndexSearcher(dir);
@@ -280,7 +280,7 @@
dirName = fullDir(dirName);
- Directory dir = FSDirectory.getDirectory(dirName);
+ Directory dir = FSDirectory.getDirectory(dirName, null, false);
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
writer.setUseCompoundFile(doCFS);
@@ -311,7 +311,7 @@
rmDir(outputDir);
try {
- Directory dir = FSDirectory.getDirectory(fullDir(outputDir));
+ Directory dir = FSDirectory.getDirectory(fullDir(outputDir), null, false);
boolean autoCommit = 0 == pass;
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java Sat Nov 10 09:51:00 2007
@@ -62,7 +62,7 @@
super.setUp();
File file = new File(System.getProperty("tempDir"), "testIndex");
_TestUtil.rmDir(file);
- dir = FSDirectory.getDirectory(file);
+ dir = FSDirectory.getDirectory(file, null, false);
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java Sat Nov 10 09:51:00 2007
@@ -147,7 +147,7 @@
if (tempDir == null)
throw new IOException("java.io.tmpdir undefined, cannot run test");
File indexDir = new File(tempDir, "lucenetestindex");
- Directory rd = FSDirectory.getDirectory(indexDir);
+ Directory rd = FSDirectory.getDirectory(indexDir, null, false);
IndexThread.id = 0;
IndexThread.idStack.clear();
IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create);
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java Sat Nov 10 09:51:00 2007
@@ -81,7 +81,7 @@
// test with a single index: index1
File indexDir1 = new File(tempDir, "lucenetestindex1");
- Directory dir1 = FSDirectory.getDirectory(indexDir1);
+ Directory dir1 = FSDirectory.getDirectory(indexDir1, null, false);
norms = new ArrayList();
modifiedNorms = new ArrayList();
@@ -99,14 +99,14 @@
numDocNorms = 0;
File indexDir2 = new File(tempDir, "lucenetestindex2");
- Directory dir2 = FSDirectory.getDirectory(indexDir2);
+ Directory dir2 = FSDirectory.getDirectory(indexDir2, null, false);
createIndex(dir2);
doTestNorms(dir2);
// add index1 and index2 to a third index: index3
File indexDir3 = new File(tempDir, "lucenetestindex3");
- Directory dir3 = FSDirectory.getDirectory(indexDir3);
+ Directory dir3 = FSDirectory.getDirectory(indexDir3, null, false);
createIndex(dir3);
IndexWriter iw = new IndexWriter(dir3,anlzr,false);
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java Sat Nov 10 09:51:00 2007
@@ -178,7 +178,7 @@
// FSDir
String tempDir = System.getProperty("java.io.tmpdir");
File dirPath = new File(tempDir, "lucene.test.stress");
- directory = FSDirectory.getDirectory(dirPath);
+ directory = FSDirectory.getDirectory(dirPath, null, false);
runStressTest(directory, true, null);
directory.close();
@@ -188,7 +188,7 @@
directory.close();
// With ConcurrentMergeScheduler, in FSDir
- directory = FSDirectory.getDirectory(dirPath);
+ directory = FSDirectory.getDirectory(dirPath, null, false);
runStressTest(directory, true, new ConcurrentMergeScheduler());
directory.close();
@@ -198,7 +198,7 @@
directory.close();
// With ConcurrentMergeScheduler and autoCommit=false, in FSDir
- directory = FSDirectory.getDirectory(dirPath);
+ directory = FSDirectory.getDirectory(dirPath, null, false);
runStressTest(directory, false, new ConcurrentMergeScheduler());
directory.close();
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java Sat Nov 10 09:51:00 2007
@@ -149,7 +149,7 @@
throw new IOException("tempDir undefined, cannot run test");
String dirName = tempDir + "/luceneTestThreadedOptimize";
- directory = FSDirectory.getDirectory(dirName);
+ directory = FSDirectory.getDirectory(dirName, null, false);
runTest(directory, false, null);
runTest(directory, true, null);
runTest(directory, false, new ConcurrentMergeScheduler());
Modified: lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java Sat Nov 10 09:51:00 2007
@@ -320,7 +320,7 @@
}
public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws IOException {
- FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory);
+ FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory, false);
// First create a 1 doc index:
IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true);