You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/10/15 11:58:19 UTC
svn commit: r1708760 [1/4] - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/analysis/common/src/test/org/apache/lucene/analysis/core/
lucene/analysis/common/src/test/org/apache/lucene/analysis/...
Author: mikemccand
Date: Thu Oct 15 09:58:18 2015
New Revision: 1708760
URL: http://svn.apache.org/viewvc?rev=1708760&view=rev
Log:
LUCENE-6829: OfflineSorter now uses Directory API; add Directory.createTempOutput and IndexOutput.getName
Removed:
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/bkdtree/BKDTreeDocValuesConsumer.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/bkdtree/BKDTreeDocValuesFormat.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/bkdtree/BKDTreeWriter.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/bkdtree/OfflineLatLonReader.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/bkdtree/OfflineLatLonWriter.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/rangetree/OfflineSliceReader.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/rangetree/OfflineSliceWriter.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/rangetree/RangeTreeDocValuesConsumer.java
lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/rangetree/RangeTreeWriter.java
lucene/dev/trunk/lucene/sandbox/src/test/org/apache/lucene/bkdtree/TestBKDTree.java
lucene/dev/trunk/lucene/sandbox/src/test/org/apache/lucene/rangetree/TestRangeTree.java
lucene/dev/trunk/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/BKD3DTreeWriter.java
lucene/dev/trunk/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/Geo3DDocValuesConsumer.java
lucene/dev/trunk/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/Geo3DDocValuesFormat.java
lucene/dev/trunk/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/OfflineReader.java
lucene/dev/trunk/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/OfflineWriter.java
lucene/dev/trunk/lucene/spatial3d/src/test/org/apache/lucene/bkdtree3d/TestGeo3DPointField.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingLookupFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Oct 15 09:58:18 2015
@@ -59,6 +59,13 @@ API Changes
* LUCENE-6706: PayloadTermQuery and PayloadNearQuery have been removed.
Instead, use PayloadScoreQuery to wrap any SpanQuery. (Alan Woodward)
+* LUCENE-6829: OfflineSorter, and the classes that use it (suggesters,
+ hunspell) now do all temporary file IO via Directory instead of
+ directly through java's temp dir. Directory.createTempOutput
+ creates a uniquely named IndexOutput, and the new
+ IndexOutput.getName returns its name (Dawid Weiss, Robert Muir, Mike
+ McCandless)
+
Changes in Runtime Behavior
* LUCENE-6789: IndexSearcher's default Similarity is changed to BM25Similarity.
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java Thu Oct 15 09:58:18 2015
@@ -17,28 +17,6 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
-import org.apache.lucene.store.ByteArrayDataOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.IntsRefBuilder;
-import org.apache.lucene.util.OfflineSorter;
-import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;
-import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.RegExp;
-import org.apache.lucene.util.fst.Builder;
-import org.apache.lucene.util.fst.CharSequenceOutputs;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.IntSequenceOutputs;
-import org.apache.lucene.util.fst.Outputs;
-import org.apache.lucene.util.fst.Util;
-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
@@ -53,6 +31,7 @@ import java.nio.charset.CodingErrorActio
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -67,6 +46,31 @@ import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
+import org.apache.lucene.util.OfflineSorter;
+import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;
+import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.RegExp;
+import org.apache.lucene.util.fst.Builder;
+import org.apache.lucene.util.fst.CharSequenceOutputs;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.IntSequenceOutputs;
+import org.apache.lucene.util.fst.Outputs;
+import org.apache.lucene.util.fst.Util;
+
/**
* In-memory structure for the dictionary (.dic) and affix (.aff)
* data of a hunspell dictionary.
@@ -139,7 +143,7 @@ public class Dictionary {
// when set, some words have exceptional stems, and the last entry is a pointer to stemExceptions
boolean hasStemExceptions;
- private final Path tempDir = OfflineSorter.getDefaultTempDir(); // TODO: make this configurable?
+ private final Path tempPath = getDefaultTempDir(); // TODO: make this configurable?
boolean ignoreCase;
boolean complexPrefixes;
@@ -167,19 +171,21 @@ public class Dictionary {
String language;
// true if case algorithms should use alternate (Turkish/Azeri) mapping
boolean alternateCasing;
-
+
/**
* Creates a new Dictionary containing the information read from the provided InputStreams to hunspell affix
* and dictionary files.
* You have to close the provided InputStreams yourself.
*
+ * @param tempDir Directory to use for offline sorting
+ * @param tempFileNamePrefix prefix to use to generate temp file names
* @param affix InputStream for reading the hunspell affix file (won't be closed).
* @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @throws IOException Can be thrown while reading from the InputStreams
* @throws ParseException Can be thrown if the content of the files does not meet expected formats
*/
- public Dictionary(InputStream affix, InputStream dictionary) throws IOException, ParseException {
- this(affix, Collections.singletonList(dictionary), false);
+ public Dictionary(Directory tempDir, String tempFileNamePrefix, InputStream affix, InputStream dictionary) throws IOException, ParseException {
+ this(tempDir, tempFileNamePrefix, affix, Collections.singletonList(dictionary), false);
}
/**
@@ -187,18 +193,20 @@ public class Dictionary {
* and dictionary files.
* You have to close the provided InputStreams yourself.
*
+ * @param tempDir Directory to use for offline sorting
+ * @param tempFileNamePrefix prefix to use to generate temp file names
* @param affix InputStream for reading the hunspell affix file (won't be closed).
* @param dictionaries InputStream for reading the hunspell dictionary files (won't be closed).
* @throws IOException Can be thrown while reading from the InputStreams
* @throws ParseException Can be thrown if the content of the files does not meet expected formats
*/
- public Dictionary(InputStream affix, List<InputStream> dictionaries, boolean ignoreCase) throws IOException, ParseException {
+ public Dictionary(Directory tempDir, String tempFileNamePrefix, InputStream affix, List<InputStream> dictionaries, boolean ignoreCase) throws IOException, ParseException {
this.ignoreCase = ignoreCase;
this.needsInputCleaning = ignoreCase;
this.needsOutputCleaning = false; // set if we have an OCONV
flagLookup.add(new BytesRef()); // no flags -> ord 0
- Path aff = Files.createTempFile(tempDir, "affix", "aff");
+ Path aff = Files.createTempFile(tempPath, "affix", "aff");
OutputStream out = new BufferedOutputStream(Files.newOutputStream(aff));
InputStream aff1 = null;
InputStream aff2 = null;
@@ -224,7 +232,7 @@ public class Dictionary {
// read dictionary entries
IntSequenceOutputs o = IntSequenceOutputs.getSingleton();
Builder<IntsRef> b = new Builder<>(FST.INPUT_TYPE.BYTE4, o);
- readDictionaryFiles(dictionaries, decoder, b);
+ readDictionaryFiles(tempDir, tempFileNamePrefix, dictionaries, decoder, b);
words = b.finish();
aliases = null; // no longer needed
morphAliases = null; // no longer needed
@@ -766,7 +774,7 @@ public class Dictionary {
return Math.max(pos1, pos2);
}
}
-
+
/**
* Reads the dictionary file through the provided InputStreams, building up the words map
*
@@ -774,13 +782,13 @@ public class Dictionary {
* @param decoder CharsetDecoder used to decode the contents of the file
* @throws IOException Can be thrown while reading from the file
*/
- private void readDictionaryFiles(List<InputStream> dictionaries, CharsetDecoder decoder, Builder<IntsRef> words) throws IOException {
+ private void readDictionaryFiles(Directory tempDir, String tempFileNamePrefix, List<InputStream> dictionaries, CharsetDecoder decoder, Builder<IntsRef> words) throws IOException {
BytesRefBuilder flagsScratch = new BytesRefBuilder();
IntsRefBuilder scratchInts = new IntsRefBuilder();
StringBuilder sb = new StringBuilder();
-
- Path unsorted = Files.createTempFile(tempDir, "unsorted", "dat");
+
+ IndexOutput unsorted = tempDir.createTempOutput(tempFileNamePrefix, "dat", IOContext.DEFAULT);
try (ByteSequencesWriter writer = new ByteSequencesWriter(unsorted)) {
for (InputStream dictionary : dictionaries) {
BufferedReader lines = new BufferedReader(new InputStreamReader(dictionary, decoder));
@@ -823,9 +831,8 @@ public class Dictionary {
}
}
}
- Path sorted = Files.createTempFile(tempDir, "sorted", "dat");
-
- OfflineSorter sorter = new OfflineSorter(new Comparator<BytesRef>() {
+
+ OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix, new Comparator<BytesRef>() {
BytesRef scratch1 = new BytesRef();
BytesRef scratch2 = new BytesRef();
@@ -862,21 +869,23 @@ public class Dictionary {
}
}
});
+
+ String sorted;
boolean success = false;
try {
- sorter.sort(unsorted, sorted);
+ sorted = sorter.sort(unsorted.getName());
success = true;
} finally {
if (success) {
- Files.delete(unsorted);
+ tempDir.deleteFile(unsorted.getName());
} else {
- IOUtils.deleteFilesIgnoringExceptions(unsorted);
+ IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName());
}
}
boolean success2 = false;
- ByteSequencesReader reader = new ByteSequencesReader(sorted);
- try {
+
+ try (ByteSequencesReader reader = new ByteSequencesReader(tempDir.openInput(sorted, IOContext.READONCE))) {
BytesRefBuilder scratchLine = new BytesRefBuilder();
// TODO: the flags themselves can be double-chars (long) or also numeric
@@ -956,11 +965,10 @@ public class Dictionary {
words.add(scratchInts.get(), currentOrds.get());
success2 = true;
} finally {
- IOUtils.closeWhileHandlingException(reader);
if (success2) {
- Files.delete(sorted);
+ tempDir.deleteFile(sorted);
} else {
- IOUtils.deleteFilesIgnoringExceptions(sorted);
+ IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted);
}
}
}
@@ -1245,4 +1253,33 @@ public class Dictionary {
public boolean getIgnoreCase() {
return ignoreCase;
}
+
+ private static Path DEFAULT_TEMP_DIR;
+
+ /** Used by test framework */
+ public static void setDefaultTempDir(Path tempDir) {
+ DEFAULT_TEMP_DIR = tempDir;
+ }
+
+ /**
+ * Returns the default temporary directory. By default, java.io.tmpdir. If not accessible
+ * or not available, an IOException is thrown
+ */
+ synchronized static Path getDefaultTempDir() throws IOException {
+ if (DEFAULT_TEMP_DIR == null) {
+ // Lazy init
+ String tempDirPath = System.getProperty("java.io.tmpdir");
+ if (tempDirPath == null) {
+ throw new IOException("Java has no temporary folder property (java.io.tmpdir)?");
+ }
+ Path tempDirectory = Paths.get(tempDirPath);
+ if (Files.isWritable(tempDirectory) == false) {
+ throw new IOException("Java's temporary folder not present or writeable?: "
+ + tempDirectory.toAbsolutePath());
+ }
+ DEFAULT_TEMP_DIR = tempDirectory;
+ }
+
+ return DEFAULT_TEMP_DIR;
+ }
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java Thu Oct 15 09:58:18 2015
@@ -19,6 +19,8 @@ package org.apache.lucene.analysis.hunsp
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
@@ -28,6 +30,8 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.IOUtils;
/**
@@ -48,6 +52,7 @@ import org.apache.lucene.util.IOUtils;
public class HunspellStemFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
private static final String PARAM_DICTIONARY = "dictionary";
private static final String PARAM_AFFIX = "affix";
+ // NOTE: this one is currently unused?:
private static final String PARAM_RECURSION_CAP = "recursionCap";
private static final String PARAM_IGNORE_CASE = "ignoreCase";
private static final String PARAM_LONGEST_ONLY = "longestOnly";
@@ -91,7 +96,12 @@ public class HunspellStemFilterFactory e
}
affix = loader.openResource(affixFile);
- this.dictionary = new Dictionary(affix, dictionaries, ignoreCase);
+ Path tempPath = Files.createTempDirectory(Dictionary.getDefaultTempDir(), "Hunspell");
+ try (Directory tempDir = FSDirectory.open(tempPath)) {
+ this.dictionary = new Dictionary(tempDir, "hunspell", affix, dictionaries, ignoreCase);
+ } finally {
+ IOUtils.rm(tempPath);
+ }
} catch (ParseException e) {
throw new IOException("Unable to load hunspell data! [dictionary=" + dictionaries + ",affix=" + affixFile + "]", e);
} finally {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Thu Oct 15 09:58:18 2015
@@ -85,6 +85,7 @@ import org.apache.lucene.analysis.synony
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
+import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.CharsRef;
@@ -435,7 +436,7 @@ public class TestRandomChains extends Ba
InputStream affixStream = TestHunspellStemFilter.class.getResourceAsStream("simple.aff");
InputStream dictStream = TestHunspellStemFilter.class.getResourceAsStream("simple.dic");
try {
- return new Dictionary(affixStream, dictStream);
+ return new Dictionary(new RAMDirectory(), "dictionary", affixStream, dictStream);
} catch (Exception ex) {
Rethrow.rethrow(ex);
return null; // unreachable code
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java Thu Oct 15 09:58:18 2015
@@ -24,6 +24,7 @@ import java.text.ParseException;
import java.util.Arrays;
import java.util.List;
+import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -61,7 +62,7 @@ public abstract class StemmerTestBase ex
}
try {
- Dictionary dictionary = new Dictionary(affixStream, Arrays.asList(dictStreams), ignoreCase);
+ Dictionary dictionary = new Dictionary(new RAMDirectory(), "dictionary", affixStream, Arrays.asList(dictStreams), ignoreCase);
stemmer = new Stemmer(dictionary);
} finally {
IOUtils.closeWhileHandlingException(affixStream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java Thu Oct 15 09:58:18 2015
@@ -24,6 +24,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -51,8 +53,11 @@ public class Test64kAffixes extends Luce
dictWriter.write("1\ndrink/2\n");
dictWriter.close();
- try (InputStream affStream = Files.newInputStream(affix); InputStream dictStream = Files.newInputStream(dict)) {
- Dictionary dictionary = new Dictionary(affStream, dictStream);
+ try (InputStream affStream = Files.newInputStream(affix); InputStream dictStream = Files.newInputStream(dict); Directory tempDir2 = newDirectory()) {
+ if (tempDir2 instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) tempDir2).setEnableVirusScanner(false);
+ }
+ Dictionary dictionary = new Dictionary(tempDir2, "dictionary", affStream, dictStream);
Stemmer stemmer = new Stemmer(dictionary);
// drinks should still stem to drink
List<CharsRef> stems = stemmer.stem("drinks");
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java Thu Oct 15 09:58:18 2015
@@ -22,7 +22,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import org.apache.lucene.analysis.hunspell.Dictionary;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
@@ -165,14 +166,14 @@ public class TestAllDictionaries extends
IOUtils.rm(tmp);
Files.createDirectory(tmp);
- try (InputStream in = Files.newInputStream(f)) {
+ try (InputStream in = Files.newInputStream(f); Directory tempDir = getDirectory()) {
TestUtil.unzip(in, tmp);
Path dicEntry = tmp.resolve(tests[i+1]);
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
InputStream affix = Files.newInputStream(affEntry)) {
- Dictionary dic = new Dictionary(affix, dictionary);
+ Dictionary dic = new Dictionary(tempDir, "dictionary", affix, dictionary);
System.out.println(tests[i] + "\t" + RamUsageTester.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageTester.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageTester.humanSizeOf(dic.flagLookup) + ", " +
@@ -204,11 +205,20 @@ public class TestAllDictionaries extends
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
- InputStream affix = Files.newInputStream(affEntry)) {
- new Dictionary(affix, dictionary);
+ InputStream affix = Files.newInputStream(affEntry);
+ Directory tempDir = getDirectory()) {
+ new Dictionary(tempDir, "dictionary", affix, dictionary);
}
}
}
}
}
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
+ }
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java Thu Oct 15 09:58:18 2015
@@ -22,12 +22,13 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import org.apache.lucene.analysis.hunspell.Dictionary;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.apache.lucene.util.RamUsageTester;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.junit.Ignore;
/**
@@ -186,8 +187,12 @@ public class TestAllDictionaries2 extend
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
- InputStream affix = Files.newInputStream(affEntry)) {
- Dictionary dic = new Dictionary(affix, dictionary);
+ InputStream affix = Files.newInputStream(affEntry);
+ Directory tempDir = newDirectory()) {
+ if (tempDir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
+ }
+ Dictionary dic = new Dictionary(tempDir, "dictionary", affix, dictionary);
System.out.println(tests[i] + "\t" + RamUsageTester.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageTester.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageTester.humanSizeOf(dic.flagLookup) + ", " +
@@ -219,8 +224,12 @@ public class TestAllDictionaries2 extend
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
- InputStream affix = Files.newInputStream(affEntry)) {
- new Dictionary(affix, dictionary);
+ InputStream affix = Files.newInputStream(affEntry);
+ Directory tempDir = newDirectory()) {
+ if (tempDir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
+ }
+ new Dictionary(tempDir, "dictionary", affix, dictionary);
}
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java Thu Oct 15 09:58:18 2015
@@ -24,9 +24,10 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.LuceneTestCase;
@@ -41,8 +42,9 @@ public class TestDictionary extends Luce
public void testSimpleDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("simple.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -63,13 +65,15 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
public void testCompressedDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Directory tempDir = getDirectory();
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -80,13 +84,15 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
public void testCompressedBeforeSetDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed-before-set.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
+ Directory tempDir = getDirectory();
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -97,13 +103,15 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
public void testCompressedEmptyAliasDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed-empty-alias.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
+ Directory tempDir = getDirectory();
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -114,15 +122,17 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
// malformed rule causes ParseException
public void testInvalidData() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("broken.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
try {
- new Dictionary(affixStream, dictStream);
+ new Dictionary(tempDir, "dictionary", affixStream, dictStream);
fail("didn't get expected exception");
} catch (ParseException expected) {
assertTrue(expected.getMessage().startsWith("The affix file contains a rule with less than four elements"));
@@ -131,15 +141,17 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
// malformed flags causes ParseException
public void testInvalidFlags() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("broken-flags.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
try {
- new Dictionary(affixStream, dictStream);
+ new Dictionary(tempDir, "dictionary", affixStream, dictStream);
fail("didn't get expected exception");
} catch (Exception expected) {
assertTrue(expected.getMessage().startsWith("expected only one flag"));
@@ -147,6 +159,7 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
private class CloseCheckInputStream extends FilterInputStream {
@@ -170,21 +183,22 @@ public class TestDictionary extends Luce
public void testResourceCleanup() throws Exception {
CloseCheckInputStream affixStream = new CloseCheckInputStream(getClass().getResourceAsStream("compressed.aff"));
CloseCheckInputStream dictStream = new CloseCheckInputStream(getClass().getResourceAsStream("compressed.dic"));
+ Directory tempDir = getDirectory();
- new Dictionary(affixStream, dictStream);
+ new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertFalse(affixStream.isClosed());
assertFalse(dictStream.isClosed());
affixStream.close();
dictStream.close();
+ tempDir.close();
assertTrue(affixStream.isClosed());
assertTrue(dictStream.isClosed());
}
-
public void testReplacements() throws Exception {
Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
Builder<CharsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE2, outputs);
@@ -244,4 +258,12 @@ public class TestDictionary extends Luce
assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8"));
assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8"));
}
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
+ }
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java Thu Oct 15 09:58:18 2015
@@ -27,10 +27,10 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.hunspell.Dictionary;
-import org.apache.lucene.analysis.hunspell.HunspellStemFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -43,11 +43,14 @@ public class TestHunspellStemFilter exte
// no multiple try-with to workaround bogus VerifyError
InputStream affixStream = TestStemmer.class.getResourceAsStream("simple.aff");
InputStream dictStream = TestStemmer.class.getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
+
try {
- dictionary = new Dictionary(affixStream, dictStream);
+ dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
} finally {
IOUtils.closeWhileHandlingException(affixStream, dictStream);
}
+ tempDir.close();
}
@AfterClass
@@ -107,8 +110,9 @@ public class TestHunspellStemFilter exte
// no multiple try-with to workaround bogus VerifyError
InputStream affixStream = TestStemmer.class.getResourceAsStream("simple.aff");
InputStream dictStream = TestStemmer.class.getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
try {
- d = new Dictionary(affixStream, Collections.singletonList(dictStream), true);
+ d = new Dictionary(tempDir, "dictionary", affixStream, Collections.singletonList(dictStream), true);
} finally {
IOUtils.closeWhileHandlingException(affixStream, dictStream);
}
@@ -121,5 +125,14 @@ public class TestHunspellStemFilter exte
};
checkOneTerm(a, "NoChAnGy", "NoChAnGy");
a.close();
+ tempDir.close();
+ }
+
+ private static Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java Thu Oct 15 09:58:18 2015
@@ -140,6 +140,9 @@ public class SimpleTextCompoundFormat ex
@Override
public IndexOutput createOutput(String name, IOContext context) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) { throw new UnsupportedOperationException(); }
@Override
public void sync(Collection<String> names) { throw new UnsupportedOperationException(); }
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java Thu Oct 15 09:58:18 2015
@@ -29,12 +29,12 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.IOUtils;
+import java.io.FileNotFoundException;
+import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.io.FileNotFoundException;
-import java.io.IOException;
/**
* Class for accessing a compound stream.
@@ -172,6 +172,11 @@ final class Lucene50CompoundReader exten
public IndexOutput createOutput(String name, IOContext context) throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ throw new UnsupportedOperationException();
+ }
@Override
public void sync(Collection<String> names) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Thu Oct 15 09:58:18 2015
@@ -32,8 +32,8 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
-import java.util.Map.Entry;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -57,10 +57,10 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.LockValidatingDirectoryWrapper;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.RateLimitedIndexOutput;
import org.apache.lucene.store.TrackingDirectoryWrapper;
-import org.apache.lucene.store.LockValidatingDirectoryWrapper;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -2615,7 +2615,7 @@ public class IndexWriter implements Clos
SegmentCommitInfo infoPerCommit = new SegmentCommitInfo(info, 0, -1L, -1L, -1L);
info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
- trackingDir.getCreatedFiles().clear();
+ trackingDir.clearCreatedFiles();
setDiagnostics(info, SOURCE_ADDINDEXES_READERS);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java Thu Oct 15 09:58:18 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.Random;
/**
* Base implementation for a concrete {@link Directory} that uses a {@link LockFactory} for locking.
@@ -31,6 +32,22 @@ public abstract class BaseDirectory exte
* this Directory instance). */
protected final LockFactory lockFactory;
+ /** Subclasses can use this to generate temp file name candidates */
+ protected static final Random tempFileRandom;
+
+ static {
+ String prop = System.getProperty("tests.seed");
+ int seed;
+ if (prop != null) {
+ // So if there is a test failure that relied on temp file names,
+ //we remain reproducible based on the test seed:
+ seed = prop.hashCode();
+ } else {
+ seed = (int) System.currentTimeMillis();
+ }
+ tempFileRandom = new Random(seed);
+ }
+
/** Sole constructor. */
protected BaseDirectory(LockFactory lockFactory) {
super();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java Thu Oct 15 09:58:18 2015
@@ -17,9 +17,9 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.Closeable;
import java.nio.file.NoSuchFileException;
import java.util.Collection; // for javadocs
@@ -70,8 +70,12 @@ public abstract class Directory implemen
/** Creates a new, empty file in the directory with the given name.
Returns a stream writing this file. */
- public abstract IndexOutput createOutput(String name, IOContext context)
- throws IOException;
+ public abstract IndexOutput createOutput(String name, IOContext context) throws IOException;
+
+ /** Creates a new, empty file for writing in the directory, with a
+ * temporary file name derived from prefix and suffix. Use
+ * {@link IndexOutput#getName} to see what name was used. */
+ public abstract IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException;
/**
* Ensure that any writes to these files are moved to
@@ -120,8 +124,7 @@ public abstract class Directory implemen
/** Closes the store. */
@Override
- public abstract void close()
- throws IOException;
+ public abstract void close() throws IOException;
@Override
public String toString() {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Thu Oct 15 09:58:18 2015
@@ -19,11 +19,14 @@ package org.apache.lucene.store;
import java.io.FilterOutputStream;
import java.io.IOException;
+import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.DirectoryStream;
+import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
+import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
-import java.nio.channels.ClosedChannelException; // javadoc @link
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -220,11 +223,23 @@ public abstract class FSDirectory extend
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
-
ensureCanWrite(name);
return new FSIndexOutput(name);
}
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ ensureOpen();
+ while (true) {
+ String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
+ try {
+ return new FSIndexOutput(name, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+ } catch (FileAlreadyExistsException faee) {
+ // Retry with next random name
+ }
+ }
+ }
+
protected void ensureCanWrite(String name) throws IOException {
Files.deleteIfExists(directory.resolve(name)); // delete existing, if any
}
@@ -273,7 +288,11 @@ public abstract class FSDirectory extend
static final int CHUNK_SIZE = 8192;
public FSIndexOutput(String name) throws IOException {
- super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", new FilterOutputStream(Files.newOutputStream(directory.resolve(name))) {
+ this(name, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
+ }
+
+ FSIndexOutput(String name, OpenOption... options) throws IOException {
+ super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", name, new FilterOutputStream(Files.newOutputStream(directory.resolve(name), options)) {
// This implementation ensures, that we never write more than CHUNK_SIZE bytes:
@Override
public void write(byte[] b, int offset, int length) throws IOException {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java Thu Oct 15 09:58:18 2015
@@ -22,13 +22,12 @@ import java.nio.file.AtomicMoveNotSuppor
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.HashSet;
import org.apache.lucene.util.IOUtils;
-
/**
* Expert: A Directory instance that switches files between
* two other Directory instances.
@@ -156,6 +155,11 @@ public class FileSwitchDirectory extends
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ return getDirectory("."+suffix).createTempOutput(prefix, suffix, context);
+ }
+
+ @Override
public void sync(Collection<String> names) throws IOException {
List<String> primaryNames = new ArrayList<>();
List<String> secondaryNames = new ArrayList<>();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java Thu Oct 15 09:58:18 2015
@@ -74,6 +74,11 @@ public class FilterDirectory extends Dir
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ return in.createTempOutput(prefix, suffix, context);
+ }
+
+ @Override
public void sync(Collection<String> names) throws IOException {
in.sync(names);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java Thu Oct 15 09:58:18 2015
@@ -31,15 +31,27 @@ import java.io.IOException;
*/
public abstract class IndexOutput extends DataOutput implements Closeable {
+ /** Full description of this output, e.g. which class such as {@code FSIndexOutput}, and the full path to the file */
private final String resourceDescription;
+ /** Just the name part from {@code resourceDescription} */
+ private final String name;
+
/** Sole constructor. resourceDescription should be non-null, opaque string
* describing this resource; it's returned from {@link #toString}. */
- protected IndexOutput(String resourceDescription) {
+ protected IndexOutput(String resourceDescription, String name) {
if (resourceDescription == null) {
throw new IllegalArgumentException("resourceDescription must not be null");
}
this.resourceDescription = resourceDescription;
+ this.name = name;
+ }
+
+ /** Returns the name used to create this {@code IndexOutput}. This is especially useful when using
+ * {@link Directory#createTempOutput}. */
+ // TODO: can we somehow use this as the default resource description or something?
+ public String getName() {
+ return name;
}
/** Closes this stream to further operations. */
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java Thu Oct 15 09:58:18 2015
@@ -37,8 +37,8 @@ public class OutputStreamIndexOutput ext
* @param bufferSize the buffer size in bytes used to buffer writes internally.
* @throws IllegalArgumentException if the given buffer size is less or equal to <tt>0</tt>
*/
- public OutputStreamIndexOutput(String resourceDescription, OutputStream out, int bufferSize) {
- super(resourceDescription);
+ public OutputStreamIndexOutput(String resourceDescription, String name, OutputStream out, int bufferSize) {
+ super(resourceDescription, name);
this.os = new BufferedOutputStream(new CheckedOutputStream(out, crc), bufferSize);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java Thu Oct 15 09:58:18 2015
@@ -17,8 +17,8 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
@@ -26,12 +26,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
-
/**
* A memory-resident {@link Directory} implementation. Locking
* implementation is by default the {@link SingleInstanceLockFactory}.
@@ -111,10 +111,7 @@ public class RAMDirectory extends BaseDi
// and do not synchronize or anything stronger. it's great for testing!
// NOTE: fileMap.keySet().toArray(new String[0]) is broken in non Sun JDKs,
// and the code below is resilient to map changes during the array population.
- Set<String> fileNames = fileMap.keySet();
- List<String> names = new ArrayList<>(fileNames.size());
- for (String name : fileNames) names.add(name);
- return names.toArray(new String[names.size()]);
+ return fileMap.keySet().toArray(new String[fileMap.size()]);
}
public final boolean fileNameExists(String name) {
@@ -150,9 +147,6 @@ public class RAMDirectory extends BaseDi
return Accountables.namedAccountables("file", fileMap);
}
- /** Removes an existing file in the directory.
- * @throws IOException if the file does not exist
- */
@Override
public void deleteFile(String name) throws IOException {
ensureOpen();
@@ -165,7 +159,6 @@ public class RAMDirectory extends BaseDi
}
}
- /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
@@ -179,6 +172,22 @@ public class RAMDirectory extends BaseDi
return new RAMOutputStream(name, file, true);
}
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ ensureOpen();
+
+ // Make the file first...
+ RAMFile file = newRAMFile();
+
+ // ... then try to find a unique name for it:
+ while (true) {
+ String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
+ if (fileMap.putIfAbsent(name, file) == null) {
+ return new RAMOutputStream(name, file, true);
+ }
+ }
+ }
+
/**
* Returns a new {@link RAMFile} for storing data. This method can be
* overridden to return different {@link RAMFile} impls, that e.g. override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java Thu Oct 15 09:58:18 2015
@@ -57,7 +57,7 @@ public class RAMOutputStream extends Ind
/** Creates this, with specified name. */
public RAMOutputStream(String name, RAMFile f, boolean checksum) {
- super("RAMOutputStream(name=\"" + name + "\")");
+ super("RAMOutputStream(name=\"" + name + "\")", name);
file = f;
// make sure that we switch to the
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java Thu Oct 15 09:58:18 2015
@@ -38,7 +38,7 @@ public final class RateLimitedIndexOutpu
private long currentMinPauseCheckBytes;
public RateLimitedIndexOutput(final RateLimiter rateLimiter, final IndexOutput delegate) {
- super("RateLimitedIndexOutput(" + delegate + ")");
+ super("RateLimitedIndexOutput(" + delegate + ")", delegate.getName());
this.delegate = delegate;
this.rateLimiter = rateLimiter;
this.currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java Thu Oct 15 09:58:18 2015
@@ -46,6 +46,14 @@ public final class TrackingDirectoryWrap
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
+ throws IOException {
+ IndexOutput tempOutput = in.createTempOutput(prefix, suffix, context);
+ createdFileNames.add(tempOutput.getName());
+ return tempOutput;
+ }
+
+ @Override
public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
in.copyFrom(from, src, dest, context);
createdFileNames.add(dest);
@@ -60,10 +68,12 @@ public final class TrackingDirectoryWrap
}
}
- // maybe clone before returning.... all callers are
- // cloning anyway....
+ /** NOTE: returns a copy of the created files. */
public Set<String> getCreatedFiles() {
- return createdFileNames;
+ return new HashSet<>(createdFileNames);
}
+ public void clearCreatedFiles() {
+ createdFileNames.clear();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Thu Oct 15 09:58:18 2015
@@ -190,7 +190,7 @@ public final class IOUtils {
* <p>
* Note that the files should not be null.
*/
- public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
+ public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) {
for (String name : files) {
try {
dir.deleteFile(name);
@@ -199,6 +199,42 @@ public final class IOUtils {
}
}
}
+
+ public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
+ deleteFilesIgnoringExceptions(dir, Arrays.asList(files));
+ }
+
+ /**
+ * Deletes all given file names. Some of the
+ * file names may be null; they are
+ * ignored. After everything is deleted, the method either
+ * throws the first exception it hit while deleting, or
+ * completes normally if there were no exceptions.
+ *
+ * @param dir Directory to delete files from
+ * @param files file names to delete
+ */
+ public static void deleteFiles(Directory dir, Collection<String> files) throws IOException {
+ Throwable th = null;
+ for (String name : files) {
+ if (name != null) {
+ try {
+ dir.deleteFile(name);
+ } catch (Throwable t) {
+ addSuppressed(th, t);
+ if (th == null) {
+ th = t;
+ }
+ }
+ }
+ }
+
+ reThrow(th);
+ }
+
+ public static void deleteFiles(Directory dir, String... files) throws IOException {
+ deleteFiles(dir, Arrays.asList(files));
+ }
/**
* Deletes all given files, suppressing all thrown IOExceptions.
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java Thu Oct 15 09:58:18 2015
@@ -17,24 +17,20 @@ package org.apache.lucene.util;
* limitations under the License.
*/
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
import java.io.Closeable;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.TrackingDirectoryWrapper;
+
/**
* On-disk sorting of byte arrays. Each byte array (entry) is a composed of the following
* fields:
@@ -43,14 +39,12 @@ import java.util.Locale;
* <li>exactly the above count of bytes for the sequence to be sorted.
* </ul>
*
- * @see #sort(Path, Path)
+ * @see #sort(String)
* @lucene.experimental
* @lucene.internal
*/
public final class OfflineSorter {
- private static Path DEFAULT_TEMP_DIR;
-
/** Convenience constant for megabytes */
public final static long MB = 1024 * 1024;
/** Convenience constant for gigabytes */
@@ -72,6 +66,10 @@ public final class OfflineSorter {
*/
public final static int MAX_TEMPFILES = 128;
+ private final Directory dir;
+
+ private final String tempFileNamePrefix;
+
/**
* A bit more descriptive unit for constructors.
*
@@ -142,7 +140,7 @@ public final class OfflineSorter {
/** number of partition merges */
public int mergeRounds;
/** number of lines of data read */
- public int lines;
+ public int lineCount;
/** time spent merging sorted partitions (in milliseconds) */
public long mergeTime;
/** time spent sorting data (in milliseconds) */
@@ -162,17 +160,16 @@ public final class OfflineSorter {
return String.format(Locale.ROOT,
"time=%.2f sec. total (%.2f reading, %.2f sorting, %.2f merging), lines=%d, temp files=%d, merges=%d, soft ram limit=%.2f MB",
totalTime / 1000.0d, readTime / 1000.0d, sortTime / 1000.0d, mergeTime / 1000.0d,
- lines, tempMergeFiles, mergeRounds,
+ lineCount, tempMergeFiles, mergeRounds,
(double) bufferSize / MB);
}
}
private final BufferSize ramBufferSize;
- private final Path tempDirectory;
private final Counter bufferBytesUsed = Counter.newCounter();
private final BytesRefArray buffer = new BytesRefArray(bufferBytesUsed);
- private SortInfo sortInfo;
+ SortInfo sortInfo;
private int maxTempFiles;
private final Comparator<BytesRef> comparator;
@@ -182,27 +179,25 @@ public final class OfflineSorter {
/**
* Defaults constructor.
*
- * @see #getDefaultTempDir()
* @see BufferSize#automatic()
*/
- public OfflineSorter() throws IOException {
- this(DEFAULT_COMPARATOR, BufferSize.automatic(), getDefaultTempDir(), MAX_TEMPFILES);
+ public OfflineSorter(Directory dir, String tempFileNamePrefix) throws IOException {
+ this(dir, tempFileNamePrefix, DEFAULT_COMPARATOR, BufferSize.automatic(), MAX_TEMPFILES);
}
/**
* Defaults constructor with a custom comparator.
*
- * @see #getDefaultTempDir()
* @see BufferSize#automatic()
*/
- public OfflineSorter(Comparator<BytesRef> comparator) throws IOException {
- this(comparator, BufferSize.automatic(), getDefaultTempDir(), MAX_TEMPFILES);
+ public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator<BytesRef> comparator) throws IOException {
+ this(dir, tempFileNamePrefix, comparator, BufferSize.automatic(), MAX_TEMPFILES);
}
/**
* All-details constructor.
*/
- public OfflineSorter(Comparator<BytesRef> comparator, BufferSize ramBufferSize, Path tempDirectory, int maxTempfiles) {
+ public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator<BytesRef> comparator, BufferSize ramBufferSize, int maxTempfiles) {
if (ramBufferSize.bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE) {
throw new IllegalArgumentException(MIN_BUFFER_SIZE_MSG + ": " + ramBufferSize.bytes);
}
@@ -212,160 +207,129 @@ public final class OfflineSorter {
}
this.ramBufferSize = ramBufferSize;
- this.tempDirectory = tempDirectory;
this.maxTempFiles = maxTempfiles;
this.comparator = comparator;
+ this.dir = dir;
+ this.tempFileNamePrefix = tempFileNamePrefix;
+ }
+
+ /** Returns the {@link Directory} we use to create temp files. */
+ public Directory getDirectory() {
+ return dir;
+ }
+
+ /** Returns the temp file name prefix passed to {@link Directory#createTempOutput} to generate temporary files. */
+ public String getTempFileNamePrefix() {
+ return tempFileNamePrefix;
}
/**
- * Sort input to output, explicit hint for the buffer size. The amount of allocated
- * memory may deviate from the hint (may be smaller or larger).
+ * Sort input to a new temp file, returning its name.
*/
- public SortInfo sort(Path input, Path output) throws IOException {
+ public String sort(String inputFileName) throws IOException {
+
sortInfo = new SortInfo();
sortInfo.totalTime = System.currentTimeMillis();
- // NOTE: don't remove output here: its existence (often created by the caller
- // up above using Files.createTempFile) prevents another concurrent caller
- // of this API (from a different thread) from incorrectly re-using this file name
-
- ArrayList<Path> merges = new ArrayList<>();
- boolean success3 = false;
- try {
- ByteSequencesReader is = new ByteSequencesReader(input);
- boolean success = false;
- try {
- int lines = 0;
- while ((lines = readPartition(is)) > 0) {
- merges.add(sortPartition(lines));
- sortInfo.tempMergeFiles++;
- sortInfo.lines += lines;
-
- // Handle intermediate merges.
- if (merges.size() == maxTempFiles) {
- Path intermediate = Files.createTempFile(tempDirectory, "sort", "intermediate");
- boolean success2 = false;
- try {
- mergePartitions(merges, intermediate);
- success2 = true;
- } finally {
- if (success2) {
- IOUtils.deleteFilesIfExist(merges);
- } else {
- IOUtils.deleteFilesIgnoringExceptions(merges);
- }
- merges.clear();
- merges.add(intermediate);
- }
- sortInfo.tempMergeFiles++;
- }
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(is);
- } else {
- IOUtils.closeWhileHandlingException(is);
+ List<String> segments = new ArrayList<>();
+
+ // So we can remove any partially written temp files on exception:
+ TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
+
+ boolean success = false;
+ try (ByteSequencesReader is = new ByteSequencesReader(dir.openInput(inputFileName, IOContext.READONCE))) {
+
+ int lineCount;
+ while ((lineCount = readPartition(is)) > 0) {
+ segments.add(sortPartition(trackingDir));
+ sortInfo.tempMergeFiles++;
+ sortInfo.lineCount += lineCount;
+
+ // Handle intermediate merges.
+ if (segments.size() == maxTempFiles) {
+ mergePartitions(trackingDir, segments);
}
}
- // One partition, try to rename or copy if unsuccessful.
- if (merges.size() == 1) {
- Files.move(merges.get(0), output, StandardCopyOption.REPLACE_EXISTING);
- } else {
- // otherwise merge the partitions with a priority queue.
- mergePartitions(merges, output);
+ // Merge the partitions to the output file with a priority queue.
+ if (segments.size() > 1) {
+ mergePartitions(trackingDir, segments);
}
- success3 = true;
- } finally {
- if (success3) {
- IOUtils.deleteFilesIfExist(merges);
+
+ String result;
+ if (segments.isEmpty()) {
+ try (IndexOutput out = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
+ result = out.getName();
+ }
} else {
- IOUtils.deleteFilesIgnoringExceptions(merges);
- IOUtils.deleteFilesIgnoringExceptions(output);
+ result = segments.get(0);
}
- }
- sortInfo.totalTime = (System.currentTimeMillis() - sortInfo.totalTime);
- return sortInfo;
- }
+ // We should be explicitly removing all intermediate files ourselves unless there is an exception:
+ assert trackingDir.getCreatedFiles().size() == 1 && trackingDir.getCreatedFiles().contains(result);
- /** Used by test framework */
- static void setDefaultTempDir(Path tempDir) {
- DEFAULT_TEMP_DIR = tempDir;
- }
+ sortInfo.totalTime = (System.currentTimeMillis() - sortInfo.totalTime);
+ success = true;
- /**
- * Returns the default temporary directory. By default, java.io.tmpdir. If not accessible
- * or not available, an IOException is thrown
- */
- public synchronized static Path getDefaultTempDir() throws IOException {
- if (DEFAULT_TEMP_DIR == null) {
- // Lazy init
- String tempDirPath = System.getProperty("java.io.tmpdir");
- if (tempDirPath == null) {
- throw new IOException("Java has no temporary folder property (java.io.tmpdir)?");
- }
- Path tempDirectory = Paths.get(tempDirPath);
- if (Files.isWritable(tempDirectory) == false) {
- throw new IOException("Java's temporary folder not present or writeable?: "
- + tempDirectory.toAbsolutePath());
+ return result;
+
+ } finally {
+ if (success == false) {
+ IOUtils.deleteFilesIgnoringExceptions(trackingDir, trackingDir.getCreatedFiles());
}
- DEFAULT_TEMP_DIR = tempDirectory;
}
-
- return DEFAULT_TEMP_DIR;
}
/** Sort a single partition in-memory. */
- protected Path sortPartition(int len) throws IOException {
+ protected String sortPartition(TrackingDirectoryWrapper trackingDir) throws IOException {
BytesRefArray data = this.buffer;
- Path tempFile = Files.createTempFile(tempDirectory, "sort", "partition");
- long start = System.currentTimeMillis();
- sortInfo.sortTime += (System.currentTimeMillis() - start);
-
- final ByteSequencesWriter out = new ByteSequencesWriter(tempFile);
- BytesRef spare;
- try {
+ try (IndexOutput tempFile = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
+ ByteSequencesWriter out = new ByteSequencesWriter(tempFile);
+ BytesRef spare;
+
+ long start = System.currentTimeMillis();
BytesRefIterator iter = buffer.iterator(comparator);
- while((spare = iter.next()) != null) {
+ sortInfo.sortTime += (System.currentTimeMillis() - start);
+
+ while ((spare = iter.next()) != null) {
assert spare.length <= Short.MAX_VALUE;
out.write(spare);
}
- out.close();
-
// Clean up the buffer for the next partition.
data.clear();
- return tempFile;
- } finally {
- IOUtils.close(out);
+
+ return tempFile.getName();
}
}
- /** Merge a list of sorted temporary files (partitions) into an output file */
- void mergePartitions(List<Path> merges, Path outputFile) throws IOException {
+ /** Merge a list of sorted temporary files (partitions) into an output file. Note that this closes the
+ * incoming {@link IndexOutput}. */
+ void mergePartitions(Directory trackingDir, List<String> segments) throws IOException {
long start = System.currentTimeMillis();
- ByteSequencesWriter out = new ByteSequencesWriter(outputFile);
-
- PriorityQueue<FileAndTop> queue = new PriorityQueue<FileAndTop>(merges.size()) {
+ PriorityQueue<FileAndTop> queue = new PriorityQueue<FileAndTop>(segments.size()) {
@Override
protected boolean lessThan(FileAndTop a, FileAndTop b) {
return comparator.compare(a.current.get(), b.current.get()) < 0;
}
};
- ByteSequencesReader [] streams = new ByteSequencesReader [merges.size()];
- try {
+ ByteSequencesReader[] streams = new ByteSequencesReader[segments.size()];
+
+ String newSegmentName = null;
+
+ try (IndexOutput out = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT)) {
+ newSegmentName = out.getName();
+ ByteSequencesWriter writer = new ByteSequencesWriter(out);
+
// Open streams and read the top for each file
- for (int i = 0; i < merges.size(); i++) {
- streams[i] = new ByteSequencesReader(merges.get(i));
- byte line[] = streams[i].read();
- if (line != null) {
- queue.insertWithOverflow(new FileAndTop(i, line));
- }
+ for (int i = 0; i < segments.size(); i++) {
+ streams[i] = new ByteSequencesReader(dir.openInput(segments.get(i), IOContext.READONCE));
+ byte[] line = streams[i].read();
+ assert line != null;
+ queue.insertWithOverflow(new FileAndTop(i, line));
}
// Unix utility sort() uses ordered array of files to pick the next line from, updating
@@ -374,7 +338,7 @@ public final class OfflineSorter {
// so it shouldn't make much of a difference (didn't check).
FileAndTop top;
while ((top = queue.top()) != null) {
- out.write(top.current.bytes(), 0, top.current.length());
+ writer.write(top.current.bytes(), 0, top.current.length());
if (!streams[top.fd].read(top.current)) {
queue.pop();
} else {
@@ -385,14 +349,15 @@ public final class OfflineSorter {
sortInfo.mergeTime += System.currentTimeMillis() - start;
sortInfo.mergeRounds++;
} finally {
- // The logic below is: if an exception occurs in closing out, it has a priority over exceptions
- // happening in closing streams.
- try {
- IOUtils.close(streams);
- } finally {
- IOUtils.close(out);
- }
+ IOUtils.close(streams);
}
+
+ IOUtils.deleteFiles(trackingDir, segments);
+
+ segments.clear();
+ segments.add(newSegmentName);
+
+ sortInfo.tempMergeFiles++;
}
/** Read in a single partition of data */
@@ -428,18 +393,11 @@ public final class OfflineSorter {
* Complementary to {@link ByteSequencesReader}.
*/
public static class ByteSequencesWriter implements Closeable {
- private final DataOutput os;
-
- /** Constructs a ByteSequencesWriter to the provided Path */
- public ByteSequencesWriter(Path path) throws IOException {
- this(new DataOutputStream(
- new BufferedOutputStream(
- Files.newOutputStream(path))));
- }
+ private final IndexOutput out;
/** Constructs a ByteSequencesWriter to the provided DataOutput */
- public ByteSequencesWriter(DataOutput os) {
- this.os = os;
+ public ByteSequencesWriter(IndexOutput out) {
+ this.out = out;
}
/**
@@ -455,7 +413,7 @@ public final class OfflineSorter {
* Writes a byte array.
* @see #write(byte[], int, int)
*/
- public void write(byte [] bytes) throws IOException {
+ public void write(byte[] bytes) throws IOException {
write(bytes, 0, bytes.length);
}
@@ -465,25 +423,23 @@ public final class OfflineSorter {
* The length is written as a <code>short</code>, followed
* by the bytes.
*/
- public void write(byte [] bytes, int off, int len) throws IOException {
+ public void write(byte[] bytes, int off, int len) throws IOException {
assert bytes != null;
assert off >= 0 && off + len <= bytes.length;
assert len >= 0;
if (len > Short.MAX_VALUE) {
throw new IllegalArgumentException("len must be <= " + Short.MAX_VALUE + "; got " + len);
}
- os.writeShort(len);
- os.write(bytes, off, len);
+ out.writeShort((short) len);
+ out.writeBytes(bytes, off, len);
}
/**
- * Closes the provided {@link DataOutput} if it is {@link Closeable}.
+ * Closes the provided {@link IndexOutput}.
*/
@Override
public void close() throws IOException {
- if (os instanceof Closeable) {
- ((Closeable) os).close();
- }
+ out.close();
}
}
@@ -492,18 +448,11 @@ public final class OfflineSorter {
* Complementary to {@link ByteSequencesWriter}.
*/
public static class ByteSequencesReader implements Closeable {
- private final DataInput is;
+ private final IndexInput in;
- /** Constructs a ByteSequencesReader from the provided Path */
- public ByteSequencesReader(Path path) throws IOException {
- this(new DataInputStream(
- new BufferedInputStream(
- Files.newInputStream(path))));
- }
-
- /** Constructs a ByteSequencesReader from the provided DataInput */
- public ByteSequencesReader(DataInput is) {
- this.is = is;
+ /** Constructs a ByteSequencesReader from the provided IndexInput */
+ public ByteSequencesReader(IndexInput in) {
+ this.in = in;
}
/**
@@ -517,14 +466,14 @@ public final class OfflineSorter {
public boolean read(BytesRefBuilder ref) throws IOException {
short length;
try {
- length = is.readShort();
+ length = in.readShort();
} catch (EOFException e) {
return false;
}
ref.grow(length);
ref.setLength(length);
- is.readFully(ref.bytes(), 0, length);
+ in.readBytes(ref.bytes(), 0, length);
return true;
}
@@ -540,25 +489,23 @@ public final class OfflineSorter {
public byte[] read() throws IOException {
short length;
try {
- length = is.readShort();
+ length = in.readShort();
} catch (EOFException e) {
return null;
}
assert length >= 0 : "Sanity: sequence length < 0: " + length;
- byte [] result = new byte [length];
- is.readFully(result);
+ byte[] result = new byte[length];
+ in.readBytes(result, 0, length);
return result;
}
/**
- * Closes the provided {@link DataInput} if it is {@link Closeable}.
+ * Closes the provided {@link IndexInput}.
*/
@Override
public void close() throws IOException {
- if (is instanceof Closeable) {
- ((Closeable) is).close();
- }
+ in.close();
}
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java?rev=1708760&r1=1708759&r2=1708760&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java Thu Oct 15 09:58:18 2015
@@ -283,7 +283,7 @@ public class TestCodecUtil extends Lucen
final IndexOutput output = new RAMOutputStream(file, false);
AtomicLong fakeChecksum = new AtomicLong();
// wrap the index input where we control the checksum for mocking
- IndexOutput fakeOutput = new IndexOutput("fake") {
+ IndexOutput fakeOutput = new IndexOutput("fake", "fake") {
@Override
public void close() throws IOException {
output.close();