You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/10/19 15:38:44 UTC
svn commit: r1709414 [1/5] - in /lucene/dev/branches/lucene6835: ./ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/analysis/common/src/test/org/apache/lucene/analysis/core/ l...
Author: mikemccand
Date: Mon Oct 19 13:38:42 2015
New Revision: 1709414
URL: http://svn.apache.org/viewvc?rev=1709414&view=rev
Log:
LUCENE-6835: merge trunk
Added:
lucene/dev/branches/lucene6835/lucene/licenses/antlr4-runtime-4.5.1-1.jar.sha1
- copied unchanged from r1709413, lucene/dev/trunk/lucene/licenses/antlr4-runtime-4.5.1-1.jar.sha1
lucene/dev/branches/lucene6835/solr/licenses/antlr4-runtime-4.5.1-1.jar.sha1
- copied unchanged from r1709413, lucene/dev/trunk/solr/licenses/antlr4-runtime-4.5.1-1.jar.sha1
lucene/dev/branches/lucene6835/solr/licenses/presto-parser-0.122.jar.sha1
- copied unchanged from r1709413, lucene/dev/trunk/solr/licenses/presto-parser-0.122.jar.sha1
Removed:
lucene/dev/branches/lucene6835/lucene/licenses/antlr4-runtime-4.5.jar.sha1
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
lucene/dev/branches/lucene6835/solr/licenses/antlr4-runtime-4.5.jar.sha1
lucene/dev/branches/lucene6835/solr/licenses/presto-parser-0.108.jar.sha1
Modified:
lucene/dev/branches/lucene6835/ (props changed)
lucene/dev/branches/lucene6835/lucene/ (props changed)
lucene/dev/branches/lucene6835/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6835/lucene/analysis/ (props changed)
lucene/dev/branches/lucene6835/lucene/analysis/common/ (props changed)
lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
lucene/dev/branches/lucene6835/lucene/classification/ (props changed)
lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
lucene/dev/branches/lucene6835/lucene/codecs/ (props changed)
lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
lucene/dev/branches/lucene6835/lucene/core/ (props changed)
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/Multiset.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
lucene/dev/branches/lucene6835/lucene/expressions/ (props changed)
lucene/dev/branches/lucene6835/lucene/expressions/build.xml
lucene/dev/branches/lucene6835/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptBaseVisitor.java
lucene/dev/branches/lucene6835/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptLexer.java
lucene/dev/branches/lucene6835/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptParser.java
lucene/dev/branches/lucene6835/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptVisitor.java
lucene/dev/branches/lucene6835/lucene/facet/ (props changed)
lucene/dev/branches/lucene6835/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
lucene/dev/branches/lucene6835/lucene/ivy-versions.properties (contents, props changed)
lucene/dev/branches/lucene6835/lucene/licenses/ (props changed)
lucene/dev/branches/lucene6835/lucene/misc/ (props changed)
lucene/dev/branches/lucene6835/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
lucene/dev/branches/lucene6835/lucene/queries/ (props changed)
lucene/dev/branches/lucene6835/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadScoreQuery.java
lucene/dev/branches/lucene6835/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadExplanations.java
lucene/dev/branches/lucene6835/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
lucene/dev/branches/lucene6835/lucene/sandbox/ (props changed)
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/bkdtree/BKDTreeDocValuesConsumer.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/bkdtree/BKDTreeDocValuesFormat.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/bkdtree/BKDTreeWriter.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/bkdtree/OfflineLatLonReader.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/bkdtree/OfflineLatLonWriter.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/rangetree/OfflineSliceReader.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/rangetree/OfflineSliceWriter.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/rangetree/RangeTreeDocValuesConsumer.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/java/org/apache/lucene/rangetree/RangeTreeWriter.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/test/org/apache/lucene/bkdtree/TestBKDTree.java
lucene/dev/branches/lucene6835/lucene/sandbox/src/test/org/apache/lucene/rangetree/TestRangeTree.java
lucene/dev/branches/lucene6835/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/BKD3DTreeWriter.java
lucene/dev/branches/lucene6835/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/Geo3DDocValuesConsumer.java
lucene/dev/branches/lucene6835/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/Geo3DDocValuesFormat.java
lucene/dev/branches/lucene6835/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/OfflineReader.java
lucene/dev/branches/lucene6835/lucene/spatial3d/src/java/org/apache/lucene/bkdtree3d/OfflineWriter.java
lucene/dev/branches/lucene6835/lucene/spatial3d/src/test/org/apache/lucene/bkdtree3d/TestGeo3DPointField.java
lucene/dev/branches/lucene6835/lucene/suggest/ (props changed)
lucene/dev/branches/lucene6835/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
lucene/dev/branches/lucene6835/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/branches/lucene6835/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java
lucene/dev/branches/lucene6835/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
lucene/dev/branches/lucene6835/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/branches/lucene6835/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/branches/lucene6835/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
lucene/dev/branches/lucene6835/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
lucene/dev/branches/lucene6835/solr/ (props changed)
lucene/dev/branches/lucene6835/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6835/solr/bin/ (props changed)
lucene/dev/branches/lucene6835/solr/bin/solr
lucene/dev/branches/lucene6835/solr/contrib/ (props changed)
lucene/dev/branches/lucene6835/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/MockTool.java
lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java
lucene/dev/branches/lucene6835/solr/core/ (props changed)
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/search/Grouping.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/LookupFactory.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingLookupFactory.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FSTLookupFactory.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/fst/WFSTLookupFactory.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/tst/TSTLookupFactory.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java
lucene/dev/branches/lucene6835/solr/core/src/test-files/solr/collection1/conf/schema.xml
lucene/dev/branches/lucene6835/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
lucene/dev/branches/lucene6835/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
lucene/dev/branches/lucene6835/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java
lucene/dev/branches/lucene6835/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
lucene/dev/branches/lucene6835/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
lucene/dev/branches/lucene6835/solr/core/src/test/org/apache/solr/update/processor/UpdateProcessorTestBase.java
lucene/dev/branches/lucene6835/solr/licenses/ (props changed)
lucene/dev/branches/lucene6835/solr/solrj/ (props changed)
lucene/dev/branches/lucene6835/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java
Modified: lucene/dev/branches/lucene6835/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/CHANGES.txt?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6835/lucene/CHANGES.txt Mon Oct 19 13:38:42 2015
@@ -59,6 +59,13 @@ API Changes
* LUCENE-6706: PayloadTermQuery and PayloadNearQuery have been removed.
Instead, use PayloadScoreQuery to wrap any SpanQuery. (Alan Woodward)
+* LUCENE-6829: OfflineSorter, and the classes that use it (suggesters,
+ hunspell) now do all temporary file IO via Directory instead of
+ directly through java's temp dir. Directory.createTempOutput
+ creates a uniquely named IndexOutput, and the new
+ IndexOutput.getName returns its name (Dawid Weiss, Robert Muir, Mike
+ McCandless)
+
Changes in Runtime Behavior
* LUCENE-6789: IndexSearcher's default Similarity is changed to BM25Similarity.
@@ -81,6 +88,12 @@ New Features
fast, very accurate query to find all indexed points within an
earth-surface shape (Karl Wright, Mike McCandless)
+* LUCENE-6838: Added IndexSearcher#getQueryCache and #getQueryCachingPolicy.
+ (Adrien Grand)
+
+* LUCENE-6844: PayloadScoreQuery can include or exclude underlying span scores
+ from its score calculations (Bill Bell, Alan Woodward)
+
API Changes
* LUCENE-6590: Query.setBoost(), Query.getBoost() and Query.clone() are gone.
@@ -104,6 +117,9 @@ API Changes
order to let Lucene know that a Query should be used for filtering but not
scoring.
+* LUCENE-6939: SpanOrQuery.addClause is now deprecated, clauses should all be
+ provided at construction time. (Paul Elschot via Adrien Grand)
+
Optimizations
* LUCENE-6708: TopFieldCollector does not compute the score several times on the
@@ -198,8 +214,8 @@ Other
* LUCENE-6174: Improve "ant eclipse" to select right JRE for building.
(Uwe Schindler, Dawid Weiss)
-* LUCENE-6417: Upgrade ANTLR used in expressions module to version 4.5.
- (Jack Conradson, Uwe Schindler)
+* LUCENE-6417, LUCENE-6830: Upgrade ANTLR used in expressions module
+ to version 4.5.1-1. (Jack Conradson, Uwe Schindler)
* LUCENE-6729: Upgrade ASM used in expressions module to version 5.0.4.
(Uwe Schindler)
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java Mon Oct 19 13:38:42 2015
@@ -17,28 +17,6 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
-import org.apache.lucene.store.ByteArrayDataOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.IntsRefBuilder;
-import org.apache.lucene.util.OfflineSorter;
-import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;
-import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.RegExp;
-import org.apache.lucene.util.fst.Builder;
-import org.apache.lucene.util.fst.CharSequenceOutputs;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.IntSequenceOutputs;
-import org.apache.lucene.util.fst.Outputs;
-import org.apache.lucene.util.fst.Util;
-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
@@ -53,6 +31,7 @@ import java.nio.charset.CodingErrorActio
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -67,6 +46,31 @@ import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.IntsRefBuilder;
+import org.apache.lucene.util.OfflineSorter;
+import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;
+import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.RegExp;
+import org.apache.lucene.util.fst.Builder;
+import org.apache.lucene.util.fst.CharSequenceOutputs;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.IntSequenceOutputs;
+import org.apache.lucene.util.fst.Outputs;
+import org.apache.lucene.util.fst.Util;
+
/**
* In-memory structure for the dictionary (.dic) and affix (.aff)
* data of a hunspell dictionary.
@@ -139,7 +143,7 @@ public class Dictionary {
// when set, some words have exceptional stems, and the last entry is a pointer to stemExceptions
boolean hasStemExceptions;
- private final Path tempDir = OfflineSorter.getDefaultTempDir(); // TODO: make this configurable?
+ private final Path tempPath = getDefaultTempDir(); // TODO: make this configurable?
boolean ignoreCase;
boolean complexPrefixes;
@@ -167,19 +171,21 @@ public class Dictionary {
String language;
// true if case algorithms should use alternate (Turkish/Azeri) mapping
boolean alternateCasing;
-
+
/**
* Creates a new Dictionary containing the information read from the provided InputStreams to hunspell affix
* and dictionary files.
* You have to close the provided InputStreams yourself.
*
+ * @param tempDir Directory to use for offline sorting
+ * @param tempFileNamePrefix prefix to use to generate temp file names
* @param affix InputStream for reading the hunspell affix file (won't be closed).
* @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @throws IOException Can be thrown while reading from the InputStreams
* @throws ParseException Can be thrown if the content of the files does not meet expected formats
*/
- public Dictionary(InputStream affix, InputStream dictionary) throws IOException, ParseException {
- this(affix, Collections.singletonList(dictionary), false);
+ public Dictionary(Directory tempDir, String tempFileNamePrefix, InputStream affix, InputStream dictionary) throws IOException, ParseException {
+ this(tempDir, tempFileNamePrefix, affix, Collections.singletonList(dictionary), false);
}
/**
@@ -187,18 +193,20 @@ public class Dictionary {
* and dictionary files.
* You have to close the provided InputStreams yourself.
*
+ * @param tempDir Directory to use for offline sorting
+ * @param tempFileNamePrefix prefix to use to generate temp file names
* @param affix InputStream for reading the hunspell affix file (won't be closed).
* @param dictionaries InputStream for reading the hunspell dictionary files (won't be closed).
* @throws IOException Can be thrown while reading from the InputStreams
* @throws ParseException Can be thrown if the content of the files does not meet expected formats
*/
- public Dictionary(InputStream affix, List<InputStream> dictionaries, boolean ignoreCase) throws IOException, ParseException {
+ public Dictionary(Directory tempDir, String tempFileNamePrefix, InputStream affix, List<InputStream> dictionaries, boolean ignoreCase) throws IOException, ParseException {
this.ignoreCase = ignoreCase;
this.needsInputCleaning = ignoreCase;
this.needsOutputCleaning = false; // set if we have an OCONV
flagLookup.add(new BytesRef()); // no flags -> ord 0
- Path aff = Files.createTempFile(tempDir, "affix", "aff");
+ Path aff = Files.createTempFile(tempPath, "affix", "aff");
OutputStream out = new BufferedOutputStream(Files.newOutputStream(aff));
InputStream aff1 = null;
InputStream aff2 = null;
@@ -224,7 +232,7 @@ public class Dictionary {
// read dictionary entries
IntSequenceOutputs o = IntSequenceOutputs.getSingleton();
Builder<IntsRef> b = new Builder<>(FST.INPUT_TYPE.BYTE4, o);
- readDictionaryFiles(dictionaries, decoder, b);
+ readDictionaryFiles(tempDir, tempFileNamePrefix, dictionaries, decoder, b);
words = b.finish();
aliases = null; // no longer needed
morphAliases = null; // no longer needed
@@ -766,7 +774,7 @@ public class Dictionary {
return Math.max(pos1, pos2);
}
}
-
+
/**
* Reads the dictionary file through the provided InputStreams, building up the words map
*
@@ -774,13 +782,13 @@ public class Dictionary {
* @param decoder CharsetDecoder used to decode the contents of the file
* @throws IOException Can be thrown while reading from the file
*/
- private void readDictionaryFiles(List<InputStream> dictionaries, CharsetDecoder decoder, Builder<IntsRef> words) throws IOException {
+ private void readDictionaryFiles(Directory tempDir, String tempFileNamePrefix, List<InputStream> dictionaries, CharsetDecoder decoder, Builder<IntsRef> words) throws IOException {
BytesRefBuilder flagsScratch = new BytesRefBuilder();
IntsRefBuilder scratchInts = new IntsRefBuilder();
StringBuilder sb = new StringBuilder();
-
- Path unsorted = Files.createTempFile(tempDir, "unsorted", "dat");
+
+ IndexOutput unsorted = tempDir.createTempOutput(tempFileNamePrefix, "dat", IOContext.DEFAULT);
try (ByteSequencesWriter writer = new ByteSequencesWriter(unsorted)) {
for (InputStream dictionary : dictionaries) {
BufferedReader lines = new BufferedReader(new InputStreamReader(dictionary, decoder));
@@ -823,9 +831,8 @@ public class Dictionary {
}
}
}
- Path sorted = Files.createTempFile(tempDir, "sorted", "dat");
-
- OfflineSorter sorter = new OfflineSorter(new Comparator<BytesRef>() {
+
+ OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix, new Comparator<BytesRef>() {
BytesRef scratch1 = new BytesRef();
BytesRef scratch2 = new BytesRef();
@@ -862,21 +869,23 @@ public class Dictionary {
}
}
});
+
+ String sorted;
boolean success = false;
try {
- sorter.sort(unsorted, sorted);
+ sorted = sorter.sort(unsorted.getName());
success = true;
} finally {
if (success) {
- Files.delete(unsorted);
+ tempDir.deleteFile(unsorted.getName());
} else {
- IOUtils.deleteFilesIgnoringExceptions(unsorted);
+ IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName());
}
}
boolean success2 = false;
- ByteSequencesReader reader = new ByteSequencesReader(sorted);
- try {
+
+ try (ByteSequencesReader reader = new ByteSequencesReader(tempDir.openInput(sorted, IOContext.READONCE))) {
BytesRefBuilder scratchLine = new BytesRefBuilder();
// TODO: the flags themselves can be double-chars (long) or also numeric
@@ -956,11 +965,10 @@ public class Dictionary {
words.add(scratchInts.get(), currentOrds.get());
success2 = true;
} finally {
- IOUtils.closeWhileHandlingException(reader);
if (success2) {
- Files.delete(sorted);
+ tempDir.deleteFile(sorted);
} else {
- IOUtils.deleteFilesIgnoringExceptions(sorted);
+ IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted);
}
}
}
@@ -1245,4 +1253,33 @@ public class Dictionary {
public boolean getIgnoreCase() {
return ignoreCase;
}
+
+ private static Path DEFAULT_TEMP_DIR;
+
+ /** Used by test framework */
+ public static void setDefaultTempDir(Path tempDir) {
+ DEFAULT_TEMP_DIR = tempDir;
+ }
+
+ /**
+ * Returns the default temporary directory. By default, java.io.tmpdir. If not accessible
+ * or not available, an IOException is thrown
+ */
+ synchronized static Path getDefaultTempDir() throws IOException {
+ if (DEFAULT_TEMP_DIR == null) {
+ // Lazy init
+ String tempDirPath = System.getProperty("java.io.tmpdir");
+ if (tempDirPath == null) {
+ throw new IOException("Java has no temporary folder property (java.io.tmpdir)?");
+ }
+ Path tempDirectory = Paths.get(tempDirPath);
+ if (Files.isWritable(tempDirectory) == false) {
+ throw new IOException("Java's temporary folder not present or writeable?: "
+ + tempDirectory.toAbsolutePath());
+ }
+ DEFAULT_TEMP_DIR = tempDirectory;
+ }
+
+ return DEFAULT_TEMP_DIR;
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilterFactory.java Mon Oct 19 13:38:42 2015
@@ -19,6 +19,8 @@ package org.apache.lucene.analysis.hunsp
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
@@ -28,6 +30,8 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.IOUtils;
/**
@@ -48,6 +52,7 @@ import org.apache.lucene.util.IOUtils;
public class HunspellStemFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
private static final String PARAM_DICTIONARY = "dictionary";
private static final String PARAM_AFFIX = "affix";
+ // NOTE: this one is currently unused?:
private static final String PARAM_RECURSION_CAP = "recursionCap";
private static final String PARAM_IGNORE_CASE = "ignoreCase";
private static final String PARAM_LONGEST_ONLY = "longestOnly";
@@ -91,7 +96,12 @@ public class HunspellStemFilterFactory e
}
affix = loader.openResource(affixFile);
- this.dictionary = new Dictionary(affix, dictionaries, ignoreCase);
+ Path tempPath = Files.createTempDirectory(Dictionary.getDefaultTempDir(), "Hunspell");
+ try (Directory tempDir = FSDirectory.open(tempPath)) {
+ this.dictionary = new Dictionary(tempDir, "hunspell", affix, dictionaries, ignoreCase);
+ } finally {
+ IOUtils.rm(tempPath);
+ }
} catch (ParseException e) {
throw new IOException("Unable to load hunspell data! [dictionary=" + dictionaries + ",affix=" + affixFile + "]", e);
} finally {
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Mon Oct 19 13:38:42 2015
@@ -85,6 +85,7 @@ import org.apache.lucene.analysis.synony
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
+import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.CharsRef;
@@ -435,7 +436,7 @@ public class TestRandomChains extends Ba
InputStream affixStream = TestHunspellStemFilter.class.getResourceAsStream("simple.aff");
InputStream dictStream = TestHunspellStemFilter.class.getResourceAsStream("simple.dic");
try {
- return new Dictionary(affixStream, dictStream);
+ return new Dictionary(new RAMDirectory(), "dictionary", affixStream, dictStream);
} catch (Exception ex) {
Rethrow.rethrow(ex);
return null; // unreachable code
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java Mon Oct 19 13:38:42 2015
@@ -24,6 +24,7 @@ import java.text.ParseException;
import java.util.Arrays;
import java.util.List;
+import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -61,7 +62,7 @@ public abstract class StemmerTestBase ex
}
try {
- Dictionary dictionary = new Dictionary(affixStream, Arrays.asList(dictStreams), ignoreCase);
+ Dictionary dictionary = new Dictionary(new RAMDirectory(), "dictionary", affixStream, Arrays.asList(dictStreams), ignoreCase);
stemmer = new Stemmer(dictionary);
} finally {
IOUtils.closeWhileHandlingException(affixStream);
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java Mon Oct 19 13:38:42 2015
@@ -24,6 +24,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -51,8 +53,11 @@ public class Test64kAffixes extends Luce
dictWriter.write("1\ndrink/2\n");
dictWriter.close();
- try (InputStream affStream = Files.newInputStream(affix); InputStream dictStream = Files.newInputStream(dict)) {
- Dictionary dictionary = new Dictionary(affStream, dictStream);
+ try (InputStream affStream = Files.newInputStream(affix); InputStream dictStream = Files.newInputStream(dict); Directory tempDir2 = newDirectory()) {
+ if (tempDir2 instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) tempDir2).setEnableVirusScanner(false);
+ }
+ Dictionary dictionary = new Dictionary(tempDir2, "dictionary", affStream, dictStream);
Stemmer stemmer = new Stemmer(dictionary);
// drinks should still stem to drink
List<CharsRef> stems = stemmer.stem("drinks");
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java Mon Oct 19 13:38:42 2015
@@ -22,7 +22,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import org.apache.lucene.analysis.hunspell.Dictionary;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
@@ -165,14 +166,14 @@ public class TestAllDictionaries extends
IOUtils.rm(tmp);
Files.createDirectory(tmp);
- try (InputStream in = Files.newInputStream(f)) {
+ try (InputStream in = Files.newInputStream(f); Directory tempDir = getDirectory()) {
TestUtil.unzip(in, tmp);
Path dicEntry = tmp.resolve(tests[i+1]);
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
InputStream affix = Files.newInputStream(affEntry)) {
- Dictionary dic = new Dictionary(affix, dictionary);
+ Dictionary dic = new Dictionary(tempDir, "dictionary", affix, dictionary);
System.out.println(tests[i] + "\t" + RamUsageTester.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageTester.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageTester.humanSizeOf(dic.flagLookup) + ", " +
@@ -204,11 +205,20 @@ public class TestAllDictionaries extends
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
- InputStream affix = Files.newInputStream(affEntry)) {
- new Dictionary(affix, dictionary);
+ InputStream affix = Files.newInputStream(affEntry);
+ Directory tempDir = getDirectory()) {
+ new Dictionary(tempDir, "dictionary", affix, dictionary);
}
}
}
}
}
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java Mon Oct 19 13:38:42 2015
@@ -22,12 +22,13 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import org.apache.lucene.analysis.hunspell.Dictionary;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.apache.lucene.util.RamUsageTester;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.junit.Ignore;
/**
@@ -186,8 +187,12 @@ public class TestAllDictionaries2 extend
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
- InputStream affix = Files.newInputStream(affEntry)) {
- Dictionary dic = new Dictionary(affix, dictionary);
+ InputStream affix = Files.newInputStream(affEntry);
+ Directory tempDir = newDirectory()) {
+ if (tempDir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
+ }
+ Dictionary dic = new Dictionary(tempDir, "dictionary", affix, dictionary);
System.out.println(tests[i] + "\t" + RamUsageTester.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageTester.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageTester.humanSizeOf(dic.flagLookup) + ", " +
@@ -219,8 +224,12 @@ public class TestAllDictionaries2 extend
Path affEntry = tmp.resolve(tests[i+2]);
try (InputStream dictionary = Files.newInputStream(dicEntry);
- InputStream affix = Files.newInputStream(affEntry)) {
- new Dictionary(affix, dictionary);
+ InputStream affix = Files.newInputStream(affEntry);
+ Directory tempDir = newDirectory()) {
+ if (tempDir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
+ }
+ new Dictionary(tempDir, "dictionary", affix, dictionary);
}
}
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java Mon Oct 19 13:38:42 2015
@@ -24,9 +24,10 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.LuceneTestCase;
@@ -41,8 +42,9 @@ public class TestDictionary extends Luce
public void testSimpleDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("simple.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -63,13 +65,15 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
public void testCompressedDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Directory tempDir = getDirectory();
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -80,13 +84,15 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
public void testCompressedBeforeSetDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed-before-set.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
+ Directory tempDir = getDirectory();
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -97,13 +103,15 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
public void testCompressedEmptyAliasDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed-empty-alias.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
+ Directory tempDir = getDirectory();
- Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
@@ -114,15 +122,17 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
// malformed rule causes ParseException
public void testInvalidData() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("broken.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
try {
- new Dictionary(affixStream, dictStream);
+ new Dictionary(tempDir, "dictionary", affixStream, dictStream);
fail("didn't get expected exception");
} catch (ParseException expected) {
assertTrue(expected.getMessage().startsWith("The affix file contains a rule with less than four elements"));
@@ -131,15 +141,17 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
// malformed flags causes ParseException
public void testInvalidFlags() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("broken-flags.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
try {
- new Dictionary(affixStream, dictStream);
+ new Dictionary(tempDir, "dictionary", affixStream, dictStream);
fail("didn't get expected exception");
} catch (Exception expected) {
assertTrue(expected.getMessage().startsWith("expected only one flag"));
@@ -147,6 +159,7 @@ public class TestDictionary extends Luce
affixStream.close();
dictStream.close();
+ tempDir.close();
}
private class CloseCheckInputStream extends FilterInputStream {
@@ -170,21 +183,22 @@ public class TestDictionary extends Luce
public void testResourceCleanup() throws Exception {
CloseCheckInputStream affixStream = new CloseCheckInputStream(getClass().getResourceAsStream("compressed.aff"));
CloseCheckInputStream dictStream = new CloseCheckInputStream(getClass().getResourceAsStream("compressed.dic"));
+ Directory tempDir = getDirectory();
- new Dictionary(affixStream, dictStream);
+ new Dictionary(tempDir, "dictionary", affixStream, dictStream);
assertFalse(affixStream.isClosed());
assertFalse(dictStream.isClosed());
affixStream.close();
dictStream.close();
+ tempDir.close();
assertTrue(affixStream.isClosed());
assertTrue(dictStream.isClosed());
}
-
public void testReplacements() throws Exception {
Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
Builder<CharsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE2, outputs);
@@ -244,4 +258,12 @@ public class TestDictionary extends Luce
assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8"));
assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8"));
}
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java Mon Oct 19 13:38:42 2015
@@ -27,10 +27,10 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.hunspell.Dictionary;
-import org.apache.lucene.analysis.hunspell.HunspellStemFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -43,11 +43,14 @@ public class TestHunspellStemFilter exte
// no multiple try-with to workaround bogus VerifyError
InputStream affixStream = TestStemmer.class.getResourceAsStream("simple.aff");
InputStream dictStream = TestStemmer.class.getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
+
try {
- dictionary = new Dictionary(affixStream, dictStream);
+ dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
} finally {
IOUtils.closeWhileHandlingException(affixStream, dictStream);
}
+ tempDir.close();
}
@AfterClass
@@ -107,8 +110,9 @@ public class TestHunspellStemFilter exte
// no multiple try-with to workaround bogus VerifyError
InputStream affixStream = TestStemmer.class.getResourceAsStream("simple.aff");
InputStream dictStream = TestStemmer.class.getResourceAsStream("simple.dic");
+ Directory tempDir = getDirectory();
try {
- d = new Dictionary(affixStream, Collections.singletonList(dictStream), true);
+ d = new Dictionary(tempDir, "dictionary", affixStream, Collections.singletonList(dictStream), true);
} finally {
IOUtils.closeWhileHandlingException(affixStream, dictStream);
}
@@ -121,5 +125,14 @@ public class TestHunspellStemFilter exte
};
checkOneTerm(a, "NoChAnGy", "NoChAnGy");
a.close();
+ tempDir.close();
+ }
+
+ private static Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
}
}
Modified: lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java (original)
+++ lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java Mon Oct 19 13:38:42 2015
@@ -214,7 +214,6 @@ public class BooleanPerceptronClassifier
}
}
tokenStream.end();
- tokenStream.close();
}
double score = 1 - Math.exp(-1 * Math.abs(threshold - output.doubleValue()) / threshold);
Modified: lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java (original)
+++ lucene/dev/branches/lucene6835/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java Mon Oct 19 13:38:42 2015
@@ -222,7 +222,6 @@ public class SimpleNaiveBayesClassifier
result.add(charTermAttribute.toString());
}
tokenStream.end();
- tokenStream.close();
}
}
return result.toArray(new String[result.size()]);
Modified: lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java (original)
+++ lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java Mon Oct 19 13:38:42 2015
@@ -140,6 +140,9 @@ public class SimpleTextCompoundFormat ex
@Override
public IndexOutput createOutput(String name, IOContext context) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) { throw new UnsupportedOperationException(); }
@Override
public void sync(Collection<String> names) { throw new UnsupportedOperationException(); }
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java Mon Oct 19 13:38:42 2015
@@ -172,6 +172,11 @@ final class Lucene50CompoundReader exten
public IndexOutput createOutput(String name, IOContext context) throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ throw new UnsupportedOperationException();
+ }
@Override
public void sync(Collection<String> names) {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Mon Oct 19 13:38:42 2015
@@ -32,8 +32,8 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
-import java.util.Map.Entry;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -2620,7 +2620,7 @@ public class IndexWriter implements Clos
SegmentCommitInfo infoPerCommit = new SegmentCommitInfo(info, 0, -1L, -1L, -1L);
info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
- trackingDir.getCreatedFiles().clear();
+ trackingDir.clearCreatedFiles();
setDiagnostics(info, SOURCE_ADDINDEXES_READERS);
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java Mon Oct 19 13:38:42 2015
@@ -269,6 +269,17 @@ public class IndexSearcher {
}
/**
+ * Return the query cache of this {@link IndexSearcher}. This will be either
+ * the {@link #getDefaultQueryCache() default query cache} or the query cache
+ * that was last set through {@link #setQueryCache(QueryCache)}. A return
+ * value of {@code null} indicates that caching is disabled.
+ * @lucene.experimental
+ */
+ public QueryCache getQueryCache() {
+ return queryCache;
+ }
+
+ /**
* Set the {@link QueryCachingPolicy} to use for query caching.
* This method should be called <b>before</b> starting using this
* {@link IndexSearcher}.
@@ -280,6 +291,16 @@ public class IndexSearcher {
}
/**
+ * Return the query cache of this {@link IndexSearcher}. This will be either
+ * the {@link #getDefaultQueryCachingPolicy() default policy} or the policy
+ * that was last set through {@link #setQueryCachingPolicy(QueryCachingPolicy)}.
+ * @lucene.experimental
+ */
+ public QueryCachingPolicy getQueryCachingPolicy() {
+ return queryCachingPolicy;
+ }
+
+ /**
* Expert: Creates an array of leaf slices each holding a subset of the given leaves.
* Each {@link LeafSlice} is executed in a single thread. By default there
* will be one {@link LeafSlice} per leaf ({@link org.apache.lucene.index.LeafReaderContext}).
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/Multiset.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/Multiset.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/Multiset.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/Multiset.java Mon Oct 19 13:38:42 2015
@@ -87,6 +87,7 @@ final class Multiset<T> extends Abstract
}
@Override
+ @SuppressWarnings("unchecked")
public boolean remove(Object o) {
final Integer count = map.get(o);
if (count == null) {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java Mon Oct 19 13:38:42 2015
@@ -29,6 +29,8 @@ import org.apache.lucene.search.TopTerms
import java.io.IOException;
import java.util.Objects;
+import java.util.List;
+import java.util.ArrayList;
/**
* Wraps any {@link MultiTermQuery} as a {@link SpanQuery},
@@ -156,14 +158,14 @@ public class SpanMultiTermQueryWrapper<Q
* @see #setRewriteMethod
*/
public final static SpanRewriteMethod SCORING_SPAN_QUERY_REWRITE = new SpanRewriteMethod() {
- private final ScoringRewrite<SpanOrQuery> delegate = new ScoringRewrite<SpanOrQuery>() {
+ private final ScoringRewrite<List<SpanQuery>> delegate = new ScoringRewrite<List<SpanQuery>>() {
@Override
- protected SpanOrQuery getTopLevelBuilder() {
- return new SpanOrQuery();
+ protected List<SpanQuery> getTopLevelBuilder() {
+ return new ArrayList<SpanQuery>();
}
- protected Query build(SpanOrQuery builder) {
- return builder;
+ protected Query build(List<SpanQuery> builder) {
+ return new SpanOrQuery(builder.toArray(new SpanQuery[builder.size()]));
}
@Override
@@ -172,9 +174,9 @@ public class SpanMultiTermQueryWrapper<Q
}
@Override
- protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost, TermContext states) {
+ protected void addClause(List<SpanQuery> topLevel, Term term, int docCount, float boost, TermContext states) {
final SpanTermQuery q = new SpanTermQuery(term, states);
- topLevel.addClause(q);
+ topLevel.add(q);
}
};
@@ -196,33 +198,33 @@ public class SpanMultiTermQueryWrapper<Q
* @see #setRewriteMethod
*/
public static final class TopTermsSpanBooleanQueryRewrite extends SpanRewriteMethod {
- private final TopTermsRewrite<SpanOrQuery> delegate;
+ private final TopTermsRewrite<List<SpanQuery>> delegate;
/**
* Create a TopTermsSpanBooleanQueryRewrite for
* at most <code>size</code> terms.
*/
public TopTermsSpanBooleanQueryRewrite(int size) {
- delegate = new TopTermsRewrite<SpanOrQuery>(size) {
+ delegate = new TopTermsRewrite<List<SpanQuery>>(size) {
@Override
protected int getMaxSize() {
return Integer.MAX_VALUE;
}
@Override
- protected SpanOrQuery getTopLevelBuilder() {
- return new SpanOrQuery();
+ protected List<SpanQuery> getTopLevelBuilder() {
+ return new ArrayList<SpanQuery>();
}
@Override
- protected Query build(SpanOrQuery builder) {
- return builder;
+ protected Query build(List<SpanQuery> builder) {
+ return new SpanOrQuery(builder.toArray(new SpanQuery[builder.size()]));
}
@Override
- protected void addClause(SpanOrQuery topLevel, Term term, int docFreq, float boost, TermContext states) {
+ protected void addClause(List<SpanQuery> topLevel, Term term, int docFreq, float boost, TermContext states) {
final SpanTermQuery q = new SpanTermQuery(term, states);
- topLevel.addClause(q);
+ topLevel.add(q);
}
};
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Mon Oct 19 13:38:42 2015
@@ -53,7 +53,7 @@ public final class SpanOrQuery extends S
}
/** Adds a clause to this query */
- public final void addClause(SpanQuery clause) {
+ private final void addClause(SpanQuery clause) {
if (field == null) {
field = clause.getField();
} else if (clause.getField() != null && !clause.getField().equals(field)) {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java Mon Oct 19 13:38:42 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.Random;
/**
* Base implementation for a concrete {@link Directory} that uses a {@link LockFactory} for locking.
@@ -31,6 +32,22 @@ public abstract class BaseDirectory exte
* this Directory instance). */
protected final LockFactory lockFactory;
+ /** Subclasses can use this to generate temp file name candidates */
+ protected static final Random tempFileRandom;
+
+ static {
+ String prop = System.getProperty("tests.seed");
+ int seed;
+ if (prop != null) {
+ // So if there is a test failure that relied on temp file names,
+ //we remain reproducible based on the test seed:
+ seed = prop.hashCode();
+ } else {
+ seed = (int) System.currentTimeMillis();
+ }
+ tempFileRandom = new Random(seed);
+ }
+
/** Sole constructor. */
protected BaseDirectory(LockFactory lockFactory) {
super();
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java Mon Oct 19 13:38:42 2015
@@ -17,9 +17,9 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.Closeable;
import java.nio.file.NoSuchFileException;
import java.util.Collection; // for javadocs
@@ -69,8 +69,12 @@ public abstract class Directory implemen
/** Creates a new, empty file in the directory with the given name.
Returns a stream writing this file. */
- public abstract IndexOutput createOutput(String name, IOContext context)
- throws IOException;
+ public abstract IndexOutput createOutput(String name, IOContext context) throws IOException;
+
+ /** Creates a new, empty file for writing in the directory, with a
+ * temporary file name derived from prefix and suffix. Use
+ * {@link IndexOutput#getName} to see what name was used. */
+ public abstract IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException;
/**
* Ensure that any writes to these files are moved to
@@ -119,8 +123,7 @@ public abstract class Directory implemen
/** Closes the store. */
@Override
- public abstract void close()
- throws IOException;
+ public abstract void close() throws IOException;
@Override
public String toString() {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Mon Oct 19 13:38:42 2015
@@ -22,10 +22,13 @@ import java.io.FilterOutputStream;
import java.io.IOException;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.DirectoryStream;
+import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
+import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -236,6 +239,19 @@ public abstract class FSDirectory extend
return new FSIndexOutput(name);
}
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ ensureOpen();
+ while (true) {
+ String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
+ try {
+ return new FSIndexOutput(name, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+ } catch (FileAlreadyExistsException faee) {
+ // Retry with next random name
+ }
+ }
+ }
+
protected void ensureCanWrite(String name) throws IOException {
Files.deleteIfExists(directory.resolve(name)); // delete existing, if any
}
@@ -348,7 +364,11 @@ public abstract class FSDirectory extend
static final int CHUNK_SIZE = 8192;
public FSIndexOutput(String name) throws IOException {
- super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", new FilterOutputStream(Files.newOutputStream(directory.resolve(name))) {
+ this(name, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
+ }
+
+ FSIndexOutput(String name, OpenOption... options) throws IOException {
+ super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", name, new FilterOutputStream(Files.newOutputStream(directory.resolve(name), options)) {
// This implementation ensures, that we never write more than CHUNK_SIZE bytes:
@Override
public void write(byte[] b, int offset, int length) throws IOException {
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java Mon Oct 19 13:38:42 2015
@@ -22,13 +22,12 @@ import java.nio.file.AtomicMoveNotSuppor
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.HashSet;
import org.apache.lucene.util.IOUtils;
-
/**
* Expert: A Directory instance that switches files between
* two other Directory instances.
@@ -169,6 +168,11 @@ public class FileSwitchDirectory extends
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ return getDirectory("."+suffix).createTempOutput(prefix, suffix, context);
+ }
+
+ @Override
public void sync(Collection<String> names) throws IOException {
List<String> primaryNames = new ArrayList<>();
List<String> secondaryNames = new ArrayList<>();
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java Mon Oct 19 13:38:42 2015
@@ -74,6 +74,11 @@ public class FilterDirectory extends Dir
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ return in.createTempOutput(prefix, suffix, context);
+ }
+
+ @Override
public void sync(Collection<String> names) throws IOException {
in.sync(names);
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java Mon Oct 19 13:38:42 2015
@@ -31,15 +31,27 @@ import java.io.IOException;
*/
public abstract class IndexOutput extends DataOutput implements Closeable {
+ /** Full description of this output, e.g. which class such as {@code FSIndexOutput}, and the full path to the file */
private final String resourceDescription;
+ /** Just the name part from {@code resourceDescription} */
+ private final String name;
+
/** Sole constructor. resourceDescription should be non-null, opaque string
* describing this resource; it's returned from {@link #toString}. */
- protected IndexOutput(String resourceDescription) {
+ protected IndexOutput(String resourceDescription, String name) {
if (resourceDescription == null) {
throw new IllegalArgumentException("resourceDescription must not be null");
}
this.resourceDescription = resourceDescription;
+ this.name = name;
+ }
+
+ /** Returns the name used to create this {@code IndexOutput}. This is especially useful when using
+ * {@link Directory#createTempOutput}. */
+ // TODO: can we somehow use this as the default resource description or something?
+ public String getName() {
+ return name;
}
/** Closes this stream to further operations. */
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/OutputStreamIndexOutput.java Mon Oct 19 13:38:42 2015
@@ -37,8 +37,8 @@ public class OutputStreamIndexOutput ext
* @param bufferSize the buffer size in bytes used to buffer writes internally.
* @throws IllegalArgumentException if the given buffer size is less or equal to <tt>0</tt>
*/
- public OutputStreamIndexOutput(String resourceDescription, OutputStream out, int bufferSize) {
- super(resourceDescription);
+ public OutputStreamIndexOutput(String resourceDescription, String name, OutputStream out, int bufferSize) {
+ super(resourceDescription, name);
this.os = new BufferedOutputStream(new CheckedOutputStream(out, crc), bufferSize);
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java Mon Oct 19 13:38:42 2015
@@ -26,12 +26,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
-
/**
* A memory-resident {@link Directory} implementation. Locking
* implementation is by default the {@link SingleInstanceLockFactory}.
@@ -111,10 +111,7 @@ public class RAMDirectory extends BaseDi
// and do not synchronize or anything stronger. it's great for testing!
// NOTE: fileMap.keySet().toArray(new String[0]) is broken in non Sun JDKs,
// and the code below is resilient to map changes during the array population.
- Set<String> fileNames = fileMap.keySet();
- List<String> names = new ArrayList<>(fileNames.size());
- for (String name : fileNames) names.add(name);
- return names.toArray(new String[names.size()]);
+ return fileMap.keySet().toArray(new String[fileMap.size()]);
}
public final boolean fileNameExists(String name) {
@@ -150,9 +147,6 @@ public class RAMDirectory extends BaseDi
return Accountables.namedAccountables("file", fileMap);
}
- /** Removes an existing file in the directory.
- * @throws IOException if the file does not exist
- */
@Override
public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
@@ -168,7 +162,6 @@ public class RAMDirectory extends BaseDi
}
}
- /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
@@ -182,6 +175,22 @@ public class RAMDirectory extends BaseDi
return new RAMOutputStream(name, file, true);
}
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ ensureOpen();
+
+ // Make the file first...
+ RAMFile file = newRAMFile();
+
+ // ... then try to find a unique name for it:
+ while (true) {
+ String name = prefix + tempFileRandom.nextInt(Integer.MAX_VALUE) + "." + suffix;
+ if (fileMap.putIfAbsent(name, file) == null) {
+ return new RAMOutputStream(name, file, true);
+ }
+ }
+ }
+
/**
* Returns a new {@link RAMFile} for storing data. This method can be
* overridden to return different {@link RAMFile} impls, that e.g. override
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java Mon Oct 19 13:38:42 2015
@@ -57,7 +57,7 @@ public class RAMOutputStream extends Ind
/** Creates this, with specified name. */
public RAMOutputStream(String name, RAMFile f, boolean checksum) {
- super("RAMOutputStream(name=\"" + name + "\")");
+ super("RAMOutputStream(name=\"" + name + "\")", name);
file = f;
// make sure that we switch to the
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java Mon Oct 19 13:38:42 2015
@@ -38,7 +38,7 @@ public final class RateLimitedIndexOutpu
private long currentMinPauseCheckBytes;
public RateLimitedIndexOutput(final RateLimiter rateLimiter, final IndexOutput delegate) {
- super("RateLimitedIndexOutput(" + delegate + ")");
+ super("RateLimitedIndexOutput(" + delegate + ")", delegate.getName());
this.delegate = delegate;
this.rateLimiter = rateLimiter;
this.currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java Mon Oct 19 13:38:42 2015
@@ -49,6 +49,14 @@ public final class TrackingDirectoryWrap
}
@Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
+ throws IOException {
+ IndexOutput tempOutput = in.createTempOutput(prefix, suffix, context);
+ createdFileNames.add(tempOutput.getName());
+ return tempOutput;
+ }
+
+ @Override
public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
in.copyFrom(from, src, dest, context);
createdFileNames.add(dest);
@@ -63,10 +71,12 @@ public final class TrackingDirectoryWrap
}
}
- // maybe clone before returning.... all callers are
- // cloning anyway....
+ /** NOTE: returns a copy of the created files. */
public Set<String> getCreatedFiles() {
- return createdFileNames;
+ return new HashSet<>(createdFileNames);
}
+ public void clearCreatedFiles() {
+ createdFileNames.clear();
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Mon Oct 19 13:38:42 2015
@@ -38,8 +38,10 @@ import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -190,13 +192,43 @@ public final class IOUtils {
* <p>
* Note that the files should not be null.
*/
- public static void deleteFilesIgnoringExceptions(Directory dir, String... names) {
+ public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) {
try {
- dir.deleteFiles(Arrays.asList(names));
+ dir.deleteFiles(files);
} catch (Throwable ignored) {
// ignore
}
}
+
+ public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
+ deleteFilesIgnoringExceptions(dir, Arrays.asList(files));
+ }
+
+ /**
+ * Deletes all given file names. Some of the
+ * file names may be null; they are
+ * ignored. After everything is deleted, the method either
+ * throws the first exception it hit while deleting, or
+ * completes normally if there were no exceptions.
+ *
+ * @param dir Directory to delete files from
+ * @param names file names to delete
+ */
+ public static void deleteFiles(Directory dir, Collection<String> names) throws IOException {
+ Set<String> nonNullNames = new HashSet<>();
+ for(String name : names) {
+ if (name != null) {
+ nonNullNames.add(name);
+ }
+ }
+ if (names.isEmpty() == false) {
+ dir.deleteFiles(names);
+ }
+ }
+
+ public static void deleteFiles(Directory dir, String... files) throws IOException {
+ deleteFiles(dir, Arrays.asList(files));
+ }
/**
* Deletes all given files, suppressing all thrown IOExceptions.