You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/08/13 07:20:49 UTC
svn commit: r1513349 [1/2] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/idea/solr/core/src/java/
dev-tools/idea/solr/core/src/test/ dev-tools/maven/solr/core/src/java/
dev-tools/maven/solr/core/src/test/ lucene/ lucene/analysis/ lucene...
Author: cm
Date: Tue Aug 13 05:20:47 2013
New Revision: 1513349
URL: http://svn.apache.org/r1513349
Log:
Merged changes from trunk (r1512578 through r1513348)
Added:
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/join/
- copied from r1513348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParserPlugin.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParserPlugin.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java
lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/search/join/
- copied from r1513348, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
- copied unchanged from r1513348, lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
Removed:
lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig-termindex.xml
Modified:
lucene/dev/branches/lucene4956/ (props changed)
lucene/dev/branches/lucene4956/dev-tools/ (props changed)
lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/java/solr-core.iml
lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/test/solr-core-tests.iml
lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/java/pom.xml.template
lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/test/pom.xml.template
lucene/dev/branches/lucene4956/lucene/ (props changed)
lucene/dev/branches/lucene4956/lucene/BUILD.txt (props changed)
lucene/dev/branches/lucene4956/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene4956/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/lucene4956/lucene/LICENSE.txt (props changed)
lucene/dev/branches/lucene4956/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/lucene4956/lucene/NOTICE.txt (props changed)
lucene/dev/branches/lucene4956/lucene/README.txt (props changed)
lucene/dev/branches/lucene4956/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/lucene4956/lucene/analysis/ (props changed)
lucene/dev/branches/lucene4956/lucene/analysis/common/ (props changed)
lucene/dev/branches/lucene4956/lucene/backwards/ (props changed)
lucene/dev/branches/lucene4956/lucene/benchmark/ (props changed)
lucene/dev/branches/lucene4956/lucene/build.xml (props changed)
lucene/dev/branches/lucene4956/lucene/classification/ (props changed)
lucene/dev/branches/lucene4956/lucene/codecs/ (props changed)
lucene/dev/branches/lucene4956/lucene/common-build.xml (props changed)
lucene/dev/branches/lucene4956/lucene/core/ (props changed)
lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
lucene/dev/branches/lucene4956/lucene/demo/ (props changed)
lucene/dev/branches/lucene4956/lucene/facet/ (props changed)
lucene/dev/branches/lucene4956/lucene/grouping/ (props changed)
lucene/dev/branches/lucene4956/lucene/highlighter/ (props changed)
lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
lucene/dev/branches/lucene4956/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/lucene4956/lucene/join/ (props changed)
lucene/dev/branches/lucene4956/lucene/licenses/ (props changed)
lucene/dev/branches/lucene4956/lucene/memory/ (props changed)
lucene/dev/branches/lucene4956/lucene/misc/ (props changed)
lucene/dev/branches/lucene4956/lucene/module-build.xml (contents, props changed)
lucene/dev/branches/lucene4956/lucene/queries/ (props changed)
lucene/dev/branches/lucene4956/lucene/queryparser/ (props changed)
lucene/dev/branches/lucene4956/lucene/replicator/ (props changed)
lucene/dev/branches/lucene4956/lucene/sandbox/ (props changed)
lucene/dev/branches/lucene4956/lucene/site/ (props changed)
lucene/dev/branches/lucene4956/lucene/spatial/ (props changed)
lucene/dev/branches/lucene4956/lucene/suggest/ (props changed)
lucene/dev/branches/lucene4956/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/lucene4956/lucene/tools/ (props changed)
lucene/dev/branches/lucene4956/solr/ (props changed)
lucene/dev/branches/lucene4956/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene4956/solr/LICENSE.txt (props changed)
lucene/dev/branches/lucene4956/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene4956/solr/README.txt (props changed)
lucene/dev/branches/lucene4956/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/lucene4956/solr/build.xml (props changed)
lucene/dev/branches/lucene4956/solr/cloud-dev/ (props changed)
lucene/dev/branches/lucene4956/solr/common-build.xml (contents, props changed)
lucene/dev/branches/lucene4956/solr/contrib/ (props changed)
lucene/dev/branches/lucene4956/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/velocity/numFound.vm (props changed)
lucene/dev/branches/lucene4956/solr/core/ (props changed)
lucene/dev/branches/lucene4956/solr/core/ivy.xml
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema.xml
lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema15.xml
lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/core/TestConfig.java
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/schema/TestManagedSchema.java
lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
lucene/dev/branches/lucene4956/solr/example/ (props changed)
lucene/dev/branches/lucene4956/solr/licenses/ (props changed)
lucene/dev/branches/lucene4956/solr/scripts/ (props changed)
lucene/dev/branches/lucene4956/solr/site/ (props changed)
lucene/dev/branches/lucene4956/solr/solrj/ (props changed)
lucene/dev/branches/lucene4956/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
lucene/dev/branches/lucene4956/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
lucene/dev/branches/lucene4956/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
lucene/dev/branches/lucene4956/solr/test-framework/ (props changed)
lucene/dev/branches/lucene4956/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
lucene/dev/branches/lucene4956/solr/webapp/ (props changed)
Modified: lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/java/solr-core.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/java/solr-core.iml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/java/solr-core.iml (original)
+++ lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/java/solr-core.iml Tue Aug 13 05:20:47 2013
@@ -25,5 +25,6 @@
<orderEntry type="module" module-name="analysis-common" />
<orderEntry type="module" module-name="lucene-core" />
<orderEntry type="module" module-name="queryparser" />
+ <orderEntry type="module" module-name="join" />
</component>
</module>
Modified: lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/test/solr-core-tests.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/test/solr-core-tests.iml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/test/solr-core-tests.iml (original)
+++ lucene/dev/branches/lucene4956/dev-tools/idea/solr/core/src/test/solr-core-tests.iml Tue Aug 13 05:20:47 2013
@@ -27,5 +27,6 @@
<orderEntry type="module" scope="TEST" module-name="suggest" />
<orderEntry type="module" scope="TEST" module-name="spatial" />
<orderEntry type="module" scope="TEST" module-name="misc" />
+ <orderEntry type="module" scope="TEST" module-name="join" />
</component>
</module>
Modified: lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/java/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/java/pom.xml.template?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/java/pom.xml.template (original)
+++ lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/java/pom.xml.template Tue Aug 13 05:20:47 2013
@@ -95,6 +95,11 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-join</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${project.version}</version>
<exclusions>
Modified: lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/test/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/test/pom.xml.template?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/test/pom.xml.template (original)
+++ lucene/dev/branches/lucene4956/dev-tools/maven/solr/core/src/test/pom.xml.template Tue Aug 13 05:20:47 2013
@@ -198,6 +198,12 @@
<artifactId>jetty-util</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>dom4j</groupId>
+ <artifactId>dom4j</artifactId>
+ <version>1.6.1</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<sourceDirectory/>
Modified: lucene/dev/branches/lucene4956/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/CHANGES.txt?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4956/lucene/CHANGES.txt Tue Aug 13 05:20:47 2013
@@ -38,12 +38,6 @@ New Features
* SOLR-3359: Added analyzer attribute/property to SynonymFilterFactory.
(Ryo Onodera via Koji Sekiguchi)
-Bugs
-
-* LUCENE-5160: Handle the case where reading from a file or FileChannel returns -1, which
- could happen in rare cases where something happens to the file between the time we start the
- read loop (where we check the length) and when we actually do the read. (gsingers, yonik, Robert Muir, Uwe Schindler)
-
Optimizations
* LUCENE-4848: Use Java 7 NIO2-FileChannel instead of RandomAccessFile
@@ -112,6 +106,15 @@ Bug Fixes
seek/lookup which can cause sideeffects if done on a cached FST root arc.
(Simon Willnauer)
+* LUCENE-5160: Handle the case where reading from a file or FileChannel returns -1,
+ which could happen in rare cases where something happens to the file between the
+ time we start the read loop (where we check the length) and when we actually do
+ the read. (gsingers, yonik, Robert Muir, Uwe Schindler)
+
+* LUCENE-5166: PostingsHighlighter would throw IOOBE if a term spanned the maxLength
+ boundary, made it into the top-N and went to the formatter.
+ (Manuel Amoabeng, Michael McCandless, Robert Muir)
+
API Changes
* LUCENE-5094: Add ramBytesUsed() to MultiDocValues.OrdinalMap.
@@ -147,6 +150,11 @@ API Changes
* LUCENE-5156: CompressingTermVectors TermsEnum no longer supports ord().
(Robert Muir)
+* LUCENE-5161, LUCENE-5164: Fix default chunk sizes in FSDirectory to not be
+ unnecessarily large (now 8192 bytes); also use chunking when writing to index
+ files. FSDirectory#setReadChunkSize() is now deprecated and will be removed
+ in Lucene 5.0. (Uwe Schindler, Robert Muir, gsingers)
+
Optimizations
* LUCENE-5088: Added TermFilter to filter docs by a specific term.
Modified: lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java Tue Aug 13 05:20:47 2013
@@ -23,7 +23,7 @@ import java.io.IOException;
/** Base implementation class for buffered {@link IndexInput}. */
public abstract class BufferedIndexInput extends IndexInput {
- /** Default buffer size set to 1024*/
+ /** Default buffer size set to {@value #BUFFER_SIZE}. */
public static final int BUFFER_SIZE = 1024;
// The normal read buffer size defaults to 1024, but
@@ -33,7 +33,7 @@ public abstract class BufferedIndexInput
// BufferedIndexInputs created during merging. See
// LUCENE-888 for details.
/**
- * A buffer size for merges set to 4096
+ * A buffer size for merges set to {@value #MERGE_BUFFER_SIZE}.
*/
public static final int MERGE_BUFFER_SIZE = 4096;
@@ -115,15 +115,14 @@ public abstract class BufferedIndexInput
@Override
public final void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
-
- if(len <= (bufferLength-bufferPosition)){
+ int available = bufferLength - bufferPosition;
+ if(len <= available){
// the buffer contains enough data to satisfy this request
if(len>0) // to allow b to be null if len is 0...
System.arraycopy(buffer, bufferPosition, b, offset, len);
bufferPosition+=len;
} else {
// the buffer does not have enough data. First serve all we've got.
- int available = bufferLength - bufferPosition;
if(available > 0){
System.arraycopy(buffer, bufferPosition, b, offset, available);
offset += available;
Modified: lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Tue Aug 13 05:20:47 2013
@@ -31,6 +31,7 @@ import java.util.concurrent.Future;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
/**
* Base class for Directory implementations that store index
@@ -111,17 +112,8 @@ import org.apache.lucene.util.Constants;
*/
public abstract class FSDirectory extends Directory {
- /**
- * Default read chunk size. This is a conditional default: on 32bit JVMs, it defaults to 100 MB. On 64bit JVMs, it's
- * <code>Integer.MAX_VALUE</code>.
- *
- * @see #setReadChunkSize
- */
- public static final int DEFAULT_READ_CHUNK_SIZE = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : 100 * 1024 * 1024;
-
protected final File directory; // The underlying filesystem directory
protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed
- private int chunkSize = DEFAULT_READ_CHUNK_SIZE; // LUCENE-1566
// returns the canonical version of the directory, creating it if it doesn't exist.
private static File getCanonicalPath(File file) throws IOException {
@@ -355,68 +347,38 @@ public abstract class FSDirectory extend
}
/**
- * Sets the maximum number of bytes read at once from the
- * underlying file during {@link IndexInput#readBytes}.
- * The default value is {@link #DEFAULT_READ_CHUNK_SIZE};
- *
- * <p> This was introduced due to <a
- * href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6478546">Sun
- * JVM Bug 6478546</a>, which throws an incorrect
- * OutOfMemoryError when attempting to read too many bytes
- * at once. It only happens on 32bit JVMs with a large
- * maximum heap size.</p>
- *
- * <p>Changes to this value will not impact any
- * already-opened {@link IndexInput}s. You should call
- * this before attempting to open an index on the
- * directory.</p>
- *
- * <p> <b>NOTE</b>: This value should be as large as
- * possible to reduce any possible performance impact. If
- * you still encounter an incorrect OutOfMemoryError,
- * trying lowering the chunk size.</p>
- */
- public final void setReadChunkSize(int chunkSize) {
- // LUCENE-1566
- if (chunkSize <= 0) {
- throw new IllegalArgumentException("chunkSize must be positive");
- }
- if (!Constants.JRE_IS_64BIT) {
- this.chunkSize = chunkSize;
- }
- }
-
- /**
- * The maximum number of bytes to read at once from the
- * underlying file during {@link IndexInput#readBytes}.
- * @see #setReadChunkSize
- */
- public final int getReadChunkSize() {
- // LUCENE-1566
- return chunkSize;
- }
-
- /**
* Writes output with {@link RandomAccessFile#write(byte[], int, int)}
*/
protected static class FSIndexOutput extends BufferedIndexOutput {
+ /**
+ * The maximum chunk size is 8192 bytes, because {@link RandomAccessFile} mallocs
+ * a native buffer outside of stack if the write buffer size is larger.
+ */
+ private static final int CHUNK_SIZE = 8192;
+
private final FSDirectory parent;
private final String name;
private final RandomAccessFile file;
private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once
public FSIndexOutput(FSDirectory parent, String name) throws IOException {
+ super(CHUNK_SIZE);
this.parent = parent;
this.name = name;
file = new RandomAccessFile(new File(parent.directory, name), "rw");
isOpen = true;
}
- /** output methods: */
@Override
- public void flushBuffer(byte[] b, int offset, int size) throws IOException {
+ protected void flushBuffer(byte[] b, int offset, int size) throws IOException {
assert isOpen;
- file.write(b, offset, size);
+ while (size > 0) {
+ final int toWrite = Math.min(CHUNK_SIZE, size);
+ file.write(b, offset, toWrite);
+ offset += toWrite;
+ size -= toWrite;
+ }
+ assert size == 0;
}
@Override
@@ -424,21 +386,14 @@ public abstract class FSDirectory extend
parent.onIndexOutputClosed(name);
// only close the file if it has not been closed yet
if (isOpen) {
- boolean success = false;
+ IOException priorE = null;
try {
super.close();
- success = true;
+ } catch (IOException ioe) {
+ priorE = ioe;
} finally {
isOpen = false;
- if (!success) {
- try {
- file.close();
- } catch (Throwable t) {
- // Suppress so we don't mask original exception
- }
- } else {
- file.close();
- }
+ IOUtils.closeWhileHandlingException(priorE, file);
}
}
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java Tue Aug 13 05:20:47 2013
@@ -20,6 +20,7 @@ package org.apache.lucene.store;
import java.io.File;
import java.io.EOFException;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
@@ -79,7 +80,7 @@ public class NIOFSDirectory extends FSDi
ensureOpen();
File path = new File(getDirectory(), name);
FileChannel fc = FileChannel.open(path.toPath(), StandardOpenOption.READ);
- return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context, getReadChunkSize());
+ return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context);
}
@Override
@@ -98,7 +99,7 @@ public class NIOFSDirectory extends FSDi
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new NIOFSIndexInput("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
- length, BufferedIndexInput.bufferSize(context), getReadChunkSize());
+ length, BufferedIndexInput.bufferSize(context));
}
};
}
@@ -107,12 +108,15 @@ public class NIOFSDirectory extends FSDi
* Reads bytes with {@link FileChannel#read(ByteBuffer, long)}
*/
protected static class NIOFSIndexInput extends BufferedIndexInput {
+ /**
+ * The maximum chunk size for reads of 16384 bytes.
+ */
+ private static final int CHUNK_SIZE = 16384;
+
/** the file channel we will read from */
protected final FileChannel channel;
/** is this instance a clone and hence does not own the file to close it */
boolean isClone = false;
- /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
- protected final int chunkSize;
/** start offset: non-zero in the slice case */
protected final long off;
/** end offset (start+length) */
@@ -120,18 +124,16 @@ public class NIOFSDirectory extends FSDi
private ByteBuffer byteBuf; // wraps the buffer for NIO
- public NIOFSIndexInput(String resourceDesc, FileChannel fc, IOContext context, int chunkSize) throws IOException {
+ public NIOFSIndexInput(String resourceDesc, FileChannel fc, IOContext context) throws IOException {
super(resourceDesc, context);
this.channel = fc;
- this.chunkSize = chunkSize;
this.off = 0L;
this.end = fc.size();
}
- public NIOFSIndexInput(String resourceDesc, FileChannel fc, long off, long length, int bufferSize, int chunkSize) {
+ public NIOFSIndexInput(String resourceDesc, FileChannel fc, long off, long length, int bufferSize) {
super(resourceDesc, bufferSize);
this.channel = fc;
- this.chunkSize = chunkSize;
this.off = off;
this.end = off + length;
this.isClone = true;
@@ -164,24 +166,18 @@ public class NIOFSDirectory extends FSDi
@Override
protected void readInternal(byte[] b, int offset, int len) throws IOException {
-
final ByteBuffer bb;
// Determine the ByteBuffer we should use
- if (b == buffer && 0 == offset) {
+ if (b == buffer) {
// Use our own pre-wrapped byteBuf:
assert byteBuf != null;
- byteBuf.clear();
- byteBuf.limit(len);
bb = byteBuf;
+ byteBuf.clear().position(offset);
} else {
bb = ByteBuffer.wrap(b, offset, len);
}
- int readOffset = bb.position();
- int readLength = bb.limit() - readOffset;
- assert readLength == len;
-
long pos = getFilePointer() + off;
if (pos + len > end) {
@@ -189,33 +185,20 @@ public class NIOFSDirectory extends FSDi
}
try {
+ int readLength = len;
while (readLength > 0) {
- final int limit;
- if (readLength > chunkSize) {
- // LUCENE-1566 - work around JVM Bug by breaking
- // very large reads into chunks
- limit = readOffset + chunkSize;
- } else {
- limit = readOffset + readLength;
- }
- bb.limit(limit);
- int i = channel.read(bb, pos);
- if (i < 0){//be defensive here, even though we checked before hand, something could have changed
- throw new EOFException("read past EOF: " + this + " off: " + offset + " len: " + len + " pos: " + pos + " limit: " + limit + " end: " + end);
+ final int toRead = Math.min(CHUNK_SIZE, readLength);
+ bb.limit(bb.position() + toRead);
+ assert bb.remaining() == toRead;
+ final int i = channel.read(bb, pos);
+ if (i < 0) { // be defensive here, even though we checked before hand, something could have changed
+ throw new EOFException("read past EOF: " + this + " off: " + offset + " len: " + len + " pos: " + pos + " chunkLen: " + toRead + " end: " + end);
}
+ assert i > 0 : "FileChannel.read with non zero-length bb.remaining() must always read at least one byte (FileChannel is in blocking mode, see spec of ReadableByteChannel)";
pos += i;
- readOffset += i;
readLength -= i;
}
- } catch (OutOfMemoryError e) {
- // propagate OOM up and add a hint for 32bit VM Users hitting the bug
- // with a large chunk size in the fast path.
- final OutOfMemoryError outOfMemoryError = new OutOfMemoryError(
- "OutOfMemoryError likely caused by the Sun VM Bug described in "
- + "https://issues.apache.org/jira/browse/LUCENE-1566; try calling FSDirectory.setReadChunkSize "
- + "with a value smaller than the current chunk size (" + chunkSize + ")");
- outOfMemoryError.initCause(e);
- throw outOfMemoryError;
+ assert readLength == 0;
} catch (IOException ioe) {
throw new IOException(ioe.getMessage() + ": " + this, ioe);
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java Tue Aug 13 05:20:47 2013
@@ -56,7 +56,7 @@ public class SimpleFSDirectory extends F
ensureOpen();
final File path = new File(directory, name);
RandomAccessFile raf = new RandomAccessFile(path, "r");
- return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context, getReadChunkSize());
+ return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);
}
@Override
@@ -75,7 +75,7 @@ public class SimpleFSDirectory extends F
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new SimpleFSIndexInput("SimpleFSIndexInput(" + sliceDescription + " in path=\"" + file.getPath() + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
- length, BufferedIndexInput.bufferSize(context), getReadChunkSize());
+ length, BufferedIndexInput.bufferSize(context));
}
};
}
@@ -85,29 +85,31 @@ public class SimpleFSDirectory extends F
* {@link RandomAccessFile#read(byte[], int, int)}.
*/
protected static class SimpleFSIndexInput extends BufferedIndexInput {
+ /**
+ * The maximum chunk size is 8192 bytes, because {@link RandomAccessFile} mallocs
+ * a native buffer outside of stack if the read buffer size is larger.
+ */
+ private static final int CHUNK_SIZE = 8192;
+
/** the file channel we will read from */
protected final RandomAccessFile file;
/** is this instance a clone and hence does not own the file to close it */
boolean isClone = false;
- /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
- protected final int chunkSize;
/** start offset: non-zero in the slice case */
protected final long off;
/** end offset (start+length) */
protected final long end;
- public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, IOContext context, int chunkSize) throws IOException {
+ public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, IOContext context) throws IOException {
super(resourceDesc, context);
this.file = file;
- this.chunkSize = chunkSize;
this.off = 0L;
this.end = file.length();
}
- public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize, int chunkSize) {
+ public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize) {
super(resourceDesc, bufferSize);
this.file = file;
- this.chunkSize = chunkSize;
this.off = off;
this.end = off + length;
this.isClone = true;
@@ -146,29 +148,16 @@ public class SimpleFSDirectory extends F
}
try {
- do {
- final int readLength;
- if (total + chunkSize > len) {
- readLength = len - total;
- } else {
- // LUCENE-1566 - work around JVM Bug by breaking very large reads into chunks
- readLength = chunkSize;
- }
- final int i = file.read(b, offset + total, readLength);
- if (i < 0){//be defensive here, even though we checked before hand, something could have changed
- throw new EOFException("read past EOF: " + this + " off: " + offset + " len: " + len + " total: " + total + " readLen: " + readLength + " end: " + end);
+ while (total < len) {
+ final int toRead = Math.min(CHUNK_SIZE, len - total);
+ final int i = file.read(b, offset + total, toRead);
+ if (i < 0) { // be defensive here, even though we checked before hand, something could have changed
+ throw new EOFException("read past EOF: " + this + " off: " + offset + " len: " + len + " total: " + total + " chunkLen: " + toRead + " end: " + end);
}
+ assert i > 0 : "RandomAccessFile.read with non zero-length toRead must always read at least one byte";
total += i;
- } while (total < len);
- } catch (OutOfMemoryError e) {
- // propagate OOM up and add a hint for 32bit VM Users hitting the bug
- // with a large chunk size in the fast path.
- final OutOfMemoryError outOfMemoryError = new OutOfMemoryError(
- "OutOfMemoryError likely caused by the Sun VM Bug described in "
- + "https://issues.apache.org/jira/browse/LUCENE-1566; try calling FSDirectory.setReadChunkSize "
- + "with a value smaller than the current chunk size (" + chunkSize + ")");
- outOfMemoryError.initCause(e);
- throw outOfMemoryError;
+ }
+ assert total == len;
} catch (IOException ioe) {
throw new IOException(ioe.getMessage() + ": " + this, ioe);
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java Tue Aug 13 05:20:47 2013
@@ -84,25 +84,6 @@ public class TestBufferedIndexInput exte
public void testReadBytes() throws Exception {
MyBufferedIndexInput input = new MyBufferedIndexInput();
runReadBytes(input, BufferedIndexInput.BUFFER_SIZE, random());
-
- // This tests the workaround code for LUCENE-1566 where readBytesInternal
- // provides a workaround for a JVM Bug that incorrectly raises a OOM Error
- // when a large byte buffer is passed to a file read.
- // NOTE: this does only test the chunked reads and NOT if the Bug is triggered.
- //final int tmpFileSize = 1024 * 1024 * 5;
- final int inputBufferSize = 128;
- File tmpInputFile = _TestUtil.createTempFile("IndexInput", "tmpFile", TEMP_DIR);
- tmpInputFile.deleteOnExit();
- writeBytes(tmpInputFile, TEST_FILE_LENGTH);
-
- // run test with chunk size of 10 bytes
- runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")",
- new RandomAccessFile(tmpInputFile, "r"), newIOContext(random()), 10), inputBufferSize, random());
-
- // run test with chunk size of 10 bytes
- runReadBytesAndClose(new NIOFSIndexInput("NIOFSIndexInput(path=\"" + tmpInputFile + "\")",
- FileChannel.open(tmpInputFile.toPath(), StandardOpenOption.READ), newIOContext(random()), 10),
- inputBufferSize, random());
}
private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r) throws IOException {
@@ -211,6 +192,7 @@ public class TestBufferedIndexInput exte
private static byte byten(long n){
return (byte)(n*n%256);
}
+
private static class MyBufferedIndexInput extends BufferedIndexInput {
private long pos;
private long len;
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java Tue Aug 13 05:20:47 2013
@@ -134,52 +134,66 @@ public class TestDirectory extends Lucen
// Test that different instances of FSDirectory can coexist on the same
// path, can read, write, and lock files.
public void testDirectInstantiation() throws Exception {
- File path = _TestUtil.getTempDir("testDirectInstantiation");
-
- int sz = 3;
- Directory[] dirs = new Directory[sz];
+ final File path = _TestUtil.getTempDir("testDirectInstantiation");
+
+ final byte[] largeBuffer = new byte[random().nextInt(256*1024)], largeReadBuffer = new byte[largeBuffer.length];
+ for (int i = 0; i < largeBuffer.length; i++) {
+ largeBuffer[i] = (byte) i; // automatically loops with modulo
+ }
- dirs[0] = new SimpleFSDirectory(path, null);
- dirs[1] = new NIOFSDirectory(path, null);
- dirs[2] = new MMapDirectory(path, null);
+ final FSDirectory[] dirs = new FSDirectory[] {
+ new SimpleFSDirectory(path, null),
+ new NIOFSDirectory(path, null),
+ new MMapDirectory(path, null)
+ };
- for (int i=0; i<sz; i++) {
- Directory dir = dirs[i];
+ for (int i=0; i<dirs.length; i++) {
+ FSDirectory dir = dirs[i];
dir.ensureOpen();
String fname = "foo." + i;
String lockname = "foo" + i + ".lck";
IndexOutput out = dir.createOutput(fname, newIOContext(random()));
out.writeByte((byte)i);
+ out.writeBytes(largeBuffer, largeBuffer.length);
out.close();
- for (int j=0; j<sz; j++) {
- Directory d2 = dirs[j];
+ for (int j=0; j<dirs.length; j++) {
+ FSDirectory d2 = dirs[j];
d2.ensureOpen();
assertTrue(d2.fileExists(fname));
- assertEquals(1, d2.fileLength(fname));
+ assertEquals(1 + largeBuffer.length, d2.fileLength(fname));
- // don't test read on MMapDirectory, since it can't really be
- // closed and will cause a failure to delete the file.
- if (d2 instanceof MMapDirectory) continue;
+ // don't do read tests if unmapping is not supported!
+ if (d2 instanceof MMapDirectory && !((MMapDirectory) d2).getUseUnmap())
+ continue;
IndexInput input = d2.openInput(fname, newIOContext(random()));
assertEquals((byte)i, input.readByte());
+ // read array with buffering enabled
+ Arrays.fill(largeReadBuffer, (byte)0);
+ input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, true);
+ assertArrayEquals(largeBuffer, largeReadBuffer);
+ // read again without using buffer
+ input.seek(1L);
+ Arrays.fill(largeReadBuffer, (byte)0);
+ input.readBytes(largeReadBuffer, 0, largeReadBuffer.length, false);
+ assertArrayEquals(largeBuffer, largeReadBuffer);
input.close();
}
// delete with a different dir
- dirs[(i+1)%sz].deleteFile(fname);
+ dirs[(i+1)%dirs.length].deleteFile(fname);
- for (int j=0; j<sz; j++) {
- Directory d2 = dirs[j];
+ for (int j=0; j<dirs.length; j++) {
+ FSDirectory d2 = dirs[j];
assertFalse(d2.fileExists(fname));
}
Lock lock = dir.makeLock(lockname);
assertTrue(lock.obtain());
- for (int j=0; j<sz; j++) {
- Directory d2 = dirs[j];
+ for (int j=0; j<dirs.length; j++) {
+ FSDirectory d2 = dirs[j];
Lock lock2 = d2.makeLock(lockname);
try {
assertFalse(lock2.obtain(1));
@@ -191,13 +205,13 @@ public class TestDirectory extends Lucen
lock.release();
// now lock with different dir
- lock = dirs[(i+1)%sz].makeLock(lockname);
+ lock = dirs[(i+1)%dirs.length].makeLock(lockname);
assertTrue(lock.obtain());
lock.release();
}
- for (int i=0; i<sz; i++) {
- Directory dir = dirs[i];
+ for (int i=0; i<dirs.length; i++) {
+ FSDirectory dir = dirs[i];
dir.ensureOpen();
dir.close();
assertFalse(dir.isOpen);
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java Tue Aug 13 05:20:47 2013
@@ -506,6 +506,13 @@ public class PostingsHighlighter {
throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
}
int end = dp.endOffset();
+ // LUCENE-5166: this hit would span the content limit... however more valid
+ // hits may exist (they are sorted by start). so we pretend like we never
+ // saw this term, it won't cause a passage to be added to passageQueue or anything.
+ assert EMPTY.startOffset() == Integer.MAX_VALUE;
+ if (start < contentLength && end > contentLength) {
+ continue;
+ }
if (start >= current.endOffset) {
if (current.startOffset >= 0) {
// finalize current
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java Tue Aug 13 05:20:47 2013
@@ -87,6 +87,81 @@ public class TestPostingsHighlighter ext
dir.close();
}
+ public void testFormatWithMatchExceedingContentLength() throws Exception {
+
+ int maxLength = 17;
+ String bodyText = "123 5678 01234 TEST";
+
+ final Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ final FieldType fieldType = new FieldType(TextField.TYPE_STORED);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ final Field body = new Field("body", bodyText, fieldType);
+
+ Document doc = new Document();
+ doc.add(body);
+
+ iw.addDocument(doc);
+
+ IndexReader ir = iw.getReader();
+ iw.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+
+ Query query = new TermQuery(new Term("body", "test"));
+
+ TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER);
+ assertEquals(1, topDocs.totalHits);
+
+ PostingsHighlighter highlighter = new PostingsHighlighter(maxLength);
+ String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
+
+
+ assertEquals(1, snippets.length);
+ // LUCENE-5166: no snippet
+ assertEquals("123 5678 01234 TE", snippets[0]);
+
+ ir.close();
+ dir.close();
+ }
+
+ // simple test highlighting last word.
+ public void testHighlightLastWord() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
+ offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ Field body = new Field("body", "", offsetsType);
+ Document doc = new Document();
+ doc.add(body);
+
+ body.setStringValue("This is a test");
+ iw.addDocument(doc);
+
+ IndexReader ir = iw.getReader();
+ iw.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ PostingsHighlighter highlighter = new PostingsHighlighter();
+ Query query = new TermQuery(new Term("body", "test"));
+ TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER);
+ assertEquals(1, topDocs.totalHits);
+ String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
+ assertEquals(1, snippets.length);
+ assertEquals("This is a <b>test</b>", snippets[0]);
+
+ ir.close();
+ dir.close();
+ }
+
// simple test with one sentence documents.
public void testOneSentence() throws Exception {
Directory dir = newDirectory();
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java Tue Aug 13 05:20:47 2013
@@ -173,6 +173,8 @@ public class TestPostingsHighlighterRank
assertTrue(p.getNumMatches() > 0);
assertTrue(p.getStartOffset() >= 0);
assertTrue(p.getStartOffset() <= content.length());
+ assertTrue(p.getEndOffset() >= p.getStartOffset());
+ assertTrue(p.getEndOffset() <= content.length());
// we use a very simple analyzer. so we can assert the matches are correct
int lastMatchStart = -1;
for (int i = 0; i < p.getNumMatches(); i++) {
Modified: lucene/dev/branches/lucene4956/lucene/module-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/module-build.xml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/module-build.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/module-build.xml Tue Aug 13 05:20:47 2013
@@ -132,6 +132,17 @@
</ant>
<property name="queryparser-javadocs.uptodate" value="true"/>
</target>
+
+ <property name="join.jar" value="${common.dir}/build/join/lucene-join-${version}.jar"/>
+ <target name="check-join-uptodate" unless="join.uptodate">
+ <module-uptodate name="join" jarfile="${join.jar}" property="join.uptodate"/>
+ </target>
+ <target name="jar-join" unless="join.uptodate" depends="check-join-uptodate">
+ <ant dir="${common.dir}/join" target="jar-core" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <property name="join.uptodate" value="true"/>
+ </target>
<property name="analyzers-common.jar" value="${common.dir}/build/analysis/common/lucene-analyzers-common-${version}.jar"/>
<target name="check-analyzers-common-uptodate" unless="analyzers-common.uptodate">
Modified: lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Tue Aug 13 05:20:47 2013
@@ -20,6 +20,7 @@ package org.apache.lucene.util;
import java.io.*;
import java.lang.annotation.*;
import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.*;
@@ -1135,8 +1136,8 @@ public abstract class LuceneTestCase ext
FSDirectory d = null;
try {
d = CommandLineUtil.newFSDirectory(clazz, file);
- } catch (Exception e) {
- d = FSDirectory.open(file);
+ } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
+ Rethrow.rethrow(e);
}
return d;
}
Modified: lucene/dev/branches/lucene4956/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/CHANGES.txt?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4956/solr/CHANGES.txt Tue Aug 13 05:20:47 2013
@@ -113,12 +113,15 @@ Bug Fixes
* SOLR-5121: zkcli usage help for makepath doesn't match actual command.
(Daniel Collins via Mark Miller)
+
+* SOLR-5119: Managed schema problems after adding fields via Schema Rest API.
+ (Nils Kübler, Steve Rowe)
Optimizations
----------------------
* SOLR-5044: Admin UI - Note on Core-Admin about directories while creating
- core (seffkes)
+ core (steffkes)
Other Changes
----------------------
@@ -178,6 +181,7 @@ Upgrading from Solr 4.3.0
* LUCENE-5063: ByteField and ShortField have been deprecated and will be removed
in 5.0. If you are still using these field types, you should migrate your
fields to TrieIntField.
+
Detailed Change List
----------------------
@@ -258,6 +262,16 @@ New Features
* SOLR-4943: Add a new system wide info admin handler that exposes the system info
that could previously only be retrieved using a SolrCore. (Mark Miller)
+* SOLR-3076: Block joins. Documents and their sub-documents must be indexed
+ as a block.
+ {!parent which=<allParents>}<someChildren> takes in a query that matches child
+ documents and results in matches on their parents.
+ {!child of=<allParents>}<someParents> takes in a query that matches some parent
+ documents and results in matches on their children.
+ (Mikhail Khludnev, Vadim Kirilchuk, Alan Woodward, Tom Burton-West, Mike McCandless,
+ hossman, yonik)
+
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/lucene4956/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/common-build.xml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/common-build.xml (original)
+++ lucene/dev/branches/lucene4956/solr/common-build.xml Tue Aug 13 05:20:47 2013
@@ -83,6 +83,7 @@
<pathelement location="${grouping.jar}"/>
<pathelement location="${queries.jar}"/>
<pathelement location="${queryparser.jar}"/>
+ <pathelement location="${join.jar}"/>
</path>
<path id="solr.base.classpath">
@@ -142,7 +143,7 @@
<target name="prep-lucene-jars"
depends="jar-lucene-core, jar-analyzers-phonetic, jar-analyzers-arirang, jar-analyzers-kuromoji, jar-codecs, jar-suggest, jar-highlighter, jar-memory,
- jar-misc, jar-spatial, jar-grouping, jar-queries, jar-queryparser">
+ jar-misc, jar-spatial, jar-grouping, jar-queries, jar-queryparser, jar-join">
<property name="solr.deps.compiled" value="true"/>
</target>
Modified: lucene/dev/branches/lucene4956/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/ivy.xml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/ivy.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/ivy.xml Tue Aug 13 05:20:47 2013
@@ -41,6 +41,7 @@
<dependency org="org.restlet.jee" name="org.restlet" rev="2.1.1" conf="compile->*"/>
<dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="2.1.1" conf="compile->*"/>
<dependency org="joda-time" name="joda-time" rev="2.2" conf="compile->*"/>
+ <dependency org="dom4j" name="dom4j" rev="1.6.1" transitive="false"/>
<dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" conf="test->*"/>
<dependency org="org.easymock" name="easymock" rev="3.0" conf="test->*"/>
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java Tue Aug 13 05:20:47 2013
@@ -16,50 +16,51 @@ package org.apache.solr.handler.loader;
* limitations under the License.
*/
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.apache.solr.update.AddUpdateCommand;
-import org.apache.solr.update.CommitUpdateCommand;
-import org.apache.solr.update.RollbackUpdateCommand;
-import org.apache.solr.update.DeleteUpdateCommand;
-import org.apache.solr.util.xslt.TransformerProvider;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.ContentStreamBase;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.XMLErrorLogger;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.XMLErrorLogger;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.handler.RequestHandlerUtils;
import org.apache.solr.handler.UpdateRequestHandler;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+import org.apache.solr.update.RollbackUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.util.EmptyEntityResolver;
-import org.apache.commons.io.IOUtils;
+import org.apache.solr.util.xslt.TransformerProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
import org.xml.sax.XMLReader;
-import javax.xml.stream.XMLStreamReader;
-import javax.xml.stream.XMLStreamException;
+import javax.xml.parsers.SAXParserFactory;
import javax.xml.stream.FactoryConfigurationError;
-import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.sax.SAXSource;
-import javax.xml.parsers.SAXParserFactory;
-
import java.io.ByteArrayInputStream;
-import java.io.InputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -381,6 +382,7 @@ public class XMLLoader extends ContentSt
float boost = 1.0f;
boolean isNull = false;
String update = null;
+ Collection<SolrInputDocument> subDocs = null;
Map<String, Map<String, Object>> updateMap = null;
boolean complete = false;
while (!complete) {
@@ -395,9 +397,14 @@ public class XMLLoader extends ContentSt
case XMLStreamConstants.END_ELEMENT:
if ("doc".equals(parser.getLocalName())) {
+ if (subDocs != null && !subDocs.isEmpty()) {
+ doc.addChildDocuments(subDocs);
+ subDocs = null;
+ }
complete = true;
break;
} else if ("field".equals(parser.getLocalName())) {
+ // should I warn in some text has been found too
Object v = isNull ? null : text.toString();
if (update != null) {
if (updateMap == null) updateMap = new HashMap<String, Map<String, Object>>();
@@ -425,34 +432,43 @@ public class XMLLoader extends ContentSt
}
doc.addField(name, v, boost);
boost = 1.0f;
+ // field is over
+ name = null;
}
break;
case XMLStreamConstants.START_ELEMENT:
text.setLength(0);
String localName = parser.getLocalName();
- if (!"field".equals(localName)) {
- log.warn("unexpected XML tag doc/" + localName);
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "unexpected XML tag doc/" + localName);
+ if ("doc".equals(localName)) {
+ if (subDocs == null)
+ subDocs = Lists.newArrayList();
+ subDocs.add(readDoc(parser));
}
- boost = 1.0f;
- update = null;
- isNull = false;
- String attrVal = "";
- for (int i = 0; i < parser.getAttributeCount(); i++) {
- attrName = parser.getAttributeLocalName(i);
- attrVal = parser.getAttributeValue(i);
- if ("name".equals(attrName)) {
- name = attrVal;
- } else if ("boost".equals(attrName)) {
- boost = Float.parseFloat(attrVal);
- } else if ("null".equals(attrName)) {
- isNull = StrUtils.parseBoolean(attrVal);
- } else if ("update".equals(attrName)) {
- update = attrVal;
- } else {
- log.warn("Unknown attribute doc/field/@" + attrName);
+ else {
+ if (!"field".equals(localName)) {
+ log.warn("unexpected XML tag doc/" + localName);
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "unexpected XML tag doc/" + localName);
+ }
+ boost = 1.0f;
+ update = null;
+ isNull = false;
+ String attrVal = "";
+ for (int i = 0; i < parser.getAttributeCount(); i++) {
+ attrName = parser.getAttributeLocalName(i);
+ attrVal = parser.getAttributeValue(i);
+ if ("name".equals(attrName)) {
+ name = attrVal;
+ } else if ("boost".equals(attrName)) {
+ boost = Float.parseFloat(attrVal);
+ } else if ("null".equals(attrName)) {
+ isNull = StrUtils.parseBoolean(attrVal);
+ } else if ("update".equals(attrName)) {
+ update = attrVal;
+ } else {
+ log.warn("Unknown attribute doc/field/@" + attrName);
+ }
}
}
break;
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Tue Aug 13 05:20:47 2013
@@ -113,7 +113,7 @@ public class DefaultSolrHighlighter exte
formatters.put("", fmt);
formatters.put(null, fmt);
- // Load the formatters
+ // Load the encoders
SolrEncoder enc = solrCore.initPlugins(info.getChildren("encoder"), encoders,SolrEncoder.class,null);
if (enc == null) enc = new DefaultEncoder();
encoders.put("", enc);
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java Tue Aug 13 05:20:47 2013
@@ -389,6 +389,9 @@ public final class ManagedIndexSchema ex
newSchema.uniqueKeyField = uniqueKeyField;
newSchema.uniqueKeyFieldName = uniqueKeyFieldName;
newSchema.uniqueKeyFieldType = uniqueKeyFieldType;
+
+ // After the schema is persisted, resourceName is the same as managedSchemaResourceName
+ newSchema.resourceName = managedSchemaResourceName;
if (includeFieldDataStructures) {
// These need new collections, since addFields() can add members to them
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/QParserPlugin.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/QParserPlugin.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/search/QParserPlugin.java Tue Aug 13 05:20:47 2013
@@ -20,6 +20,8 @@ import org.apache.solr.common.params.Sol
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.join.BlockJoinChildQParserPlugin;
+import org.apache.solr.search.join.BlockJoinParentQParserPlugin;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import java.net.URL;
@@ -47,7 +49,9 @@ public abstract class QParserPlugin impl
JoinQParserPlugin.NAME, JoinQParserPlugin.class,
SurroundQParserPlugin.NAME, SurroundQParserPlugin.class,
SwitchQParserPlugin.NAME, SwitchQParserPlugin.class,
- MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class
+ MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class,
+ BlockJoinParentQParserPlugin.NAME, BlockJoinParentQParserPlugin.class,
+ BlockJoinChildQParserPlugin.NAME, BlockJoinChildQParserPlugin.class
};
/** return a {@link QParser} */
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java Tue Aug 13 05:20:47 2013
@@ -18,6 +18,7 @@
package org.apache.solr.update;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexDocument;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
@@ -27,10 +28,15 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
/**
*
*/
-public class AddUpdateCommand extends UpdateCommand {
+public class AddUpdateCommand extends UpdateCommand implements Iterable<IndexDocument> {
// optional id in "internal" indexed form... if it is needed and not supplied,
// it will be obtained from the doc.
private BytesRef indexedId;
@@ -143,8 +149,64 @@ public class AddUpdateCommand extends Up
}
return id;
}
-
- @Override
+
+ public boolean isBlock() {
+ return solrDoc.hasChildDocuments();
+ }
+
+ @Override
+ public Iterator<IndexDocument> iterator() {
+ return new Iterator<IndexDocument>() {
+ Iterator<SolrInputDocument> iter;
+
+ {
+ List<SolrInputDocument> all = flatten(solrDoc);
+
+ SchemaField uniq = req.getSchema().getUniqueKeyField();
+ String idField = getHashableId();
+
+ for (SolrInputDocument sdoc : all) {
+ sdoc.setField("_root_", idField); // should this be a string or the same type as the ID?
+ // TODO: if possible concurrent modification exception (if SolrInputDocument not cloned and is being forwarded to replicas)
+ // then we could add this field to the generated lucene document instead.
+ }
+
+ iter = all.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public IndexDocument next() {
+ return DocumentBuilder.toDocument(iter.next(), req.getSchema());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ private List<SolrInputDocument> flatten(SolrInputDocument root) {
+ List<SolrInputDocument> unwrappedDocs = new ArrayList<SolrInputDocument>();
+ recUnwrapp(unwrappedDocs, root);
+ Collections.reverse(unwrappedDocs);
+ return unwrappedDocs;
+ }
+
+ private void recUnwrapp(List<SolrInputDocument> unwrappedDocs, SolrInputDocument currentDoc) {
+ unwrappedDocs.add(currentDoc);
+ for (SolrInputDocument child : currentDoc.getChildDocuments()) {
+ recUnwrapp(unwrappedDocs, child);
+ }
+ }
+
+
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
sb.append(",id=").append(getPrintableId());
@@ -153,4 +215,6 @@ public class AddUpdateCommand extends Up
sb.append('}');
return sb.toString();
}
- }
+
+
+}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Tue Aug 13 05:20:47 2013
@@ -20,16 +20,6 @@
package org.apache.solr.update;
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -56,12 +46,22 @@ import org.apache.solr.schema.IndexSchem
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.FunctionRangeQuery;
import org.apache.solr.search.QParser;
-import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.function.ValueSourceRangeFilter;
import org.apache.solr.util.RefCounted;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+
/**
* <code>DirectUpdateHandler2</code> implements an UpdateHandler where documents are added
* directly to the main Lucene index as opposed to adding to a separate smaller index.
@@ -199,19 +199,23 @@ public class DirectUpdateHandler2 extend
// normal update
Term updateTerm;
- Term idTerm = new Term(idField.getName(), cmd.getIndexedId());
+ Term idTerm = new Term(cmd.isBlock() ? "_root_" : idField.getName(), cmd.getIndexedId());
boolean del = false;
if (cmd.updateTerm == null) {
updateTerm = idTerm;
} else {
+ // this is only used by the dedup update processor
del = true;
updateTerm = cmd.updateTerm;
}
-
- Document luceneDocument = cmd.getLuceneDocument();
- // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer);
- writer.updateDocument(updateTerm, luceneDocument,
- schema.getAnalyzer());
+
+ if (cmd.isBlock()) {
+ writer.updateDocuments(updateTerm, cmd, schema.getAnalyzer());
+ } else {
+ Document luceneDocument = cmd.getLuceneDocument();
+ // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer);
+ writer.updateDocument(updateTerm, luceneDocument, schema.getAnalyzer());
+ }
// SolrCore.verbose("updateDocument",updateTerm,"DONE");
if (del) { // ensure id remains unique
@@ -234,7 +238,12 @@ public class DirectUpdateHandler2 extend
} else {
// allow duplicates
- writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer());
+ if (cmd.isBlock()) {
+ writer.addDocuments(cmd, schema.getAnalyzer());
+ } else {
+ writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer());
+ }
+
if (ulog != null) ulog.add(cmd);
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema.xml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema.xml Tue Aug 13 05:20:47 2013
@@ -441,6 +441,8 @@
<fields>
<field name="id" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="_root_" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
+
<field name="signatureField" type="string" indexed="true" stored="false"/>
<field name="uuid" type="uuid" stored="true" />
<field name="name" type="nametext" indexed="true" stored="true"/>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema15.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema15.xml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema15.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/schema15.xml Tue Aug 13 05:20:47 2013
@@ -523,13 +523,15 @@
<field name="uniq3" type="string" indexed="true" stored="true"/>
<field name="nouniq" type="string" indexed="true" stored="true" multiValued="true"/>
- <!-- for versioning -->
- <field name="_version_" type="long" indexed="true" stored="true"/>
-
-
+
<field name="copyfield_source" type="string" indexed="true" stored="true" multiValued="true"/>
+ <!-- for versioning -->
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ <!-- points to the root document of a block of nested documents -->
+ <field name="_root_" type="string" indexed="true" stored="true"/>
+
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false"/>
@@ -560,6 +562,7 @@
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_tt" type="text" indexed="true" stored="true"/>
+ <dynamicField name="*_ws" type="nametext" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>
Modified: lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Tue Aug 13 05:20:47 2013
@@ -120,6 +120,12 @@
initialSize="512"
autowarmCount="0"/>
+ <cache name="perSegFilter"
+ class="solr.search.LRUCache"
+ size="10"
+ initialSize="0"
+ autowarmCount="10" />
+
<!-- If true, stored fields that are not requested will be loaded lazily.
-->
<enableLazyFieldLoading>true</enableLazyFieldLoading>
@@ -553,3 +559,4 @@
</updateRequestProcessorChain>
</config>
+
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Tue Aug 13 05:20:47 2013
@@ -18,6 +18,7 @@
package org.apache.solr;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.index.LogDocMergePolicy;
import org.noggit.JSONUtil;
import org.noggit.ObjectBuilder;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
@@ -54,6 +55,11 @@ public class TestGroupingSearch extends
@BeforeClass
public static void beforeTests() throws Exception {
+ // force LogDocMergePolicy so that we get a predictable doc order
+ // when doing unsorted group collection
+ System.setProperty("solr.tests.mergePolicy",
+ LogDocMergePolicy.class.getName());
+
System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
initCore("solrconfig.xml", "schema12.xml");
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Tue Aug 13 05:20:47 2013
@@ -121,6 +121,8 @@ public class FullSolrCloudDistribCmdsTes
results = query(cloudClient);
assertEquals(2, results.getResults().getNumFound());
+ docId = testIndexQueryDeleteHierarchical(docId);
+
testIndexingWithSuss();
// TODO: testOptimisticUpdate(results);
@@ -235,6 +237,75 @@ public class FullSolrCloudDistribCmdsTes
assertEquals(0, query(cloudClient).getResults().getNumFound());
}
+ private long testIndexQueryDeleteHierarchical(long docId) throws Exception {
+ //index
+ int topDocsNum = atLeast(10);
+ int childsNum = atLeast(10);
+ for (int i = 0; i < topDocsNum; ++i) {
+ UpdateRequest uReq = new UpdateRequest();
+ SolrInputDocument topDocument = new SolrInputDocument();
+ topDocument.addField("id", docId++);
+ topDocument.addField("type_s", "parent");
+ topDocument.addField(i + "parent_f1_s", "v1");
+ topDocument.addField(i + "parent_f2_s", "v2");
+
+
+ for (int index = 0; index < childsNum; ++index) {
+ docId = addChildren("child", topDocument, index, false, docId);
+ }
+
+ uReq.add(topDocument);
+ uReq.process(cloudClient);
+ uReq.process(controlClient);
+ }
+
+ commit();
+ checkShardConsistency();
+ assertDocCounts(VERBOSE);
+
+ //query
+ // parents
+ SolrQuery query = new SolrQuery("type_s:parent");
+ QueryResponse results = cloudClient.query(query);
+ assertEquals(topDocsNum, results.getResults().getNumFound());
+
+ //childs
+ query = new SolrQuery("type_s:child");
+ results = cloudClient.query(query);
+ assertEquals(topDocsNum * childsNum, results.getResults().getNumFound());
+
+ //grandchilds
+ query = new SolrQuery("type_s:grand");
+ results = cloudClient.query(query);
+ //each topDoc has t childs where each child has x = 0 + 2 + 4 + ..(t-1)*2 grands
+ //x = 2 * (1 + 2 + 3 +.. (t-1)) => arithmetic summ of t-1
+ //x = 2 * ((t-1) * t / 2) = t * (t - 1)
+ assertEquals(topDocsNum * childsNum * (childsNum - 1), results.getResults().getNumFound());
+
+ //delete
+ del("*:*");
+ commit();
+
+ return docId;
+ }
+
+ private long addChildren(String prefix, SolrInputDocument topDocument, int childIndex, boolean lastLevel, long docId) {
+ SolrInputDocument childDocument = new SolrInputDocument();
+ childDocument.addField("id", docId++);
+ childDocument.addField("type_s", prefix);
+ for (int index = 0; index < childIndex; ++index) {
+ childDocument.addField(childIndex + prefix + index + "_s", childIndex + "value"+ index);
+ }
+
+ if (!lastLevel) {
+ for (int i = 0; i < childIndex * 2; ++i) {
+ docId = addChildren("grand", childDocument, i, true, docId);
+ }
+ }
+ topDocument.addChildDocument(childDocument);
+ return docId;
+ }
+
private void testIndexingWithSuss() throws Exception {
ConcurrentUpdateSolrServer suss = new ConcurrentUpdateSolrServer(
((HttpSolrServer) clients.get(0)).getBaseURL(), 3, 1);
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/core/TestConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/core/TestConfig.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/core/TestConfig.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/core/TestConfig.java Tue Aug 13 05:20:47 2013
@@ -18,12 +18,16 @@
package org.apache.solr.core;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.handler.admin.ShowFileRequestHandler;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.SolrIndexConfig;
import org.apache.solr.util.RefCounted;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.IndexSchemaFactory;
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Node;
@@ -38,7 +42,7 @@ public class TestConfig extends SolrTest
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-termindex.xml","schema-reversed.xml");
+ initCore("solrconfig-test-misc.xml","schema-reversed.xml");
}
@Test
@@ -91,16 +95,6 @@ public class TestConfig extends SolrTest
assertEquals("prefix-proptwo-suffix", node.getTextContent());
}
- @Test
- public void testLucene23Upgrades() throws Exception {
- double bufferSize = solrConfig.indexConfig.ramBufferSizeMB;
- assertTrue(bufferSize + " does not equal: " + 100, bufferSize == 100);
- String mergePolicy = solrConfig.indexConfig.mergePolicyInfo.className;
- assertEquals(TieredMergePolicy.class.getName(), mergePolicy);
- String mergeSched = solrConfig.indexConfig.mergeSchedulerInfo.className;
- assertTrue(mergeSched + " is not equal to " + SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME, mergeSched.equals(SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME) == true);
- }
-
// sometime if the config referes to old things, it must be replaced with new stuff
@Test
public void testAutomaticDeprecationSupport() {
@@ -124,6 +118,14 @@ public class TestConfig extends SolrTest
assertEquals("default LockType", SolrIndexConfig.LOCK_TYPE_NATIVE, sic.lockType);
assertEquals("default useCompoundFile", false, sic.useCompoundFile);
+ IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema("schema.xml", solrConfig);
+ IndexWriterConfig iwc = sic.toIndexWriterConfig(indexSchema);
+
+ assertNotNull("null mp", iwc.getMergePolicy());
+ assertTrue("mp is not TMP", iwc.getMergePolicy() instanceof TieredMergePolicy);
+
+ assertNotNull("null ms", iwc.getMergeScheduler());
+ assertTrue("ms is not CMS", iwc.getMergeScheduler() instanceof ConcurrentMergeScheduler);
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/schema/TestManagedSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/schema/TestManagedSchema.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/schema/TestManagedSchema.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/schema/TestManagedSchema.java Tue Aug 13 05:20:47 2013
@@ -421,4 +421,24 @@ public class TestManagedSchema extends A
assertNotNull(newNewSchema.getUniqueKeyField());
assertEquals("str", newNewSchema.getUniqueKeyField().getName());
}
+
+ public void testAddFieldThenReload() throws Exception {
+ deleteCore();
+ File managedSchemaFile = new File(tmpConfDir, "managed-schema");
+ assertTrue(managedSchemaFile.delete()); // Delete managed-schema so it won't block parsing a new schema
+ initCore("solrconfig-mutable-managed-schema.xml", "schema-one-field-no-dynamic-field.xml", tmpSolrHome.getPath());
+
+ String fieldName = "new_text_field";
+ assertNull("Field '" + fieldName + "' is present in the schema",
+ h.getCore().getLatestSchema().getFieldOrNull(fieldName));
+
+ Map<String,Object> options = new HashMap<String,Object>();
+ IndexSchema oldSchema = h.getCore().getLatestSchema();
+ String fieldType = "text";
+ SchemaField newField = oldSchema.newField(fieldName, fieldType, options);
+ IndexSchema newSchema = oldSchema.addField(newField);
+ h.getCore().setLatestSchema(newSchema);
+
+ h.reload();
+ }
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java Tue Aug 13 05:20:47 2013
@@ -298,6 +298,13 @@ public class QueryEqualityTest extends S
}
}
+ public void testBlockJoin() throws Exception {
+ assertQueryEquals("parent", "{!parent which=foo_s:parent}dude",
+ "{!parent which=foo_s:parent}dude");
+ assertQueryEquals("child", "{!child of=foo_s:parent}dude",
+ "{!child of=foo_s:parent}dude");
+ }
+
public void testQuerySurround() throws Exception {
assertQueryEquals("surround", "{!surround}and(apache,solr)",
"and(apache,solr)", "apache AND solr");
Modified: lucene/dev/branches/lucene4956/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java?rev=1513349&r1=1513348&r2=1513349&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java (original)
+++ lucene/dev/branches/lucene4956/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java Tue Aug 13 05:20:47 2013
@@ -133,6 +133,11 @@ public class ClientUtils
}
}
}
+
+ for (SolrInputDocument childDocument : doc.getChildDocuments()) {
+ writeXML(childDocument, writer);
+ }
+
writer.write("</doc>");
}