You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/05/13 14:55:22 UTC
svn commit: r1481833 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
dev-tools/maven/lucene/replicator/ lucene/ lucene/core/
lucene/core/src/java/org/apache/lucene/index/ lucene/licenses/
lucene/replicator/ lucene/replicator/lib/ lucene/test-framewo...
Author: shaie
Date: Mon May 13 12:55:21 2013
New Revision: 1481833
URL: http://svn.apache.org/r1481833
Log:
LUCENE-4975: Add Replication module to Lucene
Added:
lucene/dev/branches/branch_4x/dev-tools/maven/lucene/replicator/
- copied from r1481804, lucene/dev/trunk/dev-tools/maven/lucene/replicator/
lucene/dev/branches/branch_4x/lucene/licenses/httpclient-4.2.3.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/httpclient-4.2.3.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/httpclient-LICENSE-ASL.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/httpclient-LICENSE-ASL.txt
lucene/dev/branches/branch_4x/lucene/licenses/httpclient-NOTICE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/httpclient-NOTICE.txt
lucene/dev/branches/branch_4x/lucene/licenses/httpcore-4.2.2.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/httpcore-4.2.2.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/httpcore-LICENSE-ASL.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/httpcore-LICENSE-ASL.txt
lucene/dev/branches/branch_4x/lucene/licenses/httpcore-NOTICE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/httpcore-NOTICE.txt
lucene/dev/branches/branch_4x/lucene/licenses/javax.servlet-api-LICENSE-CDDL.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/javax.servlet-api-LICENSE-CDDL.txt
lucene/dev/branches/branch_4x/lucene/licenses/javax.servlet-api-NOTICE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/javax.servlet-api-NOTICE.txt
lucene/dev/branches/branch_4x/lucene/licenses/jcl-over-slf4j-1.6.6.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jcl-over-slf4j-1.6.6.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/jcl-over-slf4j-LICENSE-BSD_LIKE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jcl-over-slf4j-LICENSE-BSD_LIKE.txt
lucene/dev/branches/branch_4x/lucene/licenses/jcl-over-slf4j-NOTICE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jcl-over-slf4j-NOTICE.txt
lucene/dev/branches/branch_4x/lucene/licenses/jetty-LICENSE-ASL.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-LICENSE-ASL.txt
lucene/dev/branches/branch_4x/lucene/licenses/jetty-NOTICE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-NOTICE.txt
lucene/dev/branches/branch_4x/lucene/licenses/jetty-continuation-8.1.10.v20130312.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-continuation-8.1.10.v20130312.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/jetty-http-8.1.10.v20130312.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-http-8.1.10.v20130312.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/jetty-io-8.1.10.v20130312.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-io-8.1.10.v20130312.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/jetty-server-8.1.10.v20130312.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-server-8.1.10.v20130312.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/jetty-servlet-8.1.10.v20130312.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-servlet-8.1.10.v20130312.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/jetty-util-8.1.10.v20130312.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/jetty-util-8.1.10.v20130312.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/servlet-api-3.0.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/servlet-api-3.0.jar.sha1
lucene/dev/branches/branch_4x/lucene/licenses/slf4j-LICENSE-BSD_LIKE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/slf4j-LICENSE-BSD_LIKE.txt
lucene/dev/branches/branch_4x/lucene/licenses/slf4j-NOTICE.txt
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/slf4j-NOTICE.txt
lucene/dev/branches/branch_4x/lucene/licenses/slf4j-api-1.6.6.jar.sha1
- copied unchanged from r1481804, lucene/dev/trunk/lucene/licenses/slf4j-api-1.6.6.jar.sha1
lucene/dev/branches/branch_4x/lucene/replicator/ (props changed)
- copied from r1481804, lucene/dev/trunk/lucene/replicator/
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (contents, props changed)
lucene/dev/branches/branch_4x/lucene/replicator/lib/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1481833&r1=1481832&r2=1481833&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Mon May 13 12:55:21 2013
@@ -149,6 +149,9 @@ New Features
* LUCENE-4979: LiveFieldFields can work with any ReferenceManager, not
just ReferenceManager<IndexSearcher> (Mike McCandless).
+* LUCENE-4975: Added a new Replicator module which can replicate index
+ revisions between server and client. (Shai Erera, Mike McCandless)
+
Build
* LUCENE-4987: Upgrade randomized testing to version 2.0.10:
Modified: lucene/dev/branches/branch_4x/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/build.xml?rev=1481833&r1=1481832&r2=1481833&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/build.xml (original)
+++ lucene/dev/branches/branch_4x/lucene/build.xml Mon May 13 12:55:21 2013
@@ -160,7 +160,13 @@
</target>
<target name="check-licenses" depends="compile-tools,resolve,load-custom-tasks" description="Validate license stuff.">
- <license-check-macro dir="${basedir}" licensedir="${common.dir}/licenses" />
+ <license-check-macro dir="${basedir}" licensedir="${common.dir}/licenses">
+ <additional-filters>
+ <replaceregex pattern="jetty([^/]+)$" replace="jetty" flags="gi" />
+ <replaceregex pattern="slf4j-([^/]+)$" replace="slf4j" flags="gi" />
+ <replaceregex pattern="(bcmail|bcprov)-([^/]+)$" replace="\1" flags="gi" />
+ </additional-filters>
+ </license-check-macro>
</target>
<target name="check-forbidden-apis" depends="compile-tools,compile-test,install-forbidden-apis,-forbidden-apis-classpath,-check-forbidden-jdk-apis,-check-forbidden-test-apis,-check-system-out" description="Check forbidden API calls in compiled class files"/>
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1481833&r1=1481832&r2=1481833&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Mon May 13 12:55:21 2013
@@ -327,7 +327,7 @@ public abstract class DirectoryReader ex
// corrupt first commit, but it's too deadly to make
// this logic "smarter" and risk accidentally returning
// false due to various cases like file description
- // exhaustion, access denited, etc., because in that
+ // exhaustion, access denied, etc., because in that
// case IndexWriter may delete the entire index. It's
// safer to err towards "index exists" than try to be
// smart about detecting not-yet-fully-committed or
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1481833&r1=1481832&r2=1481833&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java Mon May 13 12:55:21 2013
@@ -199,7 +199,10 @@ public final class IndexFileNames {
return filename;
}
- // All files created by codecs much match this pattern (we
- // check this in SegmentInfo.java):
- static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*");
+ /**
+ * All files created by codecs much match this pattern (checked in
+ * SegmentInfo).
+ */
+ public static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*");
+
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1481833&r1=1481832&r2=1481833&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Mon May 13 12:55:21 2013
@@ -246,6 +246,39 @@ public final class SegmentInfos implemen
}
}
+ /**
+ * A utility for writing the {@link IndexFileNames#SEGMENTS_GEN} file to a
+ * {@link Directory}.
+ *
+ * <p>
+ * <b>NOTE:</b> this is an internal utility which is kept public so that it's
+ * accessible by code from other packages. You should avoid calling this
+ * method unless you're absolutely sure what you're doing!
+ *
+ * @lucene.internal
+ */
+ public static void writeSegmentsGen(Directory dir, long generation) {
+ try {
+ IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
+ try {
+ genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT);
+ genOutput.writeLong(generation);
+ genOutput.writeLong(generation);
+ } finally {
+ genOutput.close();
+ dir.sync(Collections.singleton(IndexFileNames.SEGMENTS_GEN));
+ }
+ } catch (Throwable t) {
+ // It's OK if we fail to write this file since it's
+ // used only as one of the retry fallbacks.
+ try {
+ dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
+ } catch (Throwable t2) {
+ // Ignore; this file is only used in a retry
+ // fallback on init.
+ }
+ }
+ }
/**
* Get the next segments_N filename that will be written.
@@ -963,27 +996,7 @@ public final class SegmentInfos implemen
}
lastGeneration = generation;
-
- try {
- IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
- try {
- genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT);
- genOutput.writeLong(generation);
- genOutput.writeLong(generation);
- } finally {
- genOutput.close();
- dir.sync(Collections.singleton(IndexFileNames.SEGMENTS_GEN));
- }
- } catch (Throwable t) {
- // It's OK if we fail to write this file since it's
- // used only as one of the retry fallbacks.
- try {
- dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
- } catch (Throwable t2) {
- // Ignore; this file is only used in a retry
- // fallback on init.
- }
- }
+ writeSegmentsGen(dir, generation);
}
/** Writes & syncs to the Directory dir, taking care to
Modified: lucene/dev/branches/branch_4x/lucene/module-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/module-build.xml?rev=1481833&r1=1481832&r2=1481833&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/module-build.xml (original)
+++ lucene/dev/branches/branch_4x/lucene/module-build.xml Mon May 13 12:55:21 2013
@@ -220,7 +220,29 @@
</ant>
<property name="facet-javadocs.uptodate" value="true"/>
</target>
-
+
+ <property name="replicator.jar" value="${common.dir}/build/replicator/lucene-replicator-${version}.jar"/>
+ <target name="check-replicator-uptodate" unless="replicator.uptodate">
+ <module-uptodate name="replicator" jarfile="${replicator.jar}" property="replicator.uptodate"/>
+ </target>
+ <target name="jar-replicator" unless="replicator.uptodate" depends="check-replicator-uptodate">
+ <ant dir="${common.dir}/replicator" target="jar-core" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <property name="replicator.uptodate" value="true"/>
+ </target>
+
+ <property name="replicator-javadoc.jar" value="${common.dir}/build/replicator/lucene-replicator-${version}-javadoc.jar"/>
+ <target name="check-replicator-javadocs-uptodate" unless="replicator-javadocs.uptodate">
+ <module-uptodate name="replicator" jarfile="${replicator-javadoc.jar}" property="replicator-javadocs.uptodate"/>
+ </target>
+ <target name="javadocs-replicator" unless="replicator-javadocs.uptodate" depends="check-replicator-javadocs-uptodate">
+ <ant dir="${common.dir}/replicator" target="javadocs" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <property name="replicator-javadocs.uptodate" value="true"/>
+ </target>
+
<property name="analyzers-icu.jar" value="${common.dir}/build/analysis/icu/lucene-analyzers-icu-${version}.jar"/>
<target name="check-analyzers-icu-uptodate" unless="analyzers-icu.uptodate">
<module-uptodate name="analysis/icu" jarfile="${analyzers-icu.jar}" property="analyzers-icu.uptodate"/>
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1481833&r1=1481832&r2=1481833&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Mon May 13 12:55:21 2013
@@ -372,7 +372,7 @@ public class MockDirectoryWrapper extend
System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")"));
new Throwable().printStackTrace(System.out);
}
- throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")"));
+ throw new IOException("a random IOException" + (message == null ? "" : " (" + message + ")"));
}
}
@@ -383,9 +383,9 @@ public class MockDirectoryWrapper extend
new Throwable().printStackTrace(System.out);
}
if (randomState.nextBoolean()) {
- throw new IOException("a random IOException");
+ throw new IOException("a random IOException (" + name + ")");
} else {
- throw new FileNotFoundException("a random IOException");
+ throw new FileNotFoundException("a random IOException (" + name + ")");
}
}
}
@@ -505,7 +505,7 @@ public class MockDirectoryWrapper extend
if (throttling == Throttling.ALWAYS ||
(throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0) && !(delegate instanceof RateLimitedDirectoryWrapper)) {
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockDirectoryWrapper: throttling indexOutput");
+ System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")");
}
return throttledOutput.newFromDelegate(io);
} else {