You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/05/13 13:57:24 UTC
svn commit: r1481804 [2/4] - in /lucene/dev/trunk:
dev-tools/maven/lucene/replicator/ lucene/
lucene/core/src/java/org/apache/lucene/index/ lucene/licenses/
lucene/replicator/ lucene/replicator/lib/ lucene/replicator/src/
lucene/replicator/src/java/ lu...
Added: lucene/dev/trunk/lucene/licenses/jetty-LICENSE-ASL.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-LICENSE-ASL.txt?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-LICENSE-ASL.txt (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-LICENSE-ASL.txt Mon May 13 11:57:22 2013
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
Added: lucene/dev/trunk/lucene/licenses/jetty-NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-NOTICE.txt?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-NOTICE.txt (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-NOTICE.txt Mon May 13 11:57:22 2013
@@ -0,0 +1,111 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
+<html><head>
+
+
+<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
+<title>Eclipse.org Software User Agreement</title>
+</head><body lang="EN-US" link="blue" vlink="purple">
+<h2>Eclipse Foundation Software User Agreement</h2>
+<p>March 17, 2005</p>
+
+<h3>Usage Of Content</h3>
+
+<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
+ (COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
+ CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
+ OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
+ NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
+ CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
+
+<h3>Applicable Licenses</h3>
+
+<p>Unless otherwise indicated, all Content made available by the
+Eclipse Foundation is provided to you under the terms and conditions of
+the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
+provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
+ For purposes of the EPL, "Program" will mean the Content.</p>
+
+<p>Content includes, but is not limited to, source code, object code,
+documentation and other files maintained in the Eclipse.org CVS
+repository ("Repository") in CVS modules ("Modules") and made available
+as downloadable archives ("Downloads").</p>
+
+<ul>
+ <li>Content may be structured and packaged into modules to
+facilitate delivering, extending, and upgrading the Content. Typical
+modules may include plug-ins ("Plug-ins"), plug-in fragments
+("Fragments"), and features ("Features").</li>
+ <li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java™ ARchive) in a directory named "plugins".</li>
+ <li>A
+Feature is a bundle of one or more Plug-ins and/or Fragments and
+associated material. Each Feature may be packaged as a sub-directory in
+a directory named "features". Within a Feature, files named
+"feature.xml" may contain a list of the names and version numbers of
+the Plug-ins and/or Fragments associated with that Feature.</li>
+ <li>Features
+may also include other Features ("Included Features"). Within a
+Feature, files named "feature.xml" may contain a list of the names and
+version numbers of Included Features.</li>
+</ul>
+
+<p>The terms and conditions governing Plug-ins and Fragments should be
+contained in files named "about.html" ("Abouts"). The terms and
+conditions governing Features and
+Included Features should be contained in files named "license.html"
+("Feature Licenses"). Abouts and Feature Licenses may be located in any
+directory of a Download or Module
+including, but not limited to the following locations:</p>
+
+<ul>
+ <li>The top-level (root) directory</li>
+ <li>Plug-in and Fragment directories</li>
+ <li>Inside Plug-ins and Fragments packaged as JARs</li>
+ <li>Sub-directories of the directory named "src" of certain Plug-ins</li>
+ <li>Feature directories</li>
+</ul>
+
+<p>Note: if a Feature made available by the Eclipse Foundation is
+installed using the Eclipse Update Manager, you must agree to a license
+("Feature Update License") during the
+installation process. If the Feature contains Included Features, the
+Feature Update License should either provide you with the terms and
+conditions governing the Included Features or
+inform you where you can locate them. Feature Update Licenses may be
+found in the "license" property of files named "feature.properties"
+found within a Feature.
+Such Abouts, Feature Licenses, and Feature Update Licenses contain the
+terms and conditions (or references to such terms and conditions) that
+govern your use of the associated Content in
+that directory.</p>
+
+<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
+TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
+CONDITIONS. SOME OF THESE
+OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
+
+<ul>
+ <li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
+ <li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
+ <li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
+ <li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
+ <li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
+ <li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
+</ul>
+
+<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
+CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
+or Feature Update License is provided, please
+contact the Eclipse Foundation to determine what terms and conditions
+govern that particular Content.</p>
+
+<h3>Cryptography</h3>
+
+<p>Content may contain encryption software. The country in which you
+are currently may have restrictions on the import, possession, and use,
+and/or re-export to another country, of encryption software. BEFORE
+using any encryption software, please check the country's laws,
+regulations and policies concerning the import, possession, or use, and
+re-export of encryption software, to see if this is permitted.</p>
+
+<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
+</body></html>
\ No newline at end of file
Added: lucene/dev/trunk/lucene/licenses/jetty-continuation-8.1.10.v20130312.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-continuation-8.1.10.v20130312.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-continuation-8.1.10.v20130312.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-continuation-8.1.10.v20130312.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+c0e26574ddcac7a86486f19a8b3782657acfd961
Added: lucene/dev/trunk/lucene/licenses/jetty-http-8.1.10.v20130312.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-http-8.1.10.v20130312.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-http-8.1.10.v20130312.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-http-8.1.10.v20130312.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+d9eb53007e04d6338f12f3ded60fad1f7bfcb40e
Added: lucene/dev/trunk/lucene/licenses/jetty-io-8.1.10.v20130312.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-io-8.1.10.v20130312.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-io-8.1.10.v20130312.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-io-8.1.10.v20130312.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+e829c768f2b9de5d9fae3bc0aba3996bd0344f56
Added: lucene/dev/trunk/lucene/licenses/jetty-server-8.1.10.v20130312.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-server-8.1.10.v20130312.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-server-8.1.10.v20130312.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-server-8.1.10.v20130312.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+13ca9587bc1645f8fac89454b15252a2ad5bdcf5
Added: lucene/dev/trunk/lucene/licenses/jetty-servlet-8.1.10.v20130312.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-servlet-8.1.10.v20130312.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-servlet-8.1.10.v20130312.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-servlet-8.1.10.v20130312.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+98f8029fe7236e9c66381c04f292b5319f47ca84
Added: lucene/dev/trunk/lucene/licenses/jetty-util-8.1.10.v20130312.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/jetty-util-8.1.10.v20130312.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/jetty-util-8.1.10.v20130312.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/jetty-util-8.1.10.v20130312.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+d198a8ad8ea20b4fb74c781175c48500ec2b8b7a
Added: lucene/dev/trunk/lucene/licenses/servlet-api-3.0.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/servlet-api-3.0.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/servlet-api-3.0.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/servlet-api-3.0.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+0aaaa85845fb5c59da00193f06b8e5278d8bf3f8
Added: lucene/dev/trunk/lucene/licenses/slf4j-LICENSE-BSD_LIKE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/slf4j-LICENSE-BSD_LIKE.txt?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/slf4j-LICENSE-BSD_LIKE.txt (added)
+++ lucene/dev/trunk/lucene/licenses/slf4j-LICENSE-BSD_LIKE.txt Mon May 13 11:57:22 2013
@@ -0,0 +1,21 @@
+Copyright (c) 2004-2008 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Added: lucene/dev/trunk/lucene/licenses/slf4j-NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/slf4j-NOTICE.txt?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/slf4j-NOTICE.txt (added)
+++ lucene/dev/trunk/lucene/licenses/slf4j-NOTICE.txt Mon May 13 11:57:22 2013
@@ -0,0 +1,25 @@
+=========================================================================
+== SLF4J Notice -- http://www.slf4j.org/license.html ==
+=========================================================================
+
+Copyright (c) 2004-2008 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Added: lucene/dev/trunk/lucene/licenses/slf4j-api-1.6.6.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/licenses/slf4j-api-1.6.6.jar.sha1?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/licenses/slf4j-api-1.6.6.jar.sha1 (added)
+++ lucene/dev/trunk/lucene/licenses/slf4j-api-1.6.6.jar.sha1 Mon May 13 11:57:22 2013
@@ -0,0 +1 @@
+ce53b0a0e2cfbb27e8a59d38f79a18a5c6a8d2b0
Modified: lucene/dev/trunk/lucene/module-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/module-build.xml?rev=1481804&r1=1481803&r2=1481804&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/module-build.xml (original)
+++ lucene/dev/trunk/lucene/module-build.xml Mon May 13 11:57:22 2013
@@ -220,7 +220,29 @@
</ant>
<property name="facet-javadocs.uptodate" value="true"/>
</target>
-
+
+ <property name="replicator.jar" value="${common.dir}/build/replicator/lucene-replicator-${version}.jar"/>
+ <target name="check-replicator-uptodate" unless="replicator.uptodate">
+ <module-uptodate name="replicator" jarfile="${replicator.jar}" property="replicator.uptodate"/>
+ </target>
+ <target name="jar-replicator" unless="replicator.uptodate" depends="check-replicator-uptodate">
+ <ant dir="${common.dir}/replicator" target="jar-core" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <property name="replicator.uptodate" value="true"/>
+ </target>
+
+ <property name="replicator-javadoc.jar" value="${common.dir}/build/replicator/lucene-replicator-${version}-javadoc.jar"/>
+ <target name="check-replicator-javadocs-uptodate" unless="replicator-javadocs.uptodate">
+ <module-uptodate name="replicator" jarfile="${replicator-javadoc.jar}" property="replicator-javadocs.uptodate"/>
+ </target>
+ <target name="javadocs-replicator" unless="replicator-javadocs.uptodate" depends="check-replicator-javadocs-uptodate">
+ <ant dir="${common.dir}/replicator" target="javadocs" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <property name="replicator-javadocs.uptodate" value="true"/>
+ </target>
+
<property name="analyzers-icu.jar" value="${common.dir}/build/analysis/icu/lucene-analyzers-icu-${version}.jar"/>
<target name="check-analyzers-icu-uptodate" unless="analyzers-icu.uptodate">
<module-uptodate name="analysis/icu" jarfile="${analyzers-icu.jar}" property="analyzers-icu.uptodate"/>
Added: lucene/dev/trunk/lucene/replicator/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/build.xml?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/build.xml (added)
+++ lucene/dev/trunk/lucene/replicator/build.xml Mon May 13 11:57:22 2013
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project name="replicator" default="default" xmlns:ivy="antlib:org.apache.ivy.ant">
+
+ <description>
+ Files replication utility
+ </description>
+
+ <import file="../module-build.xml"/>
+
+ <path id="classpath">
+ <fileset dir="lib" />
+ <pathelement path="${facet.jar}"/>
+ <path refid="base.classpath"/>
+ </path>
+
+ <target name="resolve" depends="common.resolve">
+ <sequential>
+ <!-- servlet-api.jar -->
+ <ivy:retrieve conf="servlet" log="download-only" type="orbit" pattern="lib/servlet-api-3.0.jar"/>
+ </sequential>
+ </target>
+
+ <target name="init" depends="module-build.init,jar-facet"/>
+
+ <target name="javadocs" depends="javadocs-facet,compile-core">
+ <invoke-module-javadoc>
+ <links>
+ <link href="../facet"/>
+ </links>
+ </invoke-module-javadoc>
+ </target>
+
+</project>
Added: lucene/dev/trunk/lucene/replicator/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/ivy.xml?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/ivy.xml (added)
+++ lucene/dev/trunk/lucene/replicator/ivy.xml Mon May 13 11:57:22 2013
@@ -0,0 +1,50 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!DOCTYPE ivy-module [
+ <!ENTITY jetty.version "8.1.10.v20130312">
+]>
+<ivy-module version="2.0">
+ <info organisation="org.apache.lucene" module="replicator"/>
+
+ <configurations>
+ <conf name="http" description="httpclient jars"/>
+ <conf name="jetty" description="jetty jars"/>
+ <conf name="start" description="jetty start jar"/>
+ <conf name="servlet" description="servlet-api jar"/>
+ <conf name="logging" description="logging setup"/>
+ </configurations>
+
+ <dependencies>
+ <dependency org="org.apache.httpcomponents" name="httpclient" rev="4.2.3" transitive="false" conf="http->default"/>
+ <dependency org="org.apache.httpcomponents" name="httpcore" rev="4.2.2" transitive="false" conf="http->default"/>
+ <dependency org="org.eclipse.jetty" name="jetty-server" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
+ <dependency org="org.eclipse.jetty" name="jetty-servlet" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
+ <dependency org="org.eclipse.jetty" name="jetty-util" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
+ <dependency org="org.eclipse.jetty" name="jetty-io" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
+ <dependency org="org.eclipse.jetty" name="jetty-continuation" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
+ <dependency org="org.eclipse.jetty" name="jetty-http" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
+ <dependency org="org.slf4j" name="slf4j-api" rev="1.6.6" transitive="false" conf="logging->default"/>
+ <dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.6.6" transitive="false" conf="logging->default"/>
+ <dependency org="org.eclipse.jetty.orbit" name="javax.servlet" rev="3.0.0.v201112011016" transitive="false" conf="servlet->default">
+ <artifact name="javax.servlet" type="orbit" ext="jar"/>
+ </dependency>
+ <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
+ </dependencies>
+
+</ivy-module>
Added: lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java (added)
+++ lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java Mon May 13 11:57:22 2013
@@ -0,0 +1,191 @@
+package org.apache.lucene.replicator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.InfoStream;
+
+/**
+ * A {@link ReplicationHandler} for replication of an index and taxonomy pair.
+ * See {@link IndexReplicationHandler} for more detail. This handler ensures
+ * that the search and taxonomy indexes are replicated in a consistent way.
+ * <p>
+ * <b>NOTE:</b> if you intend to recreate a taxonomy index, you should make sure
+ * to reopen an IndexSearcher and TaxonomyReader pair via the provided callback,
+ * to guarantee that both indexes are in sync. This handler does not prevent
+ * replicating such index and taxonomy pairs, and if they are reopened by a
+ * different thread, unexpected errors can occur, as well as inconsistency
+ * between the taxonomy and index readers.
+ *
+ * @see IndexReplicationHandler
+ *
+ * @lucene.experimental
+ */
+public class IndexAndTaxonomyReplicationHandler implements ReplicationHandler {
+
+ /**
+ * The component used to log messages to the {@link InfoStream#getDefault()
+ * default} {@link InfoStream}.
+ */
+ public static final String INFO_STREAM_COMPONENT = "IndexAndTaxonomyReplicationHandler";
+
+ private final Directory indexDir;
+ private final Directory taxoDir;
+ private final Callable<Boolean> callback;
+
+ private volatile Map<String,List<RevisionFile>> currentRevisionFiles;
+ private volatile String currentVersion;
+ private volatile InfoStream infoStream = InfoStream.getDefault();
+
+ /**
+ * Constructor with the given index directory and callback to notify when the
+ * indexes were updated.
+ */
+ public IndexAndTaxonomyReplicationHandler(Directory indexDir, Directory taxoDir, Callable<Boolean> callback)
+ throws IOException {
+ this.callback = callback;
+ this.indexDir = indexDir;
+ this.taxoDir = taxoDir;
+ currentRevisionFiles = null;
+ currentVersion = null;
+ final boolean indexExists = DirectoryReader.indexExists(indexDir);
+ final boolean taxoExists = DirectoryReader.indexExists(taxoDir);
+ if (indexExists != taxoExists) {
+ throw new IllegalStateException("search and taxonomy indexes must either both exist or not: index=" + indexExists
+ + " taxo=" + taxoExists);
+ }
+ if (indexExists) { // both indexes exist
+ final IndexCommit indexCommit = IndexReplicationHandler.getLastCommit(indexDir);
+ final IndexCommit taxoCommit = IndexReplicationHandler.getLastCommit(taxoDir);
+ currentRevisionFiles = IndexAndTaxonomyRevision.revisionFiles(indexCommit, taxoCommit);
+ currentVersion = IndexAndTaxonomyRevision.revisionVersion(indexCommit, taxoCommit);
+ final InfoStream infoStream = InfoStream.getDefault();
+ if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
+ infoStream.message(INFO_STREAM_COMPONENT, "constructor(): currentVersion=" + currentVersion
+ + " currentRevisionFiles=" + currentRevisionFiles);
+ infoStream.message(INFO_STREAM_COMPONENT, "constructor(): indexCommit=" + indexCommit
+ + " taxoCommit=" + taxoCommit);
+ }
+ }
+ }
+
+ @Override
+ public String currentVersion() {
+ return currentVersion;
+ }
+
+ @Override
+ public Map<String,List<RevisionFile>> currentRevisionFiles() {
+ return currentRevisionFiles;
+ }
+
+ @Override
+ public void revisionReady(String version, Map<String,List<RevisionFile>> revisionFiles,
+ Map<String,List<String>> copiedFiles, Map<String,Directory> sourceDirectory) throws IOException {
+ Directory taxoClientDir = sourceDirectory.get(IndexAndTaxonomyRevision.TAXONOMY_SOURCE);
+ Directory indexClientDir = sourceDirectory.get(IndexAndTaxonomyRevision.INDEX_SOURCE);
+ List<String> taxoFiles = copiedFiles.get(IndexAndTaxonomyRevision.TAXONOMY_SOURCE);
+ List<String> indexFiles = copiedFiles.get(IndexAndTaxonomyRevision.INDEX_SOURCE);
+ String taxoSegmentsFile = IndexReplicationHandler.getSegmentsFile(taxoFiles, true);
+ String indexSegmentsFile = IndexReplicationHandler.getSegmentsFile(indexFiles, false);
+
+ boolean success = false;
+ try {
+ // copy taxonomy files before index files
+ IndexReplicationHandler.copyFiles(taxoClientDir, taxoDir, taxoFiles);
+ IndexReplicationHandler.copyFiles(indexClientDir, indexDir, indexFiles);
+
+ // fsync all copied files (except segmentsFile)
+ if (!taxoFiles.isEmpty()) {
+ taxoDir.sync(taxoFiles);
+ }
+ indexDir.sync(indexFiles);
+
+ // now copy and fsync segmentsFile, taxonomy first because it is ok if a
+ // reader sees a more advanced taxonomy than the index.
+ if (taxoSegmentsFile != null) {
+ taxoClientDir.copy(taxoDir, taxoSegmentsFile, taxoSegmentsFile, IOContext.READONCE);
+ }
+ indexClientDir.copy(indexDir, indexSegmentsFile, indexSegmentsFile, IOContext.READONCE);
+
+ if (taxoSegmentsFile != null) {
+ taxoDir.sync(Collections.singletonList(taxoSegmentsFile));
+ }
+ indexDir.sync(Collections.singletonList(indexSegmentsFile));
+
+ success = true;
+ } finally {
+ if (!success) {
+ taxoFiles.add(taxoSegmentsFile); // add it back so it gets deleted too
+ IndexReplicationHandler.cleanupFilesOnFailure(taxoDir, taxoFiles);
+ indexFiles.add(indexSegmentsFile); // add it back so it gets deleted too
+ IndexReplicationHandler.cleanupFilesOnFailure(indexDir, indexFiles);
+ }
+ }
+
+ // all files have been successfully copied + sync'd. update the handler's state
+ currentRevisionFiles = revisionFiles;
+ currentVersion = version;
+
+ if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
+ infoStream.message(INFO_STREAM_COMPONENT, "revisionReady(): currentVersion=" + currentVersion
+ + " currentRevisionFiles=" + currentRevisionFiles);
+ }
+
+ // update the segments.gen file
+ IndexReplicationHandler.writeSegmentsGen(taxoSegmentsFile, taxoDir);
+ IndexReplicationHandler.writeSegmentsGen(indexSegmentsFile, indexDir);
+
+ // Cleanup the index directory from old and unused index files.
+ // NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have
+ // side-effects, e.g. if it hits sudden IO errors while opening the index
+ // (and can end up deleting the entire index). It is not our job to protect
+ // against those errors, app will probably hit them elsewhere.
+ IndexReplicationHandler.cleanupOldIndexFiles(indexDir, indexSegmentsFile);
+ IndexReplicationHandler.cleanupOldIndexFiles(taxoDir, taxoSegmentsFile);
+
+ // successfully updated the index, notify the callback that the index is
+ // ready.
+ if (callback != null) {
+ try {
+ callback.call();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /** Sets the {@link InfoStream} to use for logging messages. */
+ public void setInfoStream(InfoStream infoStream) {
+ if (infoStream == null) {
+ infoStream = InfoStream.NO_OUTPUT;
+ }
+ this.infoStream = infoStream;
+ }
+
+}
Added: lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java (added)
+++ lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java Mon May 13 11:57:22 2013
@@ -0,0 +1,219 @@
+package org.apache.lucene.replicator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexDeletionPolicy;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.SnapshotDeletionPolicy;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+/**
+ * A {@link Revision} of a single index and taxonomy index files which comprises
+ * the list of files from both indexes. This revision should be used whenever a
+ * pair of search and taxonomy indexes need to be replicated together to
+ * guarantee consistency of both on the replicating (client) side.
+ *
+ * @see IndexRevision
+ *
+ * @lucene.experimental
+ */
+public class IndexAndTaxonomyRevision implements Revision {
+
+ /**
+ * A {@link DirectoryTaxonomyWriter} which sets the underlying
+ * {@link IndexWriter}'s {@link IndexDeletionPolicy} to
+ * {@link SnapshotDeletionPolicy}.
+ */
+ public static final class SnapshotDirectoryTaxonomyWriter extends DirectoryTaxonomyWriter {
+
+ private SnapshotDeletionPolicy sdp;
+ private IndexWriter writer;
+
+ /**
+ * @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory,
+ * IndexWriterConfig.OpenMode, TaxonomyWriterCache)
+ */
+ public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode, TaxonomyWriterCache cache)
+ throws IOException {
+ super(directory, openMode, cache);
+ }
+
+ /** @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory, IndexWriterConfig.OpenMode) */
+ public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode) throws IOException {
+ super(directory, openMode);
+ }
+
+ /** @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory) */
+ public SnapshotDirectoryTaxonomyWriter(Directory d) throws IOException {
+ super(d);
+ }
+
+ @Override
+ protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
+ IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
+ conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
+ return conf;
+ }
+
+ @Override
+ protected IndexWriter openIndexWriter(Directory directory, IndexWriterConfig config) throws IOException {
+ writer = super.openIndexWriter(directory, config);
+ // must set it here because IndexWriter clones the config
+ sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+ return writer;
+ }
+
+ /** Returns the {@link SnapshotDeletionPolicy} used by the underlying {@link IndexWriter}. */
+ public SnapshotDeletionPolicy getDeletionPolicy() {
+ return sdp;
+ }
+
+ /** Returns the {@link IndexWriter} used by this {@link DirectoryTaxonomyWriter}. */
+ public IndexWriter getIndexWriter() {
+ return writer;
+ }
+
+ }
+
+ private static final int RADIX = 16;
+
+ public static final String INDEX_SOURCE = "index";
+ public static final String TAXONOMY_SOURCE = "taxo";
+
+ private final IndexWriter indexWriter;
+ private final SnapshotDirectoryTaxonomyWriter taxoWriter;
+ private final IndexCommit indexCommit, taxoCommit;
+ private final SnapshotDeletionPolicy indexSDP, taxoSDP;
+ private final String version;
+ private final Map<String,List<RevisionFile>> sourceFiles;
+
+ /** Returns a singleton map of the revision files from the given {@link IndexCommit}. */
+ public static Map<String, List<RevisionFile>> revisionFiles(IndexCommit indexCommit, IndexCommit taxoCommit)
+ throws IOException {
+ HashMap<String,List<RevisionFile>> files = new HashMap<String,List<RevisionFile>>();
+ files.put(INDEX_SOURCE, IndexRevision.revisionFiles(indexCommit).values().iterator().next());
+ files.put(TAXONOMY_SOURCE, IndexRevision.revisionFiles(taxoCommit).values().iterator().next());
+ return files;
+ }
+
+ /**
+ * Returns a String representation of a revision's version from the given
+ * {@link IndexCommit}s of the search and taxonomy indexes.
+ */
+ public static String revisionVersion(IndexCommit indexCommit, IndexCommit taxoCommit) {
+ return Long.toString(indexCommit.getGeneration(), RADIX) + ":" + Long.toString(taxoCommit.getGeneration(), RADIX);
+ }
+
+ /**
+ * Constructor over the given {@link IndexWriter}. Uses the last
+ * {@link IndexCommit} found in the {@link Directory} managed by the given
+ * writer.
+ */
+ public IndexAndTaxonomyRevision(IndexWriter indexWriter, SnapshotDirectoryTaxonomyWriter taxoWriter)
+ throws IOException {
+ IndexDeletionPolicy delPolicy = indexWriter.getConfig().getIndexDeletionPolicy();
+ if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
+ throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
+ }
+ this.indexWriter = indexWriter;
+ this.taxoWriter = taxoWriter;
+ this.indexSDP = (SnapshotDeletionPolicy) delPolicy;
+ this.taxoSDP = taxoWriter.getDeletionPolicy();
+ this.indexCommit = indexSDP.snapshot();
+ this.taxoCommit = taxoSDP.snapshot();
+ this.version = revisionVersion(indexCommit, taxoCommit);
+ this.sourceFiles = revisionFiles(indexCommit, taxoCommit);
+ }
+
+ @Override
+ public int compareTo(String version) {
+ final String[] parts = version.split(":");
+ final long indexGen = Long.parseLong(parts[0], RADIX);
+ final long taxoGen = Long.parseLong(parts[1], RADIX);
+ final long indexCommitGen = indexCommit.getGeneration();
+ final long taxoCommitGen = taxoCommit.getGeneration();
+
+ // if the index generation is not the same as this commit's generation,
+ // compare by it. Otherwise, compare by the taxonomy generation.
+ if (indexCommitGen < indexGen) {
+ return -1;
+ } else if (indexCommitGen > indexGen) {
+ return 1;
+ } else {
+ return taxoCommitGen < taxoGen ? -1 : (taxoCommitGen > taxoGen ? 1 : 0);
+ }
+ }
+
+ @Override
+ public int compareTo(Revision o) {
+ IndexAndTaxonomyRevision other = (IndexAndTaxonomyRevision) o;
+ int cmp = indexCommit.compareTo(other.indexCommit);
+ return cmp != 0 ? cmp : taxoCommit.compareTo(other.taxoCommit);
+ }
+
+ @Override
+ public String getVersion() {
+ return version;
+ }
+
+ @Override
+ public Map<String,List<RevisionFile>> getSourceFiles() {
+ return sourceFiles;
+ }
+
+ @Override
+ public InputStream open(String source, String fileName) throws IOException {
+ assert source.equals(INDEX_SOURCE) || source.equals(TAXONOMY_SOURCE) : "invalid source; expected=(" + INDEX_SOURCE
+ + " or " + TAXONOMY_SOURCE + ") got=" + source;
+ IndexCommit ic = source.equals(INDEX_SOURCE) ? indexCommit : taxoCommit;
+ return new IndexInputInputStream(ic.getDirectory().openInput(fileName, IOContext.READONCE));
+ }
+
+ @Override
+ public void release() throws IOException {
+ try {
+ indexSDP.release(indexCommit);
+ } finally {
+ taxoSDP.release(taxoCommit);
+ }
+
+ try {
+ indexWriter.deleteUnusedFiles();
+ } finally {
+ taxoWriter.getIndexWriter().deleteUnusedFiles();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "IndexAndTaxonomyRevision version=" + version + " files=" + sourceFiles;
+ }
+
+}
Added: lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java (added)
+++ lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java Mon May 13 11:57:22 2013
@@ -0,0 +1,92 @@
+package org.apache.lucene.replicator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.lucene.store.IndexInput;
+
+/**
+ * An {@link InputStream} which wraps an {@link IndexInput}.
+ *
+ * @lucene.experimental
+ */
+public final class IndexInputInputStream extends InputStream {
+
+ private final IndexInput in;
+
+ private long remaining;
+
+ public IndexInputInputStream(IndexInput in) {
+ this.in = in;
+ remaining = in.length();
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (remaining == 0) {
+ return -1;
+ } else {
+ --remaining;
+ return in.readByte();
+ }
+ }
+
+ @Override
+ public int available() throws IOException {
+ return (int) in.length();
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (remaining == 0) {
+ return -1;
+ }
+ if (remaining < len) {
+ len = (int) remaining;
+ }
+ in.readBytes(b, off, len);
+ remaining -= len;
+ return len;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (remaining == 0) {
+ return -1;
+ }
+ if (remaining < n) {
+ n = remaining;
+ }
+ in.seek(in.getFilePointer() + n);
+ remaining -= n;
+ return n;
+ }
+
+}
\ No newline at end of file
Added: lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java (added)
+++ lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java Mon May 13 11:57:22 2013
@@ -0,0 +1,308 @@
+package org.apache.lucene.replicator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.regex.Matcher;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexNotFoundException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.InfoStream;
+
+/**
+ * A {@link ReplicationHandler} for replication of an index. Implements
+ * {@link #revisionReady} by copying the files pointed by the client resolver to
+ * the index {@link Directory} and then touches the index with
+ * {@link IndexWriter} to make sure any unused files are deleted.
+ * <p>
+ * <b>NOTE:</b> this handler assumes that {@link IndexWriter} is not opened by
+ * another process on the index directory. In fact, opening an
+ * {@link IndexWriter} on the same directory to which files are copied can lead
+ * to undefined behavior, where some or all the files will be deleted, override
+ * other files or simply create a mess. When you replicate an index, it is best
+ * if the index is never modified by {@link IndexWriter}, except the one that is
+ * open on the source index, from which you replicate.
+ * <p>
+ * This handler notifies the application via a provided {@link Callable} when an
+ * updated index commit was made available for it.
+ *
+ * @lucene.experimental
+ */
+public class IndexReplicationHandler implements ReplicationHandler {
+
+ /**
+ * The component used to log messages to the {@link InfoStream#getDefault()
+ * default} {@link InfoStream}.
+ */
+ public static final String INFO_STREAM_COMPONENT = "IndexReplicationHandler";
+
+ private final Directory indexDir;
+ private final Callable<Boolean> callback;
+
+ private volatile Map<String,List<RevisionFile>> currentRevisionFiles;
+ private volatile String currentVersion;
+ private volatile InfoStream infoStream = InfoStream.getDefault();
+
+ /**
+ * Returns the last {@link IndexCommit} found in the {@link Directory}, or
+ * {@code null} if there are no commits.
+ */
+ public static IndexCommit getLastCommit(Directory dir) throws IOException {
+ try {
+ if (DirectoryReader.indexExists(dir)) {
+ List<IndexCommit> commits = DirectoryReader.listCommits(dir);
+ // listCommits guarantees that we get at least one commit back, or
+ // IndexNotFoundException which we handle below
+ return commits.get(commits.size() - 1);
+ }
+ } catch (IndexNotFoundException e) {
+ // ignore the exception and return null
+ }
+ return null;
+ }
+
+ /**
+ * Verifies that the last file is segments_N and fails otherwise. It also
+ * removes and returns the file from the list, because it needs to be handled
+ * last, after all files. This is important in order to guarantee that if a
+ * reader sees the new segments_N, all other segment files are already on
+ * stable storage.
+ * <p>
+ * The reason why the code fails instead of putting segments_N file last is
+ * that this indicates an error in the Revision implementation.
+ */
+ public static String getSegmentsFile(List<String> files, boolean allowEmpty) {
+ if (files.isEmpty()) {
+ if (allowEmpty) {
+ return null;
+ } else {
+ throw new IllegalStateException("empty list of files not allowed");
+ }
+ }
+
+ String segmentsFile = files.remove(files.size() - 1);
+ if (!segmentsFile.startsWith(IndexFileNames.SEGMENTS) || segmentsFile.equals(IndexFileNames.SEGMENTS_GEN)) {
+ throw new IllegalStateException("last file to copy+sync must be segments_N but got " + segmentsFile
+ + "; check your Revision implementation!");
+ }
+ return segmentsFile;
+ }
+
+ /**
+ * Cleanup the index directory by deleting all given files. Called when file
+ * copy or sync failed.
+ */
+ public static void cleanupFilesOnFailure(Directory dir, List<String> files) {
+ for (String file : files) {
+ try {
+ if (dir.fileExists(file)) {
+ dir.deleteFile(file);
+ }
+ } catch (Throwable t) {
+ // suppress any exception because if we're here, it means copy
+ // failed, and we must cleanup after ourselves.
+ }
+ }
+ }
+
+ /**
+ * Cleans up the index directory from old index files. This method uses the
+ * last commit found by {@link #getLastCommit(Directory)}. If it matches the
+ * expected segmentsFile, then all files not referenced by this commit point
+ * are deleted.
+ * <p>
+ * <b>NOTE:</b> this method does a best effort attempt to clean the index
+ * directory. It suppresses any exceptions that occur, as this can be retried
+ * the next time.
+ */
+ public static void cleanupOldIndexFiles(Directory dir, String segmentsFile) {
+ try {
+ IndexCommit commit = getLastCommit(dir);
+ // commit == null means weird IO errors occurred, ignore them
+ // if there were any IO errors reading the expected commit point (i.e.
+ // segments files mismatch), then ignore that commit either.
+ if (commit != null && commit.getSegmentsFileName().equals(segmentsFile)) {
+ Set<String> commitFiles = new HashSet<String>();
+ commitFiles.addAll(commit.getFileNames());
+ commitFiles.add(IndexFileNames.SEGMENTS_GEN);
+ Matcher matcher = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+ for (String file : dir.listAll()) {
+ if (!commitFiles.contains(file)
+ && (matcher.reset(file).matches() || file.startsWith(IndexFileNames.SEGMENTS))) {
+ try {
+ dir.deleteFile(file);
+ } catch (Throwable t) {
+ // suppress, it's just a best effort
+ }
+ }
+ }
+ }
+ } catch (Throwable t) {
+ // ignore any errors that happens during this state and only log it. this
+ // cleanup will have a chance to succeed the next time we get a new
+ // revision.
+ }
+ }
+
+ /**
+ * Copies the files from the source directory to the target one, if they are
+ * not the same.
+ */
+ public static void copyFiles(Directory source, Directory target, List<String> files) throws IOException {
+ if (!source.equals(target)) {
+ for (String file : files) {
+ source.copy(target, file, file, IOContext.READONCE);
+ }
+ }
+ }
+
+ /**
+ * Writes {@link IndexFileNames#SEGMENTS_GEN} file to the directory, reading
+ * the generation from the given {@code segmentsFile}. If it is {@code null},
+ * this method deletes segments.gen from the directory.
+ */
+ public static void writeSegmentsGen(String segmentsFile, Directory dir) {
+ if (segmentsFile != null) {
+ SegmentInfos.writeSegmentsGen(dir, SegmentInfos.generationFromSegmentsFileName(segmentsFile));
+ } else {
+ try {
+ if (dir.fileExists(IndexFileNames.SEGMENTS_GEN)) {
+ dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
+ }
+ } catch (Throwable t) {
+ // suppress any errors while deleting this file.
+ }
+ }
+ }
+
+ /**
+ * Constructor with the given index directory and callback to notify when the
+ * indexes were updated.
+ */
+ public IndexReplicationHandler(Directory indexDir, Callable<Boolean> callback) throws IOException {
+ this.callback = callback;
+ this.indexDir = indexDir;
+ currentRevisionFiles = null;
+ currentVersion = null;
+ if (DirectoryReader.indexExists(indexDir)) {
+ final List<IndexCommit> commits = DirectoryReader.listCommits(indexDir);
+ final IndexCommit commit = commits.get(commits.size() - 1);
+ currentRevisionFiles = IndexRevision.revisionFiles(commit);
+ currentVersion = IndexRevision.revisionVersion(commit);
+ final InfoStream infoStream = InfoStream.getDefault();
+ if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
+ infoStream.message(INFO_STREAM_COMPONENT, "constructor(): currentVersion=" + currentVersion
+ + " currentRevisionFiles=" + currentRevisionFiles);
+ infoStream.message(INFO_STREAM_COMPONENT, "constructor(): commit=" + commit);
+ }
+ }
+ }
+
+ @Override
+ public String currentVersion() {
+ return currentVersion;
+ }
+
+ @Override
+ public Map<String,List<RevisionFile>> currentRevisionFiles() {
+ return currentRevisionFiles;
+ }
+
+ @Override
+ public void revisionReady(String version, Map<String,List<RevisionFile>> revisionFiles,
+ Map<String,List<String>> copiedFiles, Map<String,Directory> sourceDirectory) throws IOException {
+ if (revisionFiles.size() > 1) {
+ throw new IllegalArgumentException("this handler handles only a single source; got " + revisionFiles.keySet());
+ }
+
+ Directory clientDir = sourceDirectory.values().iterator().next();
+ List<String> files = copiedFiles.values().iterator().next();
+ String segmentsFile = getSegmentsFile(files, false);
+
+ boolean success = false;
+ try {
+ // copy files from the client to index directory
+ copyFiles(clientDir, indexDir, files);
+
+ // fsync all copied files (except segmentsFile)
+ indexDir.sync(files);
+
+ // now copy and fsync segmentsFile
+ clientDir.copy(indexDir, segmentsFile, segmentsFile, IOContext.READONCE);
+ indexDir.sync(Collections.singletonList(segmentsFile));
+
+ success = true;
+ } finally {
+ if (!success) {
+ files.add(segmentsFile); // add it back so it gets deleted too
+ cleanupFilesOnFailure(indexDir, files);
+ }
+ }
+
+ // all files have been successfully copied + sync'd. update the handler's state
+ currentRevisionFiles = revisionFiles;
+ currentVersion = version;
+
+ if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
+ infoStream.message(INFO_STREAM_COMPONENT, "revisionReady(): currentVersion=" + currentVersion
+ + " currentRevisionFiles=" + currentRevisionFiles);
+ }
+
+ // update the segments.gen file
+ writeSegmentsGen(segmentsFile, indexDir);
+
+ // Cleanup the index directory from old and unused index files.
+ // NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have
+ // side-effects, e.g. if it hits sudden IO errors while opening the index
+ // (and can end up deleting the entire index). It is not our job to protect
+ // against those errors, app will probably hit them elsewhere.
+ cleanupOldIndexFiles(indexDir, segmentsFile);
+
+ // successfully updated the index, notify the callback that the index is
+ // ready.
+ if (callback != null) {
+ try {
+ callback.call();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /** Sets the {@link InfoStream} to use for logging messages. */
+ public void setInfoStream(InfoStream infoStream) {
+ if (infoStream == null) {
+ infoStream = InfoStream.NO_OUTPUT;
+ }
+ this.infoStream = infoStream;
+ }
+
+}
Added: lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java (added)
+++ lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java Mon May 13 11:57:22 2013
@@ -0,0 +1,150 @@
+package org.apache.lucene.replicator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexDeletionPolicy;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SnapshotDeletionPolicy;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+/**
+ * A {@link Revision} of a single index files which comprises the list of files
+ * that are part of the current {@link IndexCommit}. To ensure the files are not
+ * deleted by {@link IndexWriter} for as long as this revision stays alive (i.e.
+ * until {@link #release()}), the current commit point is snapshotted, using
+ * {@link SnapshotDeletionPolicy} (this means that the given writer's
+ * {@link IndexWriterConfig#getIndexDeletionPolicy() config} should return
+ * {@link SnapshotDeletionPolicy}).
+ * <p>
+ * When this revision is {@link #release() released}, it releases the obtained
+ * snapshot as well as calls {@link IndexWriter#deleteUnusedFiles()} so that the
+ * snapshotted files are deleted (if they are no longer needed).
+ *
+ * @lucene.experimental
+ */
+public class IndexRevision implements Revision {
+
+ private static final int RADIX = 16;
+ private static final String SOURCE = "index";
+
+ private final IndexWriter writer;
+ private final IndexCommit commit;
+ private final SnapshotDeletionPolicy sdp;
+ private final String version;
+ private final Map<String,List<RevisionFile>> sourceFiles;
+
+ // returns a RevisionFile with some metadata
+ private static RevisionFile newRevisionFile(String file, Directory dir) throws IOException {
+ RevisionFile revFile = new RevisionFile(file);
+ revFile.size = dir.fileLength(file);
+ return revFile;
+ }
+
+ /** Returns a singleton map of the revision files from the given {@link IndexCommit}. */
+ public static Map<String,List<RevisionFile>> revisionFiles(IndexCommit commit) throws IOException {
+ Collection<String> commitFiles = commit.getFileNames();
+ List<RevisionFile> revisionFiles = new ArrayList<RevisionFile>(commitFiles.size());
+ String segmentsFile = commit.getSegmentsFileName();
+ Directory dir = commit.getDirectory();
+
+ for (String file : commitFiles) {
+ if (!file.equals(segmentsFile)) {
+ revisionFiles.add(newRevisionFile(file, dir));
+ }
+ }
+ revisionFiles.add(newRevisionFile(segmentsFile, dir)); // segments_N must be last
+ return Collections.singletonMap(SOURCE, revisionFiles);
+ }
+
+ /**
+ * Returns a String representation of a revision's version from the given
+ * {@link IndexCommit}.
+ */
+ public static String revisionVersion(IndexCommit commit) {
+ return Long.toString(commit.getGeneration(), RADIX);
+ }
+
+ /**
+ * Constructor over the given {@link IndexWriter}. Uses the last
+ * {@link IndexCommit} found in the {@link Directory} managed by the given
+ * writer.
+ */
+ public IndexRevision(IndexWriter writer) throws IOException {
+ IndexDeletionPolicy delPolicy = writer.getConfig().getIndexDeletionPolicy();
+ if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
+ throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
+ }
+ this.writer = writer;
+ this.sdp = (SnapshotDeletionPolicy) delPolicy;
+ this.commit = sdp.snapshot();
+ this.version = revisionVersion(commit);
+ this.sourceFiles = revisionFiles(commit);
+ }
+
+ @Override
+ public int compareTo(String version) {
+ long gen = Long.parseLong(version, RADIX);
+ long commitGen = commit.getGeneration();
+ return commitGen < gen ? -1 : (commitGen > gen ? 1 : 0);
+ }
+
+ @Override
+ public int compareTo(Revision o) {
+ IndexRevision other = (IndexRevision) o;
+ return commit.compareTo(other.commit);
+ }
+
+ @Override
+ public String getVersion() {
+ return version;
+ }
+
+ @Override
+ public Map<String,List<RevisionFile>> getSourceFiles() {
+ return sourceFiles;
+ }
+
+ @Override
+ public InputStream open(String source, String fileName) throws IOException {
+ assert source.equals(SOURCE) : "invalid source; expected=" + SOURCE + " got=" + source;
+ return new IndexInputInputStream(commit.getDirectory().openInput(fileName, IOContext.READONCE));
+ }
+
+ @Override
+ public void release() throws IOException {
+ sdp.release(commit);
+ writer.deleteUnusedFiles();
+ }
+
+ @Override
+ public String toString() {
+ return "IndexRevision version=" + version + " files=" + sourceFiles;
+ }
+
+}
Added: lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java (added)
+++ lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java Mon May 13 11:57:22 2013
@@ -0,0 +1,247 @@
+package org.apache.lucene.replicator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.store.AlreadyClosedException;
+
+/**
+ * A {@link Replicator} implementation for use by the side that publishes
+ * {@link Revision}s, as well for clients to {@link #checkForUpdate(String)
+ * check for updates}. When a client needs to be updated, it is returned a
+ * {@link SessionToken} through which it can
+ * {@link #obtainFile(String, String, String) obtain} the files of that
+ * revision. As long as a revision is being replicated, this replicator
+ * guarantees that it will not be {@link Revision#release() released}.
+ * <p>
+ * Replication sessions expire by default after
+ * {@link #DEFAULT_SESSION_EXPIRATION_THRESHOLD}, and the threshold can be
+ * configured through {@link #setExpirationThreshold(long)}.
+ *
+ * @lucene.experimental
+ */
+public class LocalReplicator implements Replicator {
+
+ private static class RefCountedRevision {
+ private final AtomicInteger refCount = new AtomicInteger(1);
+ public final Revision revision;
+
+ public RefCountedRevision(Revision revision) {
+ this.revision = revision;
+ }
+
+ public void decRef() throws IOException {
+ if (refCount.get() <= 0) {
+ throw new IllegalStateException("this revision is already released");
+ }
+
+ final int rc = refCount.decrementAndGet();
+ if (rc == 0) {
+ boolean success = false;
+ try {
+ revision.release();
+ success = true;
+ } finally {
+ if (!success) {
+ // Put reference back on failure
+ refCount.incrementAndGet();
+ }
+ }
+ } else if (rc < 0) {
+ throw new IllegalStateException("too many decRef calls: refCount is " + rc + " after decrement");
+ }
+ }
+
+ public void incRef() {
+ refCount.incrementAndGet();
+ }
+
+ }
+
+ private static class ReplicationSession {
+ public final SessionToken session;
+ public final RefCountedRevision revision;
+ private volatile long lastAccessTime;
+
+ ReplicationSession(SessionToken session, RefCountedRevision revision) {
+ this.session = session;
+ this.revision = revision;
+ lastAccessTime = System.currentTimeMillis();
+ }
+
+ boolean isExpired(long expirationThreshold) {
+ return lastAccessTime < (System.currentTimeMillis() - expirationThreshold);
+ }
+
+ void markAccessed() {
+ lastAccessTime = System.currentTimeMillis();
+ }
+ }
+
+ /** Threshold for expiring inactive sessions. Defaults to 30 minutes. */
+ public static final long DEFAULT_SESSION_EXPIRATION_THRESHOLD = 1000 * 60 * 30;
+
+ private long expirationThresholdMilllis = LocalReplicator.DEFAULT_SESSION_EXPIRATION_THRESHOLD;
+
+ private volatile RefCountedRevision currentRevision;
+ private volatile boolean closed = false;
+
+ private final AtomicInteger sessionToken = new AtomicInteger(0);
+ private final Map<String, ReplicationSession> sessions = new HashMap<String, ReplicationSession>();
+
+ private void checkExpiredSessions() throws IOException {
+ // make a "to-delete" list so we don't risk deleting from the map while iterating it
+ final ArrayList<ReplicationSession> toExpire = new ArrayList<ReplicationSession>();
+ for (ReplicationSession token : sessions.values()) {
+ if (token.isExpired(expirationThresholdMilllis)) {
+ toExpire.add(token);
+ }
+ }
+ for (ReplicationSession token : toExpire) {
+ releaseSession(token.session.id);
+ }
+ }
+
+ private void releaseSession(String sessionID) throws IOException {
+ ReplicationSession session = sessions.remove(sessionID);
+ // if we're called concurrently by close() and release(), could be that one
+ // thread beats the other to release the session.
+ if (session != null) {
+ session.revision.decRef();
+ }
+ }
+
+ /** Ensure that replicator is still open, or throw {@link AlreadyClosedException} otherwise. */
+ protected final synchronized void ensureOpen() {
+ if (closed) {
+ throw new AlreadyClosedException("This replicator has already been closed");
+ }
+ }
+
+ @Override
+ public synchronized SessionToken checkForUpdate(String currentVersion) {
+ ensureOpen();
+ if (currentRevision == null) { // no published revisions yet
+ return null;
+ }
+
+ if (currentVersion != null && currentRevision.revision.compareTo(currentVersion) <= 0) {
+ // currentVersion is newer or equal to latest published revision
+ return null;
+ }
+
+ // currentVersion is either null or older than latest published revision
+ currentRevision.incRef();
+ final String sessionID = Integer.toString(sessionToken.incrementAndGet());
+ final SessionToken sessionToken = new SessionToken(sessionID, currentRevision.revision);
+ final ReplicationSession timedSessionToken = new ReplicationSession(sessionToken, currentRevision);
+ sessions.put(sessionID, timedSessionToken);
+ return sessionToken;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (!closed) {
+ // release all managed revisions
+ for (ReplicationSession session : sessions.values()) {
+ session.revision.decRef();
+ }
+ sessions.clear();
+ closed = true;
+ }
+ }
+
+ /**
+ * Returns the expiration threshold.
+ *
+ * @see #setExpirationThreshold(long)
+ */
+ public long getExpirationThreshold() {
+ return expirationThresholdMilllis;
+ }
+
+ @Override
+ public synchronized InputStream obtainFile(String sessionID, String source, String fileName) throws IOException {
+ ensureOpen();
+ ReplicationSession session = sessions.get(sessionID);
+ if (session != null && session.isExpired(expirationThresholdMilllis)) {
+ releaseSession(sessionID);
+ session = null;
+ }
+ // session either previously expired, or we just expired it
+ if (session == null) {
+ throw new SessionExpiredException("session (" + sessionID + ") expired while obtaining file: source=" + source
+ + " file=" + fileName);
+ }
+ sessions.get(sessionID).markAccessed();
+ return session.revision.revision.open(source, fileName);
+ }
+
+ @Override
+ public synchronized void publish(Revision revision) throws IOException {
+ ensureOpen();
+ if (currentRevision != null) {
+ int compare = revision.compareTo(currentRevision.revision);
+ if (compare == 0) {
+ // same revision published again, ignore but release it
+ revision.release();
+ return;
+ }
+
+ if (compare < 0) {
+ revision.release();
+ throw new IllegalArgumentException("Cannot publish an older revision: rev=" + revision + " current="
+ + currentRevision);
+ }
+ }
+
+ // swap revisions
+ final RefCountedRevision oldRevision = currentRevision;
+ currentRevision = new RefCountedRevision(revision);
+ if (oldRevision != null) {
+ oldRevision.decRef();
+ }
+
+ // check for expired sessions
+ checkExpiredSessions();
+ }
+
+ @Override
+ public synchronized void release(String sessionID) throws IOException {
+ ensureOpen();
+ releaseSession(sessionID);
+ }
+
+ /**
+ * Modify session expiration time - if a replication session is inactive that
+ * long it is automatically expired, and further attempts to operate within
+ * this session will throw a {@link SessionExpiredException}.
+ */
+ public synchronized void setExpirationThreshold(long expirationThreshold) throws IOException {
+ ensureOpen();
+ this.expirationThresholdMilllis = expirationThreshold;
+ checkExpiredSessions();
+ }
+
+}
Added: lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java?rev=1481804&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java (added)
+++ lucene/dev/trunk/lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java Mon May 13 11:57:22 2013
@@ -0,0 +1,77 @@
+package org.apache.lucene.replicator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.replicator.ReplicationClient.SourceDirectoryFactory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+
+/**
+ * A {@link SourceDirectoryFactory} which returns {@link FSDirectory} under a
+ * dedicated session directory. When a session is over, the entire directory is
+ * deleted.
+ *
+ * @lucene.experimental
+ */
+public class PerSessionDirectoryFactory implements SourceDirectoryFactory {
+
+ private final File workDir;
+
+ /** Constructor with the given sources mapping. */
+ public PerSessionDirectoryFactory(File workDir) {
+ this.workDir = workDir;
+ }
+
+ private void rm(File file) throws IOException {
+ if (file.isDirectory()) {
+ for (File f : file.listFiles()) {
+ rm(f);
+ }
+ }
+
+ // This should be either an empty directory, or a file
+ if (!file.delete() && file.exists()) {
+ throw new IOException("failed to delete " + file);
+ }
+ }
+
+ @Override
+ public Directory getDirectory(String sessionID, String source) throws IOException {
+ File sessionDir = new File(workDir, sessionID);
+ if (!sessionDir.exists() && !sessionDir.mkdirs()) {
+ throw new IOException("failed to create session directory " + sessionDir);
+ }
+ File sourceDir = new File(sessionDir, source);
+ if (!sourceDir.mkdirs()) {
+ throw new IOException("failed to create source directory " + sourceDir);
+ }
+ return FSDirectory.open(sourceDir);
+ }
+
+ @Override
+ public void cleanupSession(String sessionID) throws IOException {
+ if (sessionID.isEmpty()) { // protect against deleting workDir entirely!
+ throw new IllegalArgumentException("sessionID cannot be empty");
+ }
+ rm(new File(workDir, sessionID));
+ }
+
+}