You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2023/01/14 16:14:51 UTC
[lucene] branch main updated: remove non-NRT replication support (#12038)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 8ca05967e8f remove non-NRT replication support (#12038)
8ca05967e8f is described below
commit 8ca05967e8faaf89747a349eaaaf2a67532f2dcb
Author: Robert Muir <rm...@apache.org>
AuthorDate: Sat Jan 14 11:14:46 2023 -0500
remove non-NRT replication support (#12038)
Remove non-NRT replication support in 10.x (to be deprecated in 9.5)
---
NOTICE.txt | 3 -
lucene/CHANGES.txt | 3 +
lucene/licenses/commons-logging-1.2.jar.sha1 | 1 -
lucene/licenses/commons-logging-LICENSE-ASL.txt | 202 ---------
lucene/licenses/commons-logging-NOTICE.txt | 6 -
lucene/licenses/httpclient-4.5.13.jar.sha1 | 1 -
lucene/licenses/httpclient-LICENSE-ASL.txt | 182 --------
lucene/licenses/httpclient-NOTICE.txt | 8 -
lucene/licenses/httpcore-4.4.13.jar.sha1 | 1 -
lucene/licenses/httpcore-LICENSE-ASL.txt | 182 --------
lucene/licenses/httpcore-NOTICE.txt | 8 -
lucene/licenses/javax.servlet-LICENSE-CDDL.txt | 124 ------
lucene/licenses/javax.servlet-NOTICE.txt | 2 -
lucene/licenses/javax.servlet-api-3.1.0.jar.sha1 | 1 -
lucene/licenses/jetty-LICENSE-ASL.txt | 202 ---------
lucene/licenses/jetty-NOTICE.txt | 111 -----
.../jetty-continuation-9.4.41.v20210516.jar.sha1 | 1 -
.../licenses/jetty-http-9.4.41.v20210516.jar.sha1 | 1 -
lucene/licenses/jetty-io-9.4.41.v20210516.jar.sha1 | 1 -
.../jetty-server-9.4.41.v20210516.jar.sha1 | 1 -
.../jetty-servlet-9.4.41.v20210516.jar.sha1 | 1 -
.../licenses/jetty-util-9.4.41.v20210516.jar.sha1 | 1 -
.../jetty-util-ajax-9.4.41.v20210516.jar.sha1 | 1 -
lucene/replicator/build.gradle | 14 -
lucene/replicator/src/java/module-info.java | 6 -
.../IndexAndTaxonomyReplicationHandler.java | 213 ---------
.../replicator/IndexAndTaxonomyRevision.java | 223 ----------
.../lucene/replicator/IndexInputInputStream.java | 89 ----
.../lucene/replicator/IndexReplicationHandler.java | 292 -------------
.../apache/lucene/replicator/IndexRevision.java | 142 ------
.../apache/lucene/replicator/LocalReplicator.java | 251 -----------
.../replicator/PerSessionDirectoryFactory.java | 58 ---
.../lucene/replicator/ReplicationClient.java | 412 ------------------
.../org/apache/lucene/replicator/Replicator.java | 68 ---
.../org/apache/lucene/replicator/Revision.java | 67 ---
.../org/apache/lucene/replicator/RevisionFile.java | 56 ---
.../lucene/replicator/SessionExpiredException.java | 50 ---
.../org/apache/lucene/replicator/SessionToken.java | 105 -----
.../lucene/replicator/http/HttpClientBase.java | 293 -------------
.../lucene/replicator/http/HttpReplicator.java | 109 -----
.../lucene/replicator/http/ReplicationService.java | 202 ---------
.../lucene/replicator/http/package-info.java | 19 -
.../org/apache/lucene/replicator/package-info.java | 71 ---
.../lucene/replicator/ReplicatorTestCase.java | 158 -------
.../TestIndexAndTaxonomyReplicationClient.java | 481 ---------------------
.../replicator/TestIndexAndTaxonomyRevision.java | 173 --------
.../replicator/TestIndexReplicationClient.java | 356 ---------------
.../lucene/replicator/TestIndexRevision.java | 156 -------
.../lucene/replicator/TestLocalReplicator.java | 200 ---------
.../apache/lucene/replicator/TestSessionToken.java | 62 ---
.../lucene/replicator/http/ReplicationServlet.java | 47 --
.../lucene/replicator/http/TestHttpReplicator.java | 165 -------
versions.lock | 11 -
versions.props | 5 +-
54 files changed, 4 insertions(+), 5594 deletions(-)
diff --git a/NOTICE.txt b/NOTICE.txt
index 89cc62480a6..be308278f13 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -105,9 +105,6 @@ developed by Dawid Weiss and Marcin Miłkowski
(https://github.com/morfologik/morfologik-stemming) and uses
data from the BSD-licensed dictionary of Polish (SGJP, http://sgjp.pl/morfeusz/).
-Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original
-source code for this can be found at http://www.eclipse.org/jetty/downloads.php
-
===========================================================================
Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration
===========================================================================
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b925b755f44..b5a0c47eff0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -150,6 +150,9 @@ API Changes
can only be disabled globally by passing the following sysprop on Java command line:
"-Dorg.apache.lucene.store.MMapDirectory.enableUnmapHack=false" (Uwe Schindler)
+* GITHUB#12038: Deprecate non-NRT replication support.
+ Please migrate to org.apache.lucene.replicator.nrt instead. (Robert Muir)
+
New Features
---------------------
* GITHUB#11795: Add ByteWritesTrackingDirectoryWrapper to expose metrics for bytes merged, flushed, and overall
diff --git a/lucene/licenses/commons-logging-1.2.jar.sha1 b/lucene/licenses/commons-logging-1.2.jar.sha1
deleted file mode 100644
index c3146883508..00000000000
--- a/lucene/licenses/commons-logging-1.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4bfc12adfe4842bf07b657f0369c4cb522955686
diff --git a/lucene/licenses/commons-logging-LICENSE-ASL.txt b/lucene/licenses/commons-logging-LICENSE-ASL.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/lucene/licenses/commons-logging-LICENSE-ASL.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lucene/licenses/commons-logging-NOTICE.txt b/lucene/licenses/commons-logging-NOTICE.txt
deleted file mode 100644
index a37977d45a1..00000000000
--- a/lucene/licenses/commons-logging-NOTICE.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-Apache Commons Logging
-Copyright 2003-2013 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
diff --git a/lucene/licenses/httpclient-4.5.13.jar.sha1 b/lucene/licenses/httpclient-4.5.13.jar.sha1
deleted file mode 100644
index ece7cdea6ea..00000000000
--- a/lucene/licenses/httpclient-4.5.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada
diff --git a/lucene/licenses/httpclient-LICENSE-ASL.txt b/lucene/licenses/httpclient-LICENSE-ASL.txt
deleted file mode 100644
index 2c41ec88f61..00000000000
--- a/lucene/licenses/httpclient-LICENSE-ASL.txt
+++ /dev/null
@@ -1,182 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
-This project contains annotations derived from JCIP-ANNOTATIONS
-Copyright (c) 2005 Brian Goetz and Tim Peierls.
-See http://www.jcip.net and the Creative Commons Attribution License
-(http://creativecommons.org/licenses/by/2.5)
-
diff --git a/lucene/licenses/httpclient-NOTICE.txt b/lucene/licenses/httpclient-NOTICE.txt
deleted file mode 100644
index 3a98f4542f5..00000000000
--- a/lucene/licenses/httpclient-NOTICE.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache HttpComponents Client
-Copyright 1999-2011 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
-
-This project contains annotations derived from JCIP-ANNOTATIONS
-Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
\ No newline at end of file
diff --git a/lucene/licenses/httpcore-4.4.13.jar.sha1 b/lucene/licenses/httpcore-4.4.13.jar.sha1
deleted file mode 100644
index cffee0313f5..00000000000
--- a/lucene/licenses/httpcore-4.4.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-853b96d3afbb7bf8cc303fe27ee96836a10c1834
diff --git a/lucene/licenses/httpcore-LICENSE-ASL.txt b/lucene/licenses/httpcore-LICENSE-ASL.txt
deleted file mode 100644
index 2c41ec88f61..00000000000
--- a/lucene/licenses/httpcore-LICENSE-ASL.txt
+++ /dev/null
@@ -1,182 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
-This project contains annotations derived from JCIP-ANNOTATIONS
-Copyright (c) 2005 Brian Goetz and Tim Peierls.
-See http://www.jcip.net and the Creative Commons Attribution License
-(http://creativecommons.org/licenses/by/2.5)
-
diff --git a/lucene/licenses/httpcore-NOTICE.txt b/lucene/licenses/httpcore-NOTICE.txt
deleted file mode 100644
index 3a98f4542f5..00000000000
--- a/lucene/licenses/httpcore-NOTICE.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache HttpComponents Client
-Copyright 1999-2011 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
-
-This project contains annotations derived from JCIP-ANNOTATIONS
-Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
\ No newline at end of file
diff --git a/lucene/licenses/javax.servlet-LICENSE-CDDL.txt b/lucene/licenses/javax.servlet-LICENSE-CDDL.txt
deleted file mode 100644
index 9f023738d47..00000000000
--- a/lucene/licenses/javax.servlet-LICENSE-CDDL.txt
+++ /dev/null
@@ -1,124 +0,0 @@
-COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0
-
-1. Definitions.
-
- 1.1. Contributor. means each individual or entity that creates or contributes to the creation of Modifications.
-
- 1.2. Contributor Version. means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor.
-
- 1.3. Covered Software. means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof.
-
- 1.4. Executable. means the Covered Software in any form other than Source Code.
-
- 1.5. Initial Developer. means the individual or entity that first makes Original Software available under this License.
-
- 1.6. Larger Work. means a work which combines Covered Software or portions thereof with code not governed by the terms of this License.
-
- 1.7. License. means this document.
-
- 1.8. Licensable. means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein.
-
- 1.9. Modifications. means the Source Code and Executable form of any of the following:
-
- A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications;
-
- B. Any new file that contains any part of the Original Software or previous Modification; or
-
- C. Any new file that is contributed or otherwise made available under the terms of this License.
-
- 1.10. Original Software. means the Source Code and Executable form of computer software code that is originally released under this License.
-
- 1.11. Patent Claims. means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor.
-
- 1.12. Source Code. means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code.
-
- 1.13. You. (or .Your.) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, .You. includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, .control. means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outs [...]
-
-2. License Grants.
-
- 2.1. The Initial Developer Grant.
-
- Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license:
-
- (a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and
-
- (b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof).
-
- (c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License.
-
- (d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices.
-
- 2.2. Contributor Grant.
-
- Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license:
-
- (a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and
-
- (b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination).
-
- (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party.
-
- (d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that [...]
-
-3. Distribution Obligations.
-
- 3.1. Availability of Source Code.
- Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Soft [...]
-
- 3.2. Modifications.
- The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License.
-
- 3.3. Required Notices.
- You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer.
-
- 3.4. Application of Additional Terms.
- You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients. rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that an [...]
-
- 3.5. Distribution of Executable Versions.
- You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient.s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable [...]
-
- 3.6. Larger Works.
- You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software.
-
-4. Versions of the License.
-
- 4.1. New Versions.
- Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License.
-
- 4.2. Effect of New Versions.
- You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under [...]
-
- 4.3. Modified Versions.
- When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License.
-
-5. DISCLAIMER OF WARRANTY.
-
- COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN .AS IS. BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME [...]
-
-6. TERMINATION.
-
- 6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive.
-
- 6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as .Participant.) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rig [...]
-
- 6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination.
-
-7. LIMITATION OF LIABILITY.
-
- UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND AL [...]
-
-8. U.S. GOVERNMENT END USERS.
-
- The Covered Software is a .commercial item,. as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of .commercial computer software. (as that term is defined at 48 C.F.R. ? 252.227-7014(a)(1)) and .commercial computer software documentation. as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth [...]
-
-9. MISCELLANEOUS.
-
- This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction.s conflict-of-law provisions. Any litigati [...]
-
-10. RESPONSIBILITY FOR CLAIMS.
-
- As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability.
-
- NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)
-
- The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California.
diff --git a/lucene/licenses/javax.servlet-NOTICE.txt b/lucene/licenses/javax.servlet-NOTICE.txt
deleted file mode 100644
index 5473a3467e6..00000000000
--- a/lucene/licenses/javax.servlet-NOTICE.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-javax.servlet-*.jar is under the CDDL license, the original source
-code for this can be found at http://www.eclipse.org/jetty/downloads.php
diff --git a/lucene/licenses/javax.servlet-api-3.1.0.jar.sha1 b/lucene/licenses/javax.servlet-api-3.1.0.jar.sha1
deleted file mode 100644
index a269ca04f74..00000000000
--- a/lucene/licenses/javax.servlet-api-3.1.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3cd63d075497751784b2fa84be59432f4905bf7c
diff --git a/lucene/licenses/jetty-LICENSE-ASL.txt b/lucene/licenses/jetty-LICENSE-ASL.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/lucene/licenses/jetty-LICENSE-ASL.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lucene/licenses/jetty-NOTICE.txt b/lucene/licenses/jetty-NOTICE.txt
deleted file mode 100644
index 4cbd7722999..00000000000
--- a/lucene/licenses/jetty-NOTICE.txt
+++ /dev/null
@@ -1,111 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
-<html><head>
-
-
-<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>Eclipse.org Software User Agreement</title>
-</head><body lang="EN-US" link="blue" vlink="purple">
-<h2>Eclipse Foundation Software User Agreement</h2>
-<p>March 17, 2005</p>
-
-<h3>Usage Of Content</h3>
-
-<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
- (COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
- CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
- OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
- NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
- CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
-
-<h3>Applicable Licenses</h3>
-
-<p>Unless otherwise indicated, all Content made available by the
-Eclipse Foundation is provided to you under the terms and conditions of
-the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
-provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
- For purposes of the EPL, "Program" will mean the Content.</p>
-
-<p>Content includes, but is not limited to, source code, object code,
-documentation and other files maintained in the Eclipse.org CVS
-repository ("Repository") in CVS modules ("Modules") and made available
-as downloadable archives ("Downloads").</p>
-
-<ul>
- <li>Content may be structured and packaged into modules to
-facilitate delivering, extending, and upgrading the Content. Typical
-modules may include plug-ins ("Plug-ins"), plug-in fragments
-("Fragments"), and features ("Features").</li>
- <li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java™ ARchive) in a directory named "plugins".</li>
- <li>A
-Feature is a bundle of one or more Plug-ins and/or Fragments and
-associated material. Each Feature may be packaged as a sub-directory in
-a directory named "features". Within a Feature, files named
-"feature.xml" may contain a list of the names and version numbers of
-the Plug-ins and/or Fragments associated with that Feature.</li>
- <li>Features
-may also include other Features ("Included Features"). Within a
-Feature, files named "feature.xml" may contain a list of the names and
-version numbers of Included Features.</li>
-</ul>
-
-<p>The terms and conditions governing Plug-ins and Fragments should be
-contained in files named "about.html" ("Abouts"). The terms and
-conditions governing Features and
-Included Features should be contained in files named "license.html"
-("Feature Licenses"). Abouts and Feature Licenses may be located in any
-directory of a Download or Module
-including, but not limited to the following locations:</p>
-
-<ul>
- <li>The top-level (root) directory</li>
- <li>Plug-in and Fragment directories</li>
- <li>Inside Plug-ins and Fragments packaged as JARs</li>
- <li>Sub-directories of the directory named "src" of certain Plug-ins</li>
- <li>Feature directories</li>
-</ul>
-
-<p>Note: if a Feature made available by the Eclipse Foundation is
-installed using the Eclipse Update Manager, you must agree to a license
-("Feature Update License") during the
-installation process. If the Feature contains Included Features, the
-Feature Update License should either provide you with the terms and
-conditions governing the Included Features or
-inform you where you can locate them. Feature Update Licenses may be
-found in the "license" property of files named "feature.properties"
-found within a Feature.
-Such Abouts, Feature Licenses, and Feature Update Licenses contain the
-terms and conditions (or references to such terms and conditions) that
-govern your use of the associated Content in
-that directory.</p>
-
-<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
-TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
-CONDITIONS. SOME OF THESE
-OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
-
-<ul>
- <li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
- <li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
- <li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
- <li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
- <li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
- <li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
-</ul>
-
-<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
-CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
-or Feature Update License is provided, please
-contact the Eclipse Foundation to determine what terms and conditions
-govern that particular Content.</p>
-
-<h3>Cryptography</h3>
-
-<p>Content may contain encryption software. The country in which you
-are currently may have restrictions on the import, possession, and use,
-and/or re-export to another country, of encryption software. BEFORE
-using any encryption software, please check the country's laws,
-regulations and policies concerning the import, possession, or use, and
-re-export of encryption software, to see if this is permitted.</p>
-
-<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
-</body></html>
\ No newline at end of file
diff --git a/lucene/licenses/jetty-continuation-9.4.41.v20210516.jar.sha1 b/lucene/licenses/jetty-continuation-9.4.41.v20210516.jar.sha1
deleted file mode 100644
index 7bd5e5a78c5..00000000000
--- a/lucene/licenses/jetty-continuation-9.4.41.v20210516.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e5859cddb2cac51fd8339668bd35ccac106cb28f
diff --git a/lucene/licenses/jetty-http-9.4.41.v20210516.jar.sha1 b/lucene/licenses/jetty-http-9.4.41.v20210516.jar.sha1
deleted file mode 100644
index 36812d204b4..00000000000
--- a/lucene/licenses/jetty-http-9.4.41.v20210516.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0d460bece4dd9666b46cbd18f8d7fd31cf02ecd9
diff --git a/lucene/licenses/jetty-io-9.4.41.v20210516.jar.sha1 b/lucene/licenses/jetty-io-9.4.41.v20210516.jar.sha1
deleted file mode 100644
index d88fbd5f52a..00000000000
--- a/lucene/licenses/jetty-io-9.4.41.v20210516.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-820eea368623939c2113902b1ca7a98186f64a73
diff --git a/lucene/licenses/jetty-server-9.4.41.v20210516.jar.sha1 b/lucene/licenses/jetty-server-9.4.41.v20210516.jar.sha1
deleted file mode 100644
index dd66f473319..00000000000
--- a/lucene/licenses/jetty-server-9.4.41.v20210516.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-25b1963b0a1c56202ec37046adc55861815107ce
diff --git a/lucene/licenses/jetty-servlet-9.4.41.v20210516.jar.sha1 b/lucene/licenses/jetty-servlet-9.4.41.v20210516.jar.sha1
deleted file mode 100644
index 6ff3ebd2180..00000000000
--- a/lucene/licenses/jetty-servlet-9.4.41.v20210516.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ea45368ea7fd04026038f89e6910f17f70939641
diff --git a/lucene/licenses/jetty-util-9.4.41.v20210516.jar.sha1 b/lucene/licenses/jetty-util-9.4.41.v20210516.jar.sha1
deleted file mode 100644
index 9870323ef5c..00000000000
--- a/lucene/licenses/jetty-util-9.4.41.v20210516.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-548c76ea00d7eb3e2bcea273174e5d030639d109
diff --git a/lucene/licenses/jetty-util-ajax-9.4.41.v20210516.jar.sha1 b/lucene/licenses/jetty-util-ajax-9.4.41.v20210516.jar.sha1
deleted file mode 100644
index 23af8c84736..00000000000
--- a/lucene/licenses/jetty-util-ajax-9.4.41.v20210516.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d4c1d66fc62796a17548e6c344fbf89b5889f873
diff --git a/lucene/replicator/build.gradle b/lucene/replicator/build.gradle
index 20585023046..abf30ca189c 100644
--- a/lucene/replicator/build.gradle
+++ b/lucene/replicator/build.gradle
@@ -22,20 +22,6 @@ description = 'Lucene index files replication utility'
dependencies {
moduleApi project(':lucene:core')
- moduleImplementation project(':lucene:facet')
-
- moduleImplementation('org.apache.httpcomponents:httpclient', {
- exclude group: "commons-codec", module: "commons-codec"
- })
-
- moduleImplementation 'javax.servlet:javax.servlet-api'
-
moduleTestImplementation project(':lucene:test-framework')
-
- moduleTestImplementation 'org.eclipse.jetty:jetty-server'
- moduleTestImplementation('org.eclipse.jetty:jetty-servlet', {
- exclude group: "org.eclipse.jetty", module: "jetty-security"
- })
- moduleTestImplementation 'org.eclipse.jetty:jetty-continuation'
}
diff --git a/lucene/replicator/src/java/module-info.java b/lucene/replicator/src/java/module-info.java
index 1c37c97c9da..e01cb269941 100644
--- a/lucene/replicator/src/java/module-info.java
+++ b/lucene/replicator/src/java/module-info.java
@@ -18,13 +18,7 @@
/** Lucene index files replication utility */
@SuppressWarnings({"requires-automatic"})
module org.apache.lucene.replicator {
- requires javax.servlet.api;
- requires org.apache.httpcomponents.httpclient;
- requires org.apache.httpcomponents.httpcore;
requires org.apache.lucene.core;
- requires org.apache.lucene.facet;
- exports org.apache.lucene.replicator;
- exports org.apache.lucene.replicator.http;
exports org.apache.lucene.replicator.nrt;
}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java b/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java
deleted file mode 100644
index 27bb1d43a3c..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.InfoStream;
-
-/**
- * A {@link ReplicationHandler} for replication of an index and taxonomy pair. See {@link
- * IndexReplicationHandler} for more detail. This handler ensures that the search and taxonomy
- * indexes are replicated in a consistent way.
- *
- * <p><b>NOTE:</b> if you intend to recreate a taxonomy index, you should make sure to reopen an
- * IndexSearcher and TaxonomyReader pair via the provided callback, to guarantee that both indexes
- * are in sync. This handler does not prevent replicating such index and taxonomy pairs, and if they
- * are reopened by a different thread, unexpected errors can occur, as well as inconsistency between
- * the taxonomy and index readers.
- *
- * @see IndexReplicationHandler
- * @lucene.experimental
- */
-public class IndexAndTaxonomyReplicationHandler implements ReplicationHandler {
-
- /**
- * The component used to log messages to the {@link InfoStream#getDefault() default} {@link
- * InfoStream}.
- */
- public static final String INFO_STREAM_COMPONENT = "IndexAndTaxonomyReplicationHandler";
-
- private final Directory indexDir;
- private final Directory taxoDir;
- private final Callable<Boolean> callback;
-
- private volatile Map<String, List<RevisionFile>> currentRevisionFiles;
- private volatile String currentVersion;
- private volatile InfoStream infoStream = InfoStream.getDefault();
-
- /**
- * Constructor with the given index directory and callback to notify when the indexes were
- * updated.
- */
- public IndexAndTaxonomyReplicationHandler(
- Directory indexDir, Directory taxoDir, Callable<Boolean> callback) throws IOException {
- this.callback = callback;
- this.indexDir = indexDir;
- this.taxoDir = taxoDir;
- currentRevisionFiles = null;
- currentVersion = null;
- final boolean indexExists = DirectoryReader.indexExists(indexDir);
- final boolean taxoExists = DirectoryReader.indexExists(taxoDir);
- if (indexExists != taxoExists) {
- throw new IllegalStateException(
- "search and taxonomy indexes must either both exist or not: index="
- + indexExists
- + " taxo="
- + taxoExists);
- }
- if (indexExists) { // both indexes exist
- final IndexCommit indexCommit = IndexReplicationHandler.getLastCommit(indexDir);
- final IndexCommit taxoCommit = IndexReplicationHandler.getLastCommit(taxoDir);
- currentRevisionFiles = IndexAndTaxonomyRevision.revisionFiles(indexCommit, taxoCommit);
- currentVersion = IndexAndTaxonomyRevision.revisionVersion(indexCommit, taxoCommit);
- final InfoStream infoStream = InfoStream.getDefault();
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT,
- "constructor(): currentVersion="
- + currentVersion
- + " currentRevisionFiles="
- + currentRevisionFiles);
- infoStream.message(
- INFO_STREAM_COMPONENT,
- "constructor(): indexCommit=" + indexCommit + " taxoCommit=" + taxoCommit);
- }
- }
- }
-
- @Override
- public String currentVersion() {
- return currentVersion;
- }
-
- @Override
- public Map<String, List<RevisionFile>> currentRevisionFiles() {
- return currentRevisionFiles;
- }
-
- @Override
- public void revisionReady(
- String version,
- Map<String, List<RevisionFile>> revisionFiles,
- Map<String, List<String>> copiedFiles,
- Map<String, Directory> sourceDirectory)
- throws IOException {
- Directory taxoClientDir = sourceDirectory.get(IndexAndTaxonomyRevision.TAXONOMY_SOURCE);
- Directory indexClientDir = sourceDirectory.get(IndexAndTaxonomyRevision.INDEX_SOURCE);
- List<String> taxoFiles = copiedFiles.get(IndexAndTaxonomyRevision.TAXONOMY_SOURCE);
- List<String> indexFiles = copiedFiles.get(IndexAndTaxonomyRevision.INDEX_SOURCE);
- String taxoSegmentsFile = IndexReplicationHandler.getSegmentsFile(taxoFiles, true);
- String indexSegmentsFile = IndexReplicationHandler.getSegmentsFile(indexFiles, false);
- String taxoPendingFile = taxoSegmentsFile == null ? null : "pending_" + taxoSegmentsFile;
- String indexPendingFile = "pending_" + indexSegmentsFile;
-
- boolean success = false;
- try {
- // copy taxonomy files before index files
- IndexReplicationHandler.copyFiles(taxoClientDir, taxoDir, taxoFiles);
- IndexReplicationHandler.copyFiles(indexClientDir, indexDir, indexFiles);
-
- // fsync all copied files (except segmentsFile)
- if (!taxoFiles.isEmpty()) {
- taxoDir.sync(taxoFiles);
- }
- indexDir.sync(indexFiles);
-
- // now copy, fsync, and rename segmentsFile, taxonomy first because it is ok if a
- // reader sees a more advanced taxonomy than the index.
-
- if (taxoSegmentsFile != null) {
- taxoDir.copyFrom(taxoClientDir, taxoSegmentsFile, taxoPendingFile, IOContext.READONCE);
- }
- indexDir.copyFrom(indexClientDir, indexSegmentsFile, indexPendingFile, IOContext.READONCE);
-
- if (taxoSegmentsFile != null) {
- taxoDir.sync(Collections.singletonList(taxoPendingFile));
- }
- indexDir.sync(Collections.singletonList(indexPendingFile));
-
- if (taxoSegmentsFile != null) {
- taxoDir.rename(taxoPendingFile, taxoSegmentsFile);
- taxoDir.syncMetaData();
- }
-
- indexDir.rename(indexPendingFile, indexSegmentsFile);
- indexDir.syncMetaData();
-
- success = true;
- } finally {
- if (!success) {
- if (taxoSegmentsFile != null) {
- taxoFiles.add(taxoSegmentsFile); // add it back so it gets deleted too
- taxoFiles.add(taxoPendingFile);
- }
- IndexReplicationHandler.cleanupFilesOnFailure(taxoDir, taxoFiles);
- indexFiles.add(indexSegmentsFile); // add it back so it gets deleted too
- indexFiles.add(indexPendingFile);
- IndexReplicationHandler.cleanupFilesOnFailure(indexDir, indexFiles);
- }
- }
-
- // all files have been successfully copied + sync'd. update the handler's state
- currentRevisionFiles = revisionFiles;
- currentVersion = version;
-
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT,
- "revisionReady(): currentVersion="
- + currentVersion
- + " currentRevisionFiles="
- + currentRevisionFiles);
- }
-
- // Cleanup the index directory from old and unused index files.
- // NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have
- // side-effects, e.g. if it hits sudden IO errors while opening the index
- // (and can end up deleting the entire index). It is not our job to protect
- // against those errors, app will probably hit them elsewhere.
- IndexReplicationHandler.cleanupOldIndexFiles(indexDir, indexSegmentsFile, infoStream);
- IndexReplicationHandler.cleanupOldIndexFiles(taxoDir, taxoSegmentsFile, infoStream);
-
- // successfully updated the index, notify the callback that the index is
- // ready.
- if (callback != null) {
- try {
- callback.call();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
- }
-
- /** Sets the {@link InfoStream} to use for logging messages. */
- public void setInfoStream(InfoStream infoStream) {
- if (infoStream == null) {
- infoStream = InfoStream.NO_OUTPUT;
- }
- this.infoStream = infoStream;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java b/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java
deleted file mode 100644
index 6222e11c9fc..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexDeletionPolicy;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/**
- * A {@link Revision} of a single index and taxonomy index files which comprises the list of files
- * from both indexes. This revision should be used whenever a pair of search and taxonomy indexes
- * need to be replicated together to guarantee consistency of both on the replicating (client) side.
- *
- * @see IndexRevision
- * @lucene.experimental
- */
-public class IndexAndTaxonomyRevision implements Revision {
-
- /**
- * A {@link DirectoryTaxonomyWriter} which sets the underlying {@link IndexWriter}'s {@link
- * IndexDeletionPolicy} to {@link SnapshotDeletionPolicy}.
- */
- public static final class SnapshotDirectoryTaxonomyWriter extends DirectoryTaxonomyWriter {
-
- private SnapshotDeletionPolicy sdp;
- private IndexWriter writer;
-
- /**
- * @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory, IndexWriterConfig.OpenMode,
- * TaxonomyWriterCache)
- */
- public SnapshotDirectoryTaxonomyWriter(
- Directory directory, OpenMode openMode, TaxonomyWriterCache cache) throws IOException {
- super(directory, openMode, cache);
- }
-
- /**
- * @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory, IndexWriterConfig.OpenMode)
- */
- public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode)
- throws IOException {
- super(directory, openMode);
- }
-
- /**
- * @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory)
- */
- public SnapshotDirectoryTaxonomyWriter(Directory d) throws IOException {
- super(d);
- }
-
- @Override
- protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
- IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
- sdp = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
- conf.setIndexDeletionPolicy(sdp);
- return conf;
- }
-
- @Override
- protected IndexWriter openIndexWriter(Directory directory, IndexWriterConfig config)
- throws IOException {
- writer = super.openIndexWriter(directory, config);
- return writer;
- }
-
- /** Returns the {@link SnapshotDeletionPolicy} used by the underlying {@link IndexWriter}. */
- public SnapshotDeletionPolicy getDeletionPolicy() {
- return sdp;
- }
-
- /** Returns the {@link IndexWriter} used by this {@link DirectoryTaxonomyWriter}. */
- public IndexWriter getIndexWriter() {
- return writer;
- }
- }
-
- private static final int RADIX = 16;
-
- public static final String INDEX_SOURCE = "index";
- public static final String TAXONOMY_SOURCE = "taxo";
-
- private final IndexWriter indexWriter;
- private final SnapshotDirectoryTaxonomyWriter taxoWriter;
- private final IndexCommit indexCommit, taxoCommit;
- private final SnapshotDeletionPolicy indexSDP, taxoSDP;
- private final String version;
- private final Map<String, List<RevisionFile>> sourceFiles;
-
- /** Returns a singleton map of the revision files from the given {@link IndexCommit}. */
- public static Map<String, List<RevisionFile>> revisionFiles(
- IndexCommit indexCommit, IndexCommit taxoCommit) throws IOException {
- HashMap<String, List<RevisionFile>> files = new HashMap<>();
- files.put(INDEX_SOURCE, IndexRevision.revisionFiles(indexCommit).values().iterator().next());
- files.put(TAXONOMY_SOURCE, IndexRevision.revisionFiles(taxoCommit).values().iterator().next());
- return files;
- }
-
- /**
- * Returns a String representation of a revision's version from the given {@link IndexCommit}s of
- * the search and taxonomy indexes.
- */
- public static String revisionVersion(IndexCommit indexCommit, IndexCommit taxoCommit) {
- return Long.toString(indexCommit.getGeneration(), RADIX)
- + ":"
- + Long.toString(taxoCommit.getGeneration(), RADIX);
- }
-
- /**
- * Constructor over the given {@link IndexWriter}. Uses the last {@link IndexCommit} found in the
- * {@link Directory} managed by the given writer.
- */
- public IndexAndTaxonomyRevision(
- IndexWriter indexWriter, SnapshotDirectoryTaxonomyWriter taxoWriter) throws IOException {
- IndexDeletionPolicy delPolicy = indexWriter.getConfig().getIndexDeletionPolicy();
- if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
- throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
- }
- this.indexWriter = indexWriter;
- this.taxoWriter = taxoWriter;
- this.indexSDP = (SnapshotDeletionPolicy) delPolicy;
- this.taxoSDP = taxoWriter.getDeletionPolicy();
- this.indexCommit = indexSDP.snapshot();
- this.taxoCommit = taxoSDP.snapshot();
- this.version = revisionVersion(indexCommit, taxoCommit);
- this.sourceFiles = revisionFiles(indexCommit, taxoCommit);
- }
-
- @Override
- public int compareTo(String version) {
- final String[] parts = version.split(":");
- final long indexGen = Long.parseLong(parts[0], RADIX);
- final long taxoGen = Long.parseLong(parts[1], RADIX);
- final long indexCommitGen = indexCommit.getGeneration();
- final long taxoCommitGen = taxoCommit.getGeneration();
-
- // if the index generation is not the same as this commit's generation,
- // compare by it. Otherwise, compare by the taxonomy generation.
- if (indexCommitGen < indexGen) {
- return -1;
- } else if (indexCommitGen > indexGen) {
- return 1;
- } else {
- return taxoCommitGen < taxoGen ? -1 : (taxoCommitGen > taxoGen ? 1 : 0);
- }
- }
-
- @Override
- public int compareTo(Revision o) {
- IndexAndTaxonomyRevision other = (IndexAndTaxonomyRevision) o;
- int cmp = indexCommit.compareTo(other.indexCommit);
- return cmp != 0 ? cmp : taxoCommit.compareTo(other.taxoCommit);
- }
-
- @Override
- public String getVersion() {
- return version;
- }
-
- @Override
- public Map<String, List<RevisionFile>> getSourceFiles() {
- return sourceFiles;
- }
-
- @Override
- public InputStream open(String source, String fileName) throws IOException {
- assert source.equals(INDEX_SOURCE) || source.equals(TAXONOMY_SOURCE)
- : "invalid source; expected=("
- + INDEX_SOURCE
- + " or "
- + TAXONOMY_SOURCE
- + ") got="
- + source;
- IndexCommit ic = source.equals(INDEX_SOURCE) ? indexCommit : taxoCommit;
- return new IndexInputInputStream(ic.getDirectory().openInput(fileName, IOContext.READONCE));
- }
-
- @Override
- public void release() throws IOException {
- try {
- indexSDP.release(indexCommit);
- } finally {
- taxoSDP.release(taxoCommit);
- }
-
- try {
- indexWriter.deleteUnusedFiles();
- } finally {
- taxoWriter.getIndexWriter().deleteUnusedFiles();
- }
- }
-
- @Override
- public String toString() {
- return "IndexAndTaxonomyRevision version=" + version + " files=" + sourceFiles;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java b/lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java
deleted file mode 100644
index 841e242bb9c..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import org.apache.lucene.store.IndexInput;
-
-/**
- * An {@link InputStream} which wraps an {@link IndexInput}.
- *
- * @lucene.experimental
- */
-public final class IndexInputInputStream extends InputStream {
-
- private final IndexInput in;
-
- private long remaining;
-
- public IndexInputInputStream(IndexInput in) {
- this.in = in;
- remaining = in.length();
- }
-
- @Override
- public int read() throws IOException {
- if (remaining == 0) {
- return -1;
- } else {
- --remaining;
- return in.readByte();
- }
- }
-
- @Override
- public int available() throws IOException {
- return (int) in.length();
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- return read(b, 0, b.length);
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- if (remaining == 0) {
- return -1;
- }
- if (remaining < len) {
- len = (int) remaining;
- }
- in.readBytes(b, off, len);
- remaining -= len;
- return len;
- }
-
- @Override
- public long skip(long n) throws IOException {
- if (remaining == 0) {
- return -1;
- }
- if (remaining < n) {
- n = remaining;
- }
- in.seek(in.getFilePointer() + n);
- remaining -= n;
- return n;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java b/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java
deleted file mode 100644
index e9db15696fe..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.regex.Matcher;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexNotFoundException;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.InfoStream;
-
-/**
- * A {@link ReplicationHandler} for replication of an index. Implements {@link #revisionReady} by
- * copying the files pointed by the client resolver to the index {@link Directory} and then touches
- * the index with {@link IndexWriter} to make sure any unused files are deleted.
- *
- * <p><b>NOTE:</b> this handler assumes that {@link IndexWriter} is not opened by another process on
- * the index directory. In fact, opening an {@link IndexWriter} on the same directory to which files
- * are copied can lead to undefined behavior, where some or all the files will be deleted, override
- * other files or simply create a mess. When you replicate an index, it is best if the index is
- * never modified by {@link IndexWriter}, except the one that is open on the source index, from
- * which you replicate.
- *
- * <p>This handler notifies the application via a provided {@link Callable} when an updated index
- * commit was made available for it.
- *
- * @lucene.experimental
- */
-public class IndexReplicationHandler implements ReplicationHandler {
-
- /**
- * The component used to log messages to the {@link InfoStream#getDefault() default} {@link
- * InfoStream}.
- */
- public static final String INFO_STREAM_COMPONENT = "IndexReplicationHandler";
-
- private final Directory indexDir;
- private final Callable<Boolean> callback;
-
- private volatile Map<String, List<RevisionFile>> currentRevisionFiles;
- private volatile String currentVersion;
- private volatile InfoStream infoStream = InfoStream.getDefault();
-
- /**
- * Returns the last {@link IndexCommit} found in the {@link Directory}, or {@code null} if there
- * are no commits.
- */
- public static IndexCommit getLastCommit(Directory dir) throws IOException {
- try {
- if (DirectoryReader.indexExists(dir)) {
- List<IndexCommit> commits = DirectoryReader.listCommits(dir);
- // listCommits guarantees that we get at least one commit back, or
- // IndexNotFoundException which we handle below
- return commits.get(commits.size() - 1);
- }
- } catch (
- @SuppressWarnings("unused")
- IndexNotFoundException e) {
- // ignore the exception and return null
- }
- return null;
- }
-
- /**
- * Verifies that the last file is segments_N and fails otherwise. It also removes and returns the
- * file from the list, because it needs to be handled last, after all files. This is important in
- * order to guarantee that if a reader sees the new segments_N, all other segment files are
- * already on stable storage.
- *
- * <p>The reason why the code fails instead of putting segments_N file last is that this indicates
- * an error in the Revision implementation.
- */
- public static String getSegmentsFile(List<String> files, boolean allowEmpty) {
- if (files.isEmpty()) {
- if (allowEmpty) {
- return null;
- } else {
- throw new IllegalStateException("empty list of files not allowed");
- }
- }
-
- String segmentsFile = files.remove(files.size() - 1);
- if (!segmentsFile.startsWith(IndexFileNames.SEGMENTS)) {
- throw new IllegalStateException(
- "last file to copy+sync must be segments_N but got "
- + segmentsFile
- + "; check your Revision implementation!");
- }
- return segmentsFile;
- }
-
- /**
- * Cleanup the index directory by deleting all given files. Called when file copy or sync failed.
- */
- public static void cleanupFilesOnFailure(Directory dir, List<String> files) {
- for (String file : files) {
- // suppress any exception because if we're here, it means copy
- // failed, and we must cleanup after ourselves.
- IOUtils.deleteFilesIgnoringExceptions(dir, file);
- }
- }
-
- /**
- * Cleans up the index directory from old index files. This method uses the last commit found by
- * {@link #getLastCommit(Directory)}. If it matches the expected segmentsFile, then all files not
- * referenced by this commit point are deleted.
- *
- * <p><b>NOTE:</b> this method does a best effort attempt to clean the index directory. It
- * suppresses any exceptions that occur, as this can be retried the next time.
- */
- public static void cleanupOldIndexFiles(
- Directory dir, String segmentsFile, InfoStream infoStream) {
- try {
- IndexCommit commit = getLastCommit(dir);
- // commit == null means weird IO errors occurred, ignore them
- // if there were any IO errors reading the expected commit point (i.e.
- // segments files mismatch), then ignore that commit either.
- if (commit != null && commit.getSegmentsFileName().equals(segmentsFile)) {
- Set<String> commitFiles = new HashSet<>(commit.getFileNames());
- Matcher matcher = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
- for (String file : dir.listAll()) {
- if (!commitFiles.contains(file)
- && (matcher.reset(file).matches() || file.startsWith(IndexFileNames.SEGMENTS))) {
- // suppress exceptions, it's just a best effort
- IOUtils.deleteFilesIgnoringExceptions(dir, file);
- }
- }
- }
- } catch (Throwable t) {
- // ignore any errors that happen during this state and only log it. this
- // cleanup will have a chance to succeed the next time we get a new
- // revision.
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT, "cleanupOldIndexFiles(): failed on error " + t.getMessage());
- }
- }
- }
-
- /** Copies the files from the source directory to the target one, if they are not the same. */
- public static void copyFiles(Directory source, Directory target, List<String> files)
- throws IOException {
- if (!source.equals(target)) {
- for (String file : files) {
- target.copyFrom(source, file, file, IOContext.READONCE);
- }
- }
- }
-
- /**
- * Constructor with the given index directory and callback to notify when the indexes were
- * updated.
- */
- public IndexReplicationHandler(Directory indexDir, Callable<Boolean> callback)
- throws IOException {
- this.callback = callback;
- this.indexDir = indexDir;
- currentRevisionFiles = null;
- currentVersion = null;
- if (DirectoryReader.indexExists(indexDir)) {
- final List<IndexCommit> commits = DirectoryReader.listCommits(indexDir);
- final IndexCommit commit = commits.get(commits.size() - 1);
- currentRevisionFiles = IndexRevision.revisionFiles(commit);
- currentVersion = IndexRevision.revisionVersion(commit);
- final InfoStream infoStream = InfoStream.getDefault();
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT,
- "constructor(): currentVersion="
- + currentVersion
- + " currentRevisionFiles="
- + currentRevisionFiles);
- infoStream.message(INFO_STREAM_COMPONENT, "constructor(): commit=" + commit);
- }
- }
- }
-
- @Override
- public String currentVersion() {
- return currentVersion;
- }
-
- @Override
- public Map<String, List<RevisionFile>> currentRevisionFiles() {
- return currentRevisionFiles;
- }
-
- @Override
- public void revisionReady(
- String version,
- Map<String, List<RevisionFile>> revisionFiles,
- Map<String, List<String>> copiedFiles,
- Map<String, Directory> sourceDirectory)
- throws IOException {
- if (revisionFiles.size() > 1) {
- throw new IllegalArgumentException(
- "this handler handles only a single source; got " + revisionFiles.keySet());
- }
-
- Directory clientDir = sourceDirectory.values().iterator().next();
- List<String> files = copiedFiles.values().iterator().next();
- String segmentsFile = getSegmentsFile(files, false);
- String pendingSegmentsFile = "pending_" + segmentsFile;
-
- boolean success = false;
- try {
- // copy files from the client to index directory
- copyFiles(clientDir, indexDir, files);
-
- // fsync all copied files (except segmentsFile)
- indexDir.sync(files);
-
- // now copy and fsync segmentsFile as pending, then rename (simulating lucene commit)
- indexDir.copyFrom(clientDir, segmentsFile, pendingSegmentsFile, IOContext.READONCE);
- indexDir.sync(Collections.singletonList(pendingSegmentsFile));
- indexDir.rename(pendingSegmentsFile, segmentsFile);
- indexDir.syncMetaData();
-
- success = true;
- } finally {
- if (!success) {
- files.add(segmentsFile); // add it back so it gets deleted too
- files.add(pendingSegmentsFile);
- cleanupFilesOnFailure(indexDir, files);
- }
- }
-
- // all files have been successfully copied + sync'd. update the handler's state
- currentRevisionFiles = revisionFiles;
- currentVersion = version;
-
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT,
- "revisionReady(): currentVersion="
- + currentVersion
- + " currentRevisionFiles="
- + currentRevisionFiles);
- }
-
- // Cleanup the index directory from old and unused index files.
- // NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have
- // side-effects, e.g. if it hits sudden IO errors while opening the index
- // (and can end up deleting the entire index). It is not our job to protect
- // against those errors, app will probably hit them elsewhere.
- cleanupOldIndexFiles(indexDir, segmentsFile, infoStream);
-
- // successfully updated the index, notify the callback that the index is
- // ready.
- if (callback != null) {
- try {
- callback.call();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
- }
-
- /** Sets the {@link InfoStream} to use for logging messages. */
- public void setInfoStream(InfoStream infoStream) {
- if (infoStream == null) {
- infoStream = InfoStream.NO_OUTPUT;
- }
- this.infoStream = infoStream;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java b/lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java
deleted file mode 100644
index c649e4aa7a5..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexDeletionPolicy;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/**
- * A {@link Revision} of a single index files which comprises the list of files that are part of the
- * current {@link IndexCommit}. To ensure the files are not deleted by {@link IndexWriter} for as
- * long as this revision stays alive (i.e. until {@link #release()}), the current commit point is
- * snapshotted, using {@link SnapshotDeletionPolicy} (this means that the given writer's {@link
- * IndexWriterConfig#getIndexDeletionPolicy() config} should return {@link SnapshotDeletionPolicy}).
- *
- * <p>When this revision is {@link #release() released}, it releases the obtained snapshot as well
- * as calls {@link IndexWriter#deleteUnusedFiles()} so that the snapshotted files are deleted (if
- * they are no longer needed).
- *
- * @lucene.experimental
- */
-public class IndexRevision implements Revision {
-
- private static final int RADIX = 16;
- private static final String SOURCE = "index";
-
- private final IndexWriter writer;
- private final IndexCommit commit;
- private final SnapshotDeletionPolicy sdp;
- private final String version;
- private final Map<String, List<RevisionFile>> sourceFiles;
-
- // returns a RevisionFile with some metadata
- private static RevisionFile newRevisionFile(String file, Directory dir) throws IOException {
- RevisionFile revFile = new RevisionFile(file);
- revFile.size = dir.fileLength(file);
- return revFile;
- }
-
- /** Returns a singleton map of the revision files from the given {@link IndexCommit}. */
- public static Map<String, List<RevisionFile>> revisionFiles(IndexCommit commit)
- throws IOException {
- Collection<String> commitFiles = commit.getFileNames();
- List<RevisionFile> revisionFiles = new ArrayList<>(commitFiles.size());
- String segmentsFile = commit.getSegmentsFileName();
- Directory dir = commit.getDirectory();
-
- for (String file : commitFiles) {
- if (!file.equals(segmentsFile)) {
- revisionFiles.add(newRevisionFile(file, dir));
- }
- }
- revisionFiles.add(newRevisionFile(segmentsFile, dir)); // segments_N must be last
- return Collections.singletonMap(SOURCE, revisionFiles);
- }
-
- /** Returns a String representation of a revision's version from the given {@link IndexCommit}. */
- public static String revisionVersion(IndexCommit commit) {
- return Long.toString(commit.getGeneration(), RADIX);
- }
-
- /**
- * Constructor over the given {@link IndexWriter}. Uses the last {@link IndexCommit} found in the
- * {@link Directory} managed by the given writer.
- */
- public IndexRevision(IndexWriter writer) throws IOException {
- IndexDeletionPolicy delPolicy = writer.getConfig().getIndexDeletionPolicy();
- if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
- throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
- }
- this.writer = writer;
- this.sdp = (SnapshotDeletionPolicy) delPolicy;
- this.commit = sdp.snapshot();
- this.version = revisionVersion(commit);
- this.sourceFiles = revisionFiles(commit);
- }
-
- @Override
- public int compareTo(String version) {
- long gen = Long.parseLong(version, RADIX);
- long commitGen = commit.getGeneration();
- return commitGen < gen ? -1 : (commitGen > gen ? 1 : 0);
- }
-
- @Override
- public int compareTo(Revision o) {
- IndexRevision other = (IndexRevision) o;
- return commit.compareTo(other.commit);
- }
-
- @Override
- public String getVersion() {
- return version;
- }
-
- @Override
- public Map<String, List<RevisionFile>> getSourceFiles() {
- return sourceFiles;
- }
-
- @Override
- public InputStream open(String source, String fileName) throws IOException {
- assert source.equals(SOURCE) : "invalid source; expected=" + SOURCE + " got=" + source;
- return new IndexInputInputStream(commit.getDirectory().openInput(fileName, IOContext.READONCE));
- }
-
- @Override
- public void release() throws IOException {
- sdp.release(commit);
- writer.deleteUnusedFiles();
- }
-
- @Override
- public String toString() {
- return "IndexRevision version=" + version + " files=" + sourceFiles;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java b/lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java
deleted file mode 100644
index fe519d07113..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.store.AlreadyClosedException;
-
-/**
- * A {@link Replicator} implementation for use by the side that publishes {@link Revision}s, as well
- * for clients to {@link #checkForUpdate(String) check for updates}. When a client needs to be
- * updated, it is returned a {@link SessionToken} through which it can {@link #obtainFile(String,
- * String, String) obtain} the files of that revision. As long as a revision is being replicated,
- * this replicator guarantees that it will not be {@link Revision#release() released}.
- *
- * <p>Replication sessions expire by default after {@link #DEFAULT_SESSION_EXPIRATION_THRESHOLD},
- * and the threshold can be configured through {@link #setExpirationThreshold(long)}.
- *
- * @lucene.experimental
- */
-public class LocalReplicator implements Replicator {
-
- private static class RefCountedRevision {
- private final AtomicInteger refCount = new AtomicInteger(1);
- public final Revision revision;
-
- public RefCountedRevision(Revision revision) {
- this.revision = revision;
- }
-
- public void decRef() throws IOException {
- if (refCount.get() <= 0) {
- throw new IllegalStateException("this revision is already released");
- }
-
- final int rc = refCount.decrementAndGet();
- if (rc == 0) {
- boolean success = false;
- try {
- revision.release();
- success = true;
- } finally {
- if (!success) {
- // Put reference back on failure
- refCount.incrementAndGet();
- }
- }
- } else if (rc < 0) {
- throw new IllegalStateException(
- "too many decRef calls: refCount is " + rc + " after decrement");
- }
- }
-
- public void incRef() {
- refCount.incrementAndGet();
- }
- }
-
- private static class ReplicationSession {
- public final SessionToken session;
- public final RefCountedRevision revision;
- private volatile long lastAccessTime;
-
- ReplicationSession(SessionToken session, RefCountedRevision revision) {
- this.session = session;
- this.revision = revision;
- lastAccessTime = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
- }
-
- boolean isExpired(long expirationThreshold) {
- return lastAccessTime
- < (TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS)
- - expirationThreshold);
- }
-
- void markAccessed() {
- lastAccessTime = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
- }
- }
-
- /** Threshold for expiring inactive sessions. Defaults to 30 minutes. */
- public static final long DEFAULT_SESSION_EXPIRATION_THRESHOLD = 1000 * 60 * 30;
-
- private long expirationThresholdMilllis = LocalReplicator.DEFAULT_SESSION_EXPIRATION_THRESHOLD;
-
- private volatile RefCountedRevision currentRevision;
- private volatile boolean closed = false;
-
- private final AtomicInteger sessionToken = new AtomicInteger(0);
- private final Map<String, ReplicationSession> sessions = new HashMap<>();
-
- private void checkExpiredSessions() throws IOException {
- // make a "to-delete" list so we don't risk deleting from the map while iterating it
- final ArrayList<ReplicationSession> toExpire = new ArrayList<>();
- for (ReplicationSession token : sessions.values()) {
- if (token.isExpired(expirationThresholdMilllis)) {
- toExpire.add(token);
- }
- }
- for (ReplicationSession token : toExpire) {
- releaseSession(token.session.id);
- }
- }
-
- private void releaseSession(String sessionID) throws IOException {
- ReplicationSession session = sessions.remove(sessionID);
- // if we're called concurrently by close() and release(), could be that one
- // thread beats the other to release the session.
- if (session != null) {
- session.revision.decRef();
- }
- }
-
- /** Ensure that replicator is still open, or throw {@link AlreadyClosedException} otherwise. */
- protected final synchronized void ensureOpen() {
- if (closed) {
- throw new AlreadyClosedException("This replicator has already been closed");
- }
- }
-
- @Override
- public synchronized SessionToken checkForUpdate(String currentVersion) {
- ensureOpen();
- if (currentRevision == null) { // no published revisions yet
- return null;
- }
-
- if (currentVersion != null && currentRevision.revision.compareTo(currentVersion) <= 0) {
- // currentVersion is newer or equal to latest published revision
- return null;
- }
-
- // currentVersion is either null or older than latest published revision
- currentRevision.incRef();
- final String sessionID = Integer.toString(sessionToken.incrementAndGet());
- final SessionToken sessionToken = new SessionToken(sessionID, currentRevision.revision);
- final ReplicationSession timedSessionToken =
- new ReplicationSession(sessionToken, currentRevision);
- sessions.put(sessionID, timedSessionToken);
- return sessionToken;
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (!closed) {
- // release all managed revisions
- for (ReplicationSession session : sessions.values()) {
- session.revision.decRef();
- }
- sessions.clear();
- closed = true;
- }
- }
-
- /**
- * Returns the expiration threshold.
- *
- * @see #setExpirationThreshold(long)
- */
- public long getExpirationThreshold() {
- return expirationThresholdMilllis;
- }
-
- @Override
- public synchronized InputStream obtainFile(String sessionID, String source, String fileName)
- throws IOException {
- ensureOpen();
- ReplicationSession session = sessions.get(sessionID);
- if (session != null && session.isExpired(expirationThresholdMilllis)) {
- releaseSession(sessionID);
- session = null;
- }
- // session either previously expired, or we just expired it
- if (session == null) {
- throw new SessionExpiredException(
- "session ("
- + sessionID
- + ") expired while obtaining file: source="
- + source
- + " file="
- + fileName);
- }
- sessions.get(sessionID).markAccessed();
- return session.revision.revision.open(source, fileName);
- }
-
- @Override
- public synchronized void publish(Revision revision) throws IOException {
- ensureOpen();
- if (currentRevision != null) {
- int compare = revision.compareTo(currentRevision.revision);
- if (compare == 0) {
- // same revision published again, ignore but release it
- revision.release();
- return;
- }
-
- if (compare < 0) {
- revision.release();
- throw new IllegalArgumentException(
- "Cannot publish an older revision: rev=" + revision + " current=" + currentRevision);
- }
- }
-
- // swap revisions
- final RefCountedRevision oldRevision = currentRevision;
- currentRevision = new RefCountedRevision(revision);
- if (oldRevision != null) {
- oldRevision.decRef();
- }
-
- // check for expired sessions
- checkExpiredSessions();
- }
-
- @Override
- public synchronized void release(String sessionID) throws IOException {
- ensureOpen();
- releaseSession(sessionID);
- }
-
- /**
- * Modify session expiration time - if a replication session is inactive that long it is
- * automatically expired, and further attempts to operate within this session will throw a {@link
- * SessionExpiredException}.
- */
- public synchronized void setExpirationThreshold(long expirationThreshold) throws IOException {
- ensureOpen();
- this.expirationThresholdMilllis = expirationThreshold;
- checkExpiredSessions();
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java b/lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java
deleted file mode 100644
index adb351ff901..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.apache.lucene.replicator.ReplicationClient.SourceDirectoryFactory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * A {@link SourceDirectoryFactory} which returns {@link FSDirectory} under a dedicated session
- * directory. When a session is over, the entire directory is deleted.
- *
- * @lucene.experimental
- */
-public class PerSessionDirectoryFactory implements SourceDirectoryFactory {
-
- private final Path workDir;
-
- /** Constructor with the given sources mapping. */
- public PerSessionDirectoryFactory(Path workDir) {
- this.workDir = workDir;
- }
-
- @Override
- public Directory getDirectory(String sessionID, String source) throws IOException {
- Path sessionDir = workDir.resolve(sessionID);
- Files.createDirectories(sessionDir);
- Path sourceDir = sessionDir.resolve(source);
- Files.createDirectories(sourceDir);
- return FSDirectory.open(sourceDir);
- }
-
- @Override
- public void cleanupSession(String sessionID) throws IOException {
- if (sessionID.isEmpty()) { // protect against deleting workDir entirely!
- throw new IllegalArgumentException("sessionID cannot be empty");
- }
- IOUtils.rm(workDir.resolve(sessionID));
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/ReplicationClient.java b/lucene/replicator/src/java/org/apache/lucene/replicator/ReplicationClient.java
deleted file mode 100644
index 54626051a6f..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/ReplicationClient.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.ThreadInterruptedException;
-
-/**
- * A client which monitors and obtains new revisions from a {@link Replicator}. It can be used to
- * either periodically check for updates by invoking {@link #startUpdateThread}, or manually by
- * calling {@link #updateNow()}.
- *
- * <p>Whenever a new revision is available, the {@link #requiredFiles(Map)} are copied to the {@link
- * Directory} specified by {@link PerSessionDirectoryFactory} and a handler is notified.
- *
- * @lucene.experimental
- */
-public class ReplicationClient implements Closeable {
-
- private class ReplicationThread extends Thread {
-
- private final long interval;
-
- // client uses this to stop us
- final CountDownLatch stop = new CountDownLatch(1);
-
- public ReplicationThread(long interval) {
- this.interval = interval;
- }
-
- @SuppressWarnings("synthetic-access")
- @Override
- public void run() {
- while (true) {
- long time = System.currentTimeMillis();
- updateLock.lock();
- try {
- doUpdate();
- } catch (Throwable t) {
- handleUpdateException(t);
- } finally {
- updateLock.unlock();
- }
- time = System.currentTimeMillis() - time;
-
- // adjust timeout to compensate the time spent doing the replication.
- final long timeout = interval - time;
- if (timeout > 0) {
- try {
- // this will return immediately if we were ordered to stop (count=0)
- // or the timeout has elapsed. if it returns true, it means count=0,
- // so terminate.
- if (stop.await(timeout, TimeUnit.MILLISECONDS)) {
- return;
- }
- } catch (InterruptedException e) {
- // if we were interruted, somebody wants to terminate us, so just
- // throw the exception further.
- Thread.currentThread().interrupt();
- throw new ThreadInterruptedException(e);
- }
- }
- }
- }
- }
-
- /** Handler for revisions obtained by the client. */
- public static interface ReplicationHandler {
-
- /** Returns the current revision files held by the handler. */
- public Map<String, List<RevisionFile>> currentRevisionFiles();
-
- /** Returns the current revision version held by the handler. */
- public String currentVersion();
-
- /**
- * Called when a new revision was obtained and is available (i.e. all needed files were
- * successfully copied).
- *
- * @param version the version of the {@link Revision} that was copied
- * @param revisionFiles the files contained by this {@link Revision}
- * @param copiedFiles the files that were actually copied
- * @param sourceDirectory a mapping from a source of files to the {@link Directory} they were
- * copied into
- */
- public void revisionReady(
- String version,
- Map<String, List<RevisionFile>> revisionFiles,
- Map<String, List<String>> copiedFiles,
- Map<String, Directory> sourceDirectory)
- throws IOException;
- }
-
- /**
- * Resolves a session and source into a {@link Directory} to use for copying the session files to.
- */
- public static interface SourceDirectoryFactory {
-
- /**
- * Called to denote that the replication actions for this session were finished and the
- * directory is no longer needed.
- */
- public void cleanupSession(String sessionID) throws IOException;
-
- /**
- * Returns the {@link Directory} to use for the given session and source. Implementations may
- * e.g. return different directories for different sessions, or the same directory for all
- * sessions. In that case, it is advised to clean the directory before it is used for a new
- * session.
- *
- * @see #cleanupSession(String)
- */
- public Directory getDirectory(String sessionID, String source) throws IOException;
- }
-
- /** The component name to use with {@link InfoStream#isEnabled(String)}. */
- public static final String INFO_STREAM_COMPONENT = "ReplicationThread";
-
- private final Replicator replicator;
- private final ReplicationHandler handler;
- private final SourceDirectoryFactory factory;
- private final byte[] copyBuffer = new byte[16384];
- private final Lock updateLock = new ReentrantLock();
-
- private volatile ReplicationThread updateThread;
- private volatile boolean closed = false;
- private volatile InfoStream infoStream = InfoStream.getDefault();
-
- /**
- * Constructor.
- *
- * @param replicator the {@link Replicator} used for checking for updates
- * @param handler notified when new revisions are ready
- * @param factory returns a {@link Directory} for a given source and session
- */
- public ReplicationClient(
- Replicator replicator, ReplicationHandler handler, SourceDirectoryFactory factory) {
- this.replicator = replicator;
- this.handler = handler;
- this.factory = factory;
- }
-
- private void copyBytes(IndexOutput out, InputStream in) throws IOException {
- int numBytes;
- while ((numBytes = in.read(copyBuffer)) > 0) {
- out.writeBytes(copyBuffer, 0, numBytes);
- }
- }
-
- private void doUpdate() throws IOException {
- SessionToken session = null;
- final Map<String, Directory> sourceDirectory = new HashMap<>();
- final Map<String, List<String>> copiedFiles = new HashMap<>();
- boolean notify = false;
- try {
- final String version = handler.currentVersion();
- session = replicator.checkForUpdate(version);
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT, "doUpdate(): handlerVersion=" + version + " session=" + session);
- }
- if (session == null) {
- // already up to date
- return;
- }
- Map<String, List<RevisionFile>> requiredFiles = requiredFiles(session.sourceFiles);
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(INFO_STREAM_COMPONENT, "doUpdate(): requiredFiles=" + requiredFiles);
- }
- for (Entry<String, List<RevisionFile>> e : requiredFiles.entrySet()) {
- String source = e.getKey();
- Directory dir = factory.getDirectory(session.id, source);
- sourceDirectory.put(source, dir);
- List<String> cpFiles = new ArrayList<>();
- copiedFiles.put(source, cpFiles);
- for (RevisionFile file : e.getValue()) {
- if (closed) {
- // if we're closed, abort file copy
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT,
- "doUpdate(): detected client was closed); abort file copy");
- }
- return;
- }
- InputStream in = null;
- IndexOutput out = null;
- try {
- in = replicator.obtainFile(session.id, source, file.fileName);
- out = dir.createOutput(file.fileName, IOContext.DEFAULT);
- copyBytes(out, in);
- cpFiles.add(file.fileName);
- // TODO add some validation, on size / checksum
- } finally {
- IOUtils.close(in, out);
- }
- }
- }
- // only notify if all required files were successfully obtained.
- notify = true;
- } finally {
- if (session != null) {
- try {
- replicator.release(session.id);
- } finally {
- if (!notify) { // cleanup after ourselves
- IOUtils.close(sourceDirectory.values());
- factory.cleanupSession(session.id);
- }
- }
- }
- }
-
- // notify outside the try-finally above, so the session is released sooner.
- // the handler may take time to finish acting on the copied files, but the
- // session itself is no longer needed.
- try {
- if (notify && !closed) { // no use to notify if we are closed already
- handler.revisionReady(session.version, session.sourceFiles, copiedFiles, sourceDirectory);
- }
- } finally {
- IOUtils.close(sourceDirectory.values());
- if (session != null) {
- factory.cleanupSession(session.id);
- }
- }
- }
-
- /** Throws {@link AlreadyClosedException} if the client has already been closed. */
- protected final void ensureOpen() {
- if (closed) {
- throw new AlreadyClosedException("this update client has already been closed");
- }
- }
-
- /**
- * Called when an exception is hit by the replication thread. The default implementation prints
- * the full stacktrace to the {@link InfoStream} set in {@link #setInfoStream(InfoStream)}, or the
- * {@link InfoStream#getDefault() default} one. You can override to log the exception elswhere.
- *
- * <p><b>NOTE:</b> if you override this method to throw the exception further, the replication
- * thread will be terminated. The only way to restart it is to call {@link #stopUpdateThread()}
- * followed by {@link #startUpdateThread(long, String)}.
- */
- protected void handleUpdateException(Throwable t) {
- final StringWriter sw = new StringWriter();
- t.printStackTrace(new PrintWriter(sw));
- if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
- infoStream.message(
- INFO_STREAM_COMPONENT, "an error occurred during revision update: " + sw.toString());
- }
- }
-
- /**
- * Returns the files required for replication. By default, this method returns all files that
- * exist in the new revision, but not in the handler.
- */
- protected Map<String, List<RevisionFile>> requiredFiles(
- Map<String, List<RevisionFile>> newRevisionFiles) {
- Map<String, List<RevisionFile>> handlerRevisionFiles = handler.currentRevisionFiles();
- if (handlerRevisionFiles == null) {
- return newRevisionFiles;
- }
-
- Map<String, List<RevisionFile>> requiredFiles = new HashMap<>();
- for (Entry<String, List<RevisionFile>> e : handlerRevisionFiles.entrySet()) {
- // put the handler files in a Set, for faster contains() checks later
- Set<String> handlerFiles = new HashSet<>();
- for (RevisionFile file : e.getValue()) {
- handlerFiles.add(file.fileName);
- }
-
- // make sure to preserve revisionFiles order
- ArrayList<RevisionFile> res = new ArrayList<>();
- String source = e.getKey();
- assert newRevisionFiles.containsKey(source)
- : "source not found in newRevisionFiles: " + newRevisionFiles;
- for (RevisionFile file : newRevisionFiles.get(source)) {
- if (!handlerFiles.contains(file.fileName)) {
- res.add(file);
- }
- }
- requiredFiles.put(source, res);
- }
-
- return requiredFiles;
- }
-
- @Override
- public synchronized void close() {
- if (!closed) {
- stopUpdateThread();
- closed = true;
- }
- }
-
- /**
- * Start the update thread with the specified interval in milliseconds. For debugging purposes,
- * you can optionally set the name to set on {@link Thread#setName(String)}. If you pass {@code
- * null}, a default name will be set.
- *
- * @throws IllegalStateException if the thread has already been started
- */
- public synchronized void startUpdateThread(long intervalMillis, String threadName) {
- ensureOpen();
- if (updateThread != null && updateThread.isAlive()) {
- throw new IllegalStateException(
- "cannot start an update thread when one is running, must first call 'stopUpdateThread()'");
- }
- threadName = threadName == null ? INFO_STREAM_COMPONENT : "ReplicationThread-" + threadName;
- updateThread = new ReplicationThread(intervalMillis);
- updateThread.setName(threadName);
- updateThread.start();
- // we rely on isAlive to return true in isUpdateThreadAlive, assert to be on the safe side
- assert updateThread.isAlive() : "updateThread started but not alive?";
- }
-
- /**
- * Stop the update thread. If the update thread is not running, silently does nothing. This method
- * returns after the update thread has stopped.
- */
- public synchronized void stopUpdateThread() {
- if (updateThread != null) {
- // this will trigger the thread to terminate if it awaits the lock.
- // otherwise, if it's in the middle of replication, we wait for it to
- // stop.
- updateThread.stop.countDown();
- try {
- updateThread.join();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new ThreadInterruptedException(e);
- }
- updateThread = null;
- }
- }
-
- /**
- * Returns true if the update thread is alive. The update thread is alive if it has been {@link
- * #startUpdateThread(long, String) started} and not {@link #stopUpdateThread() stopped}, as well
- * as didn't hit an error which caused it to terminate (i.e. {@link
- * #handleUpdateException(Throwable)} threw the exception further).
- */
- public synchronized boolean isUpdateThreadAlive() {
- return updateThread != null && updateThread.isAlive();
- }
-
- @Override
- public String toString() {
- String res = "ReplicationClient";
- if (updateThread != null) {
- res += " (" + updateThread.getName() + ")";
- }
- return res;
- }
-
- /**
- * Executes the update operation immediately, irregardless if an update thread is running or not.
- */
- public void updateNow() throws IOException {
- ensureOpen();
- updateLock.lock();
- try {
- doUpdate();
- } finally {
- updateLock.unlock();
- }
- }
-
- /** Sets the {@link InfoStream} to use for logging messages. */
- public void setInfoStream(InfoStream infoStream) {
- if (infoStream == null) {
- infoStream = InfoStream.NO_OUTPUT;
- }
- this.infoStream = infoStream;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/Replicator.java b/lucene/replicator/src/java/org/apache/lucene/replicator/Replicator.java
deleted file mode 100644
index 32fb68eb4d2..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/Replicator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * An interface for replicating files. Allows a producer to {@link #publish(Revision) publish}
- * {@link Revision}s and consumers to {@link #checkForUpdate(String) check for updates}. When a
- * client needs to be updated, it is given a {@link SessionToken} through which it can {@link
- * #obtainFile(String, String, String) obtain} the files of that revision. After the client has
- * finished obtaining all the files, it should {@link #release(String) release} the given session,
- * so that the files can be reclaimed if they are not needed anymore.
- *
- * <p>A client is always updated to the newest revision available. That is, if a client is on
- * revision <em>r1</em> and revisions <em>r2</em> and <em>r3</em> were published, then when the
- * cllient will next check for update, it will receive <em>r3</em>.
- *
- * @lucene.experimental
- */
-public interface Replicator extends Closeable {
-
- /**
- * Publish a new {@link Revision} for consumption by clients. It is the caller's responsibility to
- * verify that the revision files exist and can be read by clients. When the revision is no longer
- * needed, it will be {@link Revision#release() released} by the replicator.
- */
- public void publish(Revision revision) throws IOException;
-
- /**
- * Check whether the given version is up-to-date and returns a {@link SessionToken} which can be
- * used for fetching the revision files, otherwise returns {@code null}.
- *
- * <p><b>NOTE:</b> when the returned session token is no longer needed, you should call {@link
- * #release(String)} so that the session resources can be reclaimed, including the revision files.
- */
- public SessionToken checkForUpdate(String currVersion) throws IOException;
-
- /** Notify that the specified {@link SessionToken} is no longer needed by the caller. */
- public void release(String sessionID) throws IOException;
-
- /**
- * Returns an {@link InputStream} for the requested file and source in the context of the given
- * {@link SessionToken#id session}.
- *
- * <p><b>NOTE:</b> it is the caller's responsibility to close the returned stream.
- *
- * @throws SessionExpiredException if the specified session has already expired
- */
- public InputStream obtainFile(String sessionID, String source, String fileName)
- throws IOException;
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/Revision.java b/lucene/replicator/src/java/org/apache/lucene/replicator/Revision.java
deleted file mode 100644
index 20c4c8e8454..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/Revision.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.store.IndexInput;
-
-/**
- * A revision comprises lists of files that come from different sources and need to be replicated
- * together to e.g. guarantee that all resources are in sync. In most cases an application will
- * replicate a single index, and so the revision will contain files from a single source. However,
- * some applications may require to treat a collection of indexes as a single entity so that the
- * files from all sources are replicated together, to guarantee consistency between them. For
- * example, an application which indexes facets will need to replicate both the search and taxonomy
- * indexes together, to guarantee that they match at the client side.
- *
- * @lucene.experimental
- */
-public interface Revision extends Comparable<Revision> {
-
- /**
- * Compares the revision to the given version string. Behaves like {@link
- * Comparable#compareTo(Object)}.
- */
- public int compareTo(String version);
-
- /**
- * Returns a string representation of the version of this revision. The version is used by {@link
- * #compareTo(String)} as well as to serialize/deserialize revision information. Therefore it must
- * be self descriptive as well as be able to identify one revision from another.
- */
- public String getVersion();
-
- /**
- * Returns the files that comprise this revision, as a mapping from a source to a list of files.
- */
- public Map<String, List<RevisionFile>> getSourceFiles();
-
- /**
- * Returns an {@link IndexInput} for the given fileName and source. It is the caller's
- * responsibility to close the {@link IndexInput} when it has been consumed.
- */
- public InputStream open(String source, String fileName) throws IOException;
-
- /**
- * Called when this revision can be safely released, i.e. where there are no more references to
- * it.
- */
- public void release() throws IOException;
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java b/lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java
deleted file mode 100644
index 101cbde65fa..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-/**
- * Describes a file in a {@link Revision}. A file has a source, which allows a single revision to
- * contain files from multiple sources (e.g. multiple indexes).
- *
- * @lucene.experimental
- */
-public class RevisionFile {
-
- /** The name of the file. */
- public final String fileName;
-
- /** The size of the file denoted by {@link #fileName}. */
- public long size = -1;
-
- /** Constructor with the given file name. */
- public RevisionFile(String fileName) {
- if (fileName == null || fileName.isEmpty()) {
- throw new IllegalArgumentException("fileName must not be null or empty");
- }
- this.fileName = fileName;
- }
-
- @Override
- public boolean equals(Object obj) {
- RevisionFile other = (RevisionFile) obj;
- return fileName.equals(other.fileName) && size == other.size;
- }
-
- @Override
- public int hashCode() {
- return fileName.hashCode() ^ (int) (size ^ (size >>> 32));
- }
-
- @Override
- public String toString() {
- return "fileName=" + fileName + " size=" + size;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/SessionExpiredException.java b/lucene/replicator/src/java/org/apache/lucene/replicator/SessionExpiredException.java
deleted file mode 100644
index 81fa3dbc986..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/SessionExpiredException.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-
-/**
- * Exception indicating that a revision update session was expired due to lack of activity.
- *
- * @see LocalReplicator#DEFAULT_SESSION_EXPIRATION_THRESHOLD
- * @see LocalReplicator#setExpirationThreshold(long)
- * @lucene.experimental
- */
-public class SessionExpiredException extends IOException {
-
- /**
- * @see IOException#IOException(String, Throwable)
- */
- public SessionExpiredException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * @see IOException#IOException(String)
- */
- public SessionExpiredException(String message) {
- super(message);
- }
-
- /**
- * @see IOException#IOException(Throwable)
- */
- public SessionExpiredException(Throwable cause) {
- super(cause);
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/SessionToken.java b/lucene/replicator/src/java/org/apache/lucene/replicator/SessionToken.java
deleted file mode 100644
index d359ea140d9..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/SessionToken.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * Token for a replication session, for guaranteeing that source replicated files will be kept safe
- * until the replication completes.
- *
- * @see Replicator#checkForUpdate(String)
- * @see Replicator#release(String)
- * @see LocalReplicator#DEFAULT_SESSION_EXPIRATION_THRESHOLD
- * @lucene.experimental
- */
-public final class SessionToken {
-
- /**
- * ID of this session. Should be passed when releasing the session, thereby acknowledging the
- * {@link Replicator Replicator} that this session is no longer in use.
- *
- * @see Replicator#release(String)
- */
- public final String id;
-
- /**
- * @see Revision#getVersion()
- */
- public final String version;
-
- /**
- * @see Revision#getSourceFiles()
- */
- public final Map<String, List<RevisionFile>> sourceFiles;
-
- /** Constructor which deserializes from the given {@link DataInput}. */
- public SessionToken(DataInput in) throws IOException {
- this.id = in.readUTF();
- this.version = in.readUTF();
- this.sourceFiles = new HashMap<>();
- int numSources = in.readInt();
- while (numSources > 0) {
- String source = in.readUTF();
- int numFiles = in.readInt();
- List<RevisionFile> files = new ArrayList<>(numFiles);
- for (int i = 0; i < numFiles; i++) {
- String fileName = in.readUTF();
- RevisionFile file = new RevisionFile(fileName);
- file.size = in.readLong();
- files.add(file);
- }
- this.sourceFiles.put(source, files);
- --numSources;
- }
- }
-
- /** Constructor with the given id and revision. */
- public SessionToken(String id, Revision revision) {
- this.id = id;
- this.version = revision.getVersion();
- this.sourceFiles = revision.getSourceFiles();
- }
-
- /** Serialize the token data for communication between server and client. */
- public void serialize(DataOutput out) throws IOException {
- out.writeUTF(id);
- out.writeUTF(version);
- out.writeInt(sourceFiles.size());
- for (Entry<String, List<RevisionFile>> e : sourceFiles.entrySet()) {
- out.writeUTF(e.getKey());
- List<RevisionFile> files = e.getValue();
- out.writeInt(files.size());
- for (RevisionFile file : files) {
- out.writeUTF(file.fileName);
- out.writeLong(file.size);
- }
- }
- }
-
- @Override
- public String toString() {
- return "id=" + id + " version=" + version + " files=" + sourceFiles;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpClientBase.java b/lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpClientBase.java
deleted file mode 100644
index 315eff1a0ed..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpClientBase.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator.http;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.util.concurrent.Callable;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.StatusLine;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.util.EntityUtils;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.SuppressForbidden;
-
-/**
- * Base class for Http clients.
- *
- * @lucene.experimental
- */
-public abstract class HttpClientBase implements Closeable {
-
- /** Default connection timeout for this client, in milliseconds. */
- public static final int DEFAULT_CONNECTION_TIMEOUT = 1000;
-
- /** Default socket timeout for this client, in milliseconds. */
- public static final int DEFAULT_SO_TIMEOUT = 60000;
-
- // TODO compression?
-
- /** The URL stting to execute requests against. */
- protected final String url;
-
- private volatile boolean closed = false;
-
- private final CloseableHttpClient httpc;
- private final RequestConfig defaultConfig;
-
- /**
- * @param conMgr connection manager to use for this http client. <b>NOTE:</b>The provided {@link
- * HttpClientConnectionManager} will not be {@link HttpClientConnectionManager#shutdown()} by
- * this class.
- * @param defaultConfig the default {@link RequestConfig} to set on the client. If {@code null} a
- * default config is created w/ the default connection and socket timeouts.
- */
- protected HttpClientBase(
- String host,
- int port,
- String path,
- HttpClientConnectionManager conMgr,
- RequestConfig defaultConfig) {
- url = normalizedURL(host, port, path);
- if (defaultConfig == null) {
- this.defaultConfig =
- RequestConfig.custom()
- .setConnectionRequestTimeout(DEFAULT_CONNECTION_TIMEOUT)
- .setSocketTimeout(DEFAULT_SO_TIMEOUT)
- .build();
- } else {
- this.defaultConfig = defaultConfig;
- }
- httpc =
- HttpClientBuilder.create()
- .setConnectionManager(conMgr)
- .setDefaultRequestConfig(this.defaultConfig)
- .build();
- }
-
- /** Throws {@link AlreadyClosedException} if this client is already closed. */
- protected final void ensureOpen() throws AlreadyClosedException {
- if (closed) {
- throw new AlreadyClosedException("HttpClient already closed");
- }
- }
-
- /** Create a URL out of the given parameters, translate an empty/null path to '/' */
- private static String normalizedURL(String host, int port, String path) {
- if (path == null || path.length() == 0) {
- path = "/";
- }
- return "http://" + host + ":" + port + path;
- }
-
- /**
- * <b>Internal:</b> response status after invocation, and in case or error attempt to read the
- * exception sent by the server.
- */
- protected void verifyStatus(HttpResponse response) throws IOException {
- StatusLine statusLine = response.getStatusLine();
- if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
- try {
- throwKnownError(response, statusLine);
- } finally {
- EntityUtils.consumeQuietly(response.getEntity());
- }
- }
- }
-
- @SuppressForbidden(reason = "XXX: security hole")
- protected void throwKnownError(HttpResponse response, StatusLine statusLine) throws IOException {
- ObjectInputStream in = null;
- try {
- in = new ObjectInputStream(response.getEntity().getContent());
- } catch (Throwable t) {
- // the response stream is not an exception - could be an error in servlet.init().
- throw new RuntimeException("Unknown error: " + statusLine, t);
- }
-
- Throwable t;
- try {
- t = (Throwable) in.readObject();
- assert t != null;
- } catch (Throwable th) {
- throw new RuntimeException("Failed to read exception object: " + statusLine, th);
- } finally {
- in.close();
- }
- throw IOUtils.rethrowAlways(t);
- }
-
- /**
- * <b>internal:</b> execute a request and return its result The <code>params</code> argument is
- * treated as: name1,value1,name2,value2,...
- */
- protected HttpResponse executePOST(String request, HttpEntity entity, String... params)
- throws IOException {
- ensureOpen();
- HttpPost m = new HttpPost(queryString(request, params));
- m.setEntity(entity);
- HttpResponse response = httpc.execute(m);
- verifyStatus(response);
- return response;
- }
-
- /**
- * <b>internal:</b> execute a request and return its result The <code>params</code> argument is
- * treated as: name1,value1,name2,value2,...
- */
- protected HttpResponse executeGET(String request, String... params) throws IOException {
- ensureOpen();
- HttpGet m = new HttpGet(queryString(request, params));
- HttpResponse response = httpc.execute(m);
- verifyStatus(response);
- return response;
- }
-
- private String queryString(String request, String... params) throws UnsupportedEncodingException {
- StringBuilder query = new StringBuilder(url).append('/').append(request).append('?');
- if (params != null) {
- for (int i = 0; i < params.length; i += 2) {
- query
- .append(params[i])
- .append('=')
- .append(URLEncoder.encode(params[i + 1], "UTF8"))
- .append('&');
- }
- }
- return query.substring(0, query.length() - 1);
- }
-
- /** Internal utility: input stream of the provided response */
- public InputStream responseInputStream(HttpResponse response) throws IOException {
- return responseInputStream(response, false);
- }
-
- // TODO: can we simplify this Consuming !?!?!?
- /**
- * Internal utility: input stream of the provided response, which optionally consumes the
- * response's resources when the input stream is exhausted.
- */
- public InputStream responseInputStream(HttpResponse response, boolean consume)
- throws IOException {
- final HttpEntity entity = response.getEntity();
- final InputStream in = entity.getContent();
- if (!consume) {
- return in;
- }
- return new InputStream() {
- private boolean consumed = false;
-
- @Override
- public int read() throws IOException {
- final int res = in.read();
- consume(res);
- return res;
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- consume(-1);
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- final int res = in.read(b);
- consume(res);
- return res;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- final int res = in.read(b, off, len);
- consume(res);
- return res;
- }
-
- private void consume(int minusOne) {
- if (!consumed && minusOne == -1) {
- try {
- EntityUtils.consume(entity);
- } catch (
- @SuppressWarnings("unused")
- Exception e) {
- // ignored on purpose
- }
- consumed = true;
- }
- }
- };
- }
-
- /**
- * Returns true iff this instance was {@link #close() closed}, otherwise returns false. Note that
- * if you override {@link #close()}, you must call {@code super.close()}, in order for this
- * instance to be properly closed.
- */
- protected final boolean isClosed() {
- return closed;
- }
-
- /**
- * Same as {@link #doAction(HttpResponse, boolean, Callable)} but always do consume at the end.
- */
- protected <T> T doAction(HttpResponse response, Callable<T> call) throws IOException {
- return doAction(response, true, call);
- }
-
- /**
- * Do a specific action and validate after the action that the status is still OK, and if not,
- * attempt to extract the actual server side exception. Optionally release the response at exit,
- * depending on <code>consume</code> parameter.
- */
- protected <T> T doAction(HttpResponse response, boolean consume, Callable<T> call)
- throws IOException {
- Throwable th = null;
- try {
- return call.call();
- } catch (Throwable t) {
- th = t;
- } finally {
- try {
- verifyStatus(response);
- } finally {
- if (consume) {
- EntityUtils.consumeQuietly(response.getEntity());
- }
- }
- }
- throw IOUtils.rethrowAlways(th);
- }
-
- @Override
- public void close() throws IOException {
- httpc.close();
- closed = true;
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpReplicator.java b/lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpReplicator.java
deleted file mode 100644
index dd12172d983..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpReplicator.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator.http;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.concurrent.Callable;
-import org.apache.http.HttpResponse;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.lucene.replicator.Replicator;
-import org.apache.lucene.replicator.Revision;
-import org.apache.lucene.replicator.SessionToken;
-import org.apache.lucene.replicator.http.ReplicationService.ReplicationAction;
-
-/**
- * An HTTP implementation of {@link Replicator}. Assumes the API supported by {@link
- * ReplicationService}.
- *
- * @lucene.experimental
- */
-public class HttpReplicator extends HttpClientBase implements Replicator {
-
- /** Construct with specified connection manager. */
- public HttpReplicator(String host, int port, String path, HttpClientConnectionManager conMgr) {
- super(host, port, path, conMgr, null);
- }
-
- @Override
- public SessionToken checkForUpdate(String currVersion) throws IOException {
- String[] params = null;
- if (currVersion != null) {
- params = new String[] {ReplicationService.REPLICATE_VERSION_PARAM, currVersion};
- }
- final HttpResponse response = executeGET(ReplicationAction.UPDATE.name(), params);
- return doAction(
- response,
- new Callable<SessionToken>() {
- @Override
- public SessionToken call() throws Exception {
- final DataInputStream dis = new DataInputStream(responseInputStream(response));
- try {
- if (dis.readByte() == 0) {
- return null;
- } else {
- return new SessionToken(dis);
- }
- } finally {
- dis.close();
- }
- }
- });
- }
-
- @Override
- public InputStream obtainFile(String sessionID, String source, String fileName)
- throws IOException {
- String[] params =
- new String[] {
- ReplicationService.REPLICATE_SESSION_ID_PARAM, sessionID,
- ReplicationService.REPLICATE_SOURCE_PARAM, source,
- ReplicationService.REPLICATE_FILENAME_PARAM, fileName,
- };
- final HttpResponse response = executeGET(ReplicationAction.OBTAIN.name(), params);
- return doAction(
- response,
- false,
- new Callable<InputStream>() {
- @Override
- public InputStream call() throws Exception {
- return responseInputStream(response, true);
- }
- });
- }
-
- @Override
- public void publish(Revision revision) throws IOException {
- throw new UnsupportedOperationException(
- "this replicator implementation does not support remote publishing of revisions");
- }
-
- @Override
- public void release(String sessionID) throws IOException {
- String[] params = new String[] {ReplicationService.REPLICATE_SESSION_ID_PARAM, sessionID};
- final HttpResponse response = executeGET(ReplicationAction.RELEASE.name(), params);
- doAction(
- response,
- new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return null; // do not remove this call: as it is still validating for us!
- }
- });
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java b/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java
deleted file mode 100644
index a511cd47af3..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator.http;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectOutputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Locale;
-import java.util.Map;
-import java.util.StringTokenizer;
-import javax.servlet.ServletException;
-import javax.servlet.ServletOutputStream;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.apache.http.HttpStatus;
-import org.apache.lucene.replicator.Replicator;
-import org.apache.lucene.replicator.SessionToken;
-import org.apache.lucene.util.SuppressForbidden;
-
-/**
- * A server-side service for handling replication requests. The service assumes requests are sent in
- * the format <code>/<context>/<shard>/<action></code> where
- *
- * <ul>
- * <li>{@code context} is the servlet context, e.g. {@link #REPLICATION_CONTEXT}
- * <li>{@code shard} is the ID of the shard, e.g. "s1"
- * <li>{@code action} is one of {@link ReplicationAction} values
- * </ul>
- *
- * For example, to check whether there are revision updates for shard "s1" you should send the
- * request: <code>http://host:port/replicate/s1/update</code>.
- *
- * <p>This service is written like a servlet, and {@link #perform(HttpServletRequest,
- * HttpServletResponse)} takes servlet request and response accordingly, so it is quite easy to
- * embed in your application's servlet.
- *
- * @lucene.experimental
- */
-public class ReplicationService {
-
- /** Actions supported by the {@link ReplicationService}. */
- public enum ReplicationAction {
- OBTAIN,
- RELEASE,
- UPDATE
- }
-
- /** The context path for the servlet. */
- public static final String REPLICATION_CONTEXT = "/replicate";
-
- /** Request parameter name for providing the revision version. */
- public static final String REPLICATE_VERSION_PARAM = "version";
-
- /** Request parameter name for providing a session ID. */
- public static final String REPLICATE_SESSION_ID_PARAM = "sessionid";
-
- /** Request parameter name for providing the file's source. */
- public static final String REPLICATE_SOURCE_PARAM = "source";
-
- /** Request parameter name for providing the file's name. */
- public static final String REPLICATE_FILENAME_PARAM = "filename";
-
- private static final int SHARD_IDX = 0, ACTION_IDX = 1;
-
- private final Map<String, Replicator> replicators;
-
- public ReplicationService(Map<String, Replicator> replicators) {
- super();
- this.replicators = replicators;
- }
-
- /**
- * Returns the path elements that were given in the servlet request, excluding the servlet's
- * action context.
- */
- private String[] getPathElements(HttpServletRequest req) {
- String path = req.getServletPath();
- String pathInfo = req.getPathInfo();
- if (pathInfo != null) {
- path += pathInfo;
- }
- int actionLen = REPLICATION_CONTEXT.length();
- int startIdx = actionLen;
- if (path.length() > actionLen && path.charAt(actionLen) == '/') {
- ++startIdx;
- }
-
- // split the string on '/' and remove any empty elements. This is better
- // than using String.split() since the latter may return empty elements in
- // the array
- StringTokenizer stok = new StringTokenizer(path.substring(startIdx), "/");
- ArrayList<String> elements = new ArrayList<>();
- while (stok.hasMoreTokens()) {
- elements.add(stok.nextToken());
- }
- return elements.toArray(new String[0]);
- }
-
- private static String extractRequestParam(HttpServletRequest req, String paramName)
- throws ServletException {
- String param = req.getParameter(paramName);
- if (param == null) {
- throw new ServletException("Missing mandatory parameter: " + paramName);
- }
- return param;
- }
-
- private static void copy(InputStream in, OutputStream out) throws IOException {
- byte[] buf = new byte[16384];
- int numRead;
- while ((numRead = in.read(buf)) != -1) {
- out.write(buf, 0, numRead);
- }
- }
-
- /** Executes the replication task. */
- @SuppressForbidden(reason = "XXX: security hole")
- public void perform(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
- String[] pathElements = getPathElements(req);
-
- if (pathElements.length != 2) {
- throw new ServletException(
- "invalid path, must contain shard ID and action, e.g. */s1/update");
- }
-
- final ReplicationAction action;
- try {
- action = ReplicationAction.valueOf(pathElements[ACTION_IDX].toUpperCase(Locale.ENGLISH));
- } catch (IllegalArgumentException e) {
- throw new ServletException("Unsupported action provided: " + pathElements[ACTION_IDX], e);
- }
-
- final Replicator replicator = replicators.get(pathElements[SHARD_IDX]);
- if (replicator == null) {
- throw new ServletException("unrecognized shard ID " + pathElements[SHARD_IDX]);
- }
-
- // SOLR-8933 Don't close this stream.
- ServletOutputStream resOut = resp.getOutputStream();
- try {
- switch (action) {
- case OBTAIN:
- final String sessionID = extractRequestParam(req, REPLICATE_SESSION_ID_PARAM);
- final String fileName = extractRequestParam(req, REPLICATE_FILENAME_PARAM);
- final String source = extractRequestParam(req, REPLICATE_SOURCE_PARAM);
- InputStream in = replicator.obtainFile(sessionID, source, fileName);
- try {
- copy(in, resOut);
- } finally {
- in.close();
- }
- break;
- case RELEASE:
- replicator.release(extractRequestParam(req, REPLICATE_SESSION_ID_PARAM));
- break;
- case UPDATE:
- String currVersion = req.getParameter(REPLICATE_VERSION_PARAM);
- SessionToken token = replicator.checkForUpdate(currVersion);
- if (token == null) {
- resOut.write(0); // marker for null token
- } else {
- resOut.write(1);
- token.serialize(new DataOutputStream(resOut));
- }
- break;
- }
- } catch (Exception e) {
- resp.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); // propagate the failure
- try {
- /*
- * Note: it is assumed that "identified exceptions" are thrown before
- * anything was written to the stream.
- */
- ObjectOutputStream oos = new ObjectOutputStream(resOut);
- oos.writeObject(e);
- oos.flush();
- } catch (Exception e2) {
- throw new IOException("Could not serialize", e2);
- }
- } finally {
- resp.flushBuffer();
- }
- }
-}
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/http/package-info.java b/lucene/replicator/src/java/org/apache/lucene/replicator/http/package-info.java
deleted file mode 100644
index fc19fd5725a..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/http/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** HTTP replication implementation */
-package org.apache.lucene.replicator.http;
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/package-info.java b/lucene/replicator/src/java/org/apache/lucene/replicator/package-info.java
deleted file mode 100644
index 85d6e851503..00000000000
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/package-info.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Files replication framework
- *
- * <p>The <a href="Replicator.html">Replicator</a> allows replicating files between a server and
- * client(s). Producers publish <a href="Revision.html">revisions</a> and consumers update to the
- * latest revision available. <a href="ReplicationClient.html">ReplicationClient</a> is a helper
- * utility for performing the update operation. It can be invoked either <a
- * href="ReplicationClient.html#updateNow()">manually</a> or periodically by <a
- * href="ReplicationClient.html#startUpdateThread(long,+java.lang.String)">starting an update
- * thread</a>. <a href="http/HttpReplicator.html">HttpReplicator</a> can be used to replicate
- * revisions by consumers that reside on a different node than the producer.
- *
- * <p>The replication framework supports replicating any type of files, with built-in support for a
- * single search index as well as an index and taxonomy pair. For a single index, the application
- * should publish an <a href="IndexRevision.html">IndexRevision</a> and set <a
- * href="IndexReplicationHandler.html">IndexReplicationHandler</a> on the client. For an index and
- * taxonomy pair, the application should publish an <a
- * href="IndexAndTaxonomyRevision.html">IndexAndTaxonomyRevision</a> and set <a
- * href="IndexAndTaxonomyReplicationHandler.html">IndexAndTaxonomyReplicationHandler</a> on the
- * client.
- *
- * <p>When the replication client detects that there is a newer revision available, it copies the
- * files of the revision and then invokes the handler to complete the operation (e.g. copy the files
- * to the index directory, fsync them, reopen an index reader etc.). By default, only files that do
- * not exist in the handler's <a
- * href="ReplicationClient.ReplicationHandler.html#currentRevisionFiles()">current revision
- * files</a> are copied, however this can be overridden by extending the client.
- *
- * <p>An example usage of the Replicator:
- *
- * <pre class="prettyprint lang-java">
- * // ++++++++++++++ SERVER SIDE ++++++++++++++ //
- * IndexWriter publishWriter; // the writer used for indexing
- * Replicator replicator = new LocalReplicator();
- * replicator.publish(new IndexRevision(publishWriter));
- *
- * // ++++++++++++++ CLIENT SIDE ++++++++++++++ //
- * // either LocalReplictor, or HttpReplicator if client and server are on different nodes
- * Replicator replicator;
- *
- * // callback invoked after handler finished handling the revision and e.g. can reopen the reader.
- * Callable<Boolean> callback = null; // can also be null if no callback is needed
- * ReplicationHandler handler = new IndexReplicationHandler(indexDir, callback);
- * SourceDirectoryFactory factory = new PerSessionDirectoryFactory(workDir);
- * ReplicationClient client = new ReplicationClient(replicator, handler, factory);
- *
- * // invoke client manually
- * client.updateNow();
- *
- * // or, periodically
- * client.startUpdateThread(100); // check for update every 100 milliseconds
- * </pre>
- */
-package org.apache.lucene.replicator;
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java b/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java
deleted file mode 100644
index 443965745f7..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import java.util.Random;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.Handler;
-import org.eclipse.jetty.server.HttpConfiguration;
-import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.SecureRequestCustomizer;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.SslConnectionFactory;
-import org.eclipse.jetty.server.session.DefaultSessionIdManager;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.junit.AfterClass;
-
-@ThreadLeakLingering(linger = 80000) // Jetty might ignore interrupt for a minute
-public abstract class ReplicatorTestCase extends LuceneTestCase {
-
- private static HttpClientConnectionManager clientConnectionManager;
-
- @AfterClass
- public static void afterClassReplicatorTestCase() throws Exception {
- if (clientConnectionManager != null) {
- clientConnectionManager.shutdown();
- clientConnectionManager = null;
- }
- }
-
- /**
- * Returns a new {@link Server HTTP Server} instance. To obtain its port, use {@link
- * #serverPort(Server)}.
- */
- public static synchronized Server newHttpServer(Handler handler) throws Exception {
- // if this property is true, then jetty will be configured to use SSL
- // leveraging the same system properties as java to specify
- // the keystore/truststore if they are set
- //
- // This means we will use the same truststore, keystore (and keys) for
- // the server as well as any client actions taken by this JVM in
- // talking to that server, but for the purposes of testing that should
- // be good enough
- final boolean useSsl = Boolean.getBoolean("tests.jettySsl");
- final SslContextFactory.Server sslcontext = new SslContextFactory.Server();
-
- if (useSsl) {
- if (null != System.getProperty("javax.net.ssl.keyStore")) {
- sslcontext.setKeyStorePath(System.getProperty("javax.net.ssl.keyStore"));
- }
- if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
- sslcontext.setKeyStorePassword(System.getProperty("javax.net.ssl.keyStorePassword"));
- }
- if (null != System.getProperty("javax.net.ssl.trustStore")) {
- sslcontext.setKeyStorePath(System.getProperty("javax.net.ssl.trustStore"));
- }
- if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
- sslcontext.setTrustStorePassword(System.getProperty("javax.net.ssl.trustStorePassword"));
- }
- sslcontext.setNeedClientAuth(Boolean.getBoolean("tests.jettySsl.clientAuth"));
- }
-
- final QueuedThreadPool threadPool = new QueuedThreadPool();
- threadPool.setDaemon(true);
- threadPool.setMaxThreads(10000);
- threadPool.setIdleTimeout(5000);
- threadPool.setStopTimeout(30000);
-
- Server server = new Server(threadPool);
- server.setStopAtShutdown(true);
- server.manage(threadPool);
-
- final ServerConnector connector;
- if (useSsl) {
- HttpConfiguration configuration = new HttpConfiguration();
- configuration.setSecureScheme("https");
- configuration.addCustomizer(new SecureRequestCustomizer());
- @SuppressWarnings("resource")
- ServerConnector c =
- new ServerConnector(
- server,
- new SslConnectionFactory(sslcontext, "http/1.1"),
- new HttpConnectionFactory(configuration));
- connector = c;
- } else {
- @SuppressWarnings("resource")
- ServerConnector c = new ServerConnector(server, new HttpConnectionFactory());
- connector = c;
- }
-
- connector.setPort(0);
- connector.setHost("127.0.0.1");
-
- server.setConnectors(new Connector[] {connector});
- server.setSessionIdManager(
- new DefaultSessionIdManager(server, new Random(random().nextLong())));
- server.setHandler(handler);
-
- server.start();
-
- return server;
- }
-
- /** Returns a {@link Server}'s port. */
- public static int serverPort(Server server) {
- return ((ServerConnector) server.getConnectors()[0]).getLocalPort();
- }
-
- /** Returns a {@link Server}'s host. */
- public static String serverHost(Server server) {
- return ((ServerConnector) server.getConnectors()[0]).getHost();
- }
-
- /**
- * Stops the given HTTP Server instance. This method does its best to guarantee that no threads
- * will be left running following this method.
- */
- public static void stopHttpServer(Server httpServer) throws Exception {
- httpServer.stop();
- httpServer.join();
- }
-
- /**
- * Returns a {@link HttpClientConnectionManager}.
- *
- * <p><b>NOTE:</b> do not {@link HttpClientConnectionManager#shutdown()} this connection manager,
- * it will be close automatically after all tests have finished.
- */
- public static synchronized HttpClientConnectionManager getClientConnectionManager() {
- if (clientConnectionManager == null) {
- PoolingHttpClientConnectionManager ccm = new PoolingHttpClientConnectionManager();
- ccm.setDefaultMaxPerRoute(128);
- ccm.setMaxTotal(128);
- clientConnectionManager = ccm;
- }
-
- return clientConnectionManager;
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyReplicationClient.java b/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyReplicationClient.java
deleted file mode 100644
index ffd56dc4667..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyReplicationClient.java
+++ /dev/null
@@ -1,481 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.facet.DrillDownQuery;
-import org.apache.lucene.facet.FacetField;
-import org.apache.lucene.facet.Facets;
-import org.apache.lucene.facet.FacetsCollector;
-import org.apache.lucene.facet.FacetsCollectorManager;
-import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts;
-import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.index.CheckIndex;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.replicator.IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter;
-import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
-import org.apache.lucene.replicator.ReplicationClient.SourceDirectoryFactory;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.store.MockDirectoryWrapper;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.ThreadInterruptedException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestIndexAndTaxonomyReplicationClient extends ReplicatorTestCase {
-
- private static class IndexAndTaxonomyReadyCallback implements Callable<Boolean>, Closeable {
-
- private final Directory indexDir, taxoDir;
- private DirectoryReader indexReader;
- private DirectoryTaxonomyReader taxoReader;
- private FacetsConfig config;
- private long lastIndexGeneration = -1;
-
- public IndexAndTaxonomyReadyCallback(Directory indexDir, Directory taxoDir) throws IOException {
- this.indexDir = indexDir;
- this.taxoDir = taxoDir;
- config = new FacetsConfig();
- config.setHierarchical("A", true);
- if (DirectoryReader.indexExists(indexDir)) {
- indexReader = DirectoryReader.open(indexDir);
- lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
- taxoReader = new DirectoryTaxonomyReader(taxoDir);
- }
- }
-
- @Override
- public Boolean call() throws Exception {
- if (indexReader == null) {
- indexReader = DirectoryReader.open(indexDir);
- lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
- taxoReader = new DirectoryTaxonomyReader(taxoDir);
- } else {
- // verify search index
- DirectoryReader newReader = DirectoryReader.openIfChanged(indexReader);
- assertNotNull(
- "should not have reached here if no changes were made to the index", newReader);
- long newGeneration = newReader.getIndexCommit().getGeneration();
- assertTrue(
- "expected newer generation; current=" + lastIndexGeneration + " new=" + newGeneration,
- newGeneration > lastIndexGeneration);
- indexReader.close();
- indexReader = newReader;
- lastIndexGeneration = newGeneration;
- TestUtil.checkIndex(indexDir);
-
- // verify taxonomy index
- DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
- if (newTaxoReader != null) {
- taxoReader.close();
- taxoReader = newTaxoReader;
- }
- TestUtil.checkIndex(taxoDir);
-
- // verify faceted search
- int id = Integer.parseInt(indexReader.getIndexCommit().getUserData().get(VERSION_ID), 16);
- IndexSearcher searcher = new IndexSearcher(indexReader);
- FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager());
- Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
- assertEquals(1, facets.getSpecificValue("A", Integer.toString(id, 16)).intValue());
-
- DrillDownQuery drillDown = new DrillDownQuery(config);
- drillDown.add("A", Integer.toString(id, 16));
- TopDocs docs = searcher.search(drillDown, 10);
- assertEquals(1, docs.totalHits.value);
- }
- return null;
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(indexReader, taxoReader);
- }
- }
-
- private Directory publishIndexDir, publishTaxoDir;
- private MockDirectoryWrapper handlerIndexDir, handlerTaxoDir;
- private Replicator replicator;
- private SourceDirectoryFactory sourceDirFactory;
- private ReplicationClient client;
- private ReplicationHandler handler;
- private IndexWriter publishIndexWriter;
- private SnapshotDirectoryTaxonomyWriter publishTaxoWriter;
- private FacetsConfig config;
- private IndexAndTaxonomyReadyCallback callback;
- private Path clientWorkDir;
-
- private static final String VERSION_ID = "version";
-
- private void assertHandlerRevision(int expectedID, Directory dir) throws IOException {
- // loop as long as client is alive. test-framework will terminate us if
- // there's a serious bug, e.g. client doesn't really update. otherwise,
- // introducing timeouts is not good, can easily lead to false positives.
- while (client.isUpdateThreadAlive()) {
- // give client a chance to update
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- throw new ThreadInterruptedException(e);
- }
-
- try {
- DirectoryReader reader = DirectoryReader.open(dir);
- try {
- int handlerID =
- Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
- if (expectedID == handlerID) {
- return;
- }
- } finally {
- reader.close();
- }
- } catch (
- @SuppressWarnings("unused")
- Exception e) {
- // we can hit IndexNotFoundException or e.g. EOFException (on
- // segments_N) because it is being copied at the same time it is read by
- // DirectoryReader.open().
- }
- }
- }
-
- private Revision createRevision(final int id) throws IOException {
- publishIndexWriter.addDocument(newDocument(publishTaxoWriter, id));
- publishIndexWriter.setLiveCommitData(
- new HashMap<String, String>() {
- {
- put(VERSION_ID, Integer.toString(id, 16));
- }
- }.entrySet());
- publishIndexWriter.commit();
- publishTaxoWriter.commit();
- return new IndexAndTaxonomyRevision(publishIndexWriter, publishTaxoWriter);
- }
-
- private Document newDocument(TaxonomyWriter taxoWriter, int id) throws IOException {
- Document doc = new Document();
- doc.add(new FacetField("A", Integer.toString(id, 16)));
- return config.build(taxoWriter, doc);
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- publishIndexDir = newDirectory();
- publishTaxoDir = newDirectory();
- handlerIndexDir = newMockDirectory();
- handlerTaxoDir = newMockDirectory();
- clientWorkDir = createTempDir("replicationClientTest");
- sourceDirFactory = new PerSessionDirectoryFactory(clientWorkDir);
- replicator = new LocalReplicator();
- callback = new IndexAndTaxonomyReadyCallback(handlerIndexDir, handlerTaxoDir);
- handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, callback);
- client = new ReplicationClient(replicator, handler, sourceDirFactory);
-
- IndexWriterConfig conf = newIndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- publishIndexWriter = new IndexWriter(publishIndexDir, conf);
- publishTaxoWriter = new SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
- config = new FacetsConfig();
- config.setHierarchical("A", true);
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- publishIndexWriter.close();
- IOUtils.close(
- client,
- callback,
- publishTaxoWriter,
- replicator,
- publishIndexDir,
- publishTaxoDir,
- handlerIndexDir,
- handlerTaxoDir);
- super.tearDown();
- }
-
- @Test
- public void testNoUpdateThread() throws Exception {
- assertNull("no version expected at start", handler.currentVersion());
-
- // Callback validates the replicated index
- replicator.publish(createRevision(1));
- client.updateNow();
-
- // make sure updating twice, when in fact there's nothing to update, works
- client.updateNow();
-
- replicator.publish(createRevision(2));
- client.updateNow();
-
- // Publish two revisions without update, handler should be upgraded to latest
- replicator.publish(createRevision(3));
- replicator.publish(createRevision(4));
- client.updateNow();
- }
-
- @Test
- public void testRestart() throws Exception {
- replicator.publish(createRevision(1));
- client.updateNow();
-
- replicator.publish(createRevision(2));
- client.updateNow();
-
- client.stopUpdateThread();
- client.close();
- client = new ReplicationClient(replicator, handler, sourceDirFactory);
-
- // Publish two revisions without update, handler should be upgraded to latest
- replicator.publish(createRevision(3));
- replicator.publish(createRevision(4));
- client.updateNow();
- }
-
- @Test
- public void testUpdateThread() throws Exception {
- client.startUpdateThread(10, "indexTaxo");
-
- replicator.publish(createRevision(1));
- assertHandlerRevision(1, handlerIndexDir);
-
- replicator.publish(createRevision(2));
- assertHandlerRevision(2, handlerIndexDir);
-
- // Publish two revisions without update, handler should be upgraded to latest
- replicator.publish(createRevision(3));
- replicator.publish(createRevision(4));
- assertHandlerRevision(4, handlerIndexDir);
- }
-
- @Test
- public void testRecreateTaxonomy() throws Exception {
- replicator.publish(createRevision(1));
- client.updateNow();
-
- // recreate index and taxonomy
- Directory newTaxo = newDirectory();
- new DirectoryTaxonomyWriter(newTaxo).close();
- publishTaxoWriter.replaceTaxonomy(newTaxo);
- publishIndexWriter.deleteAll();
- replicator.publish(createRevision(2));
-
- client.updateNow();
- newTaxo.close();
- }
-
- /*
- * This test verifies that the client and handler do not end up in a corrupt
- * index if exceptions are thrown at any point during replication. Either when
- * a client copies files from the server to the temporary space, or when the
- * handler copies them to the index directory.
- */
- @Test
- public void testConsistencyOnExceptions() throws Exception {
- // so the handler's index isn't empty
- replicator.publish(createRevision(1));
- client.updateNow();
- client.close();
- callback.close();
-
- // wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
- final SourceDirectoryFactory in = sourceDirFactory;
- final AtomicInteger failures = new AtomicInteger(atLeast(10));
- sourceDirFactory =
- new SourceDirectoryFactory() {
-
- private long clientMaxSize = 100, handlerIndexMaxSize = 100, handlerTaxoMaxSize = 100;
- private double clientExRate = 1.0, handlerIndexExRate = 1.0, handlerTaxoExRate = 1.0;
-
- @Override
- public void cleanupSession(String sessionID) throws IOException {
- in.cleanupSession(sessionID);
- }
-
- @SuppressWarnings("synthetic-access")
- @Override
- public Directory getDirectory(String sessionID, String source) throws IOException {
- Directory dir = in.getDirectory(sessionID, source);
- if (random().nextBoolean()
- && failures.get() > 0) { // client should fail, return wrapped dir
- MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
- mdw.setRandomIOExceptionRateOnOpen(clientExRate);
- mdw.setMaxSizeInBytes(clientMaxSize);
- mdw.setRandomIOExceptionRate(clientExRate);
- mdw.setCheckIndexOnClose(false);
- clientMaxSize *= 2;
- clientExRate /= 2;
- return mdw;
- }
-
- if (failures.get() > 0 && random().nextBoolean()) { // handler should fail
- if (random().nextBoolean()) { // index dir fail
- handlerIndexDir.setMaxSizeInBytes(handlerIndexMaxSize);
- handlerIndexDir.setRandomIOExceptionRate(handlerIndexExRate);
- handlerIndexDir.setRandomIOExceptionRateOnOpen(handlerIndexExRate);
- handlerIndexMaxSize *= 2;
- handlerIndexExRate /= 2;
- } else { // taxo dir fail
- handlerTaxoDir.setMaxSizeInBytes(handlerTaxoMaxSize);
- handlerTaxoDir.setRandomIOExceptionRate(handlerTaxoExRate);
- handlerTaxoDir.setRandomIOExceptionRateOnOpen(handlerTaxoExRate);
- handlerTaxoDir.setCheckIndexOnClose(false);
- handlerTaxoMaxSize *= 2;
- handlerTaxoExRate /= 2;
- }
- } else {
- // disable all errors
- handlerIndexDir.setMaxSizeInBytes(0);
- handlerIndexDir.setRandomIOExceptionRate(0.0);
- handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
- handlerTaxoDir.setMaxSizeInBytes(0);
- handlerTaxoDir.setRandomIOExceptionRate(0.0);
- handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
- }
-
- return dir;
- }
- };
-
- handler =
- new IndexAndTaxonomyReplicationHandler(
- handlerIndexDir,
- handlerTaxoDir,
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- if (random().nextDouble() < 0.2 && failures.get() > 0) {
- throw new RuntimeException("random exception from callback");
- }
- return null;
- }
- });
-
- final AtomicBoolean failed = new AtomicBoolean();
-
- // wrap handleUpdateException so we can act on the thrown exception
- client =
- new ReplicationClient(replicator, handler, sourceDirFactory) {
- @SuppressWarnings("synthetic-access")
- @Override
- protected void handleUpdateException(Throwable t) {
- if (t instanceof IOException) {
- try {
- if (VERBOSE) {
- System.out.println("hit exception during update: " + t);
- t.printStackTrace(System.out);
- }
-
- // test that the index can be read and also some basic statistics
- DirectoryReader reader = DirectoryReader.open(handlerIndexDir.getDelegate());
- try {
- int numDocs = reader.numDocs();
- int version =
- Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
- assertEquals(numDocs, version);
- } finally {
- reader.close();
- }
- // verify index is fully consistent
- TestUtil.checkIndex(handlerIndexDir.getDelegate());
-
- // verify taxonomy index is fully consistent (since we only add one
- // category to all documents, there's nothing much more to validate.
- ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
-
- try (CheckIndex checker = new CheckIndex(handlerTaxoDir.getDelegate())) {
- checker.setFailFast(true);
- checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
- try {
- checker.checkIndex(null);
- } catch (@SuppressWarnings("unused") IOException | RuntimeException ioe) {
- // ok: we fallback below
- }
- }
-
- } catch (IOException e) {
- failed.set(true);
- throw new RuntimeException(e);
- } catch (RuntimeException e) {
- failed.set(true);
- throw e;
- } finally {
- // count-down number of failures
- failures.decrementAndGet();
- assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
- if (VERBOSE) {
- if (failures.get() == 0) {
- System.out.println("no more failures expected");
- } else {
- System.out.println("num failures left: " + failures.get());
- }
- }
- }
- } else {
- failed.set(true);
- if (t instanceof RuntimeException) {
- throw (RuntimeException) t;
- }
- throw new RuntimeException(t);
- }
- }
- };
-
- client.startUpdateThread(10, "indexAndTaxo");
-
- final Directory baseHandlerIndexDir = handlerIndexDir.getDelegate();
- int numRevisions = atLeast(20) + 2;
- for (int i = 2; i < numRevisions && failed.get() == false; i++) {
- replicator.publish(createRevision(i));
- assertHandlerRevision(i, baseHandlerIndexDir);
- }
-
- // disable errors -- maybe randomness didn't exhaust all allowed failures,
- // and we don't want e.g. CheckIndex to hit false errors.
- handlerIndexDir.setMaxSizeInBytes(0);
- handlerIndexDir.setRandomIOExceptionRate(0.0);
- handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
- handlerTaxoDir.setMaxSizeInBytes(0);
- handlerTaxoDir.setRandomIOExceptionRate(0.0);
- handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyRevision.java b/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyRevision.java
deleted file mode 100644
index d57d30a2e42..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyRevision.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.facet.FacetField;
-import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.replicator.IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.IOUtils;
-import org.junit.Test;
-
-public class TestIndexAndTaxonomyRevision extends ReplicatorTestCase {
-
- private Document newDocument(TaxonomyWriter taxoWriter) throws IOException {
- FacetsConfig config = new FacetsConfig();
- Document doc = new Document();
- doc.add(new FacetField("A", "1"));
- return config.build(taxoWriter, doc);
- }
-
- @Test
- public void testNoCommit() throws Exception {
- Directory indexDir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter indexWriter = new IndexWriter(indexDir, conf);
-
- Directory taxoDir = newDirectory();
- SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
- // should fail when there are no commits to snapshot
- expectThrows(
- IllegalStateException.class,
- () -> {
- new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
- });
-
- indexWriter.close();
- IOUtils.close(taxoWriter, taxoDir, indexDir);
- }
-
- @Test
- public void testRevisionRelease() throws Exception {
- Directory indexDir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter indexWriter = new IndexWriter(indexDir, conf);
-
- Directory taxoDir = newDirectory();
- SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
- try {
- indexWriter.addDocument(newDocument(taxoWriter));
- indexWriter.commit();
- taxoWriter.commit();
- Revision rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
- // releasing that revision should not delete the files
- rev1.release();
- assertTrue(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
- assertTrue(slowFileExists(taxoDir, IndexFileNames.SEGMENTS + "_1"));
-
- rev1 =
- new IndexAndTaxonomyRevision(
- indexWriter, taxoWriter); // create revision again, so the files are snapshotted
- indexWriter.addDocument(newDocument(taxoWriter));
- indexWriter.commit();
- taxoWriter.commit();
- assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
- rev1.release(); // this release should trigger the delete of segments_1
- assertFalse(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
- indexWriter.close();
- } finally {
- IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
- }
- }
-
- @Test
- public void testSegmentsFileLast() throws Exception {
- Directory indexDir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter indexWriter = new IndexWriter(indexDir, conf);
-
- Directory taxoDir = newDirectory();
- SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
- try {
- indexWriter.addDocument(newDocument(taxoWriter));
- indexWriter.commit();
- taxoWriter.commit();
- Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
- Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
- assertEquals(2, sourceFiles.size());
- for (List<RevisionFile> files : sourceFiles.values()) {
- String lastFile = files.get(files.size() - 1).fileName;
- assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
- }
- indexWriter.close();
- } finally {
- IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
- }
- }
-
- @Test
- public void testOpen() throws Exception {
- Directory indexDir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter indexWriter = new IndexWriter(indexDir, conf);
-
- Directory taxoDir = newDirectory();
- SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
- try {
- indexWriter.addDocument(newDocument(taxoWriter));
- indexWriter.commit();
- taxoWriter.commit();
- Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
- for (Entry<String, List<RevisionFile>> e : rev.getSourceFiles().entrySet()) {
- String source = e.getKey();
- @SuppressWarnings("resource") // silly, both directories are closed in the end
- Directory dir = source.equals(IndexAndTaxonomyRevision.INDEX_SOURCE) ? indexDir : taxoDir;
- for (RevisionFile file : e.getValue()) {
- IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
- InputStream in = rev.open(source, file.fileName);
- assertEquals(src.length(), in.available());
- byte[] srcBytes = new byte[(int) src.length()];
- byte[] inBytes = new byte[(int) src.length()];
- int offset = 0;
- if (random().nextBoolean()) {
- int skip = random().nextInt(10);
- if (skip >= src.length()) {
- skip = 0;
- }
- in.skip(skip);
- src.seek(skip);
- offset = skip;
- }
- src.readBytes(srcBytes, offset, srcBytes.length - offset);
- in.read(inBytes, offset, inBytes.length - offset);
- assertArrayEquals(srcBytes, inBytes);
- IOUtils.close(src, in);
- }
- }
- indexWriter.close();
- } finally {
- IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
- }
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexReplicationClient.java b/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexReplicationClient.java
deleted file mode 100644
index 0b3b1388ccd..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexReplicationClient.java
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
-import org.apache.lucene.replicator.ReplicationClient.SourceDirectoryFactory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.store.MockDirectoryWrapper;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.ThreadInterruptedException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestIndexReplicationClient extends ReplicatorTestCase {
-
- private static class IndexReadyCallback implements Callable<Boolean>, Closeable {
-
- private final Directory indexDir;
- private DirectoryReader reader;
- private long lastGeneration = -1;
-
- public IndexReadyCallback(Directory indexDir) throws IOException {
- this.indexDir = indexDir;
- if (DirectoryReader.indexExists(indexDir)) {
- reader = DirectoryReader.open(indexDir);
- lastGeneration = reader.getIndexCommit().getGeneration();
- }
- }
-
- @Override
- public Boolean call() throws Exception {
- if (reader == null) {
- reader = DirectoryReader.open(indexDir);
- lastGeneration = reader.getIndexCommit().getGeneration();
- } else {
- DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
- assertNotNull(
- "should not have reached here if no changes were made to the index", newReader);
- long newGeneration = newReader.getIndexCommit().getGeneration();
- assertTrue(
- "expected newer generation; current=" + lastGeneration + " new=" + newGeneration,
- newGeneration > lastGeneration);
- reader.close();
- reader = newReader;
- lastGeneration = newGeneration;
- TestUtil.checkIndex(indexDir);
- }
- return null;
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(reader);
- }
- }
-
- private MockDirectoryWrapper publishDir, handlerDir;
- private Replicator replicator;
- private SourceDirectoryFactory sourceDirFactory;
- private ReplicationClient client;
- private ReplicationHandler handler;
- private IndexWriter publishWriter;
- private IndexReadyCallback callback;
-
- private static final String VERSION_ID = "version";
-
- private void assertHandlerRevision(int expectedID, Directory dir) throws IOException {
- // loop as long as client is alive. test-framework will terminate us if
- // there's a serious bug, e.g. client doesn't really update. otherwise,
- // introducing timeouts is not good, can easily lead to false positives.
- while (client.isUpdateThreadAlive()) {
- // give client a chance to update
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- throw new ThreadInterruptedException(e);
- }
-
- try {
- DirectoryReader reader = DirectoryReader.open(dir);
- try {
- int handlerID =
- Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
- if (expectedID == handlerID) {
- return;
- } else if (VERBOSE) {
- System.out.println(
- "expectedID="
- + expectedID
- + " actual="
- + handlerID
- + " generation="
- + reader.getIndexCommit().getGeneration());
- }
- } finally {
- reader.close();
- }
- } catch (
- @SuppressWarnings("unused")
- Exception e) {
- // we can hit IndexNotFoundException or e.g. EOFException (on
- // segments_N) because it is being copied at the same time it is read by
- // DirectoryReader.open().
- }
- }
- }
-
- private Revision createRevision(final int id) throws IOException {
- publishWriter.addDocument(new Document());
- publishWriter.setLiveCommitData(
- new HashMap<String, String>() {
- {
- put(VERSION_ID, Integer.toString(id, 16));
- }
- }.entrySet());
- publishWriter.commit();
- return new IndexRevision(publishWriter);
- }
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- publishDir = newMockDirectory();
- handlerDir = newMockDirectory();
- sourceDirFactory = new PerSessionDirectoryFactory(createTempDir("replicationClientTest"));
- replicator = new LocalReplicator();
- callback = new IndexReadyCallback(handlerDir);
- handler = new IndexReplicationHandler(handlerDir, callback);
- client = new ReplicationClient(replicator, handler, sourceDirFactory);
-
- IndexWriterConfig conf = newIndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- publishWriter = new IndexWriter(publishDir, conf);
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- publishWriter.close();
- IOUtils.close(client, callback, replicator, publishDir, handlerDir);
- super.tearDown();
- }
-
- @Test
- public void testNoUpdateThread() throws Exception {
- assertNull("no version expected at start", handler.currentVersion());
-
- // Callback validates the replicated index
- replicator.publish(createRevision(1));
- client.updateNow();
-
- replicator.publish(createRevision(2));
- client.updateNow();
-
- // Publish two revisions without update, handler should be upgraded to latest
- replicator.publish(createRevision(3));
- replicator.publish(createRevision(4));
- client.updateNow();
- }
-
- @Test
- public void testUpdateThread() throws Exception {
- client.startUpdateThread(10, "index");
-
- replicator.publish(createRevision(1));
- assertHandlerRevision(1, handlerDir);
-
- replicator.publish(createRevision(2));
- assertHandlerRevision(2, handlerDir);
-
- // Publish two revisions without update, handler should be upgraded to latest
- replicator.publish(createRevision(3));
- replicator.publish(createRevision(4));
- assertHandlerRevision(4, handlerDir);
- }
-
- @Test
- public void testRestart() throws Exception {
- replicator.publish(createRevision(1));
- client.updateNow();
-
- replicator.publish(createRevision(2));
- client.updateNow();
-
- client.stopUpdateThread();
- client.close();
- client = new ReplicationClient(replicator, handler, sourceDirFactory);
-
- // Publish two revisions without update, handler should be upgraded to latest
- replicator.publish(createRevision(3));
- replicator.publish(createRevision(4));
- client.updateNow();
- }
-
- /*
- * This test verifies that the client and handler do not end up in a corrupt
- * index if exceptions are thrown at any point during replication. Either when
- * a client copies files from the server to the temporary space, or when the
- * handler copies them to the index directory.
- */
- @Test
- public void testConsistencyOnExceptions() throws Exception {
- // so the handler's index isn't empty
- replicator.publish(createRevision(1));
- client.updateNow();
- client.close();
- callback.close();
-
- // wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
- final SourceDirectoryFactory in = sourceDirFactory;
- final AtomicInteger failures = new AtomicInteger(atLeast(10));
- sourceDirFactory =
- new SourceDirectoryFactory() {
-
- private long clientMaxSize = 100, handlerMaxSize = 100;
- private double clientExRate = 1.0, handlerExRate = 1.0;
-
- @Override
- public void cleanupSession(String sessionID) throws IOException {
- in.cleanupSession(sessionID);
- }
-
- @SuppressWarnings("synthetic-access")
- @Override
- public Directory getDirectory(String sessionID, String source) throws IOException {
- Directory dir = in.getDirectory(sessionID, source);
- if (random().nextBoolean()
- && failures.get() > 0) { // client should fail, return wrapped dir
- MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
- mdw.setRandomIOExceptionRateOnOpen(clientExRate);
- mdw.setMaxSizeInBytes(clientMaxSize);
- mdw.setRandomIOExceptionRate(clientExRate);
- mdw.setCheckIndexOnClose(false);
- clientMaxSize *= 2;
- clientExRate /= 2;
- return mdw;
- }
-
- if (failures.get() > 0 && random().nextBoolean()) { // handler should fail
- handlerDir.setMaxSizeInBytes(handlerMaxSize);
- handlerDir.setRandomIOExceptionRateOnOpen(handlerExRate);
- handlerDir.setRandomIOExceptionRate(handlerExRate);
- handlerMaxSize *= 2;
- handlerExRate /= 2;
- } else {
- // disable errors
- handlerDir.setMaxSizeInBytes(0);
- handlerDir.setRandomIOExceptionRate(0.0);
- handlerDir.setRandomIOExceptionRateOnOpen(0.0);
- }
- return dir;
- }
- };
-
- handler =
- new IndexReplicationHandler(
- handlerDir,
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- if (random().nextDouble() < 0.2 && failures.get() > 0) {
- throw new RuntimeException("random exception from callback");
- }
- return null;
- }
- });
-
- // wrap handleUpdateException so we can act on the thrown exception
- client =
- new ReplicationClient(replicator, handler, sourceDirFactory) {
- @SuppressWarnings("synthetic-access")
- @Override
- protected void handleUpdateException(Throwable t) {
- if (t instanceof IOException) {
- if (VERBOSE) {
- System.out.println("hit exception during update: " + t);
- t.printStackTrace(System.out);
- }
- try {
- // test that the index can be read and also some basic statistics
- DirectoryReader reader = DirectoryReader.open(handlerDir.getDelegate());
- try {
- int numDocs = reader.numDocs();
- int version =
- Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
- assertEquals(numDocs, version);
- } finally {
- reader.close();
- }
- // verify index consistency
- TestUtil.checkIndex(handlerDir.getDelegate());
- } catch (IOException e) {
- // exceptions here are bad, don't ignore them
- throw new RuntimeException(e);
- } finally {
- // count-down number of failures
- failures.decrementAndGet();
- assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
- if (VERBOSE) {
- if (failures.get() == 0) {
- System.out.println("no more failures expected");
- } else {
- System.out.println("num failures left: " + failures.get());
- }
- }
- }
- } else {
- if (t instanceof RuntimeException) throw (RuntimeException) t;
- throw new RuntimeException(t);
- }
- }
- };
-
- client.startUpdateThread(10, "index");
-
- final Directory baseHandlerDir = handlerDir.getDelegate();
- int numRevisions = atLeast(20);
- for (int i = 2; i < numRevisions; i++) {
- replicator.publish(createRevision(i));
- assertHandlerRevision(i, baseHandlerDir);
- }
-
- // disable errors -- maybe randomness didn't exhaust all allowed failures,
- // and we don't want e.g. CheckIndex to hit false errors.
- handlerDir.setMaxSizeInBytes(0);
- handlerDir.setRandomIOExceptionRate(0.0);
- handlerDir.setRandomIOExceptionRateOnOpen(0.0);
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexRevision.java b/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexRevision.java
deleted file mode 100644
index 9d67e7eaf75..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexRevision.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.IOUtils;
-import org.junit.Test;
-
-public class TestIndexRevision extends ReplicatorTestCase {
-
- @Test
- public void testNoSnapshotDeletionPolicy() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
- IndexWriter writer = new IndexWriter(dir, conf);
- // should fail when IndexDeletionPolicy is not Snapshot
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- new IndexRevision(writer);
- });
-
- writer.close();
- IOUtils.close(dir);
- }
-
- @Test
- public void testNoCommit() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter writer = new IndexWriter(dir, conf);
- // should fail when there are no commits to snapshot"
- expectThrows(
- IllegalStateException.class,
- () -> {
- new IndexRevision(writer);
- });
-
- writer.close();
- IOUtils.close(dir);
- }
-
- @Test
- public void testRevisionRelease() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter writer = new IndexWriter(dir, conf);
- try {
- writer.addDocument(new Document());
- writer.commit();
- Revision rev1 = new IndexRevision(writer);
- // releasing that revision should not delete the files
- rev1.release();
- assertTrue(slowFileExists(dir, IndexFileNames.SEGMENTS + "_1"));
-
- rev1 = new IndexRevision(writer); // create revision again, so the files are snapshotted
- writer.addDocument(new Document());
- writer.commit();
- assertNotNull(new IndexRevision(writer));
- rev1.release(); // this release should trigger the delete of segments_1
- assertFalse(slowFileExists(dir, IndexFileNames.SEGMENTS + "_1"));
- } finally {
- IOUtils.close(writer, dir);
- }
- }
-
- @Test
- public void testSegmentsFileLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter writer = new IndexWriter(dir, conf);
- try {
- writer.addDocument(new Document());
- writer.commit();
- Revision rev = new IndexRevision(writer);
- @SuppressWarnings("unchecked")
- Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
- assertEquals(1, sourceFiles.size());
- List<RevisionFile> files = sourceFiles.values().iterator().next();
- String lastFile = files.get(files.size() - 1).fileName;
- assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
- writer.close();
- } finally {
- IOUtils.close(dir);
- }
- }
-
- @Test
- public void testOpen() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter writer = new IndexWriter(dir, conf);
- try {
- writer.addDocument(new Document());
- writer.commit();
- Revision rev = new IndexRevision(writer);
- @SuppressWarnings("unchecked")
- Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
- String source = sourceFiles.keySet().iterator().next();
- for (RevisionFile file : sourceFiles.values().iterator().next()) {
- IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
- InputStream in = rev.open(source, file.fileName);
- assertEquals(src.length(), in.available());
- byte[] srcBytes = new byte[(int) src.length()];
- byte[] inBytes = new byte[(int) src.length()];
- int offset = 0;
- if (random().nextBoolean()) {
- int skip = random().nextInt(10);
- if (skip >= src.length()) {
- skip = 0;
- }
- in.skip(skip);
- src.seek(skip);
- offset = skip;
- }
- src.readBytes(srcBytes, offset, srcBytes.length - offset);
- in.read(inBytes, offset, inBytes.length - offset);
- assertArrayEquals(srcBytes, inBytes);
- IOUtils.close(src, in);
- }
- writer.close();
- } finally {
- IOUtils.close(dir);
- }
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/TestLocalReplicator.java b/lucene/replicator/src/test/org/apache/lucene/replicator/TestLocalReplicator.java
deleted file mode 100644
index 042ff934e32..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/TestLocalReplicator.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.NoSuchFileException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map.Entry;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.IOUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestLocalReplicator extends ReplicatorTestCase {
-
- private static final String VERSION_ID = "version";
-
- private LocalReplicator replicator;
- private Directory sourceDir;
- private IndexWriter sourceWriter;
-
- @Before
- @Override
- public void setUp() throws Exception {
- super.setUp();
- sourceDir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- sourceWriter = new IndexWriter(sourceDir, conf);
- replicator = new LocalReplicator();
- }
-
- @After
- @Override
- public void tearDown() throws Exception {
- sourceWriter.close();
- IOUtils.close(replicator, sourceDir);
- super.tearDown();
- }
-
- private Revision createRevision(final int id) throws IOException {
- sourceWriter.addDocument(new Document());
- sourceWriter.setLiveCommitData(
- new HashMap<String, String>() {
- {
- put(VERSION_ID, Integer.toString(id, 16));
- }
- }.entrySet());
- sourceWriter.commit();
- return new IndexRevision(sourceWriter);
- }
-
- @Test
- public void testCheckForUpdateNoRevisions() throws Exception {
- assertNull(replicator.checkForUpdate(null));
- }
-
- @Test
- public void testObtainFileAlreadyClosed() throws IOException {
- replicator.publish(createRevision(1));
- SessionToken res = replicator.checkForUpdate(null);
- assertNotNull(res);
- assertEquals(1, res.sourceFiles.size());
- Entry<String, List<RevisionFile>> entry = res.sourceFiles.entrySet().iterator().next();
- replicator.close();
- expectThrows(
- AlreadyClosedException.class,
- () -> {
- replicator.obtainFile(res.id, entry.getKey(), entry.getValue().get(0).fileName);
- });
- }
-
- @Test
- public void testPublishAlreadyClosed() throws IOException {
- replicator.close();
- expectThrows(
- AlreadyClosedException.class,
- () -> {
- replicator.publish(createRevision(2));
- });
- }
-
- @Test
- public void testUpdateAlreadyClosed() throws IOException {
- replicator.close();
- expectThrows(
- AlreadyClosedException.class,
- () -> {
- replicator.checkForUpdate(null);
- });
- }
-
- @Test
- public void testPublishSameRevision() throws IOException {
- Revision rev = createRevision(1);
- replicator.publish(rev);
- SessionToken res = replicator.checkForUpdate(null);
- assertNotNull(res);
- assertEquals(rev.getVersion(), res.version);
- replicator.release(res.id);
- replicator.publish(new IndexRevision(sourceWriter));
- res = replicator.checkForUpdate(res.version);
- assertNull(res);
-
- // now make sure that publishing same revision doesn't leave revisions
- // "locked", i.e. that replicator releases revisions even when they are not
- // kept
- replicator.publish(createRevision(2));
- assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
- }
-
- @Test
- public void testPublishOlderRev() throws IOException {
- replicator.publish(createRevision(1));
- Revision old = new IndexRevision(sourceWriter);
- replicator.publish(createRevision(2));
- // should fail to publish an older revision
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- replicator.publish(old);
- });
- assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
- }
-
- @Test
- public void testObtainMissingFile() throws IOException {
- replicator.publish(createRevision(1));
- SessionToken res = replicator.checkForUpdate(null);
- expectThrowsAnyOf(
- Arrays.asList(FileNotFoundException.class, NoSuchFileException.class),
- () -> {
- replicator.obtainFile(res.id, res.sourceFiles.keySet().iterator().next(), "madeUpFile");
- });
- }
-
- @Test
- public void testSessionExpiration() throws IOException, InterruptedException {
- replicator.publish(createRevision(1));
- SessionToken session = replicator.checkForUpdate(null);
- replicator.setExpirationThreshold(5); // expire quickly
- Thread.sleep(50); // sufficient for expiration
- // should fail to obtain a file for an expired session
- expectThrows(
- SessionExpiredException.class,
- () -> {
- replicator.obtainFile(
- session.id,
- session.sourceFiles.keySet().iterator().next(),
- session.sourceFiles.values().iterator().next().get(0).fileName);
- });
- }
-
- @Test
- public void testUpdateToLatest() throws IOException {
- replicator.publish(createRevision(1));
- Revision rev = createRevision(2);
- replicator.publish(rev);
- SessionToken res = replicator.checkForUpdate(null);
- assertNotNull(res);
- assertEquals(0, rev.compareTo(res.version));
- }
-
- @Test
- public void testRevisionRelease() throws Exception {
- replicator.publish(createRevision(1));
- assertTrue(slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_1"));
- replicator.publish(createRevision(2));
- // now the files of revision 1 can be deleted
- assertTrue(slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_2"));
- assertFalse(
- "segments_1 should not be found in index directory after revision is released",
- slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_1"));
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/TestSessionToken.java b/lucene/replicator/src/test/org/apache/lucene/replicator/TestSessionToken.java
deleted file mode 100644
index f6cdb509f1a..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/TestSessionToken.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.List;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.IOUtils;
-import org.junit.Test;
-
-public class TestSessionToken extends ReplicatorTestCase {
-
- @Test
- public void testSerialization() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- IndexWriter writer = new IndexWriter(dir, conf);
- writer.addDocument(new Document());
- writer.commit();
- Revision rev = new IndexRevision(writer);
-
- SessionToken session1 = new SessionToken("17", rev);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- session1.serialize(new DataOutputStream(baos));
- byte[] b = baos.toByteArray();
- SessionToken session2 = new SessionToken(new DataInputStream(new ByteArrayInputStream(b)));
- assertEquals(session1.id, session2.id);
- assertEquals(session1.version, session2.version);
- assertEquals(1, session2.sourceFiles.size());
- assertEquals(session1.sourceFiles.size(), session2.sourceFiles.size());
- assertEquals(session1.sourceFiles.keySet(), session2.sourceFiles.keySet());
- List<RevisionFile> files1 = session1.sourceFiles.values().iterator().next();
- List<RevisionFile> files2 = session2.sourceFiles.values().iterator().next();
- assertEquals(files1, files2);
-
- writer.close();
- IOUtils.close(dir);
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/http/ReplicationServlet.java b/lucene/replicator/src/test/org/apache/lucene/replicator/http/ReplicationServlet.java
deleted file mode 100644
index 99c1de32e46..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/http/ReplicationServlet.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator.http;
-
-import java.io.IOException;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-public class ReplicationServlet extends HttpServlet {
-
- private final ReplicationService service;
- private boolean respondWithError = false;
-
- public ReplicationServlet(ReplicationService service) {
- this.service = service;
- }
-
- @Override
- protected void doGet(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
- if (respondWithError) {
- resp.sendError(500, "Fake error");
- } else {
- service.perform(req, resp);
- }
- }
-
- public void setRespondWithError(boolean respondWithError) {
- this.respondWithError = respondWithError;
- }
-}
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/http/TestHttpReplicator.java b/lucene/replicator/src/test/org/apache/lucene/replicator/http/TestHttpReplicator.java
deleted file mode 100644
index d5b3ab81bab..00000000000
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/http/TestHttpReplicator.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.replicator.http;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.Collections;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.replicator.IndexReplicationHandler;
-import org.apache.lucene.replicator.IndexRevision;
-import org.apache.lucene.replicator.LocalReplicator;
-import org.apache.lucene.replicator.PerSessionDirectoryFactory;
-import org.apache.lucene.replicator.ReplicationClient;
-import org.apache.lucene.replicator.Replicator;
-import org.apache.lucene.replicator.ReplicatorTestCase;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.IOUtils;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestHttpReplicator extends ReplicatorTestCase {
- private Path clientWorkDir;
- private Replicator serverReplicator;
- private IndexWriter writer;
- private DirectoryReader reader;
- private Server server;
- private int port;
- private String host;
- private Directory serverIndexDir, handlerIndexDir;
- private ReplicationServlet replicationServlet;
-
- private void startServer() throws Exception {
- ServletHandler replicationHandler = new ServletHandler();
- ReplicationService service =
- new ReplicationService(Collections.singletonMap("s1", serverReplicator));
- replicationServlet = new ReplicationServlet(service);
- ServletHolder servlet = new ServletHolder(replicationServlet);
- replicationHandler.addServletWithMapping(
- servlet, ReplicationService.REPLICATION_CONTEXT + "/*");
- server = newHttpServer(replicationHandler);
- port = serverPort(server);
- host = serverHost(server);
- }
-
- @Before
- @Override
- public void setUp() throws Exception {
- super.setUp();
- if (VERBOSE) {
- System.setProperty("org.eclipse.jetty.LEVEL", "DEBUG"); // sets stderr logging to DEBUG level
- }
- clientWorkDir = createTempDir("httpReplicatorTest");
- handlerIndexDir = newDirectory();
- serverIndexDir = newDirectory();
- serverReplicator = new LocalReplicator();
- startServer();
-
- IndexWriterConfig conf = newIndexWriterConfig(null);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
- writer = new IndexWriter(serverIndexDir, conf);
- reader = DirectoryReader.open(writer);
- }
-
- @Override
- public void tearDown() throws Exception {
- stopHttpServer(server);
- writer.rollback();
- IOUtils.close(reader, handlerIndexDir, serverIndexDir);
- super.tearDown();
- }
-
- private void publishRevision(int id) throws IOException {
- Document doc = new Document();
- writer.addDocument(doc);
- writer.setLiveCommitData(Collections.singletonMap("ID", Integer.toString(id, 16)).entrySet());
- writer.commit();
- serverReplicator.publish(new IndexRevision(writer));
- }
-
- private void reopenReader() throws IOException {
- DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
- assertNotNull(newReader);
- reader.close();
- reader = newReader;
- }
-
- @Test
- public void testBasic() throws Exception {
- Replicator replicator =
- new HttpReplicator(
- host,
- port,
- ReplicationService.REPLICATION_CONTEXT + "/s1",
- getClientConnectionManager());
- ReplicationClient client =
- new ReplicationClient(
- replicator,
- new IndexReplicationHandler(handlerIndexDir, null),
- new PerSessionDirectoryFactory(clientWorkDir));
-
- publishRevision(1);
- client.updateNow();
- reopenReader();
- assertEquals(1, Integer.parseInt(reader.getIndexCommit().getUserData().get("ID"), 16));
-
- publishRevision(2);
- client.updateNow();
- reopenReader();
- assertEquals(2, Integer.parseInt(reader.getIndexCommit().getUserData().get("ID"), 16));
-
- client.close();
- }
-
- @Test
- public void testServerErrors() throws Exception {
- // tests the behaviour of the client when the server sends an error
- // must use BasicClientConnectionManager to test whether the client is closed correctly
- BasicHttpClientConnectionManager conMgr = new BasicHttpClientConnectionManager();
- Replicator replicator =
- new HttpReplicator(host, port, ReplicationService.REPLICATION_CONTEXT + "/s1", conMgr);
- ReplicationClient client =
- new ReplicationClient(
- replicator,
- new IndexReplicationHandler(handlerIndexDir, null),
- new PerSessionDirectoryFactory(clientWorkDir));
-
- try {
- publishRevision(5);
-
- replicationServlet.setRespondWithError(true);
- expectThrows(Exception.class, client::updateNow);
-
- replicationServlet.setRespondWithError(false);
- client.updateNow(); // now it should work
- reopenReader();
- assertEquals(5, Integer.parseInt(reader.getIndexCommit().getUserData().get("ID"), 16));
-
- client.close();
- } finally {
- replicationServlet.setRespondWithError(false);
- }
- }
-}
diff --git a/versions.lock b/versions.lock
index 14035125f8a..ad6457e6b78 100644
--- a/versions.lock
+++ b/versions.lock
@@ -3,15 +3,11 @@ com.carrotsearch:hppc:0.9.1 (1 constraints: 0c050736)
com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1 (1 constraints: 0d050e36)
com.ibm.icu:icu4j:70.1 (1 constraints: dc040a31)
commons-codec:commons-codec:1.13 (1 constraints: d904f430)
-commons-logging:commons-logging:1.2 (1 constraints: c20f9771)
io.sgr:s2-geometry-library-java:1.0.0 (1 constraints: 0305f035)
-javax.servlet:javax.servlet-api:3.1.0 (2 constraints: 88129b22)
junit:junit:4.13.1 (1 constraints: 3b05453b)
net.sourceforge.nekohtml:nekohtml:1.9.17 (1 constraints: 4405503b)
org.antlr:antlr4-runtime:4.11.1 (1 constraints: 39053f3b)
org.apache.commons:commons-compress:1.19 (1 constraints: df04fa30)
-org.apache.httpcomponents:httpclient:4.5.13 (1 constraints: 3f054e3b)
-org.apache.httpcomponents:httpcore:4.4.13 (1 constraints: 591016a2)
org.apache.opennlp:opennlp-tools:1.9.1 (1 constraints: 0d050c36)
org.carrot2:morfologik-fsa:2.1.9 (1 constraints: db0d9c36)
org.carrot2:morfologik-polish:2.1.9 (1 constraints: 0e050136)
@@ -28,11 +24,4 @@ xerces:xercesImpl:2.12.0 (1 constraints: 3705353b)
[Test dependencies]
com.carrotsearch:procfork:1.0.6 (1 constraints: 0905f635)
org.assertj:assertj-core:3.21.0 (1 constraints: 38053c3b)
-org.eclipse.jetty:jetty-continuation:9.4.41.v20210516 (1 constraints: 7907fe7c)
-org.eclipse.jetty:jetty-http:9.4.41.v20210516 (1 constraints: f60f2ccd)
-org.eclipse.jetty:jetty-io:9.4.41.v20210516 (2 constraints: 141f4566)
-org.eclipse.jetty:jetty-server:9.4.41.v20210516 (1 constraints: 7907fe7c)
-org.eclipse.jetty:jetty-servlet:9.4.41.v20210516 (1 constraints: 7907fe7c)
-org.eclipse.jetty:jetty-util:9.4.41.v20210516 (3 constraints: 422efaf1)
-org.eclipse.jetty:jetty-util-ajax:9.4.41.v20210516 (1 constraints: 64101be2)
org.locationtech.jts:jts-core:1.17.0 (1 constraints: 3b053e3b)
diff --git a/versions.props b/versions.props
index 1f5341bae6e..519aa154fce 100644
--- a/versions.props
+++ b/versions.props
@@ -5,19 +5,16 @@ com.google.errorprone:*=2.15.0
com.ibm.icu:icu4j=70.1
commons-codec:commons-codec=1.13
io.sgr:s2-geometry-library-java=1.0.0
-javax.servlet:javax.servlet-api=3.1.0
junit:junit=4.13.1
net.sourceforge.nekohtml:nekohtml=1.9.17
org.antlr:antlr4*=4.11.1
org.apache.commons:commons-compress=1.19
-org.apache.httpcomponents:httpclient=4.5.13
org.apache.opennlp:opennlp-tools=1.9.1
org.assertj:*=3.21.0
org.carrot2:morfologik-*=2.1.9
-org.eclipse.jetty:*=9.4.41.v20210516
org.hamcrest:*=2.2
org.locationtech.jts:jts-core=1.17.0
org.locationtech.spatial4j:*=0.8
org.ow2.asm:*=7.2
ua.net.nlp:morfologik-ukrainian-search=4.9.1
-xerces:xercesImpl=2.12.0
\ No newline at end of file
+xerces:xercesImpl=2.12.0