You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/08/25 03:09:52 UTC
[15/15] lucene-solr:jira/http2: Merge with master
Merge with master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/56c44139
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/56c44139
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/56c44139
Branch: refs/heads/jira/http2
Commit: 56c44139c0590daeeaece2c0bc3a0e162f763bdd
Parents: 49d9b49 f26dd13
Author: Cao Manh Dat <da...@apache.org>
Authored: Sat Aug 25 10:09:11 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Sat Aug 25 10:09:11 2018 +0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../lucene/codecs/lucene70/Lucene70Codec.java | 133 ++++++
.../apache/lucene/codecs/lucene70/package.html | 25 ++
.../services/org.apache.lucene.codecs.Codec | 1 +
.../benchmark/byTask/tasks/CreateIndexTask.java | 4 +-
.../java/org/apache/lucene/codecs/Codec.java | 2 +-
.../lucene/codecs/lucene70/Lucene70Codec.java | 175 --------
.../lucene/codecs/lucene80/Lucene80Codec.java | 177 ++++++++
.../lucene/codecs/lucene80/package-info.java | 409 +++++++++++++++++++
.../org/apache/lucene/index/PendingDeletes.java | 13 +-
.../apache/lucene/index/PendingSoftDeletes.java | 7 +-
.../apache/lucene/index/ReadersAndUpdates.java | 3 +
.../services/org.apache.lucene.codecs.Codec | 2 +-
...cene50StoredFieldsFormatHighCompression.java | 8 +-
.../lucene70/TestLucene70NormsFormat.java | 4 +-
.../org/apache/lucene/index/Test2BPoints.java | 2 +-
.../lucene/index/TestPendingSoftDeletes.java | 8 +-
.../apache/lucene/index/TestPointValues.java | 8 +-
.../TestSoftDeletesRetentionMergePolicy.java | 94 ++++-
.../org/apache/lucene/search/TestBoolean2.java | 8 +-
.../apache/lucene/search/TestPointQueries.java | 4 +-
.../directory/DirectoryTaxonomyReader.java | 6 +-
.../document/TestFloatPointNearestNeighbor.java | 2 +-
.../org/apache/lucene/search/TestNearest.java | 2 +-
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 4 +-
.../suggest/document/TestSuggestField.java | 4 +-
.../apache/lucene/geo/BaseGeoPointTestCase.java | 2 +-
.../util/TestRuleSetupAndRestoreClassEnv.java | 30 +-
.../java/org/apache/lucene/util/TestUtil.java | 4 +-
.../src/groovy/check-source-patterns.groovy | 9 +-
solr/CHANGES.txt | 9 +
.../handler/dataimport/MailEntityProcessor.java | 74 ++--
.../handler/dataimport/BinURLDataSource.java | 10 +-
.../handler/dataimport/DataImportHandler.java | 6 +-
.../solr/handler/dataimport/DataImporter.java | 24 +-
.../dataimport/DateFormatTransformer.java | 4 +-
.../solr/handler/dataimport/DocBuilder.java | 34 +-
.../dataimport/FieldReaderDataSource.java | 6 +-
.../dataimport/FieldStreamDataSource.java | 4 +-
.../solr/handler/dataimport/FileDataSource.java | 8 +-
.../solr/handler/dataimport/JdbcDataSource.java | 20 +-
.../solr/handler/dataimport/LogTransformer.java | 22 +-
.../handler/dataimport/RegexTransformer.java | 4 +-
.../handler/dataimport/SolrEntityProcessor.java | 6 +-
.../handler/dataimport/SqlEntityProcessor.java | 8 +-
.../handler/dataimport/TemplateTransformer.java | 4 +-
.../solr/handler/dataimport/URLDataSource.java | 10 +-
.../dataimport/XPathEntityProcessor.java | 24 +-
.../handler/dataimport/XPathRecordReader.java | 6 +-
.../dataimport/config/DIHConfiguration.java | 6 +-
.../TestSolrEntityProcessorEndToEnd.java | 20 +-
.../client/solrj/embedded/JettySolrRunner.java | 6 +-
.../java/org/apache/solr/cloud/LockTree.java | 6 +-
.../apache/solr/cloud/OverseerTaskQueue.java | 14 +-
.../org/apache/solr/cloud/RecoveryStrategy.java | 164 ++++----
.../apache/solr/cloud/ReplicateFromLeader.java | 6 +-
.../org/apache/solr/cloud/SolrZkServer.java | 4 +-
.../apache/solr/cloud/ZkDistributedQueue.java | 6 +-
.../cloud/autoscaling/HttpTriggerListener.java | 4 +-
.../solr/cloud/autoscaling/LoggingListener.java | 4 +-
.../solr/cloud/autoscaling/TriggerBase.java | 8 +-
.../cloud/autoscaling/TriggerEventQueue.java | 16 +-
.../apache/solr/core/HdfsDirectoryFactory.java | 50 +--
.../solr/core/IndexDeletionPolicyWrapper.java | 4 +-
.../apache/solr/core/SchemaCodecFactory.java | 4 +-
.../repository/BackupRepositoryFactory.java | 6 +-
.../org/apache/solr/handler/IndexFetcher.java | 142 +++----
.../apache/solr/handler/ReplicationHandler.java | 68 +--
.../org/apache/solr/handler/SnapShooter.java | 16 +-
.../admin/AutoscalingHistoryHandler.java | 4 +-
.../solr/handler/component/QueryComponent.java | 4 +-
.../handler/component/SpellCheckComponent.java | 24 +-
.../handler/component/SuggestComponent.java | 26 +-
.../solr/handler/export/BoolFieldWriter.java | 23 +-
.../solr/handler/export/DateFieldWriter.java | 22 +-
.../solr/handler/export/DoubleFieldWriter.java | 27 +-
.../apache/solr/handler/export/DoubleValue.java | 20 +
.../solr/handler/export/DoubleValueSortDoc.java | 9 +
.../solr/handler/export/ExportWriter.java | 2 +-
.../apache/solr/handler/export/FieldWriter.java | 2 +-
.../solr/handler/export/FloatFieldWriter.java | 27 +-
.../apache/solr/handler/export/FloatValue.java | 20 +
.../solr/handler/export/IntFieldWriter.java | 22 +-
.../apache/solr/handler/export/IntValue.java | 25 +-
.../solr/handler/export/LongFieldWriter.java | 20 +-
.../apache/solr/handler/export/LongValue.java | 20 +
.../solr/handler/export/MultiFieldWriter.java | 6 +-
.../solr/handler/export/QuadValueSortDoc.java | 13 +
.../solr/handler/export/SingleValueSortDoc.java | 7 +
.../org/apache/solr/handler/export/SortDoc.java | 10 +
.../apache/solr/handler/export/SortValue.java | 8 +
.../solr/handler/export/StringFieldWriter.java | 24 +-
.../apache/solr/handler/export/StringValue.java | 24 +-
.../solr/handler/export/TripleValueSortDoc.java | 11 +
.../metrics/reporters/ReporterClientCache.java | 6 +-
.../reporters/jmx/JmxMetricsReporter.java | 58 +--
.../reporters/solr/SolrShardReporter.java | 2 +-
.../org/apache/solr/request/SimpleFacets.java | 4 +-
.../solr/response/BinaryResponseWriter.java | 4 +-
.../solr/schema/JsonPreAnalyzedParser.java | 12 +-
.../apache/solr/schema/PreAnalyzedField.java | 10 +-
.../solr/search/SurroundQParserPlugin.java | 2 +-
.../search/stats/ExactSharedStatsCache.java | 6 +-
.../solr/search/stats/ExactStatsCache.java | 28 +-
.../apache/solr/search/stats/LRUStatsCache.java | 10 +-
.../solr/search/stats/LocalStatsCache.java | 18 +-
.../org/apache/solr/search/stats/StatsUtil.java | 22 +-
.../solr/servlet/CheckLoggingConfiguration.java | 4 +-
.../solr/spelling/DirectSolrSpellChecker.java | 6 +-
.../solr/spelling/SpellCheckCollator.java | 10 +-
.../solr/spelling/suggest/SolrSuggester.java | 34 +-
.../apache/solr/spelling/suggest/Suggester.java | 26 +-
.../suggest/jaspell/JaspellLookupFactory.java | 4 +-
.../solr/store/blockcache/BlockDirectory.java | 8 +-
.../apache/solr/store/hdfs/HdfsDirectory.java | 16 +-
.../org/apache/solr/util/stats/MetricUtils.java | 6 +-
.../apache/solr/cloud/DeleteReplicaTest.java | 8 +-
.../solr/cloud/DistribCursorPagingTest.java | 1 +
.../solr/cloud/LIROnShardRestartTest.java | 4 +-
.../solr/cloud/LIRRollingUpdatesTest.java | 4 +-
.../solr/cloud/LeaderVoteWaitTimeoutTest.java | 6 +-
.../apache/solr/cloud/MoveReplicaHDFSTest.java | 3 +-
.../cloud/SharedFSAutoReplicaFailoverTest.java | 2 +-
.../apache/solr/cloud/TestCloudConsistency.java | 4 +-
.../org/apache/solr/cloud/TestPullReplica.java | 18 +-
.../cloud/TestPullReplicaErrorHandling.java | 12 +-
.../TestSolrCloudWithDelegationTokens.java | 1 +
.../org/apache/solr/cloud/TestTlogReplica.java | 12 +-
.../apache/solr/cloud/TestWithCollection.java | 2 +
.../HdfsCollectionsAPIDistributedZkTest.java | 3 +-
.../collections/TestHdfsCloudBackupRestore.java | 3 +-
.../HdfsAutoAddReplicasIntegrationTest.java | 2 +-
.../cloud/autoscaling/ScheduledTriggerTest.java | 2 +-
.../sim/GenericDistributedQueue.java | 6 +-
.../cloud/autoscaling/sim/SimCloudManager.java | 18 +-
.../sim/SimClusterStateProvider.java | 50 +--
.../autoscaling/sim/SimDistribStateManager.java | 2 +-
.../sim/SimDistributedQueueFactory.java | 4 +-
.../autoscaling/sim/SimNodeStateProvider.java | 12 +-
.../sim/TestClusterStateProvider.java | 4 +-
.../sim/TestDistribStateManager.java | 4 +-
.../solr/handler/TestReplicationHandler.java | 2 +
.../solr/handler/export/TestExportWriter.java | 49 +++
.../solr/schema/SchemaApiFailureTest.java | 1 +
.../search/CurrencyRangeFacetCloudTest.java | 2 +-
.../solr/search/facet/RangeFacetCloudTest.java | 2 +-
.../apache/solr/update/TestHdfsUpdateLog.java | 2 +
solr/solr-ref-guide/src/analytics.adoc | 2 +-
.../src/configuring-solrconfig-xml.adoc | 4 +-
.../detecting-languages-during-indexing.adoc | 2 +-
.../solr-ref-guide/src/filter-descriptions.adoc | 8 +-
solr/solr-ref-guide/src/language-analysis.adoc | 32 +-
solr/solr-ref-guide/src/learning-to-rank.adoc | 2 +-
.../src/lib-directives-in-solrconfig.adoc | 38 --
.../src/resource-and-plugin-loading.adoc | 86 ++++
.../src/stream-decorator-reference.adoc | 153 ++++---
solr/solr-ref-guide/src/tokenizers.adoc | 2 +-
.../src/update-request-processors.adoc | 2 +-
.../cloud/autoscaling/AutoScalingConfig.java | 4 +-
.../client/solrj/cloud/autoscaling/Policy.java | 8 +-
.../solr/client/solrj/impl/HttpClientUtil.java | 12 +-
.../solrj/impl/Krb5HttpClientBuilder.java | 20 +-
.../client/solrj/io/stream/CommitStream.java | 4 +-
.../client/solrj/io/stream/DaemonStream.java | 14 +-
.../client/solrj/io/stream/ExecutorStream.java | 8 +-
.../client/solrj/io/stream/UpdateStream.java | 6 +-
.../solr/common/cloud/ZkConfigManager.java | 4 +-
.../apache/solr/common/cloud/ZkStateReader.java | 140 +++----
.../org/apache/solr/common/util/IOUtils.java | 4 +-
.../solrj/io/stream/StreamDecoratorTest.java | 4 +-
.../cloud/TestCloudCollectionsListeners.java | 1 +
171 files changed, 2267 insertions(+), 1224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/56c44139/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/56c44139/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
index e48229d,afa2ef6..813c762
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
@@@ -64,10 -49,9 +64,10 @@@ import org.slf4j.LoggerFactory
*/
public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
- private static final String LOGIN_CONFIG_PROP = "java.security.auth.login.config";
+ public static final String LOGIN_CONFIG_PROP = "java.security.auth.login.config";
+ private static final String SPNEGO_OID = "1.3.6.1.5.5.2";
- private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
++
private static Configuration jaasConfig = new SolrJaasConfiguration();
public Krb5HttpClientBuilder() {
@@@ -109,10 -93,11 +109,10 @@@
// authentication mechanism can load the credentials from the JAAS configuration.
if (useSubjectCredsVal == null) {
System.setProperty(useSubjectCredsProp, "false");
- }
- else if (!useSubjectCredsVal.toLowerCase(Locale.ROOT).equals("false")) {
+ } else if (!useSubjectCredsVal.toLowerCase(Locale.ROOT).equals("false")) {
// Don't overwrite the prop value if it's already been written to something else,
// but log because it is likely the Credentials won't be loaded correctly.
- logger.warn("System Property: " + useSubjectCredsProp + " set to: " + useSubjectCredsVal
+ log.warn("System Property: " + useSubjectCredsProp + " set to: " + useSubjectCredsVal
+ " not false. SPNego authentication may not be successful.");
}
@@@ -152,72 -137,9 +152,72 @@@
return credentialsProvider;
});
HttpClientUtil.addRequestInterceptor(bufferedEntityInterceptor);
+
+ //setup for http2
+ builder.setHttp2Configurator(http2Client -> {
+ HttpAuthenticationStore authenticationStore = new HttpAuthenticationStore();
+ authenticationStore.addAuthentication(new Authentication() {
+ @Override
+ public boolean matches(String type, URI uri, String realm) {
+ return "Negotiate".equals(type);
+ }
+
+ @Override
+ public Result authenticate(Request request, ContentResponse response, HeaderInfo headerInfo, Attributes context) {
+ String challenge = headerInfo.getBase64();
+ if (challenge == null) challenge = "";
+ byte[] input = java.util.Base64.getDecoder().decode(challenge);
+ byte[] token;
+ String authServer = request.getHost();
+ final GSSManager manager = GSSManager.getInstance();
+ try {
+ GSSName serverName = manager.createName("HTTP@" + authServer, GSSName.NT_HOSTBASED_SERVICE);
+ final GSSContext gssContext = createGSSContext(manager, new Oid(SPNEGO_OID), serverName, null);
+ if (input != null) {
+ token = gssContext.initSecContext(input, 0, input.length);
+ } else {
+ token = gssContext.initSecContext(new byte[] {}, 0, 0);
+ }
+ } catch (GSSException e) {
+ throw new IllegalArgumentException("Unable to init GSSContext", e);
+ }
+ return new Result() {
+ AtomicBoolean sentToken = new AtomicBoolean(false);
+ @Override
+ public URI getURI() {
+ // Since Kerberos is connection based authentication, sub-sequence requests won't need to resend the token in header
+ // by return null, the ProtocolHandler won't try to apply this result on sequence requests
+ return null;
+ }
+
+ @Override
+ public void apply(Request request) {
+ if (sentToken.get()) return;
+
+ final String tokenstr = java.util.Base64.getEncoder().encodeToString(token);
- if (logger.isDebugEnabled()) {
- logger.info("Sending response '" + tokenstr + "' back to the auth server");
++ if (log.isDebugEnabled()) {
++ log.info("Sending response '" + tokenstr + "' back to the auth server");
+ }
+ request.header(headerInfo.getHeader().asString(), "Negotiate "+tokenstr);
+ }
+ };
+ }
+
+ private GSSContext createGSSContext(GSSManager manager, Oid oid, GSSName serverName, final GSSCredential gssCredential) throws GSSException {
+ // Get the credentials from the JAAS configuration rather than here
+ final GSSContext gssContext = manager.createContext(serverName.canonicalize(oid), oid, gssCredential,
+ GSSContext.DEFAULT_LIFETIME);
+ gssContext.requestMutualAuth(true);
+ return gssContext;
+ }
+ });
+ http2Client.getHttpClient().setAuthenticationStore(authenticationStore);
+ http2Client.getProtocolHandlers().put(new SolrWWWAuthenticationProtocolHandler(http2Client.getHttpClient()));
+ http2Client.getProtocolHandlers().put(new SolrProxyAuthenticationProtocolHandler(http2Client.getHttpClient()));
+ });
}
} else {
- logger.warn("{} is configured without specifying system property '{}'",
+ log.warn("{} is configured without specifying system property '{}'",
getClass().getName(), LOGIN_CONFIG_PROP);
}