You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2018/11/29 18:19:25 UTC
[16/16] lucene-solr:master: SOLR-12801: Make massive improvements to
the tests.
SOLR-12801: Make massive improvements to the tests.
SOLR-12804: Remove static modifier from Overseer queue access.
SOLR-12896: Introduce more checks for shutdown and closed to improve clean close and shutdown. (Partial)
SOLR-12897: Introduce AlreadyClosedException to clean up silly close / shutdown logging. (Partial)
SOLR-12898: Replace cluster state polling with ZkStateReader#waitFor. (Partial)
SOLR-12923: The new AutoScaling tests are way too flaky and need special attention. (Partial)
SOLR-12932: ant test (without badapples=false) should pass easily for developers. (Partial)
SOLR-12933: Fix SolrCloud distributed commit.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/75b18319
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/75b18319
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/75b18319
Branch: refs/heads/master
Commit: 75b183196798232aa6f2dcaaaab117f309119053
Parents: 81c092d
Author: markrmiller <ma...@apache.org>
Authored: Thu Nov 29 11:58:18 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Thu Nov 29 11:58:51 2018 -0600
----------------------------------------------------------------------
lucene/tools/junit4/solr-tests.policy | 4 +
solr/CHANGES.txt | 26 +-
.../LegacyAbstractAnalyticsCloudTest.java | 23 +-
.../legacy/LegacyNoFacetCloudTest.java | 14 +-
.../legacy/facet/LegacyFieldFacetCloudTest.java | 7 +-
.../facet/LegacyFieldFacetExtrasCloudTest.java | 7 +-
.../legacy/facet/LegacyQueryFacetCloudTest.java | 7 +-
.../legacy/facet/LegacyRangeFacetCloudTest.java | 7 +-
.../dataimport/TestContentStreamDataSource.java | 4 +-
.../TestSolrEntityProcessorEndToEnd.java | 4 +-
.../dataimport/TestZKPropertiesWriter.java | 3 +-
.../org/apache/solr/ltr/LTRThreadModule.java | 47 +-
.../LTRFeatureLoggerTransformerFactory.java | 5 +-
.../solr/ltr/search/LTRQParserPlugin.java | 4 +-
.../org/apache/solr/ltr/TestLTROnSolrCloud.java | 3 +-
.../solr/client/solrj/embedded/JettyConfig.java | 14 +-
.../client/solrj/embedded/JettySolrRunner.java | 276 +++++--
.../org/apache/solr/cloud/ElectionContext.java | 75 +-
.../org/apache/solr/cloud/LeaderElector.java | 3 +
.../java/org/apache/solr/cloud/Overseer.java | 88 ++-
.../OverseerCollectionConfigSetProcessor.java | 18 +-
.../solr/cloud/OverseerNodePrioritizer.java | 14 +-
.../solr/cloud/OverseerTaskProcessor.java | 48 +-
.../org/apache/solr/cloud/RecoveryStrategy.java | 188 +++--
.../apache/solr/cloud/ReplicateFromLeader.java | 8 +-
.../org/apache/solr/cloud/SyncStrategy.java | 29 +-
.../org/apache/solr/cloud/ZkController.java | 151 ++--
.../apache/solr/cloud/ZkDistributedQueue.java | 9 +-
.../org/apache/solr/cloud/ZkShardTerms.java | 33 +-
.../cloud/api/collections/AddReplicaCmd.java | 3 +-
.../solr/cloud/api/collections/Assign.java | 23 +-
.../solr/cloud/api/collections/BackupCmd.java | 2 +-
.../api/collections/CreateCollectionCmd.java | 10 +-
.../cloud/api/collections/CreateShardCmd.java | 3 +-
.../api/collections/CreateSnapshotCmd.java | 2 +-
.../api/collections/DeleteCollectionCmd.java | 35 +-
.../cloud/api/collections/DeleteReplicaCmd.java | 4 +-
.../cloud/api/collections/DeleteShardCmd.java | 43 +-
.../api/collections/DeleteSnapshotCmd.java | 2 +-
.../solr/cloud/api/collections/MigrateCmd.java | 5 +-
.../OverseerCollectionMessageHandler.java | 138 ++--
.../solr/cloud/api/collections/RestoreCmd.java | 37 +-
.../cloud/api/collections/SplitShardCmd.java | 26 +-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 12 +-
.../autoscaling/OverseerTriggerThread.java | 13 +-
.../cloud/autoscaling/ScheduledTrigger.java | 4 +-
.../cloud/autoscaling/ScheduledTriggers.java | 49 +-
.../solr/cloud/autoscaling/TriggerBase.java | 7 +-
.../cloud/autoscaling/TriggerEventQueue.java | 7 +-
.../java/org/apache/solr/core/CloudConfig.java | 4 +-
.../org/apache/solr/core/CoreContainer.java | 220 ++++--
.../src/java/org/apache/solr/core/SolrCore.java | 72 +-
.../core/TransientSolrCoreCacheFactory.java | 2 +-
.../TransientSolrCoreCacheFactoryDefault.java | 2 +-
.../java/org/apache/solr/core/ZkContainer.java | 16 +-
.../solr/handler/CdcrReplicatorManager.java | 1 +
.../org/apache/solr/handler/IndexFetcher.java | 2 +-
.../apache/solr/handler/ReplicationHandler.java | 4 +-
.../admin/AutoscalingHistoryHandler.java | 2 +-
.../solr/handler/admin/CollectionsHandler.java | 93 ++-
.../solr/handler/admin/CoreAdminHandler.java | 2 +-
.../handler/admin/MetricsHistoryHandler.java | 12 +-
.../solr/handler/admin/PrepRecoveryOp.java | 202 +++--
.../component/IterativeMergeStrategy.java | 47 +-
.../solr/handler/loader/JavabinLoader.java | 8 -
.../org/apache/solr/handler/sql/SolrSchema.java | 4 +-
.../org/apache/solr/request/SimpleFacets.java | 13 +-
.../apache/solr/request/SolrRequestInfo.java | 2 +-
.../solr/security/PKIAuthenticationPlugin.java | 2 +-
.../org/apache/solr/servlet/HttpSolrCall.java | 5 +-
.../apache/solr/servlet/SolrDispatchFilter.java | 56 +-
.../org/apache/solr/update/CommitTracker.java | 2 +-
.../solr/update/DirectUpdateHandler2.java | 109 +--
.../org/apache/solr/update/SolrCoreState.java | 8 +-
.../java/org/apache/solr/update/UpdateLog.java | 16 +-
.../apache/solr/update/UpdateShardHandler.java | 21 +-
.../processor/DistributedUpdateProcessor.java | 154 +++-
.../src/java/org/apache/solr/util/SolrCLI.java | 2 +-
.../org/apache/solr/util/TestInjection.java | 91 ++-
.../src/java/org/apache/solr/util/TimeOut.java | 7 +-
.../src/test-files/solr/solr-jmxreporter.xml | 1 +
solr/core/src/test-files/solr/solr.xml | 12 +-
.../org/apache/solr/TestDistributedSearch.java | 193 +++--
.../apache/solr/TestHighlightDedupGrouping.java | 12 +-
.../org/apache/solr/TestTolerantSearch.java | 2 +-
.../org/apache/solr/cloud/AddReplicaTest.java | 26 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 47 +-
.../cloud/AssignBackwardCompatibilityTest.java | 3 +-
.../AsyncCallRequestStatusResponseTest.java | 2 +-
.../solr/cloud/BasicDistributedZk2Test.java | 6 +-
.../solr/cloud/BasicDistributedZkTest.java | 162 +++-
.../test/org/apache/solr/cloud/BasicZkTest.java | 2 +-
.../cloud/ChaosMonkeyNothingIsSafeTest.java | 58 +-
...MonkeyNothingIsSafeWithPullReplicasTest.java | 52 +-
.../solr/cloud/ChaosMonkeySafeLeaderTest.java | 6 +-
...aosMonkeySafeLeaderWithPullReplicasTest.java | 13 +-
.../solr/cloud/ChaosMonkeyShardSplitTest.java | 15 +-
.../apache/solr/cloud/CleanupOldIndexTest.java | 4 +-
.../org/apache/solr/cloud/CloudTestUtils.java | 8 +-
.../solr/cloud/ClusterStateUpdateTest.java | 6 +-
.../solr/cloud/CollectionStateFormat2Test.java | 16 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 74 +-
.../solr/cloud/ConnectionManagerTest.java | 23 +-
.../solr/cloud/CreateRoutedAliasTest.java | 20 +-
.../org/apache/solr/cloud/DeleteNodeTest.java | 1 -
.../apache/solr/cloud/DeleteReplicaTest.java | 89 ++-
.../org/apache/solr/cloud/DeleteShardTest.java | 25 +-
.../solr/cloud/DocValuesNotIndexedTest.java | 37 +-
.../org/apache/solr/cloud/ForceLeaderTest.java | 34 +-
.../FullThrottleStoppableIndexingThread.java | 18 +-
.../solr/cloud/HttpPartitionOnCommitTest.java | 14 +-
.../apache/solr/cloud/HttpPartitionTest.java | 43 +-
.../apache/solr/cloud/KerberosTestServices.java | 48 +-
.../apache/solr/cloud/LeaderElectionTest.java | 32 +-
.../cloud/LeaderFailoverAfterPartitionTest.java | 1 +
.../cloud/LeaderFailureAfterFreshStartTest.java | 4 +-
.../solr/cloud/LeaderTragicEventTest.java | 9 +-
.../solr/cloud/LeaderVoteWaitTimeoutTest.java | 82 +-
.../solr/cloud/LegacyCloudClusterPropTest.java | 15 +
.../cloud/MetricsHistoryIntegrationTest.java | 11 +-
.../solr/cloud/MockSimpleZkController.java | 36 +
.../org/apache/solr/cloud/MockSolrSource.java | 48 ++
.../org/apache/solr/cloud/MockZkController.java | 36 -
.../solr/cloud/MoveReplicaHDFSFailoverTest.java | 2 +
.../apache/solr/cloud/MoveReplicaHDFSTest.java | 4 +-
.../org/apache/solr/cloud/MoveReplicaTest.java | 32 +-
.../apache/solr/cloud/MultiThreadedOCPTest.java | 8 +-
...rriddenZkACLAndCredentialsProvidersTest.java | 2 +-
...verseerCollectionConfigSetProcessorTest.java | 245 ++++--
.../apache/solr/cloud/OverseerRolesTest.java | 10 +-
.../org/apache/solr/cloud/OverseerTest.java | 786 ++++++++++---------
.../solr/cloud/PeerSyncReplicationTest.java | 8 +-
.../solr/cloud/RecoveryAfterSoftCommitTest.java | 3 +-
.../solr/cloud/ReplaceNodeNoTargetTest.java | 2 +-
.../org/apache/solr/cloud/ReplaceNodeTest.java | 18 +-
.../solr/cloud/ReplicationFactorTest.java | 3 +-
.../solr/cloud/RestartWhileUpdatingTest.java | 2 -
.../apache/solr/cloud/RollingRestartTest.java | 2 +-
.../org/apache/solr/cloud/SSLMigrationTest.java | 3 +-
.../solr/cloud/SaslZkACLProviderTest.java | 16 +-
.../solr/cloud/ShardRoutingCustomTest.java | 17 +-
.../cloud/SharedFSAutoReplicaFailoverTest.java | 12 +-
.../org/apache/solr/cloud/SolrXmlInZkTest.java | 3 +-
.../org/apache/solr/cloud/SplitShardTest.java | 5 +-
.../org/apache/solr/cloud/SyncSliceTest.java | 6 +-
.../solr/cloud/TestAuthenticationFramework.java | 12 +-
.../apache/solr/cloud/TestCloudConsistency.java | 54 +-
.../solr/cloud/TestCloudDeleteByQuery.java | 3 +-
.../apache/solr/cloud/TestCloudRecovery.java | 63 +-
.../solr/cloud/TestCloudSearcherWarming.java | 55 +-
.../cloud/TestDeleteCollectionOnDownNodes.java | 36 +-
.../apache/solr/cloud/TestDistributedMap.java | 4 +-
.../solr/cloud/TestDownShardTolerantSearch.java | 5 +-
.../TestLeaderElectionWithEmptyReplica.java | 7 +-
.../solr/cloud/TestLeaderElectionZkExpiry.java | 2 -
.../solr/cloud/TestMiniSolrCloudClusterSSL.java | 2 +
.../org/apache/solr/cloud/TestPrepRecovery.java | 20 +-
.../org/apache/solr/cloud/TestPullReplica.java | 29 +-
.../cloud/TestPullReplicaErrorHandling.java | 11 +-
.../apache/solr/cloud/TestRandomFlRTGCloud.java | 4 +-
.../cloud/TestRandomRequestDistribution.java | 2 +-
.../solr/cloud/TestRequestForwarding.java | 2 +-
.../apache/solr/cloud/TestSegmentSorting.java | 2 +-
.../solr/cloud/TestSkipOverseerOperations.java | 94 ++-
.../TestSolrCloudWithDelegationTokens.java | 1 +
.../cloud/TestSolrCloudWithKerberosAlt.java | 3 +-
.../TestSolrCloudWithSecureImpersonation.java | 5 +-
.../TestStressCloudBlindAtomicUpdates.java | 4 +-
.../solr/cloud/TestStressInPlaceUpdates.java | 10 +-
.../apache/solr/cloud/TestStressLiveNodes.java | 3 -
.../org/apache/solr/cloud/TestTlogReplica.java | 46 +-
.../cloud/TestTolerantUpdateProcessorCloud.java | 40 +-
.../TestTolerantUpdateProcessorRandomCloud.java | 8 +-
.../org/apache/solr/cloud/TestUtilizeNode.java | 1 -
.../apache/solr/cloud/TestWithCollection.java | 48 +-
.../TlogReplayBufferedWhileIndexingTest.java | 6 +-
...MParamsZkACLAndCredentialsProvidersTest.java | 7 +-
.../org/apache/solr/cloud/ZkControllerTest.java | 16 +-
.../org/apache/solr/cloud/ZkFailoverTest.java | 16 +-
.../org/apache/solr/cloud/ZkSolrClientTest.java | 63 +-
.../solr/cloud/api/collections/AssignTest.java | 26 +-
.../CollectionsAPIAsyncDistributedZkTest.java | 69 +-
.../CollectionsAPIDistributedZkTest.java | 96 ++-
.../HdfsCollectionsAPIDistributedZkTest.java | 3 +-
.../cloud/api/collections/ShardSplitTest.java | 54 +-
.../SimpleCollectionCreateDeleteTest.java | 35 +
.../TestCollectionsAPIViaSolrCloudCluster.java | 20 +-
.../collections/TestHdfsCloudBackupRestore.java | 5 +-
.../TestLocalFSCloudBackupRestore.java | 12 +-
.../AutoAddReplicasIntegrationTest.java | 48 +-
.../AutoAddReplicasPlanActionTest.java | 103 ++-
.../autoscaling/ComputePlanActionTest.java | 45 +-
.../autoscaling/ExecutePlanActionTest.java | 40 +-
.../HdfsAutoAddReplicasIntegrationTest.java | 2 +
.../autoscaling/HttpTriggerListenerTest.java | 1 +
.../cloud/autoscaling/IndexSizeTriggerTest.java | 60 +-
.../MetricTriggerIntegrationTest.java | 4 +-
.../cloud/autoscaling/MetricTriggerTest.java | 1 +
.../NodeAddedTriggerIntegrationTest.java | 51 +-
.../cloud/autoscaling/NodeAddedTriggerTest.java | 17 +-
.../NodeLostTriggerIntegrationTest.java | 62 +-
.../cloud/autoscaling/NodeLostTriggerTest.java | 58 +-
.../NodeMarkersRegistrationTest.java | 66 +-
.../autoscaling/RestoreTriggerStateTest.java | 4 +-
.../ScheduledMaintenanceTriggerTest.java | 2 +-
.../ScheduledTriggerIntegrationTest.java | 23 +-
.../cloud/autoscaling/ScheduledTriggerTest.java | 9 +-
.../SearchRateTriggerIntegrationTest.java | 2 +
.../autoscaling/SearchRateTriggerTest.java | 31 +-
.../autoscaling/SystemLogListenerTest.java | 60 +-
.../solr/cloud/autoscaling/TestPolicyCloud.java | 12 +-
.../TriggerCooldownIntegrationTest.java | 1 +
.../autoscaling/TriggerIntegrationTest.java | 17 +-
.../cloud/autoscaling/sim/LiveNodesSet.java | 4 +
.../sim/SimClusterStateProvider.java | 48 +-
.../autoscaling/sim/SimSolrCloudTestCase.java | 33 +-
.../sim/TestSimClusterStateProvider.java | 4 +-
.../sim/TestSimComputePlanAction.java | 20 +-
.../sim/TestSimExecutePlanAction.java | 3 +-
.../autoscaling/sim/TestSimExtremeIndexing.java | 4 +-
.../autoscaling/sim/TestSimLargeCluster.java | 51 +-
.../autoscaling/sim/TestSimPolicyCloud.java | 12 +-
.../sim/TestSimTriggerIntegration.java | 117 +--
.../cloud/cdcr/BaseCdcrDistributedZkTest.java | 8 +-
.../solr/cloud/cdcr/CdcrBidirectionalTest.java | 2 -
.../solr/cloud/cdcr/CdcrBootstrapTest.java | 18 +-
.../cloud/cdcr/CdcrOpsAndBoundariesTest.java | 5 +-
.../cloud/cdcr/CdcrReplicationHandlerTest.java | 11 +-
.../apache/solr/cloud/cdcr/CdcrTestsUtil.java | 5 +-
.../cloud/cdcr/CdcrWithNodesRestartsTest.java | 13 +-
.../hdfs/HdfsChaosMonkeyNothingIsSafeTest.java | 7 +-
.../apache/solr/cloud/hdfs/StressHdfsTest.java | 7 +-
.../solr/cloud/overseer/ZkStateReaderTest.java | 9 -
.../solr/cloud/overseer/ZkStateWriterTest.java | 11 -
.../test/org/apache/solr/core/SolrCoreTest.java | 3 +
.../org/apache/solr/core/TestDynamicURP.java | 9 +-
.../apache/solr/core/TestSolrConfigHandler.java | 82 +-
.../core/snapshots/TestSolrCloudSnapshots.java | 1 +
.../core/snapshots/TestSolrCoreSnapshots.java | 1 -
.../solr/handler/TestHdfsBackupRestoreCore.java | 2 +-
.../solr/handler/TestReplicationHandler.java | 52 +-
.../handler/TestReplicationHandlerBackup.java | 4 +-
.../apache/solr/handler/TestReqParamsAPI.java | 1 +
.../apache/solr/handler/TestRestoreCore.java | 4 +-
.../solr/handler/TestSQLHandlerNonCloud.java | 2 +-
.../handler/TestSolrConfigHandlerCloud.java | 26 +-
.../solr/handler/V2ApiIntegrationTest.java | 1 +
.../admin/AutoscalingHistoryHandlerTest.java | 4 +-
.../admin/MetricsHistoryHandlerTest.java | 17 +-
.../admin/ShowFileRequestHandlerTest.java | 2 +-
.../admin/ZookeeperStatusHandlerTest.java | 2 +-
.../DistributedDebugComponentTest.java | 2 +-
.../DistributedFacetExistsSmallTest.java | 5 +-
.../org/apache/solr/metrics/JvmMetricsTest.java | 2 +-
.../reporters/solr/SolrCloudReportersTest.java | 26 +-
.../solr/request/TestRemoteStreaming.java | 2 +-
.../solr/rest/TestManagedResourceStorage.java | 4 +-
.../org/apache/solr/schema/TestBinaryField.java | 2 +-
.../solr/schema/TestBulkSchemaConcurrent.java | 3 +-
.../schema/TestManagedSchemaThreadSafety.java | 3 +-
.../solr/search/AnalyticsMergeStrategyTest.java | 2 -
.../org/apache/solr/search/TestRecovery.java | 40 +-
.../apache/solr/search/TestSolr4Spatial2.java | 2 +-
.../apache/solr/search/TestStressRecovery.java | 51 +-
.../search/join/BlockJoinFacetDistribTest.java | 2 +
.../solr/search/mlt/CloudMLTQParserTest.java | 51 +-
.../solr/search/stats/TestDistribIDF.java | 6 +-
.../solr/security/BasicAuthIntegrationTest.java | 8 +
.../solr/security/BasicAuthStandaloneTest.java | 4 +-
.../security/TestPKIAuthenticationPlugin.java | 5 +-
.../hadoop/TestDelegationWithHadoopAuth.java | 1 +
.../apache/solr/servlet/CacheHeaderTest.java | 2 +-
.../apache/solr/servlet/NoCacheHeaderTest.java | 2 +-
.../apache/solr/servlet/ResponseHeaderTest.java | 2 +-
.../solr/store/hdfs/HdfsLockFactoryTest.java | 2 +
.../apache/solr/update/SoftAutoCommitTest.java | 2 +-
.../solr/update/SolrCmdDistributorTest.java | 3 +-
.../apache/solr/update/TestHdfsUpdateLog.java | 2 -
.../solr/update/TestInPlaceUpdatesDistrib.java | 72 +-
.../TimeRoutedAliasUpdateProcessorTest.java | 29 +-
.../apache/solr/util/TestSolrCLIRunExample.java | 12 +-
.../solr/client/solrj/cloud/SocketProxy.java | 460 +++++++++++
.../solr/client/solrj/impl/CloudSolrClient.java | 29 +-
.../solr/client/solrj/impl/HttpClientUtil.java | 2 +-
.../client/solrj/impl/LBHttpSolrClient.java | 25 +-
.../client/solrj/impl/SolrClientBuilder.java | 4 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 67 +-
.../impl/ZkClientClusterStateProvider.java | 12 +-
.../solrj/impl/ZkDistribStateManager.java | 28 +-
.../solr/client/solrj/io/SolrClientCache.java | 2 +-
.../client/solrj/io/stream/FacetStream.java | 2 +-
.../client/solrj/io/stream/RandomStream.java | 2 +-
.../solr/common/AlreadyClosedException.java | 40 +
.../solr/common/cloud/ConnectionManager.java | 30 +-
.../common/cloud/DefaultConnectionStrategy.java | 3 +
.../apache/solr/common/cloud/DocCollection.java | 14 +-
.../solr/common/cloud/LiveNodesListener.java | 4 +-
.../solr/common/cloud/LiveNodesPredicate.java | 31 +
.../solr/common/cloud/LiveNodesWatcher.java | 26 +
.../org/apache/solr/common/cloud/Replica.java | 2 +-
.../apache/solr/common/cloud/SolrZkClient.java | 119 +--
.../apache/solr/common/cloud/SolrZooKeeper.java | 3 -
.../apache/solr/common/cloud/ZkCmdExecutor.java | 17 +-
.../apache/solr/common/cloud/ZkStateReader.java | 137 +++-
.../UsingSolrJRefGuideExamplesTest.java | 1 +
.../client/solrj/SolrExampleBinaryTest.java | 2 +-
.../solr/client/solrj/SolrExampleXMLTest.java | 2 +-
.../client/solrj/SolrSchemalessExampleTest.java | 2 +-
.../solr/client/solrj/TestBatchUpdate.java | 2 +-
.../solr/client/solrj/TestLBHttpSolrClient.java | 143 ++--
.../client/solrj/TestSolrJErrorHandling.java | 2 +-
.../embedded/LargeVolumeBinaryJettyTest.java | 2 +-
.../solrj/embedded/LargeVolumeJettyTest.java | 2 +-
.../solrj/embedded/SolrExampleJettyTest.java | 2 +-
.../embedded/SolrExampleStreamingTest.java | 2 +-
.../solrj/impl/BasicHttpSolrClientTest.java | 2 +-
.../client/solrj/impl/CloudSolrClientTest.java | 104 ++-
.../ConcurrentUpdateSolrClientBadInputTest.java | 2 +-
.../impl/ConcurrentUpdateSolrClientTest.java | 2 +-
.../solrj/impl/HttpSolrClientBadInputTest.java | 2 +-
.../solrj/impl/HttpSolrClientConPoolTest.java | 4 +-
.../impl/LBHttpSolrClientBadInputTest.java | 2 +-
.../impl/TestCloudSolrClientConnections.java | 2 +
.../solr/client/solrj/io/graph/GraphTest.java | 5 +-
.../solr/client/solrj/io/sql/JdbcTest.java | 3 +
.../solrj/io/stream/MathExpressionTest.java | 81 +-
.../io/stream/SelectWithEvaluatorsTest.java | 2 +-
.../solrj/io/stream/StreamDecoratorTest.java | 62 +-
.../solrj/io/stream/StreamExpressionTest.java | 10 +-
.../client/solrj/io/stream/StreamingTest.java | 4 +-
.../client/solrj/request/TestV2Request.java | 16 +-
.../solrj/response/NoOpResponseParserTest.java | 2 +-
.../cloud/TestCloudCollectionsListeners.java | 38 +-
.../cloud/TestCollectionStateWatchers.java | 24 +-
.../solr/common/cloud/TestZkConfigManager.java | 2 +-
.../solr/BaseDistributedSearchTestCase.java | 127 ++-
.../apache/solr/SolrIgnoredThreadsFilter.java | 24 +-
.../java/org/apache/solr/SolrJettyTestBase.java | 18 +-
.../java/org/apache/solr/SolrTestCaseJ4.java | 90 ++-
.../solr/cloud/AbstractDistribZkTestBase.java | 203 +++--
.../cloud/AbstractFullDistribZkTestBase.java | 421 ++++++----
.../apache/solr/cloud/AbstractZkTestCase.java | 137 +---
.../java/org/apache/solr/cloud/ChaosMonkey.java | 167 +---
.../apache/solr/cloud/MiniSolrCloudCluster.java | 266 ++++++-
.../java/org/apache/solr/cloud/SocketProxy.java | 460 -----------
.../apache/solr/cloud/SolrCloudTestCase.java | 73 +-
.../org/apache/solr/cloud/ZkTestServer.java | 383 ++++++---
.../component/TrackingShardHandlerFactory.java | 21 +-
.../apache/solr/util/BadHdfsThreadsFilter.java | 7 +
.../java/org/apache/solr/util/RestTestBase.java | 2 +-
.../java/org/apache/solr/util/TestHarness.java | 7 -
351 files changed, 7322 insertions(+), 4611 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/lucene/tools/junit4/solr-tests.policy
----------------------------------------------------------------------
diff --git a/lucene/tools/junit4/solr-tests.policy b/lucene/tools/junit4/solr-tests.policy
index 1c46a78..3333e95 100644
--- a/lucene/tools/junit4/solr-tests.policy
+++ b/lucene/tools/junit4/solr-tests.policy
@@ -90,5 +90,9 @@ grant {
permission javax.security.auth.kerberos.ServicePermission "HTTP/127.0.0.1@EXAMPLE.COM", "initiate";
permission javax.security.auth.kerberos.ServicePermission "HTTP/127.0.0.1@EXAMPLE.COM", "accept";
permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/127.0.0.1@EXAMPLE.COM\" \"krbtgt/EXAMPLE.COM@EXAMPLE.COM\"";
+
+ // java 8 accessibility requires this perm - should not after 8 I believe (rrd4j is the root reason we hit an accessibility code path)
+ permission java.awt.AWTPermission "listenToAllAWTEvents";
+ permission java.awt.AWTPermission "accessEventQueue";
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a5d1dc2..9cb681f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -131,15 +131,14 @@ New Features
----------------------
(No Changes)
-Other Changes
-----------------------
-
-* SOLR-12972: deprecate unused SolrIndexConfig.luceneVersion (Christine Poerschke)
Bug Fixes
----------------------
+
* SOLR-12546: CVSResponseWriter omits useDocValuesAsStored=true field when fl=*
(Munendra S N via Mikhail Khludnev)
+
+* SOLR-12933: Fix SolrCloud distributed commit. (Mark Miller)
Improvements
----------------------
@@ -149,6 +148,25 @@ Improvements
* SOLR-12992: When using binary format, ExportWriter to directly copy BytesRef instead of
creating new String (noble)
+* SOLR-12898: Replace cluster state polling with ZkStateReader#waitFor. (Mark Miller)
+
+* SOLR-12897: Introduce AlreadyClosedException to clean up silly close / shutdown logging. (Mark Miller)
+
+* SOLR-12896: Introduce more checks for shutdown and closed to improve clean close and shutdown. (Mark Miller)
+
+* SOLR-12804: Remove static modifier from Overseer queue access. (Mark Miller)
+
+Other Changes
+----------------------
+
+* SOLR-12972: deprecate unused SolrIndexConfig.luceneVersion (Christine Poerschke)
+
+* SOLR-12801: Make massive improvements to the tests. (Mark Miller)
+
+* SOLR-12923: The new AutoScaling tests are way too flaky and need special attention. (Mark Miller)
+
+* SOLR-12932: ant test (without badapples=false) should pass easily for developers. (Mark Miller)
+
================== 7.6.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsCloudTest.java
index f34c667..d00effd 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsCloudTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsCloudTest.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
+import java.util.concurrent.TimeoutException;
import org.apache.solr.analytics.util.AnalyticsResponseHeadings;
import org.apache.solr.analytics.util.MedianCalculator;
@@ -29,11 +30,11 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
public class LegacyAbstractAnalyticsCloudTest extends SolrCloudTestCase {
@@ -41,19 +42,23 @@ public class LegacyAbstractAnalyticsCloudTest extends SolrCloudTestCase {
protected static final int TIMEOUT = DEFAULT_TIMEOUT;
protected static final String id = "id";
- @BeforeClass
- public static void setupCollection() throws Exception {
+ @Before
+ public void setupCollection() throws Exception {
configureCluster(4)
.addConfig("conf", configset("cloud-analytics"))
.configure();
CollectionAdminRequest.createCollection(COLLECTIONORALIAS, "conf", 2, 1).process(cluster.getSolrClient());
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTIONORALIAS, cluster.getSolrClient().getZkStateReader(),
- false, true, TIMEOUT);
- cleanIndex();
+ cluster.waitForActiveCollection(COLLECTIONORALIAS, 2, 2);
+ }
+
+ @After
+ public void teardownCollection() throws Exception {
+ cluster.deleteAllCollections();
+ shutdownCluster();
}
- public static void cleanIndex() throws Exception {
+ public void cleanIndex() throws Exception {
new UpdateRequest()
.deleteByQuery("*:*")
.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
@@ -81,7 +86,7 @@ public class LegacyAbstractAnalyticsCloudTest extends SolrCloudTestCase {
}
}
- protected NamedList<Object> queryLegacyCloudAnalytics(String[] testParams) throws SolrServerException, IOException, InterruptedException {
+ protected NamedList<Object> queryLegacyCloudAnalytics(String[] testParams) throws SolrServerException, IOException, InterruptedException, TimeoutException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("q", "*:*");
params.set("indent", "true");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyNoFacetCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyNoFacetCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyNoFacetCloudTest.java
index 7489f3f..7239843 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyNoFacetCloudTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyNoFacetCloudTest.java
@@ -21,7 +21,7 @@ import java.util.List;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.util.NamedList;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
public class LegacyNoFacetCloudTest extends LegacyAbstractAnalyticsCloudTest {
@@ -57,16 +57,20 @@ public class LegacyNoFacetCloudTest extends LegacyAbstractAnalyticsCloudTest {
static ArrayList<String> stringTestStart;
static long stringMissing = 0;
- @BeforeClass
- public static void populate() throws Exception {
- cleanIndex();
-
+ @Before
+ public void populate() throws Exception {
intTestStart = new ArrayList<>();
longTestStart = new ArrayList<>();
floatTestStart = new ArrayList<>();
doubleTestStart = new ArrayList<>();
dateTestStart = new ArrayList<>();
stringTestStart = new ArrayList<>();
+ intMissing = 0;
+ longMissing = 0;
+ doubleMissing = 0;
+ floatMissing = 0;
+ dateMissing = 0;
+ stringMissing = 0;
UpdateRequest req = new UpdateRequest();
for (int j = 0; j < NUM_LOOPS; ++j) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetCloudTest.java
index 1124140..dec9059 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetCloudTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetCloudTest.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.util.NamedList;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
@@ -85,9 +85,8 @@ public class LegacyFieldFacetCloudTest extends LegacyAbstractAnalyticsFacetCloud
private static ArrayList<ArrayList<Integer>> multiDateTestStart;
private static ArrayList<Long> multiDateTestMissing;
- @BeforeClass
- public static void beforeClass() throws Exception {
- cleanIndex();
+ @Before
+ public void beforeTest() throws Exception {
//INT
intDateTestStart = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetExtrasCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetExtrasCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetExtrasCloudTest.java
index 808269a..3dac144 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetExtrasCloudTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyFieldFacetExtrasCloudTest.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.util.NamedList;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
public class LegacyFieldFacetExtrasCloudTest extends LegacyAbstractAnalyticsFacetCloudTest {
@@ -42,9 +42,8 @@ public class LegacyFieldFacetExtrasCloudTest extends LegacyAbstractAnalyticsFace
static ArrayList<ArrayList<Integer>> intDoubleTestStart;
static ArrayList<ArrayList<Integer>> intStringTestStart;
- @BeforeClass
- public static void beforeClass() throws Exception {
- cleanIndex();
+ @Before
+ public void beforeTest() throws Exception {
//INT
intLongTestStart = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyQueryFacetCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyQueryFacetCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyQueryFacetCloudTest.java
index 4c78a43..b62a819 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyQueryFacetCloudTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyQueryFacetCloudTest.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.util.NamedList;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
public class LegacyQueryFacetCloudTest extends LegacyAbstractAnalyticsFacetCloudTest {
@@ -39,9 +39,8 @@ public class LegacyQueryFacetCloudTest extends LegacyAbstractAnalyticsFacetCloud
private static ArrayList<ArrayList<Long>> longTestStart = new ArrayList<>();
private static ArrayList<ArrayList<Float>> floatTestStart = new ArrayList<>();
- @BeforeClass
- public static void beforeClass() throws Exception {
- cleanIndex();
+ @Before
+ public void beforeTest() throws Exception {
//INT
int1TestStart.add(new ArrayList<Integer>());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyRangeFacetCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyRangeFacetCloudTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyRangeFacetCloudTest.java
index 95585c4..aced62f 100644
--- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyRangeFacetCloudTest.java
+++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyRangeFacetCloudTest.java
@@ -21,7 +21,7 @@ import java.util.List;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.util.NamedList;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
@@ -44,9 +44,8 @@ public class LegacyRangeFacetCloudTest extends LegacyAbstractAnalyticsFacetCloud
static ArrayList<ArrayList<Float>> floatDoubleTestStart;
static ArrayList<ArrayList<Float>> floatDateTestStart;
- @BeforeClass
- public static void beforeClass() throws Exception {
- cleanIndex();
+ @Before
+ public void beforeTest() throws Exception {
//INT
intLongTestStart = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
index 06fd51c..9617726 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
@@ -52,7 +52,7 @@ public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCa
super.setUp();
instance = new SolrInstance("inst", null);
instance.setUp();
- jetty = createJetty(instance);
+ jetty = createAndStartJetty(instance);
}
@Override
@@ -173,7 +173,7 @@ public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCa
}
- private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
+ private JettySolrRunner createAndStartJetty(SolrInstance instance) throws Exception {
Properties nodeProperties = new Properties();
nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
index 0e9cd33..477fee1 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
@@ -127,7 +127,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe
// data source solr instance
instance = new SolrInstance();
instance.setUp();
- jetty = createJetty(instance);
+ jetty = createAndStartJetty(instance);
}
@Override
@@ -362,7 +362,7 @@ public class TestSolrEntityProcessorEndToEnd extends AbstractDataImportHandlerTe
}
}
- private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
+ private JettySolrRunner createAndStartJetty(SolrInstance instance) throws Exception {
Properties nodeProperties = new Properties();
nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
index c8727d0..14c9e98 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.solr.cloud.AbstractZkTestCase;
import org.apache.solr.cloud.ZkTestServer;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.SuppressForbidden;
@@ -62,7 +61,7 @@ public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
System.setProperty("zkHost", zkServer.getZkAddress());
System.setProperty("jetty.port", "0000");
- AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), getFile("dih/solr"),
+ zkServer.buildZooKeeper(getFile("dih/solr"),
"dataimport-solrconfig.xml", "dataimport-schema.xml");
//initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRThreadModule.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRThreadModule.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRThreadModule.java
index b8d0bda..e142610 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRThreadModule.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRThreadModule.java
@@ -18,14 +18,13 @@ package org.apache.solr.ltr;
import java.util.Iterator;
import java.util.Map;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.core.CloseHook;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.util.SolrPluginUtils;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
@@ -58,7 +57,7 @@ import org.apache.solr.util.plugin.NamedListInitializedPlugin;
* <code>totalPoolThreads</code> imposes a contention between the queries if
* <code>(totalPoolThreads < numThreadsPerRequest * total parallel queries)</code>.
*/
-final public class LTRThreadModule implements NamedListInitializedPlugin {
+final public class LTRThreadModule extends CloseHook implements NamedListInitializedPlugin {
public static LTRThreadModule getInstance(NamedList args) {
@@ -103,13 +102,10 @@ final public class LTRThreadModule implements NamedListInitializedPlugin {
// settings
private int totalPoolThreads = 1;
private int numThreadsPerRequest = 1;
- private int maxPoolSize = Integer.MAX_VALUE;
- private long keepAliveTimeSeconds = 10;
- private String threadNamePrefix = "ltrExecutor";
// implementation
private Semaphore ltrSemaphore;
- private Executor createWeightScoreExecutor;
+ private volatile ExecutorService createWeightScoreExecutor;
public LTRThreadModule() {
}
@@ -132,13 +128,6 @@ final public class LTRThreadModule implements NamedListInitializedPlugin {
} else {
ltrSemaphore = null;
}
- createWeightScoreExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(
- 0,
- maxPoolSize,
- keepAliveTimeSeconds, TimeUnit.SECONDS, // terminate idle threads after 10 sec
- new SynchronousQueue<Runnable>(), // directly hand off tasks
- new DefaultSolrThreadFactory(threadNamePrefix)
- );
}
private void validate() {
@@ -161,18 +150,6 @@ final public class LTRThreadModule implements NamedListInitializedPlugin {
this.numThreadsPerRequest = numThreadsPerRequest;
}
- public void setMaxPoolSize(int maxPoolSize) {
- this.maxPoolSize = maxPoolSize;
- }
-
- public void setKeepAliveTimeSeconds(long keepAliveTimeSeconds) {
- this.keepAliveTimeSeconds = keepAliveTimeSeconds;
- }
-
- public void setThreadNamePrefix(String threadNamePrefix) {
- this.threadNamePrefix = threadNamePrefix;
- }
-
public Semaphore createQuerySemaphore() {
return (numThreadsPerRequest > 1 ? new Semaphore(numThreadsPerRequest) : null);
}
@@ -189,4 +166,18 @@ final public class LTRThreadModule implements NamedListInitializedPlugin {
createWeightScoreExecutor.execute(command);
}
+ @Override
+ public void preClose(SolrCore core) {
+ ExecutorUtil.shutdownAndAwaitTermination(createWeightScoreExecutor);
+ }
+
+ @Override
+ public void postClose(SolrCore core) {
+
+ }
+
+ public void setExecutor(ExecutorService sharedExecutor) {
+ this.createWeightScoreExecutor = sharedExecutor;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
index 0e84009..c6c4d7b 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/response/transform/LTRFeatureLoggerTransformerFactory.java
@@ -204,7 +204,10 @@ public class LTRFeatureLoggerTransformerFactory extends TransformerFactory {
"searcher is null");
}
leafContexts = searcher.getTopReaderContext().leaves();
-
+ if (threadManager != null) {
+ threadManager.setExecutor(context.getRequest().getCore().getCoreContainer().getUpdateShardHandler().getUpdateExecutor());
+ }
+
// Setup LTRScoringQuery
scoringQuery = SolrQueryRequestContextUtils.getScoringQuery(req);
docsWereNotReranked = (scoringQuery == null);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/ltr/src/java/org/apache/solr/ltr/search/LTRQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/search/LTRQParserPlugin.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/search/LTRQParserPlugin.java
index c5db963..af99775 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/search/LTRQParserPlugin.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/search/LTRQParserPlugin.java
@@ -162,7 +162,9 @@ public class LTRQParserPlugin extends QParserPlugin implements ResourceLoaderAwa
final String fvStoreName = SolrQueryRequestContextUtils.getFvStoreName(req);
// Check if features are requested and if the model feature store and feature-transform feature store are the same
final boolean featuresRequestedFromSameStore = (modelFeatureStoreName.equals(fvStoreName) || fvStoreName == null) ? extractFeatures:false;
-
+ if (threadManager != null) {
+ threadManager.setExecutor(req.getCore().getCoreContainer().getUpdateShardHandler().getUpdateExecutor());
+ }
final LTRScoringQuery scoringQuery = new LTRScoringQuery(ltrScoringModel,
extractEFIParams(localParams),
featuresRequestedFromSameStore, threadManager);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
index 65e0e7f..85563e6 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
@@ -25,7 +25,6 @@ import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -232,7 +231,7 @@ public class TestLTROnSolrCloud extends TestRerankBase {
fail("Could not create collection. Response" + response.toString());
}
ZkStateReader zkStateReader = solrCluster.getSolrClient().getZkStateReader();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(name, zkStateReader, false, true, 100);
+ solrCluster.waitForActiveCollection(name, numShards, numShards * numReplicas);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
index 28c3cdf..748aee9 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
@@ -38,8 +38,10 @@ public class JettyConfig {
public final Map<Class<? extends Filter>, String> extraFilters;
public final SSLConfig sslConfig;
+
+ public final int portRetryTime;
- private JettyConfig(int port, String context, boolean stopAtShutdown, Long waitForLoadingCoresToFinishMs, Map<ServletHolder, String> extraServlets,
+ private JettyConfig(int port, int portRetryTime, String context, boolean stopAtShutdown, Long waitForLoadingCoresToFinishMs, Map<ServletHolder, String> extraServlets,
Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig) {
this.port = port;
this.context = context;
@@ -48,6 +50,7 @@ public class JettyConfig {
this.extraServlets = extraServlets;
this.extraFilters = extraFilters;
this.sslConfig = sslConfig;
+ this.portRetryTime = portRetryTime;
}
public static Builder builder() {
@@ -74,6 +77,7 @@ public class JettyConfig {
Map<ServletHolder, String> extraServlets = new TreeMap<>();
Map<Class<? extends Filter>, String> extraFilters = new LinkedHashMap<>();
SSLConfig sslConfig = null;
+ int portRetryTime = 60;
public Builder setPort(int port) {
this.port = port;
@@ -121,9 +125,15 @@ public class JettyConfig {
this.sslConfig = sslConfig;
return this;
}
+
+ public Builder withPortRetryTime(int portRetryTime) {
+ this.portRetryTime = portRetryTime;
+ return this;
+ }
+
public JettyConfig build() {
- return new JettyConfig(port, context, stopAtShutdown, waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig);
+ return new JettyConfig(port, portRetryTime, context, stopAtShutdown, waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
index 5fdec0f..c1d927b 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
@@ -16,18 +16,9 @@
*/
package org.apache.solr.client.solrj.embedded;
-import javax.servlet.DispatcherType;
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.net.BindException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
@@ -41,10 +32,24 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import javax.servlet.DispatcherType;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.cloud.SocketProxy;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.TimeOut;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -61,6 +66,7 @@ import org.eclipse.jetty.servlet.Source;
import org.eclipse.jetty.util.component.LifeCycle;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.ReservedThreadExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -80,8 +86,8 @@ public class JettySolrRunner {
Server server;
- FilterHolder dispatchFilter;
- FilterHolder debugFilter;
+ volatile FilterHolder dispatchFilter;
+ volatile FilterHolder debugFilter;
private boolean waitOnSolr = false;
private int jettyPort = -1;
@@ -98,6 +104,16 @@ public class JettySolrRunner {
private int proxyPort = -1;
+ private final boolean enableProxy;
+
+ private SocketProxy proxy;
+
+ private String protocol;
+
+ private String host;
+
+ private volatile boolean started = false;
+
public static class DebugFilter implements Filter {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -189,7 +205,7 @@ public class JettySolrRunner {
public JettySolrRunner(String solrHome, JettyConfig config) {
this(solrHome, new Properties(), config);
}
-
+
/**
* Construct a JettySolrRunner
*
@@ -200,10 +216,33 @@ public class JettySolrRunner {
* @param config the configuration
*/
public JettySolrRunner(String solrHome, Properties nodeProperties, JettyConfig config) {
+ this(solrHome, nodeProperties, config, false);
+ }
+ /**
+ * Construct a JettySolrRunner
+ *
+ * After construction, you must start the jetty with {@link #start()}
+ *
+ * @param solrHome the solrHome to use
+ * @param nodeProperties the container properties
+ * @param config the configuration
+ * @param enableProxy enables proxy feature to disable connections
+ */
+ public JettySolrRunner(String solrHome, Properties nodeProperties, JettyConfig config, boolean enableProxy) {
+ this.enableProxy = enableProxy;
this.solrHome = solrHome;
this.config = config;
this.nodeProperties = nodeProperties;
+
+ if (enableProxy) {
+ try {
+ proxy = new SocketProxy(0, config.sslConfig != null && config.sslConfig.isSSLMode());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ setProxyPort(proxy.getListenPort());
+ }
this.init(this.config.port);
}
@@ -213,7 +252,7 @@ public class JettySolrRunner {
QueuedThreadPool qtp = new QueuedThreadPool();
qtp.setMaxThreads(THREAD_POOL_MAX_THREADS);
qtp.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
- qtp.setStopTimeout((int) TimeUnit.MINUTES.toMillis(1));
+ qtp.setReservedThreads(0);
server = new Server(qtp);
server.manage(qtp);
server.setStopAtShutdown(config.stopAtShutdown);
@@ -246,7 +285,7 @@ public class JettySolrRunner {
connector.setPort(port);
connector.setHost("127.0.0.1");
connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
-
+ connector.setStopTimeout(0);
server.setConnectors(new Connector[] {connector});
server.setSessionIdManager(new DefaultSessionIdManager(server, new Random()));
} else {
@@ -271,10 +310,7 @@ public class JettySolrRunner {
@Override
public void lifeCycleStarting(LifeCycle arg0) {
- synchronized (JettySolrRunner.this) {
- waitOnSolr = true;
- JettySolrRunner.this.notify();
- }
+
}
@Override
@@ -306,6 +342,11 @@ public class JettySolrRunner {
dispatchFilter.setHeldClass(SolrDispatchFilter.class);
dispatchFilter.setInitParameter("excludePatterns", excludePatterns);
root.addFilter(dispatchFilter, "*", EnumSet.of(DispatcherType.REQUEST));
+
+ synchronized (JettySolrRunner.this) {
+ waitOnSolr = true;
+ JettySolrRunner.this.notify();
+ }
}
@Override
@@ -344,15 +385,19 @@ public class JettySolrRunner {
}
public String getNodeName() {
+ if (getCoreContainer() == null) {
+ return null;
+ }
return getCoreContainer().getZkController().getNodeName();
}
public boolean isRunning() {
- return server.isRunning();
+ return server.isRunning() && dispatchFilter != null && dispatchFilter.isRunning();
}
public boolean isStopped() {
- return server.isStopped();
+ return (server.isStopped() && dispatchFilter == null) || (server.isStopped() && dispatchFilter.isStopped()
+ && ((QueuedThreadPool) server.getThreadPool()).isStopped());
}
// ------------------------------------------------------------------------------------------------
@@ -382,31 +427,53 @@ public class JettySolrRunner {
// Do not let Jetty/Solr pollute the MDC for this thread
Map<String, String> prevContext = MDC.getCopyOfContextMap();
MDC.clear();
+
+ log.info("Start Jetty (original configured port={})", this.config.port);
+
try {
+ int port = reusePort && jettyPort != -1 ? jettyPort : this.config.port;
+
// if started before, make a new server
if (startedBefore) {
waitOnSolr = false;
- int port = reusePort ? jettyPort : this.config.port;
init(port);
} else {
startedBefore = true;
}
if (!server.isRunning()) {
- server.start();
+ if (config.portRetryTime > 0) {
+ retryOnPortBindFailure(config.portRetryTime, port);
+ } else {
+ server.start();
+ }
}
synchronized (JettySolrRunner.this) {
int cnt = 0;
- while (!waitOnSolr) {
+ while (!waitOnSolr || !dispatchFilter.isRunning() || getCoreContainer() == null) {
this.wait(100);
- if (cnt++ == 5) {
+ if (cnt++ == 15) {
throw new RuntimeException("Jetty/Solr unresponsive");
}
}
}
- if (config.waitForLoadingCoresToFinishMs != null && config.waitForLoadingCoresToFinishMs > 0L) waitForLoadingCoresToFinish(config.waitForLoadingCoresToFinishMs);
+ if (config.waitForLoadingCoresToFinishMs != null && config.waitForLoadingCoresToFinishMs > 0L) {
+ waitForLoadingCoresToFinish(config.waitForLoadingCoresToFinishMs);
+ }
+
+ setProtocolAndHost();
+
+ if (enableProxy) {
+ if (started) {
+ proxy.reopen();
+ } else {
+ proxy.open(getBaseUrl().toURI());
+ }
+ }
+
} finally {
+ started = true;
if (prevContext != null) {
MDC.setContextMap(prevContext);
} else {
@@ -415,6 +482,43 @@ public class JettySolrRunner {
}
}
+
+ private void setProtocolAndHost() {
+ String protocol = null;
+
+ Connector[] conns = server.getConnectors();
+ if (0 == conns.length) {
+ throw new IllegalStateException("Jetty Server has no Connectors");
+ }
+ ServerConnector c = (ServerConnector) conns[0];
+
+ protocol = c.getDefaultProtocol().startsWith("SSL") ? "https" : "http";
+
+ this.protocol = protocol;
+ this.host = c.getHost();
+ }
+
+ private void retryOnPortBindFailure(int portRetryTime, int port) throws Exception, InterruptedException {
+ TimeOut timeout = new TimeOut(portRetryTime, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+ int tryCnt = 1;
+ while (true) {
+ try {
+ log.info("Trying to start Jetty on port {} try number {} ...", port, tryCnt++);
+ server.start();
+ break;
+ } catch (BindException e) {
+ log.info("Port is in use, will try again until timeout of " + timeout);
+ server.stop();
+ Thread.sleep(3000);
+ if (!timeout.hasTimedOut()) {
+ continue;
+ }
+
+ throw e;
+ }
+ }
+ }
+
/**
* Stop the Jetty server
*
@@ -422,11 +526,33 @@ public class JettySolrRunner {
*/
public void stop() throws Exception {
// Do not let Jetty/Solr pollute the MDC for this thread
- Map<String, String> prevContext = MDC.getCopyOfContextMap();
+ Map<String,String> prevContext = MDC.getCopyOfContextMap();
MDC.clear();
try {
Filter filter = dispatchFilter.getFilter();
+ // we want to shutdown outside of jetty cutting us off
+ SolrDispatchFilter sdf = getSolrDispatchFilter();
+ Thread shutdownThead = null;
+ if (sdf != null) {
+ shutdownThead = new Thread() {
+
+ public void run() {
+ try {
+ sdf.close();
+ } catch (Throwable t) {
+ log.error("Error shutting down Solr", t);
+ }
+ }
+
+ };
+ sdf.closeOnDestroy(false);
+ shutdownThead.start();
+ }
+
+ QueuedThreadPool qtp = (QueuedThreadPool) server.getThreadPool();
+ ReservedThreadExecutor rte = qtp.getBean(ReservedThreadExecutor.class);
+
server.stop();
if (server.getState().equals(Server.FAILED)) {
@@ -438,9 +564,48 @@ public class JettySolrRunner {
}
}
- server.join();
+ // stop timeout is 0, so we will interrupt right away
+ while(!qtp.isStopped()) {
+ qtp.stop();
+ if (qtp.isStopped()) {
+ Thread.sleep(50);
+ }
+ }
+
+ // we tried to kill everything, now we wait for executor to stop
+ qtp.setStopTimeout(Integer.MAX_VALUE);
+ qtp.stop();
+ qtp.join();
+
+ if (rte != null) {
+ // we try and wait for the reserved thread executor, but it doesn't always seem to work
+ // so we actually set 0 reserved threads at creation
+
+ rte.stop();
+
+ TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+ timeout.waitFor("Timeout waiting for reserved executor to stop.", ()
+ -> rte.isStopped());
+ }
+
+ if (shutdownThead != null) {
+ shutdownThead.join();
+ }
+
+ do {
+ try {
+ server.join();
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ } while (!server.isStopped());
+
} finally {
- if (prevContext != null) {
+ if (enableProxy) {
+ proxy.close();
+ }
+
+ if (prevContext != null) {
MDC.setContextMap(prevContext);
} else {
MDC.clear();
@@ -461,15 +626,30 @@ public class JettySolrRunner {
return ((ServerConnector) conns[0]).getLocalPort();
}
+
/**
* Returns the Local Port of the jetty Server.
*
* @exception RuntimeException if there is no Connector
*/
public int getLocalPort() {
+ return getLocalPort(false);
+ }
+
+ /**
+ * Returns the Local Port of the jetty Server.
+ *
+ * @param internalPort pass true to get the true jetty port rather than the proxy port if configured
+ *
+ * @exception RuntimeException if there is no Connector
+ */
+ public int getLocalPort(boolean internalPort) {
if (jettyPort == -1) {
throw new IllegalStateException("You cannot get the port until this instance has started");
}
+ if (internalPort ) {
+ return jettyPort;
+ }
return (proxyPort != -1) ? proxyPort : jettyPort;
}
@@ -481,29 +661,27 @@ public class JettySolrRunner {
public void setProxyPort(int proxyPort) {
this.proxyPort = proxyPort;
}
-
+
/**
* Returns a base URL consisting of the protocol, host, and port for a
* Connector in use by the Jetty Server contained in this runner.
*/
public URL getBaseUrl() {
- String protocol = null;
try {
- Connector[] conns = server.getConnectors();
- if (0 == conns.length) {
- throw new IllegalStateException("Jetty Server has no Connectors");
- }
- ServerConnector c = (ServerConnector) conns[0];
- if (c.getLocalPort() < 0) {
- throw new IllegalStateException("Jetty Connector is not open: " +
- c.getLocalPort());
- }
- protocol = c.getDefaultProtocol().startsWith("SSL") ? "https" : "http";
- return new URL(protocol, c.getHost(), c.getLocalPort(), config.context);
-
+ return new URL(protocol, host, jettyPort, config.context);
+ } catch (MalformedURLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ /**
+ * Returns a base URL consisting of the protocol, host, and port for a
+ * Connector in use by the Jetty Server contained in this runner.
+ */
+ public URL getProxyBaseUrl() {
+ try {
+ return new URL(protocol, host, getLocalPort(), config.context);
} catch (MalformedURLException e) {
- throw new IllegalStateException
- ("Java could not make sense of protocol: " + protocol, e);
+ throw new RuntimeException(e);
}
}
@@ -568,7 +746,11 @@ public class JettySolrRunner {
CoreContainer cores = solrFilter.getCores();
if (cores != null) {
cores.waitForLoadingCoresToFinish(timeoutMs);
+ } else {
+ throw new IllegalStateException("The CoreContainer is not set!");
}
+ } else {
+ throw new IllegalStateException("The dispatchFilter is not set!");
}
}
@@ -583,4 +765,8 @@ public class JettySolrRunner {
this.delayValue = delay;
}
}
+
+ public SocketProxy getProxy() {
+ return proxy;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
index 6d17de4..a67ce57 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
@@ -73,6 +73,7 @@ public abstract class ElectionContext implements Closeable {
public ElectionContext(final String coreNodeName,
final String electionPath, final String leaderPath, final ZkNodeProps leaderProps, final SolrZkClient zkClient) {
+ assert zkClient != null;
this.id = coreNodeName;
this.electionPath = electionPath;
this.leaderPath = leaderPath;
@@ -116,6 +117,7 @@ class ShardLeaderElectionContextBase extends ElectionContext {
protected String collection;
protected LeaderElector leaderElector;
protected ZkStateReader zkStateReader;
+ protected ZkController zkController;
private Integer leaderZkNodeParentVersion;
// Prevents a race between cancelling and becoming leader.
@@ -123,15 +125,29 @@ class ShardLeaderElectionContextBase extends ElectionContext {
public ShardLeaderElectionContextBase(LeaderElector leaderElector,
final String shardId, final String collection, final String coreNodeName,
- ZkNodeProps props, ZkStateReader zkStateReader) {
+ ZkNodeProps props, ZkController zkController) {
super(coreNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection
+ "/leader_elect/" + shardId, ZkStateReader.getShardLeadersPath(
- collection, shardId), props, zkStateReader.getZkClient());
+ collection, shardId), props, zkController.getZkClient());
this.leaderElector = leaderElector;
+ this.zkStateReader = zkController.getZkStateReader();
this.zkClient = zkStateReader.getZkClient();
- this.zkStateReader = zkStateReader;
+ this.zkController = zkController;
this.shardId = shardId;
this.collection = collection;
+
+ String parent = new Path(leaderPath).getParent().toString();
+ ZkCmdExecutor zcmd = new ZkCmdExecutor(30000);
+ // only if /collections/{collection} exists already do we succeed in creating this path
+ log.info("make sure parent is created {}", parent);
+ try {
+ zcmd.ensureExists(parent, (byte[])null, CreateMode.PERSISTENT, zkClient, 2);
+ } catch (KeeperException e) {
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
}
@Override
@@ -171,21 +187,12 @@ class ShardLeaderElectionContextBase extends ElectionContext {
void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStartMs)
throws KeeperException, InterruptedException, IOException {
// register as leader - if an ephemeral is already there, wait to see if it goes away
-
- if (!zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection, true)) {
- log.info("Will not register as leader because collection appears to be gone.");
- return;
- }
-
- String parent = new Path(leaderPath).getParent().toString();
- ZkCmdExecutor zcmd = new ZkCmdExecutor(30000);
- // only if /collections/{collection} exists already do we succeed in creating this path
- zcmd.ensureExists(parent, (byte[])null, CreateMode.PERSISTENT, zkClient, 2);
+ String parent = new Path(leaderPath).getParent().toString();
try {
RetryUtil.retryOnThrowable(NodeExistsException.class, 60000, 5000, () -> {
synchronized (lock) {
- log.debug("Creating leader registration node {} after winning as {}", leaderPath, leaderSeqPath);
+ log.info("Creating leader registration node {} after winning as {}", leaderPath, leaderSeqPath);
List<Op> ops = new ArrayList<>(2);
// We use a multi operation to get the parent nodes version, which will
@@ -210,6 +217,9 @@ class ShardLeaderElectionContextBase extends ElectionContext {
assert leaderZkNodeParentVersion != null;
}
});
+ } catch (NoNodeException e) {
+ log.info("Will not register as leader because it seems the election is no longer taking place.");
+ return;
} catch (Throwable t) {
if (t instanceof OutOfMemoryError) {
throw (OutOfMemoryError) t;
@@ -235,7 +245,9 @@ class ShardLeaderElectionContextBase extends ElectionContext {
ZkStateReader.BASE_URL_PROP, leaderProps.get(ZkStateReader.BASE_URL_PROP),
ZkStateReader.CORE_NAME_PROP, leaderProps.get(ZkStateReader.CORE_NAME_PROP),
ZkStateReader.STATE_PROP, Replica.State.ACTIVE.toString());
- Overseer.getStateUpdateQueue(zkClient).offer(Utils.toJSON(m));
+ assert zkController != null;
+ assert zkController.getOverseer() != null;
+ zkController.getOverseer().offerStateUpdate(Utils.toJSON(m));
}
}
@@ -254,7 +266,6 @@ class ShardLeaderElectionContextBase extends ElectionContext {
final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final ZkController zkController;
private final CoreContainer cc;
private final SyncStrategy syncStrategy;
@@ -264,8 +275,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
final String shardId, final String collection,
final String coreNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) {
super(leaderElector, shardId, collection, coreNodeName, props,
- zkController.getZkStateReader());
- this.zkController = zkController;
+ zkController);
this.cc = cc;
syncStrategy = new SyncStrategy(cc);
}
@@ -304,11 +314,8 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
ActionThrottle lt;
try (SolrCore core = cc.getCore(coreName)) {
if (core == null ) {
- if (cc.isShutDown()) {
- return;
- } else {
- throw new SolrException(ErrorCode.SERVER_ERROR, "SolrCore not found:" + coreName + " in " + cc.getLoadedCoreNames());
- }
+ // shutdown or removed
+ return;
}
MDCLoggingContext.setCore(core);
lt = core.getUpdateHandler().getSolrCoreState().getLeaderThrottle();
@@ -326,7 +333,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
// Clear the leader in clusterstate. We only need to worry about this if there is actually more than one replica.
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.LEADER.toLower(),
ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP, collection);
- Overseer.getStateUpdateQueue(zkClient).offer(Utils.toJSON(m));
+ zkController.getOverseer().getStateUpdateQueue().offer(Utils.toJSON(m));
}
boolean allReplicasInLine = false;
@@ -349,13 +356,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
try (SolrCore core = cc.getCore(coreName)) {
if (core == null) {
- if (!zkController.getCoreContainer().isShutDown()) {
- cancelElection();
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "SolrCore not found:" + coreName + " in " + cc.getLoadedCoreNames());
- } else {
- return;
- }
+ return;
}
replicaType = core.getCoreDescriptor().getCloudDescriptor().getReplicaType();
@@ -698,7 +699,8 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
final class OverseerElectionContext extends ElectionContext {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final SolrZkClient zkClient;
- private Overseer overseer;
+ private final Overseer overseer;
+ private volatile boolean isClosed = false;
public OverseerElectionContext(SolrZkClient zkClient, Overseer overseer, final String zkNodeName) {
super(zkNodeName, Overseer.OVERSEER_ELECT, Overseer.OVERSEER_ELECT + "/leader", null, zkClient);
@@ -732,8 +734,10 @@ final class OverseerElectionContext extends ElectionContext {
log.warn("Wait interrupted ", e);
}
}
- if (!overseer.getZkController().isClosed() && !overseer.getZkController().getCoreContainer().isShutDown()) {
- overseer.start(id);
+ synchronized (this) {
+ if (!this.isClosed && !overseer.getZkController().getCoreContainer().isShutDown()) {
+ overseer.start(id);
+ }
}
}
@@ -744,7 +748,8 @@ final class OverseerElectionContext extends ElectionContext {
}
@Override
- public void close() {
+ public synchronized void close() {
+ this.isClosed = true;
overseer.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75b18319/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
index 46f3c88..0cc8cac 100644
--- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
+++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
@@ -26,6 +26,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.cloud.ZkController.ContextKey;
+import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
@@ -346,6 +347,8 @@ public class LeaderElector {
try {
// am I the next leader?
checkIfIamLeader(context, true);
+ } catch (AlreadyClosedException e) {
+
} catch (Exception e) {
if (!zkClient.isClosed()) {
log.warn("", e);