You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/12/12 22:41:26 UTC
svn commit: r1420992 [1/7] - in /lucene/dev/branches/branch_4x: ./
dev-tools/ lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/codecs/ lucene/core/ lucene/core/src/test/org/ap...
Author: yonik
Date: Wed Dec 12 21:41:06 2012
New Revision: 1420992
URL: http://svn.apache.org/viewvc?rev=1420992&view=rev
Log:
SOLR-2592: SOLR-1028: SOLR-3922: SOLR-3911: sync trunk with 4x
Added:
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java (with props)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesInputStream.java (with props)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesOutputStream.java (with props)
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (with props)
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestLazyCores.java (with props)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java (with props)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java (with props)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java (with props)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java (with props)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ImplicitDocRouter.java (with props)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/PlainIdRouter.java (with props)
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreContainer.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateLog.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/IndexReaderFactoryTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/SolrCoreCheckLockOnStartupTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestConfig.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestJmxIntegration.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestPropInject.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestXIncludeConfig.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/StandardRequestHandlerTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/highlight/HighlighterConfigTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestQueryUtils.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestSurroundQueryParser.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/servlet/DirectSolrConnectionTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/TestIndexingPerformance.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/UpdateParamsTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/processor/UpdateRequestProcessorFactoryTest.java
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/example/contexts/solr.xml
lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/velocity/product-doc.vm
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/SolrPing.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/Slice.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/shared/solr.xml
lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/solr.xml
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/MergeIndexesExampleTestBase.java
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/MultiCoreExampleTestBase.java
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTestBase.java
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreExampleJettyTest.java
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractZkTestCase.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/AbstractSolrTestCase.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Dec 12 21:41:06 2012
@@ -20,6 +20,10 @@ See the tutorial at http://lucene.apache
$Id$
+================== 5.0.0 ==================
+
+(No changes)
+
================== 4.1.0 ==================
Versions of Major Components
@@ -36,6 +40,12 @@ Upgrading from Solr 4.0.0-BETA
Custom java parsing plugins need to migrade from throwing the internal
ParseException to throwing SyntaxError.
+BaseDistributedSearchTestCase now randomizes the servlet context it uses when
+creating Jetty instances. Subclasses that assume a hard coded context of
+"/solr" should either be fixed to use the "String context" variable, or should
+take advantage of the new BaseDistributedSearchTestCase(String) constructor
+to explicitly specify a fixed servlet context path. See SOLR-4136 for details.
+
Detailed Change List
----------------------
@@ -63,6 +73,18 @@ New Features
* SOLR-3985: ExternalFileField caches can be reloaded on firstSearcher/
newSearcher events using the ExternalFileFieldReloader (Alan Woodward)
+* SOLR-3911: Make Directory and DirectoryFactory first class so that the majority
+ of Solr's features work with any custom implementations. (Mark Miller)
+ Additional Work:
+ - SOLR-4032: Files larger than an internal buffer size fail to replicate.
+ (Mark Miller, Markus Jelsma)
+ - SOLR-4033: Consistently use the solrconfig.xml lockType everywhere.
+ (Mark Miller, Markus Jelsma)
+
+* SOLR-1972: Add extra statistics to RequestHandlers - 5 & 15-minute reqs/sec
+ rolling averages; median, 75th, 95th, 99th, 99.9th percentile request times
+ (Alan Woodward, Shawn Heisey, Adrien Grand)
+
* SOLR-4051: Add <propertyWriter /> element to DIH's data-config.xml file,
allowing the user to specify the location, filename and Locale for
the "data-config.properties" file. Alternatively, users can specify their
@@ -95,10 +117,6 @@ New Features
* SOLR-4114: Allow creating more than one shard per instance with the
Collection API. (Per Steffensen, Mark Miller)
-* SOLR-1972: Add extra statistics to RequestHandlers - 5 & 15-minute reqs/sec
- rolling averages; median, 75th, 95th, 99th, 99.9th percentile request times
- (Alan Woodward, Shawn Heisey, Adrien Grand)
-
* SOLR-3531: Allowing configuring maxMergeSizeMB and maxCachedMB when
using NRTCachingDirectoryFactory. (Andy Laird via Mark Miller)
@@ -169,7 +187,7 @@ Bug Fixes
* SOLR-3961: Fixed error using LimitTokenCountFilterFactory
(Jack Krupansky, hossman)
-
+
* SOLR-3933: Distributed commits are not guaranteed to be ordered within a
request. (Mark Miller)
@@ -201,6 +219,10 @@ Bug Fixes
* SOLR-3998: Atomic update on uniqueKey field itself causes duplicate document.
(Eric Spencer, yonik)
+* SOLR-4001: In CachingDirectoryFactory#close, if there are still refs for a
+ Directory outstanding, we need to wait for them to be released before closing.
+ (Mark Miller)
+
* SOLR-4005: If CoreContainer fails to register a created core, it should close it.
(Mark Miller)
@@ -233,7 +255,10 @@ Bug Fixes
connection on a non-read-only database leak connections. Also expanded the
SqlEntityProcessor test to sometimes use Derby as well as HSQLDB (Derby is
one db affected by this bug). (Fenlor Sebastia, James Dyer)
-
+
+* SOLR-4064: When there is an unexpected exception while trying to run the new
+ leader process, the SolrCore will not correctly rejoin the election.
+ (Po Rui Via Mark Miller)
* SOLR-3989: SolrZkClient constructor dropped exception cause when throwing
a new RuntimeException. (Colin Bartolome, yonik)
@@ -272,7 +297,7 @@ Bug Fixes
* SOLR-3960: Fixed a bug where Distributed Grouping ignored PostFilters
(Nathan Visagan, hossman)
-
+
* SOLR-3842: DIH would not populate multivalued fields if the column name
derives from a resolved variable (James Dyer)
@@ -302,6 +327,10 @@ Bug Fixes
* SOLR-4127: Added explicit error message if users attempt Atomic document
updates with either updateLog or DistribUpdateProcessor. (hossman)
+* SOLR-4136: Fix SolrCloud behavior when using "hostContext" containing "_"
+ or"/" characters. This fix also makes SolrCloud more accepting of
+ hostContext values with leading/trailing slashes. (hossman)
+
* SOLR-4168: Ensure we are using the absolute latest index dir when getting
list of files for replication. (Mark Miller)
@@ -318,11 +347,10 @@ Other Changes
(Markus Jelsma via hossman)
* SOLR-3932: SolrCmdDistributorTest either takes 3 seconds or 3 minutes.
- (yonik, Mark Miller)
-
+ (yonik, Mark Miller)
* SOLR-3856: New tests for SqlEntityProcessor/CachedSqlEntityProcessor
- (James Dyer)
+ (James Dyer)
* SOLR-4067: ZkStateReader#getLeaderProps should not return props for a leader
that it does not think is live. (Mark Miller)
@@ -705,6 +733,11 @@ Other Changes
* SOLR-2857: The /update/json and /update/csv URLs were restored to aid
in the migration of existing clients. (yonik)
+* SOLR-3691: SimplePostTool: Mode for crawling/posting web pages
+ See http://wiki.apache.org/solr/ExtractingRequestHandler for examples (janhoy)
+
+* SOLR-3707: Upgrade Solr to Tika 1.2 (janhoy)
+
* SOLR-2747: Updated changes2html.pl to handle Solr's CHANGES.txt; added
target 'changes-to-html' to solr/build.xml.
(Steve Rowe, Robert Muir)
@@ -720,8 +753,6 @@ Other Changes
* SOLR-3780: Maven build: Make solrj tests run separately from solr-core.
(Steve Rowe)
-
-* SOLR-3707: Upgrade Solr to Tika 1.2 (janhoy)
* SOLR-3772: Optionally, on cluster startup, we can wait until we see all registered
replicas before running the leader process - or if they all do not come up,
@@ -731,9 +762,6 @@ Other Changes
running the leader sync process so that we are sure every node participates.
(Per Steffensen, Mark Miller)
-* SOLR-3691: SimplePostTool: Mode for crawling/posting web pages
- See http://wiki.apache.org/solr/ExtractingRequestHandler for examples (janhoy)
-
* SOLR-3824: Velocity: Error messages from search not displayed (janhoy)
* SOLR-3826: Test framework improvements for specifying coreName on initCore
@@ -887,11 +915,11 @@ Bug Fixes
* SOLR-3642: Correct broken check for multivalued fields in stats.facet
(Yandong Yao, hossman)
-* SOLR-3658: Adding thousands of docs with one UpdateProcessorChain instance can briefly create
- spikes of threads in the thousands. (yonik, Mark Miller)
-
* SOLR-3660: Velocity: Link to admin page broken (janhoy)
+* SOLR-3658: Adding thousands of docs with one UpdateProcessorChain instance can briefly create
+ spikes of threads in the thousands. (yonik, Mark Miller)
+
* SOLR-3656: A core reload now always uses the same dataDir. (Mark Miller, yonik)
* SOLR-3662: Core reload bugs: a reload always obtained a non-NRT searcher, which
@@ -963,7 +991,7 @@ Other Changes
Also, the configuration itself can be passed using the "dataConfig" parameter rather than
using a file (this previously worked in debug mode only). When configuration errors are
encountered, the error message is returned in XML format. (James Dyer)
-
+
* SOLR-3439: Make SolrCell easier to use out of the box. Also improves "/browse" to display
rich-text documents correctly, along with facets for author and content_type.
With the new "content" field, highlighting of body is supported. See also SOLR-3672 for
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java Wed Dec 12 21:41:06 2012
@@ -58,7 +58,9 @@ public abstract class AbstractDIHJdbcTes
try {
Class.forName("org.hsqldb.jdbcDriver").newInstance();
String oldProp = System.getProperty("derby.stream.error.field");
- System.setProperty("derby.stream.error.field", "DerbyUtil.DEV_NULL");
+ System
+ .setProperty("derby.stream.error.field",
+ "org.apache.solr.handler.dataimport.AbstractDIHJdbcTestCase$DerbyUtil.DEV_NULL");
Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance();
if (oldProp != null) {
System.setProperty("derby.stream.error.field", oldProp);
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java Wed Dec 12 21:41:06 2012
@@ -59,7 +59,7 @@ public abstract class AbstractSqlEntityP
tmpdir.mkdir();
fileLocation = tmpdir.getPath();
fileName = "the.properties";
- }
+ }
@After
public void afterSqlEntitiyProcessorTestCase() {
useSimpleCaches = false;
@@ -93,7 +93,7 @@ public abstract class AbstractSqlEntityP
for(Map.Entry<String,Object> entry : props.entrySet()) {
sb.append(" > key=" + entry.getKey() + " / value=" + entry.getValue() + "\n");
}
- log.info(sb.toString());
+ log.debug(sb.toString());
}
}
@@ -419,7 +419,7 @@ public abstract class AbstractSqlEntityP
// One second in the future ensures a change time after the last import (DIH
// uses second precision only)
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
- log.info("PEOPLE UPDATE USING TIMESTAMP: "
+ log.debug("PEOPLE UPDATE USING TIMESTAMP: "
+ new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
.format(theTime));
try {
@@ -490,7 +490,7 @@ public abstract class AbstractSqlEntityP
// One second in the future ensures a change time after the last import (DIH
// uses second precision only)
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
- log.info("COUNTRY UPDATE USING TIMESTAMP: "
+ log.debug("COUNTRY UPDATE USING TIMESTAMP: "
+ new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
.format(theTime));
try {
@@ -638,7 +638,7 @@ public abstract class AbstractSqlEntityP
sb.append("</document> \n");
sb.append("</dataConfig> \n");
String config = sb.toString().replaceAll("[']{2}", "\"");
- log.info(config);
+ log.debug(config);
return config;
}
@Override
@@ -651,7 +651,7 @@ public abstract class AbstractSqlEntityP
s.executeUpdate("create table countries(code varchar(3) not null primary key, country_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_code char(2), deleted char(1) default 'N', last_modified timestamp not null)");
s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
- log.info("INSERTING DB DATA USING TIMESTAMP: "
+ log.debug("INSERTING DB DATA USING TIMESTAMP: "
+ new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
.format(theTime));
ps = conn
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java Wed Dec 12 21:41:06 2012
@@ -40,7 +40,7 @@ public class TestNestedChildren extends
sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
sb.append(" query=\"select 1 as id, 2 as SECOND_ID, 'BEEF' as SECOND_S from sysibm.sysdummy1 WHERE 1=${FIRST.ID}\" >\n");
sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
- sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
+ sb.append(" <entity name=\"THIRD\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
sb.append(" query=\"select 1 as id, 'CHICKEN' as THIRD_S from sysibm.sysdummy1 WHERE 2=${SECOND.SECOND_ID}\" >\n");
sb.append(" <field column=\"THIRD_S\" name=\"third_s\" /> \n");
sb.append(" </entity>\n");
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Wed Dec 12 21:41:06 2012
@@ -39,46 +39,46 @@ public class TestSqlEntityProcessorDelta
}
@Test
public void testSingleEntity() throws Exception {
- log.info("testSingleEntity full-import...");
+ log.debug("testSingleEntity full-import...");
singleEntity(1);
logPropertiesFile();
changeStuff();
int c = calculateDatabaseCalls();
- log.info("testSingleEntity delta-import (" + c + " database calls expected)...");
+ log.debug("testSingleEntity delta-import (" + c + " database calls expected)...");
singleEntity(c);
validateChanges();
}
@Test
public void testWithSimpleTransformer() throws Exception {
- log.info("testWithSimpleTransformer full-import...");
+ log.debug("testWithSimpleTransformer full-import...");
simpleTransform(1);
logPropertiesFile();
changeStuff();
int c = calculateDatabaseCalls();
simpleTransform(c);
- log.info("testWithSimpleTransformer delta-import (" + c + " database calls expected)...");
+ log.debug("testWithSimpleTransformer delta-import (" + c + " database calls expected)...");
validateChanges();
}
@Test
public void testWithComplexTransformer() throws Exception {
- log.info("testWithComplexTransformer full-import...");
+ log.debug("testWithComplexTransformer full-import...");
complexTransform(1, 0);
logPropertiesFile();
changeStuff();
int c = calculateDatabaseCalls();
- log.info("testWithComplexTransformer delta-import (" + c + " database calls expected)...");
+ log.debug("testWithComplexTransformer delta-import (" + c + " database calls expected)...");
complexTransform(c, personChanges.deletedKeys.length);
validateChanges();
}
@Test
public void testChildEntities() throws Exception {
- log.info("testChildEntities full-import...");
+ log.debug("testChildEntities full-import...");
useParentDeltaQueryParam = random().nextBoolean();
- log.info("using parent delta? " + useParentDeltaQueryParam);
+ log.debug("using parent delta? " + useParentDeltaQueryParam);
withChildEntities(false, true);
logPropertiesFile();
changeStuff();
- log.info("testChildEntities delta-import...");
+ log.debug("testChildEntities delta-import...");
withChildEntities(false, false);
validateChanges();
}
@@ -148,13 +148,13 @@ public class TestSqlEntityProcessorDelta
sb.append(s).append(" ");
}
sb.append(" }");
- log.info(sb.toString());
+ log.debug(sb.toString());
}
}
private void personChangesLog()
{
if(personChanges!=null) {
- log.info("person changes { " + personChanges.toString() + " } ");
+ log.debug("person changes { " + personChanges.toString() + " } ");
}
}
@Override
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/SolrLogFormatter.java Wed Dec 12 21:41:06 2012
@@ -263,7 +263,7 @@ sb.append("(group_name=").append(tg.getN
private Map<String,Object> getCoreProps(ZkController zkController, SolrCore core) {
final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
- Replica props = zkController.getClusterState().getShardProps(collection, ZkStateReader.getCoreNodeName(zkController.getNodeName(), core.getName()));
+ Replica props = zkController.getClusterState().getReplica(collection, ZkStateReader.getCoreNodeName(zkController.getNodeName(), core.getName()));
if(props!=null) {
return props.getProperties();
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/AssignShard.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/AssignShard.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/AssignShard.java Wed Dec 12 21:41:06 2012
@@ -39,7 +39,10 @@ public class AssignShard {
numShards = 1;
}
String returnShardId = null;
- Map<String, Slice> sliceMap = state.getSlices(collection);
+ Map<String, Slice> sliceMap = state.getSlicesMap(collection);
+
+
+ // TODO: now that we create shards ahead of time, is this code needed? Esp since hash ranges aren't assigned when creating via this method?
if (sliceMap == null) {
return "shard1";
@@ -51,6 +54,8 @@ public class AssignShard {
return "shard" + (shardIdNames.size() + 1);
}
+ // TODO: don't need to sort to find shard with fewest replicas!
+
// else figure out which shard needs more replicas
final Map<String, Integer> map = new HashMap<String, Integer>();
for (String shardId : shardIdNames) {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Wed Dec 12 21:41:06 2012
@@ -263,7 +263,7 @@ final class ShardLeaderElectionContext e
ZkNodeProps leaderProps, String collection, String shardId) {
ClusterState clusterState = zkController.getZkStateReader()
.getClusterState();
- Map<String,Slice> slices = clusterState.getSlices(collection);
+ Map<String,Slice> slices = clusterState.getSlicesMap(collection);
Slice slice = slices.get(shardId);
Map<String,Replica> replicasMap = slice.getReplicasMap();
for (Map.Entry<String,Replica> shard : replicasMap.entrySet()) {
@@ -325,6 +325,7 @@ final class ShardLeaderElectionContext e
Thread.sleep(500);
slices = zkController.getClusterState().getSlice(collection, shardId);
+ // System.out.println("###### waitForReplicasToComeUp : slices=" + slices + " all=" + zkController.getClusterState().getCollectionStates() );
cnt++;
}
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/Overseer.java Wed Dec 12 21:41:06 2012
@@ -17,6 +17,7 @@ package org.apache.solr.cloud;
* the License.
*/
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -25,7 +26,10 @@ import java.util.Map;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ClosableThread;
-import org.apache.solr.common.cloud.HashPartitioner;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -229,7 +233,7 @@ public class Overseer {
//request new shardId
if (collectionExists) {
// use existing numShards
- numShards = state.getCollectionStates().get(collection).size();
+ numShards = state.getCollectionStates().get(collection).getSlices().size();
log.info("Collection already exists with " + ZkStateReader.NUM_SHARDS_PROP + "=" + numShards);
}
sliceName = AssignShard.assignShard(collection, state, numShards);
@@ -274,16 +278,23 @@ public class Overseer {
return newClusterState;
}
+ private Map<String,Object> defaultCollectionProps() {
+ HashMap<String,Object> props = new HashMap<String, Object>(2);
+ props.put(DocCollection.DOC_ROUTER, DocRouter.DEFAULT_NAME);
+ return props;
+ }
+
private ClusterState createCollection(ClusterState state, String collectionName, int numShards) {
log.info("Create collection {} with numShards {}", collectionName, numShards);
-
- HashPartitioner hp = new HashPartitioner();
- List<HashPartitioner.Range> ranges = hp.partitionRange(numShards, hp.fullRange());
+
+ DocRouter router = DocRouter.DEFAULT;
+ List<DocRouter.Range> ranges = router.partitionRange(numShards, router.fullRange());
+
+ Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>();
- Map<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String, Slice>>();
Map<String, Slice> newSlices = new LinkedHashMap<String,Slice>();
- newStates.putAll(state.getCollectionStates());
+ newCollections.putAll(state.getCollectionStates());
for (int i = 0; i < numShards; i++) {
final String sliceName = "shard" + (i+1);
@@ -292,20 +303,26 @@ public class Overseer {
newSlices.put(sliceName, new Slice(sliceName, null, sliceProps));
}
- newStates.put(collectionName, newSlices);
- ClusterState newClusterState = new ClusterState(state.getLiveNodes(), newStates);
+
+ // TODO: fill in with collection properties read from the /collections/<collectionName> node
+ Map<String,Object> collectionProps = defaultCollectionProps();
+
+ DocCollection newCollection = new DocCollection(collectionName, newSlices, collectionProps, router);
+
+ newCollections.put(collectionName, newCollection);
+ ClusterState newClusterState = new ClusterState(state.getLiveNodes(), newCollections);
return newClusterState;
}
-
+
/*
* Return an already assigned id or null if not assigned
*/
private String getAssignedId(final ClusterState state, final String nodeName,
final ZkNodeProps coreState) {
final String key = coreState.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + coreState.getStr(ZkStateReader.CORE_NAME_PROP);
- Map<String, Slice> slices = state.getSlices(coreState.getStr(ZkStateReader.COLLECTION_PROP));
+ Collection<Slice> slices = state.getSlices(coreState.getStr(ZkStateReader.COLLECTION_PROP));
if (slices != null) {
- for (Slice slice : slices.values()) {
+ for (Slice slice : slices) {
if (slice.getReplicasMap().get(key) != null) {
return slice.getName();
}
@@ -314,40 +331,49 @@ public class Overseer {
return null;
}
- private ClusterState updateSlice(ClusterState state, String collection, Slice slice) {
+ private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) {
// System.out.println("###!!!### OLD CLUSTERSTATE: " + JSONUtil.toJSON(state.getCollectionStates()));
// System.out.println("Updating slice:" + slice);
- Map<String, Map<String, Slice>> newCollections = new LinkedHashMap<String,Map<String,Slice>>(state.getCollectionStates()); // make a shallow copy
- Map<String, Slice> slices = newCollections.get(collection);
- if (slices == null) {
+ Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(state.getCollectionStates()); // make a shallow copy
+ DocCollection coll = newCollections.get(collectionName);
+ Map<String,Slice> slices;
+ Map<String,Object> props;
+ DocRouter router;
+
+ if (coll == null) {
+ // when updateSlice is called on a collection that doesn't exist, it's currently when a core is publishing itself
+ // without explicitly creating a collection. In this current case, we assume custom sharding with an "implicit" router.
slices = new HashMap<String, Slice>(1);
+ props = new HashMap<String,Object>(1);
+ props.put(DocCollection.DOC_ROUTER, ImplicitDocRouter.NAME);
+ router = new ImplicitDocRouter();
} else {
- slices = new LinkedHashMap<String, Slice>(slices); // make a shallow copy
- }
- slices.put(slice.getName(), slice);
- newCollections.put(collection, slices);
+ props = coll.getProperties();
+ router = coll.getRouter();
+ slices = new LinkedHashMap<String, Slice>(coll.getSlicesMap()); // make a shallow copy
+ }
+ slices.put(slice.getName(), slice);
+ DocCollection newCollection = new DocCollection(collectionName, slices, props, router);
+ newCollections.put(collectionName, newCollection);
// System.out.println("###!!!### NEW CLUSTERSTATE: " + JSONUtil.toJSON(newCollections));
return new ClusterState(state.getLiveNodes(), newCollections);
}
- private ClusterState setShardLeader(ClusterState state, String collection, String sliceName, String leaderUrl) {
-
- final Map<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String,Slice>>(state.getCollectionStates());
-
- Map<String, Slice> slices = newStates.get(collection);
+ private ClusterState setShardLeader(ClusterState state, String collectionName, String sliceName, String leaderUrl) {
- if(slices==null) {
- log.error("Could not mark shard leader for non existing collection:" + collection);
+ final Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(state.getCollectionStates());
+ DocCollection coll = newCollections.get(collectionName);
+ if(coll == null) {
+ log.error("Could not mark shard leader for non existing collection:" + collectionName);
return state;
}
+ Map<String, Slice> slices = coll.getSlicesMap();
// make a shallow copy and add it to the new collection
slices = new LinkedHashMap<String,Slice>(slices);
- newStates.put(collection, slices);
-
Slice slice = slices.get(sliceName);
if (slice == null) {
@@ -383,7 +409,11 @@ public class Overseer {
Slice newSlice = new Slice(slice.getName(), newReplicas, slice.getProperties());
slices.put(newSlice.getName(), newSlice);
}
- return new ClusterState(state.getLiveNodes(), newStates);
+
+
+ DocCollection newCollection = new DocCollection(coll.getName(), slices, coll.getProperties(), coll.getRouter());
+ newCollections.put(collectionName, newCollection);
+ return new ClusterState(state.getLiveNodes(), newCollections);
}
/*
@@ -394,51 +424,57 @@ public class Overseer {
final String coreNodeName = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP);
final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
- final LinkedHashMap<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String,Slice>>();
- for(String collectionName: clusterState.getCollections()) {
- if(collection.equals(collectionName)) {
- Map<String, Slice> slices = clusterState.getSlices(collection);
- LinkedHashMap<String, Slice> newSlices = new LinkedHashMap<String, Slice>();
- for(Slice slice: slices.values()) {
- if(slice.getReplicasMap().containsKey(coreNodeName)) {
- Map<String, Replica> newReplicas = slice.getReplicasCopy();
- newReplicas.remove(coreNodeName);
- if (newReplicas.size() != 0) {
- Slice newSlice = new Slice(slice.getName(), newReplicas,
- slice.getProperties());
- newSlices.put(slice.getName(), newSlice);
- }
- } else {
- newSlices.put(slice.getName(), slice);
- }
- }
- int cnt = 0;
- for (Slice slice : newSlices.values()) {
- cnt+=slice.getReplicasMap().size();
- }
- // TODO: if no nodes are left after this unload
- // remove from zk - do we have a race where Overseer
- // see's registered nodes and publishes though?
- if (cnt > 0) {
- newStates.put(collectionName, newSlices);
+ final Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(clusterState.getCollectionStates()); // shallow copy
+ DocCollection coll = newCollections.get(collection);
+ if (coll == null) {
+ // TODO: log/error that we didn't find it?
+ return clusterState;
+ }
+
+ Map<String, Slice> newSlices = new LinkedHashMap<String, Slice>();
+ for (Slice slice : coll.getSlices()) {
+ Replica replica = slice.getReplica(coreNodeName);
+ if (replica != null) {
+ Map<String, Replica> newReplicas = slice.getReplicasCopy();
+ newReplicas.remove(coreNodeName);
+ // TODO TODO TODO!!! if there are no replicas left for the slice, and the slice has no hash range, remove it
+ // if (newReplicas.size() == 0 && slice.getRange() == null) {
+ // if there are no replicas left for the slice remove it
+ if (newReplicas.size() == 0) {
+ slice = null;
} else {
- // TODO: it might be better logically to have this in ZkController
- // but for tests (it's easier) it seems better for the moment to leave CoreContainer and/or
- // ZkController out of the Overseer.
- try {
- zkClient.clean("/collections/" + collectionName);
- } catch (InterruptedException e) {
- SolrException.log(log, "Cleaning up collection in zk was interrupted:" + collectionName, e);
- Thread.currentThread().interrupt();
- } catch (KeeperException e) {
- SolrException.log(log, "Problem cleaning up collection in zk:" + collectionName, e);
- }
+ slice = new Slice(slice.getName(), newReplicas, slice.getProperties());
}
- } else {
- newStates.put(collectionName, clusterState.getSlices(collectionName));
}
+
+ if (slice != null) {
+ newSlices.put(slice.getName(), slice);
+ }
+ }
+
+ // if there are no slices left in the collection, remove it?
+ if (newSlices.size() == 0) {
+ newCollections.remove(coll.getName());
+
+ // TODO: it might be better logically to have this in ZkController
+ // but for tests (it's easier) it seems better for the moment to leave CoreContainer and/or
+ // ZkController out of the Overseer.
+ try {
+ zkClient.clean("/collections/" + collection);
+ } catch (InterruptedException e) {
+ SolrException.log(log, "Cleaning up collection in zk was interrupted:" + collection, e);
+ Thread.currentThread().interrupt();
+ } catch (KeeperException e) {
+ SolrException.log(log, "Problem cleaning up collection in zk:" + collection, e);
+ }
+
+
+ } else {
+ DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter());
+ newCollections.put(newCollection.getName(), newCollection);
}
- ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newStates);
+
+ ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections);
return newState;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Wed Dec 12 21:41:06 2012
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -259,9 +260,8 @@ public class OverseerCollectionProcessor
ShardRequest sreq = new ShardRequest();
params.set("qt", adminPath);
sreq.purpose = 1;
- // TODO: this does not work if original url had _ in it
- // We should have a master list
- String replica = nodeName.replaceAll("_", "/");
+ String replica = zkStateReader.getZkClient()
+ .getBaseUrlForNodeName(nodeName);
if (replica.startsWith("http://")) replica = replica.substring(7);
sreq.shards = new String[] {replica};
sreq.actualShards = sreq.shards;
@@ -305,13 +305,14 @@ public class OverseerCollectionProcessor
log.info("Executing Collection Cmd : " + params);
String collectionName = message.getStr("name");
- Map<String,Slice> slices = clusterState.getCollectionStates().get(collectionName);
+ DocCollection coll = clusterState.getCollection(collectionName);
- if (slices == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collectionName);
+ if (coll == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Could not find collection:" + collectionName);
}
- for (Map.Entry<String,Slice> entry : slices.entrySet()) {
+ for (Map.Entry<String,Slice> entry : coll.getSlicesMap().entrySet()) {
Slice slice = entry.getValue();
Map<String,Replica> shards = slice.getReplicasMap();
Set<Map.Entry<String,Replica>> shardEntries = shards.entrySet();
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java Wed Dec 12 21:41:06 2012
@@ -318,7 +318,7 @@ public class RecoveryStrategy extends Th
try {
CloudDescriptor cloudDesc = core.getCoreDescriptor()
.getCloudDescriptor();
- ZkNodeProps leaderprops = zkStateReader.getLeaderProps(
+ ZkNodeProps leaderprops = zkStateReader.getLeaderRetry(
cloudDesc.getCollectionName(), cloudDesc.getShardId());
String leaderBaseUrl = leaderprops.getStr(ZkStateReader.BASE_URL_PROP);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java Wed Dec 12 21:41:06 2012
@@ -19,7 +19,9 @@ package org.apache.solr.cloud;
import java.io.File;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
+import java.net.URLEncoder;
import java.net.NetworkInterface;
import java.util.Collections;
import java.util.Enumeration;
@@ -44,6 +46,7 @@ import org.apache.solr.client.solrj.requ
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
@@ -144,18 +147,31 @@ public final class ZkController {
TimeoutException, IOException {
if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null.");
this.cc = cc;
- if (localHostContext.contains("/")) {
- throw new IllegalArgumentException("localHostContext ("
- + localHostContext + ") should not contain a /");
+ if (localHostContext.startsWith("/")) {
+ // be forgiving and strip this off
+ // this allows us to support users specifying hostContext="/" in
+ // solr.xml to indicate the root context, instead of hostContext=""
+ // which means the default of "solr"
+ localHostContext = localHostContext.substring(1);
+ }
+ if (localHostContext.endsWith("/")) {
+ // be extra nice
+ localHostContext = localHostContext.substring(0,localHostContext.length()-1);
}
+
this.zkServerAddress = zkServerAddress;
this.localHostPort = locaHostPort;
this.localHostContext = localHostContext;
this.localHost = getHostAddress(localHost);
+ this.baseURL = this.localHost + ":" + this.localHostPort +
+ (this.localHostContext.isEmpty() ? "" : ("/" + this.localHostContext));
+
this.hostName = getHostNameFromAddress(this.localHost);
- this.nodeName = this.hostName + ':' + this.localHostPort + '_' + this.localHostContext;
- this.baseURL = this.localHost + ":" + this.localHostPort + "/" + this.localHostContext;
+ this.nodeName = generateNodeName(this.hostName,
+ this.localHostPort,
+ this.localHostContext);
+
this.leaderVoteWait = leaderVoteWait;
this.clientTimeout = zkClientTimeout;
zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout, zkClientConnectTimeout,
@@ -919,6 +935,10 @@ public final class ZkController {
try {
Map<String,Object> collectionProps = new HashMap<String,Object>();
+
+ // set defaults
+ collectionProps.put(DocCollection.DOC_ROUTER, "compositeId");
+
// TODO: if collection.configName isn't set, and there isn't already a conf in zk, just use that?
String defaultConfigName = System.getProperty(COLLECTION_PARAM_PREFIX+CONFIGNAME_PROP, collection);
@@ -933,8 +953,10 @@ public final class ZkController {
}
// if the config name wasn't passed in, use the default
- if (!collectionProps.containsKey(CONFIGNAME_PROP))
+ if (!collectionProps.containsKey(CONFIGNAME_PROP)) {
+ // TODO: getting the configName from the collectionPath should fail since we already know it doesn't exist?
getConfName(collection, collectionPath, collectionProps);
+ }
} else if(System.getProperty("bootstrap_confdir") != null) {
// if we are bootstrapping a collection, default the config for
@@ -958,7 +980,6 @@ public final class ZkController {
} else {
getConfName(collection, collectionPath, collectionProps);
}
-
ZkNodeProps zkProps = new ZkNodeProps(collectionProps);
zkClient.makePath(collectionPath, ZkStateReader.toJSON(zkProps), CreateMode.PERSISTENT, null, true);
@@ -1269,4 +1290,23 @@ public final class ZkController {
return cmdDistribExecutor;
}
+ /**
+ * Returns the nodeName that should be used based on the specified properties.
+ *
+ * @param hostName - must not be the empty string
+ * @param hostPort - must consist only of digits, must not be the empty string
+ * @param hostContext - should not begin or end with a slash, may be the empty string to denote the root context
+ * @lucene.experimental
+ * @see SolrZkClient#getBaseUrlForNodeName
+ */
+ static String generateNodeName(final String hostName,
+ final String hostPort,
+ final String hostContext) {
+ try {
+ return hostName + ':' + hostPort + '_' +
+ URLEncoder.encode(hostContext, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalStateException("JVM Does not seem to support UTF-8", e);
+ }
+ }
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java Wed Dec 12 21:41:06 2012
@@ -71,9 +71,13 @@ public abstract class CachingDirectoryFa
private Double maxWriteMBPerSecRead;
private Double maxWriteMBPerSecDefault;
+
+ private boolean closed;
public interface CloseListener {
- public void onClose();
+ public void postClose();
+
+ public void preClose();
}
@Override
@@ -118,6 +122,7 @@ public abstract class CachingDirectoryFa
@Override
public void close() throws IOException {
synchronized (this) {
+ this.closed = true;
for (CacheValue val : byDirectoryCache.values()) {
try {
// if there are still refs out, we have to wait for them
@@ -132,6 +137,7 @@ public abstract class CachingDirectoryFa
}
assert val.refCnt == 0 : val.refCnt;
+ log.info("Closing directory when closing factory:" + val.path);
val.directory.close();
} catch (Throwable t) {
SolrException.log(log, "Error closing directory", t);
@@ -144,27 +150,40 @@ public abstract class CachingDirectoryFa
private void close(Directory directory) throws IOException {
synchronized (this) {
+
CacheValue cacheValue = byDirectoryCache.get(directory);
if (cacheValue == null) {
throw new IllegalArgumentException("Unknown directory: " + directory
+ " " + byDirectoryCache);
}
-
- log.debug("Closing: {}", cacheValue);
+ log.info("Releasing directory:" + cacheValue.path);
cacheValue.refCnt--;
+
if (cacheValue.refCnt == 0 && cacheValue.doneWithDir) {
log.info("Closing directory:" + cacheValue.path);
- directory.close();
- byDirectoryCache.remove(directory);
- byPathCache.remove(cacheValue.path);
List<CloseListener> listeners = closeListeners.remove(directory);
if (listeners != null) {
for (CloseListener listener : listeners) {
- listener.onClose();
+ listener.preClose();
+ }
+ }
+ try {
+ log.info("Closing directory:" + cacheValue.path);
+ directory.close();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error closing directory", t);
+ }
+
+ if (listeners != null) {
+ for (CloseListener listener : listeners) {
+ listener.postClose();
}
closeListeners.remove(directory);
}
+
+ byDirectoryCache.remove(directory);
+ byPathCache.remove(cacheValue.path);
}
}
}
@@ -202,7 +221,11 @@ public abstract class CachingDirectoryFa
throws IOException {
String fullPath = new File(path).getAbsolutePath();
synchronized (this) {
- CacheValue cacheValue = byPathCache.get(fullPath);
+ if (closed) {
+ throw new RuntimeException("Already closed");
+ }
+
+ final CacheValue cacheValue = byPathCache.get(fullPath);
Directory directory = null;
if (cacheValue != null) {
directory = cacheValue.directory;
@@ -227,7 +250,7 @@ public abstract class CachingDirectoryFa
}
}
- if (directory == null || forceNew) {
+ if (directory == null || forceNew) {
directory = create(fullPath);
directory = rateLimit(directory);