You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/23 19:34:08 UTC
svn commit: r1234932 [1/5] - in /lucene/dev/branches/solrcloud: ./
dev-tools/idea/lucene/contrib/ dev-tools/maven/ dev-tools/maven/solr/
dev-tools/maven/solr/contrib/analysis-extras/
dev-tools/maven/solr/contrib/clustering/ dev-tools/maven/solr/contrib...
Author: markrmiller
Date: Mon Jan 23 18:34:04 2012
New Revision: 1234932
URL: http://svn.apache.org/viewvc?rev=1234932&view=rev
Log:
update to r1234884
Added:
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java
- copied unchanged from r1234912, lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java
- copied unchanged from r1234912, lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java
lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/MockCharFilter.java
- copied unchanged from r1234912, lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockCharFilter.java
lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWSegmentInfosFormat.java
- copied unchanged from r1234912, lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWSegmentInfosFormat.java
lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWSegmentInfosWriter.java
- copied unchanged from r1234912, lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWSegmentInfosWriter.java
lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/analysis/TestMockCharFilter.java
- copied unchanged from r1234912, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockCharFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/htmlentity.py
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/htmlentity.py
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/MS-Word 14 generated.htm
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/MS-Word 14 generated.htm
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
lucene/dev/branches/solrcloud/modules/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateHTMLStripCharFilterSupplementaryMacros.java
- copied unchanged from r1234912, lucene/dev/trunk/modules/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateHTMLStripCharFilterSupplementaryMacros.java
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/analysis/TypeTokenFilterFactory.java
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/TypeTokenFilterFactory.java
lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/stoptypes-1.txt
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/test-files/solr/conf/stoptypes-1.txt
lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/stoptypes-2.txt
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/test-files/solr/conf/stoptypes-2.txt
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestHTMLStripCharFilterFactory.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestTypeTokenFilterFactory.java
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestTypeTokenFilterFactory.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/htmlStripReaderTest.html
- copied unchanged from r1234912, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/htmlStripReaderTest.html
Modified:
lucene/dev/branches/solrcloud/ (props changed)
lucene/dev/branches/solrcloud/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/branches/solrcloud/dev-tools/maven/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/clustering/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler-extras/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/extraction/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/langid/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/uima/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/velocity/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/core/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/pom.xml.template
lucene/dev/branches/solrcloud/dev-tools/maven/solr/solrj/pom.xml.template
lucene/dev/branches/solrcloud/lucene/ (props changed)
lucene/dev/branches/solrcloud/lucene/CHANGES.txt
lucene/dev/branches/solrcloud/lucene/contrib/CHANGES.txt (props changed)
lucene/dev/branches/solrcloud/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/PostingsConsumer.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsFormat.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (props changed)
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/SentinelIntSet.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/NodeHash.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java
lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/Util.java
lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java
lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java
lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestSentinelIntSet.java
lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
lucene/dev/branches/solrcloud/modules/analysis/common/build.xml
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/charfilter/package.html
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java
lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
lucene/dev/branches/solrcloud/modules/analysis/icu/build.xml
lucene/dev/branches/solrcloud/modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
lucene/dev/branches/solrcloud/modules/analysis/kuromoji/ (props changed)
lucene/dev/branches/solrcloud/modules/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java
lucene/dev/branches/solrcloud/modules/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java
lucene/dev/branches/solrcloud/modules/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java
lucene/dev/branches/solrcloud/modules/benchmark/ (props changed)
lucene/dev/branches/solrcloud/modules/facet/ (props changed)
lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java
lucene/dev/branches/solrcloud/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
lucene/dev/branches/solrcloud/modules/join/src/java/org/apache/lucene/search/join/package.html
lucene/dev/branches/solrcloud/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/solrcloud/solr/ (props changed)
lucene/dev/branches/solrcloud/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/solrcloud/solr/LICENSE.txt (props changed)
lucene/dev/branches/solrcloud/solr/NOTICE.txt (props changed)
lucene/dev/branches/solrcloud/solr/README.txt (props changed)
lucene/dev/branches/solrcloud/solr/build.xml (props changed)
lucene/dev/branches/solrcloud/solr/client/ (props changed)
lucene/dev/branches/solrcloud/solr/cloud-dev/ (props changed)
lucene/dev/branches/solrcloud/solr/common-build.xml (props changed)
lucene/dev/branches/solrcloud/solr/contrib/ (props changed)
lucene/dev/branches/solrcloud/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler-extras/src/java/ (props changed)
lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test-files/ (props changed)
lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/branches/solrcloud/solr/contrib/uima/src/java/ (props changed)
lucene/dev/branches/solrcloud/solr/contrib/uima/src/test-files/ (props changed)
lucene/dev/branches/solrcloud/solr/core/ (props changed)
lucene/dev/branches/solrcloud/solr/core/src/java/ (props changed)
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java
lucene/dev/branches/solrcloud/solr/core/src/test/ (props changed)
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
lucene/dev/branches/solrcloud/solr/dev-tools/ (props changed)
lucene/dev/branches/solrcloud/solr/example/ (props changed)
lucene/dev/branches/solrcloud/solr/lib/ (props changed)
lucene/dev/branches/solrcloud/solr/scripts/ (props changed)
lucene/dev/branches/solrcloud/solr/site/ (props changed)
lucene/dev/branches/solrcloud/solr/site-src/ (props changed)
lucene/dev/branches/solrcloud/solr/solrj/ (props changed)
lucene/dev/branches/solrcloud/solr/solrj/src/java/ (props changed)
lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/client/ (props changed)
lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed)
lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/branches/solrcloud/solr/test-framework/ (props changed)
lucene/dev/branches/solrcloud/solr/testlogging.properties (props changed)
lucene/dev/branches/solrcloud/solr/webapp/ (props changed)
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -362,6 +362,19 @@
</dependency>
</dependencies>
</dependencyManagement>
+ <dependencies>
+ <dependency>
+ <!-- Maven 2.2.X has a bug that omits as duplicate all JUnit -->
+ <!-- dependencies from the classpath when compiling solr-clustering, -->
+ <!-- causing test compilation to fail. Maven 3.0.4 test compilation -->
+ <!-- succeeds with the exact same dependencies, so apparently the -->
+ <!-- bug has been fixed. This dependency can be removed when the -->
+ <!-- minimum Maven version is upgraded to 3.0.4+. -->
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
<build>
<directory>lucene/build/lucene-parent</directory>
<pluginManagement>
@@ -387,6 +400,11 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
</plugin>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -37,6 +37,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/clustering/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/clustering/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/clustering/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/clustering/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -37,6 +37,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler-extras/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler-extras/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler-extras/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler-extras/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -37,6 +37,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -37,6 +37,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/extraction/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/extraction/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/extraction/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/extraction/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -40,6 +40,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/langid/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/langid/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/langid/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/langid/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -41,6 +41,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/uima/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/uima/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/uima/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/uima/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -37,6 +37,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/velocity/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/velocity/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/velocity/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/contrib/velocity/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -37,6 +37,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/core/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/core/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/core/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/core/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -37,6 +37,15 @@
</properties>
<dependencies>
<dependency>
+ <!-- lucene-test-framework dependency must be declared before lucene-core -->
+ <!-- This dependency cannot be put into solr-parent, because local -->
+ <!-- dependencies are always ordered before inherited dependencies. -->
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-solrj</artifactId>
<version>${project.version}</version>
@@ -194,6 +203,12 @@
<directory>src/test-files</directory>
</testResource>
<testResource>
+ <directory>${project.build.testSourceDirectory}</directory>
+ <excludes>
+ <exclude>**/*.java</exclude>
+ </excludes>
+ </testResource>
+ <testResource>
<directory>../solrj/src/test-files</directory>
</testResource>
</testResources>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -82,12 +82,6 @@
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-test-framework</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<scope>test</scope>
Modified: lucene/dev/branches/solrcloud/dev-tools/maven/solr/solrj/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/dev-tools/maven/solr/solrj/pom.xml.template?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/dev-tools/maven/solr/solrj/pom.xml.template (original)
+++ lucene/dev/branches/solrcloud/dev-tools/maven/solr/solrj/pom.xml.template Mon Jan 23 18:34:04 2012
@@ -113,6 +113,13 @@
<skip>true</skip> <!-- Tests are run from solr-core module -->
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <skip>true</skip> <!-- This skips test compilation - tests are run from solr-core module -->
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
Modified: lucene/dev/branches/solrcloud/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/CHANGES.txt?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/solrcloud/lucene/CHANGES.txt Mon Jan 23 18:34:04 2012
@@ -785,6 +785,16 @@ New Features
(i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery()
will be passed null for any open endpoint. (Ingo Renner, Adriano
Crestani, yonik, Mike McCandless
+
+* LUCENE-3121: Add sugar reverse lookup (given an output, find the
+ input mapping to it) for FSTs that have strictly monotonic long
+ outputs (such as an ord). (Mike McCandless)
+
+* LUCENE-3671: Add TypeTokenFilter that filters tokens based on
+ their TypeAttribute. (Tommaso Teofili via Uwe Schindler)
+
+* LUCENE-3690: Added HTMLStripCharFilter, a CharFilter that strips HTML
+ markup. (Steve Rowe)
Bug fixes
@@ -804,9 +814,11 @@ Bug fixes
* LUCENE-3641: Fixed MultiReader to correctly propagate readerFinishedListeners
to clones/reopened readers. (Uwe Schindler)
-* LUCENE-3642: Fixed bugs in CharTokenizer, n-gram filters, and smart chinese
- where they would create invalid offsets in some situations, leading to problems
- in highlighting. (Max Beutel via Robert Muir)
+* LUCENE-3642, SOLR-2891, LUCENE-3717: Fixed bugs in CharTokenizer, n-gram filters,
+ compound token filters, thai word filter, icutokenizer, pattern analyzer,
+ wikipediatokenizer, and smart chinese where they would create invalid offsets in
+ some situations, leading to problems in highlighting.
+ (Max Beutel, Edwin Steiner via Robert Muir)
* LUCENE-3639: TopDocs.merge was incorrectly setting TopDocs.maxScore to
Float.MIN_VALUE when it should be Float.NaN, when there were 0
@@ -821,6 +833,12 @@ Bug fixes
* LUCENE-3605: don't sleep in a retry loop when trying to locate the
segments_N file (Robert Muir, Mike McCandless)
+* LUCENE-3711: SentinelIntSet with a small initial size can go into
+ an infinite loop when expanded. This can affect grouping using
+ TermAllGroupsCollector or TermAllGroupHeadsCollector if instantiated with a
+ non default small size. (Martijn van Groningen, yonik)
+
+
Optimizations
* LUCENE-3653: Improve concurrency in VirtualMethod and AttributeSource by
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/PostingsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/PostingsConsumer.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/PostingsConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/PostingsConsumer.java Mon Jan 23 18:34:04 2012
@@ -33,9 +33,7 @@ import org.apache.lucene.util.FixedBitSe
public abstract class PostingsConsumer {
- /** Adds a new doc in this term. If this field omits term
- * freqs & positions then termDocFreq should be ignored,
- * and, finishDoc will not be called. */
+ /** Adds a new doc in this term. */
public abstract void startDoc(int docID, int termDocFreq) throws IOException;
public static class PostingsMergeState {
@@ -52,8 +50,7 @@ public abstract class PostingsConsumer {
public abstract void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException;
/** Called when we are done adding positions & payloads
- * for each doc. Not called when the field omits term
- * freq and positions. */
+ * for each doc. */
public abstract void finishDoc() throws IOException;
/** Default merge impl: append documents, mapping around
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java Mon Jan 23 18:34:04 2012
@@ -30,9 +30,7 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfosFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
@@ -55,12 +53,8 @@ public class Lucene3xCodec extends Codec
private final FieldInfosFormat fieldInfosFormat = new Lucene3xFieldInfosFormat();
- // TODO: this should really be a different impl
- // also if we want preflex to *really* be read-only it should throw exception for the writer?
- // this way IR.commit fails on delete/undelete/setNorm/etc ?
- private final SegmentInfosFormat infosFormat = new Lucene40SegmentInfosFormat();
+ private final SegmentInfosFormat infosFormat = new Lucene3xSegmentInfosFormat();
- // TODO: this should really be a different impl
private final NormsFormat normsFormat = new Lucene3xNormsFormat();
// 3.x doesn't support docvalues
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java Mon Jan 23 18:34:04 2012
@@ -27,8 +27,12 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.store.Directory;
/**
+ * Lucene3x ReadOnly FieldInfosFromat implementation
+ * @deprecated (4.0) This is only used to read indexes created
+ * before 4.0.
* @lucene.experimental
*/
+@Deprecated
public class Lucene3xFieldInfosFormat extends FieldInfosFormat {
private final FieldInfosReader reader = new Lucene3xFieldInfosReader();
@@ -39,7 +43,7 @@ public class Lucene3xFieldInfosFormat ex
@Override
public FieldInfosWriter getFieldInfosWriter() throws IOException {
- throw new IllegalArgumentException("this codec can only be used for reading");
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsFormat.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsFormat.java Mon Jan 23 18:34:04 2012
@@ -29,10 +29,12 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.store.Directory;
/**
- * Read-Only Lucene 3.x Norms Format
- *
+ * Lucene3x ReadOnly NormsFormat implementation
+ * @deprecated (4.0) This is only used to read indexes created
+ * before 4.0.
* @lucene.experimental
*/
+@Deprecated
public class Lucene3xNormsFormat extends NormsFormat {
@@ -49,7 +51,7 @@ public class Lucene3xNormsFormat extends
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- throw new IllegalArgumentException("this codec can only be used for reading");
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xPostingsFormat.java Mon Jan 23 18:34:04 2012
@@ -57,7 +57,7 @@ public class Lucene3xPostingsFormat exte
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- throw new IllegalArgumentException("this codec can only be used for reading");
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java Mon Jan 23 18:34:04 2012
@@ -28,6 +28,13 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+/**
+ * Lucene3x ReadOnly TermVectorsFormat implementation
+ * @deprecated (4.0) This is only used to read indexes created
+ * before 4.0.
+ * @lucene.experimental
+ */
+@Deprecated
public class Lucene3xTermVectorsFormat extends TermVectorsFormat {
@Override
@@ -37,8 +44,7 @@ public class Lucene3xTermVectorsFormat e
@Override
public TermVectorsWriter vectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
- // TODO all these IAEs in preflex should be UOEs?
- throw new IllegalArgumentException("this codec can only be used for reading");
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java Mon Jan 23 18:34:04 2012
@@ -23,13 +23,10 @@ import java.util.Map;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.SegmentInfosReader;
-import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsReader;
-import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -39,88 +36,37 @@ import org.apache.lucene.store.IOContext
*/
public class Lucene40SegmentInfosReader extends SegmentInfosReader {
- // TODO: shove all backwards code to preflex!
- // this is a little tricky, because of IR.commit(), two options:
- // 1. PreFlex writes 4.x SIS format, but reads both 3.x and 4.x
- // (and maybe RW always only writes the 3.x one? for that to work well,
- // we have to move .fnx file to codec too, not too bad but more work).
- // or we just have crappier RW testing like today.
- // 2. PreFlex writes 3.x SIS format, and only reads 3.x
- // (in this case we have to move .fnx file to codec as well)
@Override
public void read(Directory directory, String segmentsFileName, ChecksumIndexInput input, SegmentInfos infos, IOContext context) throws IOException {
infos.version = input.readLong(); // read version
infos.counter = input.readInt(); // read counter
final int format = infos.getFormat();
+ assert format <= SegmentInfos.FORMAT_4_0;
for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
SegmentInfo si = readSegmentInfo(directory, format, input);
- if (si.getVersion() == null) {
- // Could be a 3.0 - try to open the doc stores - if it fails, it's a
- // 2.x segment, and an IndexFormatTooOldException will be thrown,
- // which is what we want.
- Directory dir = directory;
- if (si.getDocStoreOffset() != -1) {
- if (si.getDocStoreIsCompoundFile()) {
- dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
- si.getDocStoreSegment(), "",
- IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false);
- }
- } else if (si.getUseCompoundFile()) {
- dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
- si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
- }
-
- try {
- Lucene40StoredFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment());
- } finally {
- // If we opened the directory, close it
- if (dir != directory) dir.close();
- }
-
- // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next
- // time the segment is read, its version won't be null and we won't
- // need to open FieldsReader every time for each such segment.
- si.setVersion("3.0");
- } else if (si.getVersion().equals("2.x")) {
- // If it's a 3x index touched by 3.1+ code, then segments record their
- // version, whether they are 2.x ones or not. We detect that and throw
- // appropriate exception.
- throw new IndexFormatTooOldException("segment " + si.name + " in resource " + input, si.getVersion());
- }
+ assert si.getVersion() != null;
infos.add(si);
}
infos.userData = input.readStringStringMap();
}
- // if we make a preflex impl we can remove a lot of this hair...
public SegmentInfo readSegmentInfo(Directory dir, int format, ChecksumIndexInput input) throws IOException {
- final String version;
- if (format <= SegmentInfos.FORMAT_3_1) {
- version = input.readString();
- } else {
- version = null;
- }
+ final String version = input.readString();
final String name = input.readString();
final int docCount = input.readInt();
final long delGen = input.readLong();
+ // this is still written in 4.0 if we open a 3.x and upgrade the SI
final int docStoreOffset = input.readInt();
final String docStoreSegment;
final boolean docStoreIsCompoundFile;
- if (docStoreOffset != -1) {
+ if (docStoreOffset != -1) {
docStoreSegment = input.readString();
docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
} else {
docStoreSegment = name;
docStoreIsCompoundFile = false;
}
-
- if (format > SegmentInfos.FORMAT_4_0) {
- // pre-4.0 indexes write a byte if there is a single norms file
- byte b = input.readByte();
- assert 1 == b;
- }
-
final int numNormGen = input.readInt();
final Map<Integer,Long> normGen;
if (numNormGen == SegmentInfo.NO) {
@@ -128,61 +74,17 @@ public class Lucene40SegmentInfosReader
} else {
normGen = new HashMap<Integer, Long>();
for(int j=0;j<numNormGen;j++) {
- int fieldNumber = j;
- if (format <= SegmentInfos.FORMAT_4_0) {
- fieldNumber = input.readInt();
- }
-
- normGen.put(fieldNumber, input.readLong());
+ normGen.put(input.readInt(), input.readLong());
}
}
final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
final int delCount = input.readInt();
assert delCount <= docCount;
-
final int hasProx = input.readByte();
-
- final Codec codec;
- // note: if the codec is not available: Codec.forName will throw an exception.
- if (format <= SegmentInfos.FORMAT_4_0) {
- codec = Codec.forName(input.readString());
- } else {
- codec = Codec.forName("Lucene3x");
- }
+ final Codec codec = Codec.forName(input.readString());
final Map<String,String> diagnostics = input.readStringStringMap();
-
- final int hasVectors;
- if (format <= SegmentInfos.FORMAT_HAS_VECTORS) {
- hasVectors = input.readByte();
- } else {
- final String storesSegment;
- final String ext;
- final boolean storeIsCompoundFile;
- if (docStoreOffset != -1) {
- storesSegment = docStoreSegment;
- storeIsCompoundFile = docStoreIsCompoundFile;
- ext = IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
- } else {
- storesSegment = name;
- storeIsCompoundFile = isCompoundFile;
- ext = IndexFileNames.COMPOUND_FILE_EXTENSION;
- }
- final Directory dirToTest;
- if (storeIsCompoundFile) {
- dirToTest = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE, false);
- } else {
- dirToTest = dir;
- }
- try {
- // TODO: remove this manual file check or push to preflex codec
- hasVectors = dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, "", Lucene40TermVectorsReader.VECTORS_INDEX_EXTENSION)) ? SegmentInfo.YES : SegmentInfo.NO;
- } finally {
- if (isCompoundFile) {
- dirToTest.close();
- }
- }
- }
+ final int hasVectors = input.readByte();
return new SegmentInfo(dir, version, name, docCount, delGen, docStoreOffset,
docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java Mon Jan 23 18:34:04 2012
@@ -43,6 +43,10 @@ public class Lucene40SegmentInfosWriter
IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount())));
boolean success = false;
try {
+ /*
+ * TODO its not ideal that we write the format and the codecID inside the
+ * codec private classes but we read it in SegmentInfos.
+ */
out.writeInt(SegmentInfos.FORMAT_CURRENT); // write FORMAT
out.writeString(codecID); // write codecID
out.writeLong(infos.version);
@@ -69,7 +73,7 @@ public class Lucene40SegmentInfosWriter
output.writeString(si.name);
output.writeInt(si.docCount);
output.writeLong(si.getDelGen());
-
+ // we still need to write this in 4.0 since we can open a 3.x with shared docStores
output.writeInt(si.getDocStoreOffset());
if (si.getDocStoreOffset() != -1) {
output.writeString(si.getDocStoreSegment());
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java Mon Jan 23 18:34:04 2012
@@ -34,19 +34,24 @@ public class DocumentsWriterFlushQueue {
private final AtomicInteger ticketCount = new AtomicInteger();
private final ReentrantLock purgeLock = new ReentrantLock();
- synchronized void addDeletesAndPurge(DocumentsWriter writer,
+ void addDeletesAndPurge(DocumentsWriter writer,
DocumentsWriterDeleteQueue deleteQueue) throws IOException {
- incTickets();// first inc the ticket count - freeze opens
- // a window for #anyChanges to fail
- boolean success = false;
- try {
- queue.add(new GlobalDeletesTicket(deleteQueue.freezeGlobalBuffer(null)));
- success = true;
- } finally {
- if (!success) {
- decTickets();
+ synchronized (this) {
+ incTickets();// first inc the ticket count - freeze opens
+ // a window for #anyChanges to fail
+ boolean success = false;
+ try {
+ queue
+ .add(new GlobalDeletesTicket(deleteQueue.freezeGlobalBuffer(null)));
+ success = true;
+ } finally {
+ if (!success) {
+ decTickets();
+ }
}
}
+ // don't hold the lock on the FlushQueue when forcing the purge - this blocks and deadlocks
+ // if we hold the lock.
forcePurge(writer);
}
@@ -127,6 +132,7 @@ public class DocumentsWriterFlushQueue {
}
void forcePurge(DocumentsWriter writer) throws IOException {
+ assert !Thread.holdsLock(this);
purgeLock.lock();
try {
innerPurge(writer);
@@ -136,6 +142,7 @@ public class DocumentsWriterFlushQueue {
}
void tryPurge(DocumentsWriter writer) throws IOException {
+ assert !Thread.holdsLock(this);
if (purgeLock.tryLock()) {
try {
innerPurge(writer);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java Mon Jan 23 18:34:04 2012
@@ -404,7 +404,7 @@ public final class FieldInfos implements
public boolean hasNorms() {
for (FieldInfo fi : this) {
- if (fi.isIndexed && !fi.omitNorms) {
+ if (fi.normsPresent()) {
return true;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Mon Jan 23 18:34:04 2012
@@ -517,9 +517,8 @@ final class FreqProxTermsWriterPerField
}
}
}
-
- postingsConsumer.finishDoc();
}
+ postingsConsumer.finishDoc();
}
termsConsumer.finishTerm(text, new TermStats(numDocs, totTF));
sumTotalTermFreq += totTF;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java Mon Jan 23 18:34:04 2012
@@ -921,13 +921,7 @@ public abstract class IndexReader implem
* If this method returns an empty array, that means this
* reader is a null reader (for example a MultiReader
* that has no sub readers).
- * <p>
- * NOTE: You should not try using sub-readers returned by
- * this method to make any changes (deleteDocument,
- * etc.). While this might succeed for one composite reader
- * (like MultiReader), it will most likely lead to index
- * corruption for other readers (like DirectoryReader obtained
- * through {@link #open}. Use the parent reader directly. */
+ */
public IndexReader[] getSequentialSubReaders() {
ensureOpen();
return null;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java Mon Jan 23 18:34:04 2012
@@ -255,10 +255,8 @@ final class CompoundFileWriter implement
assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written";
seenIDs.add(id);
final DirectCFSIndexOutput out;
- if (outputTaken.compareAndSet(false, true)) {
+ if ((outputLocked = outputTaken.compareAndSet(false, true))) {
out = new DirectCFSIndexOutput(getOutput(), entry, false);
- outputLocked = true;
- success = true;
} else {
entry.dir = this.directory;
if (directory.fileExists(name)) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/SentinelIntSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/SentinelIntSet.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/SentinelIntSet.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/SentinelIntSet.java Mon Jan 23 18:34:04 2012
@@ -96,13 +96,13 @@ public class SentinelIntSet {
public int put(int key) {
int s = find(key);
if (s < 0) {
+ count++;
if (count >= rehashCount) {
rehash();
s = getSlot(key);
} else {
s = -s-1;
}
- count++;
keys[s] = key;
}
return s;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Mon Jan 23 18:34:04 2012
@@ -32,12 +32,13 @@ import org.apache.lucene.util.BytesRef;
public final class ByteSequenceOutputs extends Outputs<BytesRef> {
private final static BytesRef NO_OUTPUT = new BytesRef();
+ private final static ByteSequenceOutputs singleton = new ByteSequenceOutputs();
private ByteSequenceOutputs() {
}
public static ByteSequenceOutputs getSingleton() {
- return new ByteSequenceOutputs();
+ return singleton;
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FST.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FST.java Mon Jan 23 18:34:04 2012
@@ -274,7 +274,7 @@ public class FST<T> {
getFirstArc(arc);
final BytesReader in = getBytesReader(0);
if (targetHasArcs(arc)) {
- readFirstRealArc(arc.target, arc);
+ readFirstRealArc(arc.target, arc, in);
while(true) {
assert arc.label != END_LABEL;
if (arc.label < cachedRootArcs.length) {
@@ -666,14 +666,12 @@ public class FST<T> {
//System.out.println(" insert isFinal; nextArc=" + follow.target + " isLast=" + arc.isLast() + " output=" + outputs.outputToString(arc.output));
return arc;
} else {
- return readFirstRealArc(follow.target, arc);
+ return readFirstRealArc(follow.target, arc, getBytesReader(0));
}
}
- public Arc<T> readFirstRealArc(int address, Arc<T> arc) throws IOException {
-
- final BytesReader in = getBytesReader(address);
-
+ public Arc<T> readFirstRealArc(int address, Arc<T> arc, final BytesReader in) throws IOException {
+ in.pos = address;
arc.flags = in.readByte();
if (arc.flag(BIT_ARCS_AS_FIXED_ARRAY)) {
@@ -715,7 +713,7 @@ public class FST<T> {
// This arc went to virtual final node, ie has no outgoing arcs
return null;
}
- return readFirstRealArc(arc.nextArc, arc);
+ return readFirstRealArc(arc.nextArc, arc, getBytesReader(0));
} else {
return readNextRealArc(arc, getBytesReader(0));
}
@@ -989,14 +987,16 @@ public class FST<T> {
public final BytesReader getBytesReader(int pos) {
// TODO: maybe re-use via ThreadLocal?
- return new BytesReader(pos);
+ return new BytesReader(bytes, pos);
}
- // Non-static: reads byte[] from FST
- final class BytesReader extends DataInput {
+ /** Expert */
+ public final static class BytesReader extends DataInput {
+ final byte[] bytes;
int pos;
- public BytesReader(int pos) {
+ public BytesReader(byte[] bytes, int pos) {
+ this.bytes = bytes;
this.pos = pos;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java Mon Jan 23 18:34:04 2012
@@ -143,7 +143,7 @@ abstract class FSTEnum<T> {
// Arcs are fixed array -- use binary search to find
// the target.
- final FST<T>.BytesReader in = fst.getBytesReader(0);
+ final FST.BytesReader in = fst.getBytesReader(0);
int low = arc.arcIdx;
int high = arc.numArcs-1;
int mid = 0;
@@ -281,7 +281,7 @@ abstract class FSTEnum<T> {
// Arcs are fixed array -- use binary search to find
// the target.
- final FST<T>.BytesReader in = fst.getBytesReader(0);
+ final FST.BytesReader in = fst.getBytesReader(0);
int low = arc.arcIdx;
int high = arc.numArcs-1;
int mid = 0;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java Mon Jan 23 18:34:04 2012
@@ -32,12 +32,13 @@ import org.apache.lucene.util.IntsRef;
public final class IntSequenceOutputs extends Outputs<IntsRef> {
private final static IntsRef NO_OUTPUT = new IntsRef();
+ private final static IntSequenceOutputs singleton = new IntSequenceOutputs();
private IntSequenceOutputs() {
}
public static IntSequenceOutputs getSingleton() {
- return new IntSequenceOutputs();
+ return singleton;
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/NodeHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/NodeHash.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/NodeHash.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/NodeHash.java Mon Jan 23 18:34:04 2012
@@ -34,9 +34,8 @@ final class NodeHash<T> {
this.fst = fst;
}
- private boolean nodesEqual(Builder.UnCompiledNode<T> node, int address) throws IOException {
- final FST<T>.BytesReader in = fst.getBytesReader(0);
- fst.readFirstRealArc(address, scratchArc);
+ private boolean nodesEqual(Builder.UnCompiledNode<T> node, int address, FST.BytesReader in) throws IOException {
+ fst.readFirstRealArc(address, scratchArc, in);
if (scratchArc.bytesPerArc != 0 && node.numArcs != scratchArc.numArcs) {
return false;
}
@@ -88,10 +87,10 @@ final class NodeHash<T> {
// hash code for a frozen node
private int hash(int node) throws IOException {
final int PRIME = 31;
- final FST<T>.BytesReader in = fst.getBytesReader(0);
+ final FST.BytesReader in = fst.getBytesReader(0);
//System.out.println("hash frozen");
int h = 0;
- fst.readFirstRealArc(node, scratchArc);
+ fst.readFirstRealArc(node, scratchArc, in);
while(true) {
//System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal());
h = PRIME * h + scratchArc.label;
@@ -112,6 +111,7 @@ final class NodeHash<T> {
public int add(Builder.UnCompiledNode<T> node) throws IOException {
// System.out.println("hash: add count=" + count + " vs " + table.length);
+ final FST.BytesReader in = fst.getBytesReader(0);
final int h = hash(node);
int pos = h & mask;
int c = 0;
@@ -128,7 +128,7 @@ final class NodeHash<T> {
rehash();
}
return address;
- } else if (nodesEqual(node, v)) {
+ } else if (nodesEqual(node, v, in)) {
// same node is already here
return v;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java Mon Jan 23 18:34:04 2012
@@ -135,4 +135,9 @@ public final class PositiveIntOutputs ex
public String outputToString(Long output) {
return output.toString();
}
+
+ @Override
+ public String toString() {
+ return "PositiveIntOutputs(doShare=" + doShare + ")";
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/Util.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/Util.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/Util.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/fst/Util.java Mon Jan 23 18:34:04 2012
@@ -86,6 +86,113 @@ public final class Util {
return output;
}
}
+
+ // TODO: parameterize the FST type <T> and allow passing in a
+ // comparator; eg maybe your output is a PairOutput and
+ // one of the outputs in the pair is monotonic so you
+ // compare by that
+
+ /** Reverse lookup (lookup by output instead of by input),
+ * in the special case when your FSTs outputs are
+ * strictly ascending. This locates the input/output
+ * pair where the output is equal to the target, and will
+ * return null if that output does not exist.
+ *
+ * <p>NOTE: this only works with FST<Long>, only
+ * works when the outputs are ascending in order with
+ * the inputs and only works when you shared
+ * the outputs (pass doShare=true to {@link
+ * PositiveIntOutputs#getSingleton}).
+ * For example, simple ordinals (0, 1,
+ * 2, ...), or file offets (when appending to a file)
+ * fit this. */
+ public static IntsRef getByOutput(FST<Long> fst, long targetOutput) throws IOException {
+
+ final FST.BytesReader in = fst.getBytesReader(0);
+
+ // TODO: would be nice not to alloc this on every lookup
+ FST.Arc<Long> arc = fst.getFirstArc(new FST.Arc<Long>());
+
+ FST.Arc<Long> scratchArc = new FST.Arc<Long>();
+
+ final IntsRef result = new IntsRef();
+
+ long output = arc.output;
+ int upto = 0;
+
+ //System.out.println("reverseLookup output=" + targetOutput);
+
+ while(true) {
+ if (arc.isFinal()) {
+ final long finalOutput = output + arc.nextFinalOutput;
+ //System.out.println(" isFinal finalOutput=" + finalOutput);
+ if (finalOutput == targetOutput) {
+ result.length = upto;
+ //System.out.println(" found!");
+ return result;
+ } else if (finalOutput > targetOutput) {
+ //System.out.println(" not found!");
+ return null;
+ }
+ }
+
+ if (fst.targetHasArcs(arc)) {
+ //System.out.println(" targetHasArcs");
+ if (result.ints.length == upto) {
+ result.grow(1+upto);
+ }
+
+ fst.readFirstRealArc(arc.target, arc, in);
+
+ FST.Arc<Long> prevArc = null;
+
+ // TODO: we could do binary search here if node arcs
+ // are array'd:
+ while(true) {
+ //System.out.println(" cycle label=" + arc.label + " output=" + arc.output);
+
+ // This is the min output we'd hit if we follow
+ // this arc:
+ final long minArcOutput = output + arc.output;
+
+ if (minArcOutput == targetOutput) {
+ // Recurse on this arc:
+ //System.out.println(" match! break");
+ output = minArcOutput;
+ result.ints[upto++] = arc.label;
+ break;
+ } else if (minArcOutput > targetOutput) {
+ if (prevArc == null) {
+ // Output doesn't exist
+ return null;
+ } else {
+ // Recurse on previous arc:
+ arc.copyFrom(prevArc);
+ result.ints[upto++] = arc.label;
+ output += arc.output;
+ //System.out.println(" recurse prev label=" + (char) arc.label + " output=" + output);
+ break;
+ }
+ } else if (arc.isLast()) {
+ // Recurse on this arc:
+ output = minArcOutput;
+ //System.out.println(" recurse last label=" + (char) arc.label + " output=" + output);
+ result.ints[upto++] = arc.label;
+ break;
+ } else {
+ // Read next arc in this node:
+ prevArc = scratchArc;
+ prevArc.copyFrom(arc);
+ //System.out.println(" after copy label=" + (char) prevArc.label + " vs " + (char) arc.label);
+ fst.readNextRealArc(arc, in);
+ }
+ }
+ } else {
+ //System.out.println(" no target arcs; not found!");
+ return null;
+ }
+ }
+ }
/**
* Dumps an {@link FST} to a GraphViz's <code>dot</code> language description
@@ -356,4 +463,15 @@ public final class Util {
scratch.length = input.length;
return scratch;
}
+
+ /** Just converts IntsRef to BytesRef; you must ensure the
+ * int values fit into a byte. */
+ public static BytesRef toBytesRef(IntsRef input, BytesRef scratch) {
+ scratch.grow(input.length);
+ for(int i=0;i<input.length;i++) {
+ scratch.bytes[i] = (byte) input.ints[i+input.offset];
+ }
+ scratch.length = input.length;
+ return scratch;
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Mon Jan 23 18:34:04 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
import java.io.IOException;
import java.util.ArrayList;
@@ -289,8 +290,12 @@ public abstract class BaseTokenStreamTes
}
}
};
-
+
public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength) throws IOException {
+ checkRandomData(random, a, iterations, maxWordLength, random.nextBoolean());
+ }
+
+ public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter) throws IOException {
for (int i = 0; i < iterations; i++) {
String text;
switch(_TestUtil.nextInt(random, 0, 4)) {
@@ -311,7 +316,9 @@ public abstract class BaseTokenStreamTes
System.out.println("NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text);
}
- TokenStream ts = a.tokenStream("dummy", new StringReader(text));
+ int remainder = random.nextInt(10);
+ Reader reader = new StringReader(text);
+ TokenStream ts = a.tokenStream("dummy", useCharFilter ? new MockCharFilter(reader, remainder) : reader);
assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class));
CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = ts.hasAttribute(OffsetAttribute.class) ? ts.getAttribute(OffsetAttribute.class) : null;
@@ -339,30 +346,38 @@ public abstract class BaseTokenStreamTes
if (VERBOSE) {
System.out.println("NOTE: BaseTokenStreamTestCase: re-run analysis");
}
+ reader = new StringReader(text);
+ ts = a.tokenStream("dummy", useCharFilter ? new MockCharFilter(reader, remainder) : reader);
if (typeAtt != null && posIncAtt != null && offsetAtt != null) {
// offset + pos + type
- assertAnalyzesToReuse(a, text,
+ assertTokenStreamContents(ts,
tokens.toArray(new String[tokens.size()]),
toIntArray(startOffsets),
toIntArray(endOffsets),
types.toArray(new String[types.size()]),
- toIntArray(positions));
+ toIntArray(positions),
+ text.length());
} else if (posIncAtt != null && offsetAtt != null) {
// offset + pos
- assertAnalyzesToReuse(a, text,
+ assertTokenStreamContents(ts,
tokens.toArray(new String[tokens.size()]),
toIntArray(startOffsets),
toIntArray(endOffsets),
- toIntArray(positions));
+ null,
+ toIntArray(positions),
+ text.length());
} else if (offsetAtt != null) {
// offset
- assertAnalyzesToReuse(a, text,
+ assertTokenStreamContents(ts,
tokens.toArray(new String[tokens.size()]),
toIntArray(startOffsets),
- toIntArray(endOffsets));
+ toIntArray(endOffsets),
+ null,
+ null,
+ text.length());
} else {
// terms only
- assertAnalyzesToReuse(a, text,
+ assertTokenStreamContents(ts,
tokens.toArray(new String[tokens.size()]));
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java Mon Jan 23 18:34:04 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.preflex
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfosFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
import org.apache.lucene.util.LuceneTestCase;
@@ -33,6 +34,7 @@ public class PreFlexRWCodec extends Luce
private final NormsFormat norms = new PreFlexRWNormsFormat();
private final FieldInfosFormat fieldInfos = new PreFlexRWFieldInfosFormat();
private final TermVectorsFormat termVectors = new PreFlexRWTermVectorsFormat();
+ private final SegmentInfosFormat segmentInfos = new PreFlexRWSegmentInfosFormat();
@Override
public PostingsFormat postingsFormat() {
@@ -53,6 +55,15 @@ public class PreFlexRWCodec extends Luce
}
@Override
+ public SegmentInfosFormat segmentInfosFormat() {
+ if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
+ return segmentInfos ;
+ } else {
+ return super.segmentInfosFormat();
+ }
+ }
+
+ @Override
public FieldInfosFormat fieldInfosFormat() {
if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
return fieldInfos;
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java Mon Jan 23 18:34:04 2012
@@ -283,7 +283,8 @@ public abstract class LuceneTestCase ext
int randomVal = random.nextInt(10);
if ("Lucene3x".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal < 2)) { // preflex-only setup
- codec = new PreFlexRWCodec();
+ codec = Codec.forName("Lucene3x");
+ assert (codec instanceof PreFlexRWCodec) : "fix your classpath to have tests-framework.jar before lucene-core.jar";
PREFLEX_IMPERSONATION_IS_ACTIVE = true;
} else if ("SimpleText".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 9)) {
codec = new SimpleTextCodec();
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java?rev=1234932&r1=1234931&r2=1234932&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java Mon Jan 23 18:34:04 2012
@@ -249,7 +249,42 @@ public class _TestUtil {
}
}
- // TODO: make this more evil
+ private static final String[] HTML_CHAR_ENTITIES = {
+ "AElig", "Aacute", "Acirc", "Agrave", "Alpha", "AMP", "Aring", "Atilde",
+ "Auml", "Beta", "COPY", "Ccedil", "Chi", "Dagger", "Delta", "ETH",
+ "Eacute", "Ecirc", "Egrave", "Epsilon", "Eta", "Euml", "Gamma", "GT",
+ "Iacute", "Icirc", "Igrave", "Iota", "Iuml", "Kappa", "Lambda", "LT",
+ "Mu", "Ntilde", "Nu", "OElig", "Oacute", "Ocirc", "Ograve", "Omega",
+ "Omicron", "Oslash", "Otilde", "Ouml", "Phi", "Pi", "Prime", "Psi",
+ "QUOT", "REG", "Rho", "Scaron", "Sigma", "THORN", "Tau", "Theta",
+ "Uacute", "Ucirc", "Ugrave", "Upsilon", "Uuml", "Xi", "Yacute", "Yuml",
+ "Zeta", "aacute", "acirc", "acute", "aelig", "agrave", "alefsym",
+ "alpha", "amp", "and", "ang", "apos", "aring", "asymp", "atilde",
+ "auml", "bdquo", "beta", "brvbar", "bull", "cap", "ccedil", "cedil",
+ "cent", "chi", "circ", "clubs", "cong", "copy", "crarr", "cup",
+ "curren", "dArr", "dagger", "darr", "deg", "delta", "diams", "divide",
+ "eacute", "ecirc", "egrave", "empty", "emsp", "ensp", "epsilon",
+ "equiv", "eta", "eth", "euml", "euro", "exist", "fnof", "forall",
+ "frac12", "frac14", "frac34", "frasl", "gamma", "ge", "gt", "hArr",
+ "harr", "hearts", "hellip", "iacute", "icirc", "iexcl", "igrave",
+ "image", "infin", "int", "iota", "iquest", "isin", "iuml", "kappa",
+ "lArr", "lambda", "lang", "laquo", "larr", "lceil", "ldquo", "le",
+ "lfloor", "lowast", "loz", "lrm", "lsaquo", "lsquo", "lt", "macr",
+ "mdash", "micro", "middot", "minus", "mu", "nabla", "nbsp", "ndash",
+ "ne", "ni", "not", "notin", "nsub", "ntilde", "nu", "oacute", "ocirc",
+ "oelig", "ograve", "oline", "omega", "omicron", "oplus", "or", "ordf",
+ "ordm", "oslash", "otilde", "otimes", "ouml", "para", "part", "permil",
+ "perp", "phi", "pi", "piv", "plusmn", "pound", "prime", "prod", "prop",
+ "psi", "quot", "rArr", "radic", "rang", "raquo", "rarr", "rceil",
+ "rdquo", "real", "reg", "rfloor", "rho", "rlm", "rsaquo", "rsquo",
+ "sbquo", "scaron", "sdot", "sect", "shy", "sigma", "sigmaf", "sim",
+ "spades", "sub", "sube", "sum", "sup", "sup1", "sup2", "sup3", "supe",
+ "szlig", "tau", "there4", "theta", "thetasym", "thinsp", "thorn",
+ "tilde", "times", "trade", "uArr", "uacute", "uarr", "ucirc", "ugrave",
+ "uml", "upsih", "upsilon", "uuml", "weierp", "xi", "yacute", "yen",
+ "yuml", "zeta", "zwj", "zwnj"
+ };
+
public static String randomHtmlishString(Random random, int numElements) {
final int end = random.nextInt(numElements);
if (end == 0) {
@@ -258,17 +293,80 @@ public class _TestUtil {
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < end; i++) {
- int val = random.nextInt(10);
+ int val = random.nextInt(25);
switch(val) {
case 0: sb.append("<p>"); break;
- case 1: sb.append("</p>"); break;
- case 2: sb.append("<!--"); break;
- case 3: sb.append("-->"); break;
- case 4: sb.append("&#"); break;
- case 5: sb.append(";"); break;
- case 6: sb.append((char)_TestUtil.nextInt(random, '0', '9')); break;
- default:
- sb.append((char)_TestUtil.nextInt(random, 'a', 'z'));
+ case 1: {
+ sb.append("<");
+ sb.append(" ".substring(nextInt(random, 0, 4)));
+ sb.append(randomSimpleString(random));
+ for (int j = 0 ; j < nextInt(random, 0, 10) ; ++j) {
+ sb.append(' ');
+ sb.append(randomSimpleString(random));
+ sb.append(" ".substring(nextInt(random, 0, 1)));
+ sb.append('=');
+ sb.append(" ".substring(nextInt(random, 0, 1)));
+ sb.append("\"".substring(nextInt(random, 0, 1)));
+ sb.append(randomSimpleString(random));
+ sb.append("\"".substring(nextInt(random, 0, 1)));
+ }
+ sb.append(" ".substring(nextInt(random, 0, 4)));
+ sb.append("/".substring(nextInt(random, 0, 1)));
+ sb.append(">".substring(nextInt(random, 0, 1)));
+ break;
+ }
+ case 2: {
+ sb.append("</");
+ sb.append(" ".substring(nextInt(random, 0, 4)));
+ sb.append(randomSimpleString(random));
+ sb.append(" ".substring(nextInt(random, 0, 4)));
+ sb.append(">".substring(nextInt(random, 0, 1)));
+ break;
+ }
+ case 3: sb.append(">"); break;
+ case 4: sb.append("</p>"); break;
+ case 5: sb.append("<!--"); break;
+ case 6: sb.append("<!--#"); break;
+ case 7: sb.append("<script><!-- f('"); break;
+ case 8: sb.append("</script>"); break;
+ case 9: sb.append("<?"); break;
+ case 10: sb.append("?>"); break;
+ case 11: sb.append("\""); break;
+ case 12: sb.append("\\\""); break;
+ case 13: sb.append("'"); break;
+ case 14: sb.append("\\'"); break;
+ case 15: sb.append("-->"); break;
+ case 16: {
+ sb.append("&");
+ switch(nextInt(random, 0, 2)) {
+ case 0: sb.append(randomSimpleString(random)); break;
+ case 1: sb.append(HTML_CHAR_ENTITIES[random.nextInt(HTML_CHAR_ENTITIES.length)]); break;
+ }
+ sb.append(";".substring(nextInt(random, 0, 1)));
+ break;
+ }
+ case 17: {
+ sb.append("&#");
+ if (0 == nextInt(random, 0, 1)) {
+ sb.append(nextInt(random, 0, Integer.MAX_VALUE - 1));
+ sb.append(";".substring(nextInt(random, 0, 1)));
+ }
+ break;
+ }
+ case 18: {
+ sb.append("&#x");
+ if (0 == nextInt(random, 0, 1)) {
+ sb.append(Integer.toString(nextInt(random, 0, Integer.MAX_VALUE - 1), 16));
+ sb.append(";".substring(nextInt(random, 0, 1)));
+ }
+ break;
+ }
+
+ case 19: sb.append(";"); break;
+ case 20: sb.append(nextInt(random, 0, Integer.MAX_VALUE - 1)); break;
+ case 21: sb.append("\n");
+ case 22: sb.append(" ".substring(nextInt(random, 0, 10)));
+ default: sb.append(randomSimpleString(random));
}
}
return sb.toString();