You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/19 17:59:32 UTC
svn commit: r1363400 [2/31] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/copyright/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/maven/ dev-tools/maven/lucene/...
Modified: lucene/dev/branches/pforcodec_3892/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/build.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/build.xml Thu Jul 19 15:58:54 2012
@@ -65,7 +65,7 @@
</subant></sequential>
</target>
- <target name="resolve" description="Resolves all dependencies">
+ <target name="resolve" depends="clean-jars" description="Resolves all dependencies">
<sequential><subant target="resolve" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml" />
<fileset dir="solr" includes="build.xml" />
@@ -116,7 +116,7 @@
</sequential>
</target>
- <target name="eclipse" description="Setup Eclipse configuration" depends="resolve">
+ <target name="eclipse" depends="clean-jars, resolve" description="Setup Eclipse configuration">
<copy file="dev-tools/eclipse/dot.project" tofile=".project" overwrite="false"/>
<copy file="dev-tools/eclipse/dot.classpath" tofile=".classpath" overwrite="true"/>
<mkdir dir=".settings"/>
@@ -129,7 +129,7 @@
</echo>
</target>
- <target name="idea" description="Setup IntelliJ IDEA configuration" depends="resolve">
+ <target name="idea" depends="clean-jars, resolve" description="Setup IntelliJ IDEA configuration">
<copy todir=".">
<fileset dir="dev-tools/idea"/>
</copy>
@@ -138,6 +138,7 @@
File | Project Structure | Project | Project SDK.
</echo>
</target>
+
<target name="clean-idea"
description="Removes all IntelliJ IDEA configuration files">
<delete dir=".idea" failonerror="true"/>
@@ -148,7 +149,7 @@
</delete>
</target>
- <target name="clean" description="Clean Lucene and Solr">
+ <target name="clean" depends="clean-jars" description="Clean Lucene and Solr">
<delete dir="dist" />
<sequential>
<subant target="clean" inheritall="false" failonerror="true">
@@ -175,7 +176,7 @@
</subant>
</target>
- <target name="jar-checksums" description="Recompute SHA1 checksums for all JAR files.">
+ <target name="jar-checksums" depends="resolve" description="Recompute SHA1 checksums for all JAR files.">
<delete>
<fileset dir="${basedir}">
<include name="**/*.jar.sha1"/>
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/eclipse/dot.classpath
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/eclipse/dot.classpath?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/eclipse/dot.classpath (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/eclipse/dot.classpath Thu Jul 19 15:58:54 2012
@@ -97,12 +97,14 @@
<classpathentry kind="lib" path="lucene/sandbox/lib/jakarta-regexp-1.4.jar"/>
<classpathentry kind="lib" path="lucene/analysis/icu/lib/icu4j-4.8.1.1.jar"/>
<classpathentry kind="lib" path="lucene/analysis/phonetic/lib/commons-codec-1.6.jar"/>
- <classpathentry kind="lib" path="lucene/analysis/morfologik/lib/morfologik-fsa-1.5.2.jar"/>
- <classpathentry kind="lib" path="lucene/analysis/morfologik/lib/morfologik-polish-1.5.2.jar"/>
- <classpathentry kind="lib" path="lucene/analysis/morfologik/lib/morfologik-stemming-1.5.2.jar"/>
+ <classpathentry kind="lib" path="lucene/analysis/morfologik/lib/morfologik-fsa-1.5.3.jar"/>
+ <classpathentry kind="lib" path="lucene/analysis/morfologik/lib/morfologik-polish-1.5.3.jar"/>
+ <classpathentry kind="lib" path="lucene/analysis/morfologik/lib/morfologik-stemming-1.5.3.jar"/>
<classpathentry kind="lib" path="lucene/benchmark/lib/commons-compress-1.2.jar"/>
<classpathentry kind="lib" path="lucene/benchmark/lib/xercesImpl-2.9.1.jar"/>
+ <classpathentry kind="lib" path="lucene/benchmark/lib/nekohtml-1.9.15.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-fileupload-1.2.1.jar"/>
+ <classpathentry kind="lib" path="solr/lib/commons-cli-1.2.jar"/>
<classpathentry kind="lib" path="solr/lib/httpclient-4.1.3.jar"/>
<classpathentry kind="lib" path="solr/lib/httpcore-4.1.4.jar"/>
<classpathentry kind="lib" path="solr/lib/httpmime-4.1.3.jar"/>
@@ -115,7 +117,7 @@
<classpathentry kind="lib" path="solr/lib/slf4j-api-1.6.4.jar"/>
<classpathentry kind="lib" path="solr/lib/slf4j-jdk14-1.6.4.jar"/>
<classpathentry kind="lib" path="solr/lib/wstx-asl-3.2.7.jar"/>
- <classpathentry kind="lib" path="solr/lib/zookeeper-3.3.4.jar"/>
+ <classpathentry kind="lib" path="solr/lib/zookeeper-3.3.5.jar"/>
<classpathentry kind="lib" path="solr/example/lib/jetty-continuation-8.1.2.v20120308.jar"/>
<classpathentry kind="lib" path="solr/example/lib/jetty-deploy-8.1.2.v20120308.jar"/>
<classpathentry kind="lib" path="solr/example/lib/jetty-http-8.1.2.v20120308.jar"/>
@@ -170,6 +172,6 @@
<classpathentry kind="lib" path="solr/contrib/velocity/lib/commons-beanutils-1.7.0.jar"/>
<classpathentry kind="lib" path="solr/contrib/velocity/lib/commons-collections-3.2.1.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
- <classpathentry kind="lib" path="lucene/test-framework/lib/randomizedtesting-runner-1.5.0.jar"/>
+ <classpathentry kind="lib" path="lucene/test-framework/lib/randomizedtesting-runner-1.6.0.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/copyright/profiles_settings.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/copyright/profiles_settings.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/copyright/profiles_settings.xml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/copyright/profiles_settings.xml Thu Jul 19 15:58:54 2012
@@ -24,4 +24,4 @@
<option name="prefixLines" value="false" />
</LanguageOptions>
</settings>
-</component>
+</component>
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Ant.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Ant.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Ant.xml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Ant.xml Thu Jul 19 15:58:54 2012
@@ -7,4 +7,4 @@
<JAVADOC />
<SOURCES />
</library>
-</component>
+</component>
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/HSQLDB.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/HSQLDB.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/HSQLDB.xml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/HSQLDB.xml Thu Jul 19 15:58:54 2012
@@ -6,4 +6,4 @@
<JAVADOC />
<SOURCES />
</library>
-</component>
+</component>
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/JUnit.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/JUnit.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/JUnit.xml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/JUnit.xml Thu Jul 19 15:58:54 2012
@@ -2,9 +2,9 @@
<library name="JUnit">
<CLASSES>
<root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/junit-4.10.jar!/" />
- <root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/randomizedtesting-runner-1.5.0.jar!/" />
+ <root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/randomizedtesting-runner-1.6.0.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
-</component>
+</component>
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_extraction_library.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_extraction_library.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_extraction_library.xml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_extraction_library.xml Thu Jul 19 15:58:54 2012
@@ -7,4 +7,4 @@
<SOURCES />
<jarDirectory url="file://$PROJECT_DIR$/solr/contrib/extraction/lib" recursive="false" />
</library>
-</component>
+</component>
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_velocity_library.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_velocity_library.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_velocity_library.xml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/libraries/Solr_velocity_library.xml Thu Jul 19 15:58:54 2012
@@ -7,4 +7,4 @@
<SOURCES />
<jarDirectory url="file://$PROJECT_DIR$/solr/contrib/velocity/lib" recursive="false" />
</library>
-</component>
+</component>
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/projectCodeStyle.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/projectCodeStyle.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/projectCodeStyle.xml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/.idea/projectCodeStyle.xml Thu Jul 19 15:58:54 2012
@@ -16,6 +16,11 @@
<option name="USE_RELATIVE_INDENTS" value="false" />
</value>
</option>
+ <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="20" />
+ <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="20" />
+ <option name="PACKAGES_TO_USE_IMPORT_ON_DEMAND">
+ <value />
+ </option>
<ADDITIONAL_INDENT_OPTIONS fileType="groovy">
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="4" />
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/idea/lucene/lucene.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/idea/lucene/lucene.iml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/idea/lucene/lucene.iml (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/idea/lucene/lucene.iml Thu Jul 19 15:58:54 2012
@@ -17,6 +17,7 @@
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
+ <orderEntry type="library" name="Lucene tools library" level="project" />
<orderEntry type="library" name="Ant" level="project"/>
</component>
</module>
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/maven/lucene/benchmark/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/maven/lucene/benchmark/pom.xml.template?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/maven/lucene/benchmark/pom.xml.template (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/maven/lucene/benchmark/pom.xml.template Thu Jul 19 15:58:54 2012
@@ -90,6 +90,11 @@
<artifactId>icu4j</artifactId>
</dependency>
<dependency>
+ <groupId>net.sourceforge.nekohtml</groupId>
+ <artifactId>nekohtml</artifactId>
+ <version>1.9.15</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/maven/pom.xml.template?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/maven/pom.xml.template Thu Jul 19 15:58:54 2012
@@ -156,6 +156,11 @@
<version>1.6</version>
</dependency>
<dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
<groupId>commons-digester</groupId>
<artifactId>commons-digester</artifactId>
<version>1.7</version>
@@ -293,7 +298,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <version>3.3.4</version>
+ <version>3.3.5</version>
</dependency>
<dependency>
<groupId>org.carrot2</groupId>
@@ -303,7 +308,7 @@
<dependency>
<groupId>org.carrot2</groupId>
<artifactId>morfologik-polish</artifactId>
- <version>1.5.2</version>
+ <version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.codehaus.woodstox</groupId>
@@ -383,7 +388,7 @@
<dependency>
<groupId>com.carrotsearch.randomizedtesting</groupId>
<artifactId>randomizedtesting-runner</artifactId>
- <version>1.5.0</version>
+ <version>1.6.0</version>
</dependency>
</dependencies>
</dependencyManagement>
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/maven/solr/core/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/maven/solr/core/pom.xml.template?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/maven/solr/core/pom.xml.template (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/maven/solr/core/pom.xml.template Thu Jul 19 15:58:54 2012
@@ -139,6 +139,10 @@
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
</dependency>
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/scripts/checkJavadocLinks.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/scripts/checkJavadocLinks.py?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/scripts/checkJavadocLinks.py (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/scripts/checkJavadocLinks.py Thu Jul 19 15:58:54 2012
@@ -17,8 +17,8 @@ import traceback
import os
import sys
import re
-from HTMLParser import HTMLParser, HTMLParseError
-import urlparse
+from html.parser import HTMLParser, HTMLParseError
+import urllib.parse as urlparse
reHyperlink = re.compile(r'<a(\s+.*?)>', re.I)
reAtt = re.compile(r"""(?:\s+([a-z]+)\s*=\s*("[^"]*"|'[^']?'|[^'"\s]+))+""", re.I)
@@ -57,7 +57,7 @@ class FindHyperlinks(HTMLParser):
pass
else:
self.printFile()
- print ' WARNING: anchor "%s" appears more than once' % name
+ print(' WARNING: anchor "%s" appears more than once' % name)
else:
self.anchors.add(name)
elif href is not None:
@@ -73,8 +73,8 @@ class FindHyperlinks(HTMLParser):
def printFile(self):
if not self.printed:
- print
- print ' ' + self.baseURL
+ print()
+ print(' ' + self.baseURL)
self.printed = True
def parse(baseURL, html):
@@ -85,8 +85,8 @@ def parse(baseURL, html):
parser.close()
except HTMLParseError:
parser.printFile()
- print ' WARNING: failed to parse:'
- traceback.print_exc()
+ print(' WARNING: failed to parse %s:' % baseURL)
+ traceback.print_exc(file=sys.stdout)
failures = True
return [], []
@@ -104,8 +104,8 @@ def checkAll(dirName):
global failures
# Find/parse all HTML files first
- print
- print 'Crawl/parse...'
+ print()
+ print('Crawl/parse...')
allFiles = {}
if os.path.isfile(dirName):
@@ -128,11 +128,11 @@ def checkAll(dirName):
# deprecated-list.html can fail to escape generics types
fullPath = os.path.join(root, f)
#print ' %s' % fullPath
- allFiles[fullPath] = parse(fullPath, open('%s/%s' % (root, f)).read())
+ allFiles[fullPath] = parse(fullPath, open('%s/%s' % (root, f), encoding='UTF-8').read())
# ... then verify:
- print
- print 'Verify...'
+ print()
+ print('Verify...')
for fullPath, (links, anchors) in allFiles.items():
#print fullPath
printed = False
@@ -176,16 +176,16 @@ def checkAll(dirName):
and os.path.basename(fullPath) != 'Changes.html':
if not printed:
printed = True
- print
- print fullPath
- print ' BAD EXTERNAL LINK: %s' % link
+ print()
+ print(fullPath)
+ print(' BAD EXTERNAL LINK: %s' % link)
elif link.startswith('mailto:'):
if link.find('@lucene.apache.org') == -1 and link.find('@apache.org') != -1:
if not printed:
printed = True
- print
- print fullPath
- print ' BROKEN MAILTO (?): %s' % link
+ print()
+ print(fullPath)
+ print(' BROKEN MAILTO (?): %s' % link)
elif link.startswith('javascript:'):
# ok...?
pass
@@ -200,15 +200,15 @@ def checkAll(dirName):
if not os.path.exists(link):
if not printed:
printed = True
- print
- print fullPath
- print ' BROKEN LINK: %s' % link
+ print()
+ print(fullPath)
+ print(' BROKEN LINK: %s' % link)
elif anchor is not None and anchor not in allFiles[link][1]:
if not printed:
printed = True
- print
- print fullPath
- print ' BROKEN ANCHOR: %s' % origLink
+ print()
+ print(fullPath)
+ print(' BROKEN ANCHOR: %s' % origLink)
failures = failures or printed
@@ -216,8 +216,8 @@ def checkAll(dirName):
if __name__ == '__main__':
if checkAll(sys.argv[1]):
- print
- print 'Broken javadocs links were found!'
+ print()
+ print('Broken javadocs links were found!')
sys.exit(1)
sys.exit(0)
Modified: lucene/dev/branches/pforcodec_3892/dev-tools/scripts/smokeTestRelease.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/dev-tools/scripts/smokeTestRelease.py?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/dev-tools/scripts/smokeTestRelease.py (original)
+++ lucene/dev/branches/pforcodec_3892/dev-tools/scripts/smokeTestRelease.py Thu Jul 19 15:58:54 2012
@@ -210,16 +210,6 @@ def checkSigs(project, urlString, versio
if keysURL is None:
raise RuntimeError('%s is missing KEYS' % project)
- if not os.path.exists('%s/apache-rat-0.8.jar' % tmpDir):
- print ' downloading Apache RAT...'
- download('apache-rat-incubating-0.8-bin.tar.bz2',
- 'http://archive.apache.org/dist/incubator/rat/binaries/apache-rat-incubating-0.8-bin.tar.bz2',
- tmpDir)
- t = tarfile.open('%s/apache-rat-incubating-0.8-bin.tar.bz2' % tmpDir)
- t.extract('apache-rat-0.8/apache-rat-0.8.jar', '%s/apache-rat-0.8.jar' % tmpDir)
- else:
- print ' apache RAT already downloaded...'
-
print ' get KEYS'
download('%s.KEYS' % project, keysURL, tmpDir)
@@ -480,9 +470,6 @@ def verifyUnpacked(project, artifact, un
print ' run "ant validate"'
run('%s; ant validate' % javaExe('1.7'), '%s/validate.log' % unpackPath)
- print ' run "ant rat-sources"'
- run('%s; ant -lib "%s/apache-rat-0.8.jar/apache-rat-0.8" rat-sources' % (javaExe('1.7'), tmpDir), '%s/rat-sources.log' % unpackPath)
-
if project == 'lucene':
print ' run tests w/ Java 6...'
run('%s; ant test' % javaExe('1.6'), '%s/test.log' % unpackPath)
Modified: lucene/dev/branches/pforcodec_3892/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/CHANGES.txt?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/CHANGES.txt Thu Jul 19 15:58:54 2012
@@ -7,6 +7,120 @@ http://s.apache.org/luceneversions
======================= Lucene 5.0.0 =======================
+======================= Lucene 4.0.0-BETA =======================
+
+New features
+
+* LUCENE-4201: Added JapaneseIterationMarkCharFilter to normalize Japanese
+ iteration marks. (Robert Muir, Christian Moen)
+
+* LUCENE-3832: Added BasicAutomata.makeStringUnion method to efficiently
+ create automata from a fixed collection of UTF-8 encoded BytesRef
+ (Dawid Weiss, Robert Muir)
+
+* LUCENE-4153: Added option to fast vector highlighting via BaseFragmentsBuilder to
+ respect field boundaries in the case of highlighting for multivalued fields.
+ (Martijn van Groningen)
+
+API Changes
+
+* LUCENE-4138: update of morfologik (Polish morphological analyzer) to 1.5.3.
+ The tag attribute class has been renamed to MorphosyntacticTagsAttribute and
+ has a different API (carries a list of tags instead of a compound tag). Upgrade
+ of embedded morfologik dictionaries to version 1.9. (Dawid Weiss)
+
+* LUCENE-4178: set 'tokenized' to true on FieldType by default, so that if you
+ make a custom FieldType and set indexed = true, its analyzed by the analyzer.
+ (Robert Muir)
+
+* LUCENE-4220: Removed the buggy JavaCC-based HTML parser in the benchmark
+ module and replaced by NekoHTML. HTMLParser interface was cleaned up while
+ changing method signatures. (Uwe Schindler, Robert Muir)
+
+* LUCENE-2191: Rename Tokenizer.reset(Reader) to Tokenizer.setReader(Reader).
+ The purpose of this method was always to set a new Reader on the Tokenizer,
+ reusing the object. But the name was often confused with TokenStream.reset().
+ (Robert Muir)
+
+* LUCENE-4228: Refactored CharFilter to extend java.io.FilterReader. CharFilters
+ filter another reader and you override correct() for offset correction.
+ (Robert Muir)
+
+Optimizations
+
+* LUCENE-4171: Performance improvements to Packed64.
+ (Toke Eskildsen via Adrien Grand)
+
+* LUCENE-4184: Performance improvements to the aligned packed bits impl.
+ (Toke Eskildsen, Adrien Grand)
+
+* LUCENE-4235: Remove enforcing of Filter rewrite for NRQ queries.
+ (Uwe Schindler)
+
+Bug Fixes
+
+* LUCENE-4176: Fix AnalyzingQueryParser to analyze range endpoints as bytes,
+ so that it works correctly with Analyzers that produce binary non-UTF-8 terms
+ such as CollationAnalyzer. (Nattapong Sirilappanich via Robert Muir)
+
+* LUCENE-4209: Fix FSTCompletionLookup to close its sorter, so that it won't
+ leave temp files behind in /tmp. Fix SortedTermFreqIteratorWrapper to not
+ leave temp files behind in /tmp on Windows. Fix Sort to not leave
+ temp files behind when /tmp is a separate volume. (Uwe Schindler, Robert Muir)
+
+* LUCENE-4221: Fix overeager CheckIndex validation for term vector offsets.
+ (Robert Muir)
+
+* LUCENE-4222: TieredMergePolicy.getFloorSegmentMB was returning the
+ size in bytes not MB (Chris Fuller via Mike McCandless)
+
+* LUCENE-3505: Fix bug (Lucene 4.0alpha only) where boolean conjunctions
+ were sometimes scored incorrectly. Conjunctions of only termqueries where
+ at least one term omitted term frequencies (IndexOptions.DOCS_ONLY) would
+ be scored as if all terms omitted term frequencies. (Robert Muir)
+
+* LUCENE-2686, LUCENE-3505: Fixed BooleanQuery scorers to return correct
+ freq(). Added support for scorer navigation API (Scorer.getChildren) to
+ all queries. Made Scorer.freq() abstract.
+ (Koji Sekiguchi, Mike McCandless, Robert Muir)
+
+Build
+
+* LUCENE-4094: Support overriding file.encoding on forked test JVMs
+ (force via -Drandomized.file.encoding=XXX). (Dawid Weiss)
+
+* LUCENE-4189: Test output should include timestamps (start/end for each
+ test/ suite). Added -Dtests.timestamps=[off by default]. (Dawid Weiss)
+
+* LUCENE-4110: Report long periods of forked jvm inactivity (hung tests/ suites).
+ Added -Dtests.heartbeat=[seconds] with the default of 60 seconds.
+ (Dawid Weiss)
+
+* LUCENE-4160: Added a property to quit the tests after a given
+ number of failures has occurred. This is useful in combination
+ with -Dtests.iters=N (you can start N iterations and wait for M
+ failures, in particular M = 1). -Dtests.maxfailures=M. Alternatively,
+ specify -Dtests.failfast=true to skip all tests after the first failure.
+ (Dawid Weiss)
+
+* LUCENE-4115: JAR resolution/ cleanup should be done automatically for ant
+ clean/ eclipse/ resolve (Dawid Weiss)
+
+* LUCENE-4199, LUCENE-4202, LUCENE-4206: Add a new target "check-forbidden-apis"
+ that parses all generated .class files for use of APIs that use default
+ charset, default locale, or default timezone and fail build if violations
+ found. This ensures, that Lucene / Solr is independent on local configuration
+ options. (Uwe Schindler, Robert Muir, Dawid Weiss)
+
+* LUCENE-4217: Add the possibility to run tests with Atlassian Clover
+ loaded from IVY. A development License solely for Apache code was added in
+ the tools/ folder, but is not included in releases. (Uwe Schindler)
+
+Documentation
+
+* LUCENE-4195: Added package documentation and examples for
+ org.apache.lucene.codecs (Alan Woodward via Robert Muir)
+
======================= Lucene 4.0.0-ALPHA =======================
More information about this release, including any errata related to the
@@ -20,7 +134,7 @@ Changes in backwards compatibility polic
* LUCENE-1458, LUCENE-2111, LUCENE-2354: Changes from flexible indexing:
- - On upgrading to 3.1, if you do not fully reindex your documents,
+ - On upgrading to 4.0, if you do not fully reindex your documents,
Lucene will emulate the new flex API on top of the old index,
incurring some performance cost (up to ~10% slowdown, typically).
To prevent this slowdown, use oal.index.IndexUpgrader
@@ -29,7 +143,7 @@ Changes in backwards compatibility polic
Mixed flex/pre-flex indexes are perfectly fine -- the two
emulation layers (flex API on pre-flex index, and pre-flex API on
flex index) will remap the access as required. So on upgrading to
- 3.1 you can start indexing new documents into an existing index.
+ 4.0 you can start indexing new documents into an existing index.
To get optimal performance, use oal.index.IndexUpgrader
to upgrade your indexes to latest file format (LUCENE-3082).
@@ -283,6 +397,11 @@ Changes in backwards compatibility polic
removed, as IndexReaderContext.leaves() is now the preferred way
to access sub-readers. (Uwe Schindler)
+* LUCENE-4155: oal.util.ReaderUtil, TwoPhaseCommit, TwoPhaseCommitTool
+ classes were moved to oal.index package. oal.util.CodecUtil class was moved
+ to oal.codecs package. oal.util.DummyConcurrentLock was removed
+ (no longer used in Lucene 4.0). (Uwe Schindler)
+
Changes in Runtime Behavior
* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
@@ -989,6 +1108,11 @@ Optimizations
* LUCENE-4156: DirectoryTaxonomyWriter.getSize is no longer synchronized.
(Shai Erera, Sivan Yogev)
+* LUCENE-4163: Improve concurrency of MMapIndexInput.clone() by using
+ the new WeakIdentityMap on top of a ConcurrentHashMap to manage
+ the cloned instances. WeakIdentityMap was extended to support
+ iterating over its keys. (Uwe Schindler)
+
Bug fixes
* LUCENE-2803: The FieldCache can miss values if an entry for a reader
@@ -1062,6 +1186,13 @@ Bug fixes
* LUCENE-4114: Fix int overflow bugs in BYTES_FIXED_STRAIGHT and
BYTES_FIXED_DEREF doc values implementations (Walt Elder via Mike McCandless).
+* LUCENE-4147: Fixed thread safety issues when rollback() and commit()
+ are called simultaneously. (Simon Willnauer, Mike McCandless)
+
+* LUCENE-4165: Removed closing of the Reader used to read the affix file in
+ HunspellDictionary. Consumers are now responsible for closing all InputStreams
+ once the Dictionary has been instantiated. (Torsten Krah, Uwe Schindler, Chris Male)
+
Documentation
* LUCENE-3958: Javadocs corrections for IndexWriter.
Modified: lucene/dev/branches/pforcodec_3892/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/MIGRATE.txt?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/MIGRATE.txt (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/MIGRATE.txt Thu Jul 19 15:58:54 2012
@@ -145,7 +145,7 @@ enumeration APIs. Here are the major ch
oal.util.ReaderUtil) and then step through those readers yourself,
if you can (this is how Lucene drives searches).
- If you pass a SegmentReader to MultiFields.fiels it will simply
+ If you pass a SegmentReader to MultiFields.fields it will simply
return reader.fields(), so there is no performance hit in that
case.
@@ -334,7 +334,7 @@ based on document IDs, albeit the per-se
There are still valid use-cases where top-level readers ie. "atomic
views" on the index are desirable. Let say you want to iterate all terms
-of a complete index for auto-completion or facetting, Lucene provides
+of a complete index for auto-completion or faceting, Lucene provides
utility wrappers like SlowCompositeReaderWrapper (LUCENE-2597) emulating
an AtomicReader. Note: using "atomicity emulators" can cause serious
slowdowns due to the need to merge terms, postings, DocValues, and
@@ -574,7 +574,7 @@ you can now do this:
Also MultiTermQuery.getTermsEnum() now takes an AttributeSource. FuzzyTermsEnum
is both consumer and producer of attributes: MTQ.BoostAttribute is
added to the FuzzyTermsEnum and MTQ's rewrite mode consumes it.
- The other way round MTQ.TopTermsBooleanQueryRewrite supplys a
+ The other way round MTQ.TopTermsBooleanQueryRewrite supplies a
global AttributeSource to each segments TermsEnum. The TermsEnum is consumer
and gets the current minimum competitive boosts (MTQ.MaxNonCompetitiveBoostAttribute).
@@ -594,7 +594,7 @@ you can now do this:
* LUCENE-1076: TieredMergePolicy is now the default merge policy.
It's able to merge non-contiguous segments; this may cause problems
for applications that rely on Lucene's internal document ID
- assigment. If so, you should instead use LogByteSize/DocMergePolicy
+ assignment. If so, you should instead use LogByteSize/DocMergePolicy
during indexing.
* LUCENE-3722: Similarity methods and collection/term statistics now take
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/build.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/build.xml Thu Jul 19 15:58:54 2012
@@ -61,50 +61,50 @@
executable="${python.exe}" failonerror="true" logerror="true">
<arg value="htmlentity.py"/>
</exec>
+ <fixcrlf file="src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex" encoding="UTF-8"/>
</target>
<target name="jflex-wiki-tokenizer" depends="init,jflex-check" if="jflex.present">
<taskdef classname="jflex.anttask.JFlexTask" name="jflex">
<classpath refid="jflex.classpath"/>
</taskdef>
- <jflex file="src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex"
- outdir="src/java/org/apache/lucene/analysis/wikipedia"
- nobak="on"/>
+ <run-jflex dir="src/java/org/apache/lucene/analysis/wikipedia" name="WikipediaTokenizerImpl"/>
</target>
<target name="jflex-StandardAnalyzer" depends="init,jflex-check" if="jflex.present">
<taskdef classname="jflex.anttask.JFlexTask" name="jflex">
<classpath refid="jflex.classpath"/>
</taskdef>
-
- <jflex file="src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex"
- outdir="src/java/org/apache/lucene/analysis/standard"
- nobak="on" />
- <jflex file="src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex"
- outdir="src/java/org/apache/lucene/analysis/standard"
- nobak="on" />
- <jflex file="src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex"
- outdir="src/java/org/apache/lucene/analysis/standard/std31"
- nobak="on" />
+ <run-jflex dir="src/java/org/apache/lucene/analysis/standard" name="StandardTokenizerImpl"/>
+ <run-jflex dir="src/java/org/apache/lucene/analysis/standard" name="ClassicTokenizerImpl"/>
</target>
<target name="jflex-UAX29URLEmailTokenizer" depends="jflex-check" if="jflex.present">
<taskdef classname="jflex.anttask.JFlexTask" name="jflex">
<classpath refid="jflex.classpath"/>
</taskdef>
- <jflex file="src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex"
- outdir="src/java/org/apache/lucene/analysis/standard"
- nobak="on" />
- <jflex file="src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex"
- outdir="src/java/org/apache/lucene/analysis/standard/std31"
- nobak="on" />
- <jflex file="src/java/org/apache/lucene/analysis/standard/std34/UAX29URLEmailTokenizerImpl34.jflex"
- outdir="src/java/org/apache/lucene/analysis/standard/std34"
- nobak="on" />
+ <run-jflex dir="src/java/org/apache/lucene/analysis/standard" name="UAX29URLEmailTokenizerImpl"/>
</target>
+ <!-- Remove the inappropriate JFlex-generated constructor -->
+ <macrodef name="run-jflex">
+ <attribute name="dir"/>
+ <attribute name="name"/>
+ <sequential>
+ <jflex file="@{dir}/@{name}.jflex"
+ outdir="@{dir}"
+ nobak="on" />
+ <replaceregexp file="@{dir}/@{name}.java"
+ match="/\*\*\s*\*\s*Creates a new scanner\..*this\(new java\.io\.InputStreamReader\(in\)\);\s*\}"
+ replace="" flags="sg"/>
+ </sequential>
+ </macrodef>
+
<target name="clean-jflex">
<delete>
+ <fileset dir="src/java/org/apache/lucene/analysis/charfilter" includes="*.java">
+ <containsregexp expression="generated.*by.*JFlex"/>
+ </fileset>
<fileset dir="src/java/org/apache/lucene/analysis/wikipedia" includes="*.java">
<containsregexp expression="generated.*by.*JFlex"/>
</fileset>
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemmer.java Thu Jul 19 15:58:54 2012
@@ -1,5 +1,7 @@
package org.apache.lucene.analysis.br;
+import java.util.Locale;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -21,6 +23,7 @@ package org.apache.lucene.analysis.br;
* A stemmer for Brazilian Portuguese words.
*/
public class BrazilianStemmer {
+ private static final Locale locale = new Locale("pt", "BR");
/**
* Changed term
@@ -243,7 +246,7 @@ public class BrazilianStemmer {
return null ;
}
- value = value.toLowerCase() ;
+ value = value.toLowerCase(locale) ;
for (j=0 ; j < value.length() ; j++) {
if ((value.charAt(j) == 'á') ||
(value.charAt(j) == 'â') ||
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/BaseCharFilter.java Thu Jul 19 15:58:54 2012
@@ -17,9 +17,10 @@
package org.apache.lucene.analysis.charfilter;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.util.ArrayUtil;
+import java.io.Reader;
import java.util.Arrays;
/**
@@ -34,7 +35,7 @@ public abstract class BaseCharFilter ext
private int diffs[];
private int size = 0;
- public BaseCharFilter(CharStream in) {
+ public BaseCharFilter(Reader in) {
super(in);
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java Thu Jul 19 15:58:54 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 5/18/12 12:24 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/16/12 4:05 PM */
package org.apache.lucene.analysis.charfilter;
@@ -20,13 +20,13 @@ package org.apache.lucene.analysis.charf
*/
import java.io.IOException;
+import java.io.Reader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.util.Version;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.OpenStringBuilder;
@@ -40,8 +40,8 @@ import org.apache.lucene.analysis.util.O
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 5/18/12 12:24 PM from the specification file
- * <tt>C:/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex</tt>
+ * on 7/16/12 4:05 PM from the specification file
+ * <tt>/home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex</tt>
*/
public final class HTMLStripCharFilter extends BaseCharFilter {
@@ -30647,7 +30647,7 @@ public final class HTMLStripCharFilter e
/**
* @param source
*/
- public HTMLStripCharFilter(CharStream source) {
+ public HTMLStripCharFilter(Reader source) {
super(source);
this.zzReader = source;
}
@@ -30657,7 +30657,7 @@ public final class HTMLStripCharFilter e
* @param escapedTags Tags in this set (both start and end tags)
* will not be filtered out.
*/
- public HTMLStripCharFilter(CharStream source, Set<String> escapedTags) {
+ public HTMLStripCharFilter(Reader source, Set<String> escapedTags) {
super(source);
this.zzReader = source;
if (null != escapedTags) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex Thu Jul 19 15:58:54 2012
@@ -1,6 +1,6 @@
package org.apache.lucene.analysis.charfilter;
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -18,13 +18,13 @@ package org.apache.lucene.analysis.charf
*/
import java.io.IOException;
+import java.io.Reader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.util.Version;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.OpenStringBuilder;
@@ -173,7 +173,7 @@ InlineElment = ( [aAbBiIqQsSuU]
/**
* @param source
*/
- public HTMLStripCharFilter(CharStream source) {
+ public HTMLStripCharFilter(Reader source) {
super(source);
this.zzReader = source;
}
@@ -183,7 +183,7 @@ InlineElment = ( [aAbBiIqQsSuU]
* @param escapedTags Tags in this set (both start and end tags)
* will not be filtered out.
*/
- public HTMLStripCharFilter(CharStream source, Set<String> escapedTags) {
+ public HTMLStripCharFilter(Reader source, Set<String> escapedTags) {
super(source);
this.zzReader = source;
if (null != escapedTags) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java Thu Jul 19 15:58:54 2012
@@ -21,8 +21,7 @@ import java.io.IOException;
import java.io.Reader;
import java.util.Map;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter; // javadocs
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.RollingCharBuffer;
import org.apache.lucene.util.fst.CharSequenceOutputs;
@@ -51,8 +50,8 @@ public class MappingCharFilter extends B
private int replacementPointer;
private int inputOff;
- /** Default constructor that takes a {@link CharStream}. */
- public MappingCharFilter(NormalizeCharMap normMap, CharStream in) {
+ /** Default constructor that takes a {@link Reader}. */
+ public MappingCharFilter(NormalizeCharMap normMap, Reader in) {
super(in);
buffer.reset(in);
@@ -66,15 +65,10 @@ public class MappingCharFilter extends B
}
}
- /** Easy-use constructor that takes a {@link Reader}. */
- public MappingCharFilter(NormalizeCharMap normMap, Reader in) {
- this(normMap, CharReader.get(in));
- }
-
@Override
public void reset() throws IOException {
super.reset();
- buffer.reset(input);
+ buffer.reset(in);
replacement = null;
inputOff = 0;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java Thu Jul 19 15:58:54 2012
@@ -205,7 +205,7 @@ public final class CJKBigramFilter exten
/**
* refills buffers with new data from the current token.
*/
- private void refill() throws IOException {
+ private void refill() {
// compact buffers to keep them smallish if they become large
// just a safety check, but technically we only need the last codepoint
if (bufferLen > 64) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java Thu Jul 19 15:58:54 2012
@@ -18,6 +18,7 @@
package org.apache.lucene.analysis.compound.hyphenation;
import java.io.File;
+import java.io.PrintStream;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -463,10 +464,10 @@ public class HyphenationTree extends Ter
}
@Override
- public void printStats() {
- System.out.println("Value space size = "
+ public void printStats(PrintStream out) {
+ out.println("Value space size = "
+ Integer.toString(vspace.length()));
- super.printStats();
+ super.printStats(out);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java Thu Jul 19 15:58:54 2012
@@ -40,7 +40,7 @@ import javax.xml.parsers.SAXParserFactor
*
* This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-public class PatternParser extends DefaultHandler implements PatternConsumer {
+public class PatternParser extends DefaultHandler {
XMLReader parser;
@@ -64,7 +64,7 @@ public class PatternParser extends Defau
static final int ELEM_HYPHEN = 4;
- public PatternParser() throws HyphenationException {
+ public PatternParser() {
token = new StringBuilder();
parser = createParser();
parser.setContentHandler(this);
@@ -74,7 +74,7 @@ public class PatternParser extends Defau
}
- public PatternParser(PatternConsumer consumer) throws HyphenationException {
+ public PatternParser(PatternConsumer consumer) {
this();
this.consumer = consumer;
}
@@ -402,25 +402,4 @@ public class PatternParser extends Defau
return str.toString();
} // getLocationString(SAXParseException):String
-
- // PatternConsumer implementation for testing purposes
- public void addClass(String c) {
- System.out.println("class: " + c);
- }
-
- public void addException(String w, ArrayList<Object> e) {
- System.out.println("exception: " + w + " : " + e.toString());
- }
-
- public void addPattern(String p, String v) {
- System.out.println("pattern: " + p + " : " + v);
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length > 0) {
- PatternParser pp = new PatternParser();
- pp.setConsumer(pp);
- pp.parse(args[0]);
- }
- }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java Thu Jul 19 15:58:54 2012
@@ -17,6 +17,7 @@
package org.apache.lucene.analysis.compound.hyphenation;
+import java.io.PrintStream;
import java.util.Enumeration;
import java.util.Stack;
@@ -633,11 +634,11 @@ public class TernaryTree implements Clon
}
- public void printStats() {
- System.out.println("Number of keys = " + Integer.toString(length));
- System.out.println("Node count = " + Integer.toString(freenode));
+ public void printStats(PrintStream out) {
+ out.println("Number of keys = " + Integer.toString(length));
+ out.println("Node count = " + Integer.toString(freenode));
// System.out.println("Array length = " + Integer.toString(eq.length));
- System.out.println("Key Array length = " + Integer.toString(kv.length()));
+ out.println("Key Array length = " + Integer.toString(kv.length()));
/*
* for(int i=0; i<kv.length(); i++) if ( kv.get(i) != 0 )
@@ -647,8 +648,8 @@ public class TernaryTree implements Clon
*/
}
-
- public static void main(String[] args) throws Exception {
+/*
+ public static void main(String[] args) {
TernaryTree tt = new TernaryTree();
tt.insert("Carlos", 'C');
tt.insert("Car", 'r');
@@ -658,7 +659,8 @@ public class TernaryTree implements Clon
System.out.println((char) tt.find("Car"));
System.out.println((char) tt.find("Carlos"));
System.out.println((char) tt.find("alto"));
- tt.printStats();
+ tt.printStats(System.out);
}
+ */
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java Thu Jul 19 15:58:54 2012
@@ -94,8 +94,8 @@ public final class KeywordTokenizer exte
}
@Override
- public void reset(Reader input) throws IOException {
- super.reset(input);
+ public void setReader(Reader input) throws IOException {
+ super.setReader(input);
this.done = false;
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java Thu Jul 19 15:58:54 2012
@@ -17,7 +17,6 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@@ -122,7 +121,7 @@ public final class StopFilter extends Fi
* Returns the next input Token whose term() is not a stop word.
*/
@Override
- protected boolean accept() throws IOException {
+ protected boolean accept() {
return !stopWords.contains(termAtt.buffer(), 0, termAtt.length());
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java Thu Jul 19 15:58:54 2012
@@ -48,7 +48,7 @@ public final class TypeTokenFilter exten
* When the useWhiteList parameter is set to true then accept the token if its type is contained in the stopTypes
*/
@Override
- protected boolean accept() throws IOException {
+ protected boolean accept() {
return useWhiteList == stopTypes.contains(typeAttribute.type());
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemmer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemmer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemmer.java Thu Jul 19 15:58:54 2012
@@ -1,4 +1,7 @@
package org.apache.lucene.analysis.de;
+
+import java.util.Locale;
+
// This file is encoded in UTF-8
/*
@@ -37,6 +40,8 @@ public class GermanStemmer
* Amount of characters that are removed with <tt>substitute()</tt> while stemming.
*/
private int substCount = 0;
+
+ private static final Locale locale = new Locale("de", "DE");
/**
* Stemms the given term to an unique <tt>discriminator</tt>.
@@ -47,7 +52,7 @@ public class GermanStemmer
protected String stem( String term )
{
// Use lowercase for medium stemming.
- term = term.toLowerCase();
+ term = term.toLowerCase(locale);
if ( !isStemmable( term ) )
return term;
// Reset the StringBuilder.
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java Thu Jul 19 15:58:54 2012
@@ -289,7 +289,7 @@ public class KStemmer {
entry = new DictEntry(exceptionWords[i], true);
d.put(exceptionWords[i], entry);
} else {
- System.out.println("Warning: Entry [" + exceptionWords[i]
+ throw new RuntimeException("Warning: Entry [" + exceptionWords[i]
+ "] already in dictionary 1");
}
}
@@ -299,7 +299,7 @@ public class KStemmer {
entry = new DictEntry(directConflations[i][1], false);
d.put(directConflations[i][0], entry);
} else {
- System.out.println("Warning: Entry [" + directConflations[i][0]
+ throw new RuntimeException("Warning: Entry [" + directConflations[i][0]
+ "] already in dictionary 2");
}
}
@@ -309,7 +309,7 @@ public class KStemmer {
entry = new DictEntry(countryNationality[i][1], false);
d.put(countryNationality[i][0], entry);
} else {
- System.out.println("Warning: Entry [" + countryNationality[i][0]
+ throw new RuntimeException("Warning: Entry [" + countryNationality[i][0]
+ "] already in dictionary 3");
}
}
@@ -323,7 +323,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -333,7 +333,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -343,7 +343,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -353,7 +353,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -363,7 +363,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -373,7 +373,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -383,7 +383,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -392,7 +392,7 @@ public class KStemmer {
if (!d.containsKey(KStemData8.data[i])) {
d.put(KStemData8.data[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + KStemData8.data[i]
+ throw new RuntimeException("Warning: Entry [" + KStemData8.data[i]
+ "] already in dictionary 4");
}
}
@@ -401,7 +401,7 @@ public class KStemmer {
if (!d.containsKey(supplementDict[i])) {
d.put(supplementDict[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + supplementDict[i]
+ throw new RuntimeException("Warning: Entry [" + supplementDict[i]
+ "] already in dictionary 5");
}
}
@@ -410,7 +410,7 @@ public class KStemmer {
if (!d.containsKey(properNouns[i])) {
d.put(properNouns[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + properNouns[i]
+ throw new RuntimeException("Warning: Entry [" + properNouns[i]
+ "] already in dictionary 6");
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java Thu Jul 19 15:58:54 2012
@@ -492,10 +492,9 @@ class PorterStemmer
return dirty;
}
- /** Test program for demonstrating the Stemmer. It reads a file and
+ /* Test program for demonstrating the Stemmer. It reads a file and
* stems each word, writing the result to standard out.
* Usage: Stemmer file-name
- */
public static void main(String[] args) {
PorterStemmer s = new PorterStemmer();
@@ -542,6 +541,6 @@ class PorterStemmer
System.out.println("error reading " + args[i]);
}
}
- }
+ }*/
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
@@ -134,6 +133,6 @@ public final class PersianAnalyzer exten
*/
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new PersianCharFilter(CharReader.get(reader));
+ return new PersianCharFilter(reader);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java Thu Jul 19 15:58:54 2012
@@ -18,9 +18,9 @@ package org.apache.lucene.analysis.fa;
*/
import java.io.IOException;
+import java.io.Reader;
-import org.apache.lucene.analysis.CharStream;
-import org.apache.lucene.analysis.charfilter.CharFilter;
+import org.apache.lucene.analysis.CharFilter;
/**
* CharFilter that replaces instances of Zero-width non-joiner with an
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.charfi
*/
public class PersianCharFilter extends CharFilter {
- public PersianCharFilter(CharStream in) {
+ public PersianCharFilter(Reader in) {
super(in);
}
@@ -45,4 +45,9 @@ public class PersianCharFilter extends C
}
return charsRead;
}
+
+ @Override
+ protected int correct(int currentOff) {
+ return currentOff; // we don't change the length of the string
+ }
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java Thu Jul 19 15:58:54 2012
@@ -66,10 +66,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionary InputStream for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @throws IOException Can be thrown while reading from the InputStreams
* @throws ParseException Can be thrown if the content of the files does not meet expected formats
@@ -80,10 +81,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionary InputStream for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @throws IOException Can be thrown while reading from the InputStreams
@@ -95,10 +97,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionaries InputStreams for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionaries InputStreams for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @throws IOException Can be thrown while reading from the InputStreams
@@ -110,10 +113,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionaries InputStreams for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionaries InputStreams for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @param strictAffixParsing Affix strict parsing enabled or not (an error while reading a rule causes exception or is ignored)
@@ -194,7 +198,6 @@ public class HunspellDictionary {
flagParsingStrategy = getFlagParsingStrategy(line);
}
}
- reader.close();
}
/**
@@ -252,7 +255,7 @@ public class HunspellDictionary {
}
String condition = ruleArgs[4];
- affix.setCondition(condition, String.format(conditionPattern, condition));
+ affix.setCondition(condition, String.format(Locale.ROOT, conditionPattern, condition));
affix.setCrossProduct(crossProduct);
List<HunspellAffix> list = affixes.get(affix.getAppend());
@@ -376,7 +379,7 @@ public class HunspellDictionary {
Arrays.sort(wordForm.getFlags());
entry = line.substring(0, flagSep);
if(ignoreCase) {
- entry = entry.toLowerCase(Locale.ENGLISH);
+ entry = entry.toLowerCase(Locale.ROOT);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java Thu Jul 19 15:58:54 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.hunsp
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.Charset;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -298,13 +299,12 @@ public class HunspellStemmer {
// ================================================= Entry Point ===================================================
- /**
+ /*
* HunspellStemmer entry point. Accepts two arguments: location of affix file and location of dic file
*
* @param args Program arguments. Should contain location of affix file and location of dic file
* @throws IOException Can be thrown while reading from the files
* @throws ParseException Can be thrown while parsing the files
- */
public static void main(String[] args) throws IOException, ParseException {
boolean ignoreCase = false;
int offset = 0;
@@ -330,7 +330,7 @@ public class HunspellStemmer {
HunspellStemmer stemmer = new HunspellStemmer(dictionary);
- Scanner scanner = new Scanner(System.in);
+ Scanner scanner = new Scanner(System.in, Charset.defaultCharset().name());
System.out.print("> ");
while (scanner.hasNextLine()) {
@@ -346,12 +346,10 @@ public class HunspellStemmer {
}
}
- /**
* Prints the results of the stemming of a word
*
* @param originalWord Word that has been stemmed
* @param stems Stems of the word
- */
private static void printStemResults(String originalWord, List<Stem> stems) {
StringBuilder builder = new StringBuilder().append("stem(").append(originalWord).append(")").append("\n");
@@ -381,13 +379,12 @@ public class HunspellStemmer {
System.out.println(builder);
}
- /**
* Simple utility to check if the given String has any text
*
* @param str String to check if it has any text
* @return {@code true} if the String has text, {@code false} otherwise
- */
private static boolean hasText(String str) {
return str != null && str.length() > 0;
}
+ */
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java Thu Jul 19 15:58:54 2012
@@ -19,15 +19,13 @@ package org.apache.lucene.analysis.misce
import org.apache.lucene.analysis.TokenStream;
-import java.io.IOException;
-
/**
* An always exhausted token stream.
*/
public final class EmptyTokenStream extends TokenStream {
@Override
- public final boolean incrementToken() throws IOException {
+ public final boolean incrementToken() {
return false;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java Thu Jul 19 15:58:54 2012
@@ -17,9 +17,6 @@
package org.apache.lucene.analysis.miscellaneous;
-import java.io.IOException;
-
-import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -43,7 +40,7 @@ public final class KeepWordFilter extend
}
@Override
- public boolean accept() throws IOException {
+ public boolean accept() {
return words.contains(termAtt.buffer(), 0, termAtt.length());
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java Thu Jul 19 15:58:54 2012
@@ -17,10 +17,7 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
-import java.io.IOException;
-
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -48,7 +45,7 @@ public final class LengthFilter extends
}
@Override
- public boolean accept() throws IOException {
+ public boolean accept() {
final int len = termAtt.length();
return (len >= min && len <= max);
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java Thu Jul 19 15:58:54 2012
@@ -17,8 +17,6 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
-import java.io.IOException;
-
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
@@ -46,7 +44,7 @@ public final class SingleTokenTokenStrea
}
@Override
- public final boolean incrementToken() throws IOException {
+ public final boolean incrementToken() {
if (exhausted) {
return false;
} else {
@@ -58,7 +56,7 @@ public final class SingleTokenTokenStrea
}
@Override
- public void reset() throws IOException {
+ public void reset() {
exhausted = false;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java Thu Jul 19 15:58:54 2012
@@ -23,7 +23,6 @@ import java.io.StringReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.charfilter.BaseCharFilter;
/**
@@ -54,7 +53,7 @@ public class PatternReplaceCharFilter ex
private final String replacement;
private Reader transformedInput;
- public PatternReplaceCharFilter(Pattern pattern, String replacement, CharStream in) {
+ public PatternReplaceCharFilter(Pattern pattern, String replacement, Reader in) {
super(in);
this.pattern = pattern;
this.replacement = replacement;
@@ -64,16 +63,29 @@ public class PatternReplaceCharFilter ex
public int read(char[] cbuf, int off, int len) throws IOException {
// Buffer all input on the first call.
if (transformedInput == null) {
- StringBuilder buffered = new StringBuilder();
- char [] temp = new char [1024];
- for (int cnt = input.read(temp); cnt > 0; cnt = input.read(temp)) {
- buffered.append(temp, 0, cnt);
- }
- transformedInput = new StringReader(processPattern(buffered).toString());
+ fill();
}
return transformedInput.read(cbuf, off, len);
}
+
+ private void fill() throws IOException {
+ StringBuilder buffered = new StringBuilder();
+ char [] temp = new char [1024];
+ for (int cnt = in.read(temp); cnt > 0; cnt = in.read(temp)) {
+ buffered.append(temp, 0, cnt);
+ }
+ transformedInput = new StringReader(processPattern(buffered).toString());
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (transformedInput == null) {
+ fill();
+ }
+
+ return transformedInput.read();
+ }
@Override
protected int correct(int currentOff) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java Thu Jul 19 15:58:54 2012
@@ -84,7 +84,7 @@ public final class PatternTokenizer exte
}
@Override
- public boolean incrementToken() throws IOException {
+ public boolean incrementToken() {
if (index >= str.length()) return false;
clearAttributes();
if (group >= 0) {
@@ -130,14 +130,14 @@ public final class PatternTokenizer exte
}
@Override
- public void end() throws IOException {
+ public void end() {
final int ofs = correctOffset(str.length());
offsetAtt.setOffset(ofs, ofs);
}
@Override
- public void reset(Reader input) throws IOException {
- super.reset(input);
+ public void setReader(Reader input) throws IOException {
+ super.setReader(input);
fillBuffer(str, input);
matcher.reset(str);
index = 0;
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java Thu Jul 19 15:58:54 2012
@@ -132,7 +132,7 @@ public abstract class RSLPStemmerBase {
super(suffix, min, replacement);
for (int i = 0; i < exceptions.length; i++) {
if (!exceptions[i].endsWith(suffix))
- System.err.println("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
+ throw new RuntimeException("useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
}
this.exceptions = new CharArraySet(Version.LUCENE_50,
Arrays.asList(exceptions), false);
@@ -156,7 +156,7 @@ public abstract class RSLPStemmerBase {
super(suffix, min, replacement);
for (int i = 0; i < exceptions.length; i++) {
if (!exceptions[i].endsWith(suffix))
- System.err.println("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
+ throw new RuntimeException("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
}
this.exceptions = new char[exceptions.length][];
for (int i = 0; i < exceptions.length; i++)
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java Thu Jul 19 15:58:54 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.sinks
import java.text.DateFormat;
import java.text.ParseException;
import java.util.Date;
+import java.util.Locale;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.AttributeSource;
@@ -37,10 +38,12 @@ public class DateRecognizerSinkFilter ex
protected CharTermAttribute termAtt;
/**
- * Uses {@link java.text.SimpleDateFormat#getDateInstance()} as the {@link java.text.DateFormat} object.
+ * Uses {@link java.text.DateFormat#getDateInstance(int, Locale)
+ * DateFormat#getDateInstance(DateFormat.DEFAULT, Locale.ROOT)} as
+ * the {@link java.text.DateFormat} object.
*/
public DateRecognizerSinkFilter() {
- this(DateFormat.getDateInstance());
+ this(DateFormat.getDateInstance(DateFormat.DEFAULT, Locale.ROOT));
}
public DateRecognizerSinkFilter(DateFormat dateFormat) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java Thu Jul 19 15:58:54 2012
@@ -212,7 +212,7 @@ public final class TeeSinkTokenFilter ex
}
@Override
- public final boolean incrementToken() throws IOException {
+ public final boolean incrementToken() {
// lazy init the iterator
if (it == null) {
it = cachedStates.iterator();
@@ -228,7 +228,7 @@ public final class TeeSinkTokenFilter ex
}
@Override
- public final void end() throws IOException {
+ public final void end() {
if (finalState != null) {
restoreState(finalState);
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -114,9 +114,9 @@ public final class ClassicAnalyzer exten
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void reset(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
- super.reset(reader);
+ super.setReader(reader);
}
};
}