You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/12/21 12:40:14 UTC
svn commit: r1721134 [2/48] - in /lucene/dev/branches/lucene6835: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/lucene/analysis/morfologik/
dev-tools/scripts/ lucene/ lucene/analysis/ luc...
Modified: lucene/dev/branches/lucene6835/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/build.xml?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/build.xml (original)
+++ lucene/dev/branches/lucene6835/build.xml Mon Dec 21 11:39:57 2015
@@ -129,6 +129,7 @@
<property name="validate.baseDir" location="."/>
<groovy taskname="source-patterns" classpathref="rat.classpath"><![CDATA[
import org.apache.tools.ant.BuildException;
+ import org.apache.tools.ant.Project;
import org.apache.rat.Defaults;
import org.apache.rat.document.impl.FileDocument;
import org.apache.rat.api.MetaData;
@@ -158,7 +159,7 @@
def found = 0;
def violations = new TreeSet();
def reportViolation = { f, name ->
- task.log(name + ': ' + f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/'));
+ task.log(name + ': ' + f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/'), Project.MSG_ERR);
violations.add(name);
found++;
}
@@ -166,6 +167,7 @@
def javadocsPattern = ~$/(?sm)^\Q/**\E(.*?)\Q*/\E/$;
def lineSplitter = ~$/[\r\n]+/$;
def licenseMatcher = Defaults.createDefaultMatcher();
+ def validLoggerPattern = ~$/(?s)\b(private\s|static\s|final\s){3}+\s*Logger\s+\p{javaJavaIdentifierStart}+\s+=\s+\QLoggerFactory.getLogger(MethodHandles.lookup().lookupClass());\E/$;
ant.fileScanner{
fileset(dir: baseDir){
@@ -182,9 +184,11 @@
// excludes:
exclude(name: '**/build/**')
exclude(name: '**/dist/**')
+ exclude(name: '**/CheckLoggingConfiguration.java')
exclude(name: 'build.xml') // ourselves :-)
}
}.each{ f ->
+ task.log('Scanning file: ' + f, Project.MSG_VERBOSE);
def text = f.getText('UTF-8');
invalidPatterns.each{ pattern,name ->
if (pattern.matcher(text).find()) {
@@ -200,6 +204,11 @@
ratDocument.getMetaData().value(MetaData.RAT_URL_LICENSE_FAMILY_NAME)));
}
}
+ if (f.toString().endsWith('.java') && text.contains('org.slf4j.LoggerFactory')) {
+ if (!validLoggerPattern.matcher(text).find()) {
+ reportViolation(f, 'invalid logging pattern [not private static final, uses static class name]');
+ }
+ }
};
if (found) {
@@ -566,7 +575,7 @@ File | Project Structure | Platform Sett
int baseLen = basedir.toString().length();
// do some fake check, to verify if this is valid SVN working copy. If this fails ignore checks but log some useful message.
- task.log('Initializing working copy...');
+ task.log('Initializing working copy...', Project.MSG_INFO);
try {
wcClient.doInfo(basedir, SVNRevision.WORKING);
} catch (SVNException ex) {
@@ -590,7 +599,7 @@ File | Project Structure | Platform Sett
Set missingProps = new TreeSet(), withKeywords = new TreeSet(), unversioned = new TreeSet(), changed = new TreeSet();
- task.log('Getting all versioned and unversioned files...');
+ task.log('Getting all versioned and unversioned files...', Project.MSG_INFO);
statusClient.doStatus(basedir, SVNRevision.WORKING, SVNDepth.fromRecurse(true), false, true, false, false, {
status ->
SVNStatusType nodeStatus = status.getNodeStatus();
@@ -605,12 +614,12 @@ File | Project Structure | Platform Sett
}
} as ISVNStatusHandler, null);
- task.log('Filtering files with existing svn:eol-style...');
+ task.log('Filtering files with existing svn:eol-style...', Project.MSG_INFO);
wcClient.doGetProperty(basedir, 'svn:eol-style', SVNRevision.WORKING, SVNRevision.WORKING, true, {
file, prop -> missingProps.remove(convertRelative(file));
} as ISVNPropertyHandler);
- task.log('Filtering files with binary svn:mime-type...');
+ task.log('Filtering files with binary svn:mime-type...', Project.MSG_INFO);
wcClient.doGetProperty(basedir, 'svn:mime-type', SVNRevision.WORKING, SVNRevision.WORKING, true, {
file, prop ->
prop = SVNPropertyValue.getPropertyAsString(prop.getValue());
@@ -619,7 +628,7 @@ File | Project Structure | Platform Sett
}
} as ISVNPropertyHandler);
- task.log('Scanning for files with svn:keywords property...');
+ task.log('Scanning for files with svn:keywords property...', Project.MSG_INFO);
wcClient.doGetProperty(basedir, 'svn:keywords', SVNRevision.WORKING, SVNRevision.WORKING, true, {
file, prop -> withKeywords.add(convertRelative(file));
} as ISVNPropertyHandler);
Modified: lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs (original)
+++ lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs Mon Dec 21 11:39:57 2015
@@ -3,5 +3,5 @@ eclipse.preferences.version=1
editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=false
formatter_profile=org.eclipse.jdt.ui.default.eclipse_profile
formatter_settings_version=12
-org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?><templates><template autoinsert\="false" context\="gettercomment_context" deleted\="false" description\="Comment for getter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name\="gettercomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="settercomment_context" deleted\="false" description\="Comment for setter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.settercomment" name\="settercomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="constructorcomment_context" deleted\="false" description\="Comment for created constructors" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name\="constructorcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="true" context\="filecomment_context" deleted\="false" description\="Comment for created Java file
s" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.filecomment" name\="filecomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="typecomment_context" deleted\="false" description\="Comment for created types" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.typecomment" name\="typecomment">/**\r\n *\r\n */</template><template autoinsert\="true" context\="fieldcomment_context" deleted\="false" description\="Comment for fields" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name\="fieldcomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="methodcomment_context" deleted\="false" description\="Comment for non-overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name\="methodcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="false" context\="overridecomment_context" deleted\="false" description\="Comment for overriding methods" enabled\="true
" id\="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name\="overridecomment">/* \r\n *\r\n */</template><template autoinsert\="true" context\="delegatecomment_context" deleted\="false" description\="Comment for delegate methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name\="delegatecomment">/**\r\n * ${tags}\r\n * ${see_to_target}\r\n */</template><template autoinsert\="false" context\="newtype_context" deleted\="false" description\="Newly created files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.newtype" name\="newtype">${filecomment}\r\n${package_declaration}\r\n\r\n/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements. See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the "License"); you may not use this file except
in compliance with\r\n * the License. You may obtain a copy of the License at\r\n *\r\n * http\://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an "AS IS" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\n\r\n${typecomment}\r\n${type_declaration}</template><template autoinsert\="true" context\="classbody_context" deleted\="false" description\="Code in new class type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.classbody" name\="classbody">\r\n</template><template autoinsert\="true" context\="interfacebody_context" deleted\="false" description\="Code in new interface type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name\="interfacebody">\r\n</t
emplate><template autoinsert\="true" context\="enumbody_context" deleted\="false" description\="Code in new enum type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.enumbody" name\="enumbody">\r\n</template><template autoinsert\="true" context\="annotationbody_context" deleted\="false" description\="Code in new annotation type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name\="annotationbody">\r\n</template><template autoinsert\="false" context\="catchblock_context" deleted\="false" description\="Code in new catch blocks" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.catchblock" name\="catchblock">throw new RuntimeException();</template><template autoinsert\="false" context\="methodbody_context" deleted\="false" description\="Code in created method stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodbody" name\="methodbody">${body_statement}</template><template autoinsert\="false" context\="c
onstructorbody_context" deleted\="false" description\="Code in created constructor stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name\="constructorbody">${body_statement}</template><template autoinsert\="true" context\="getterbody_context" deleted\="false" description\="Code in created getters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.getterbody" name\="getterbody">return ${field};</template><template autoinsert\="true" context\="setterbody_context" deleted\="false" description\="Code in created setters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.setterbody" name\="setterbody">${field} \= ${param};</template></templates>
+org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?><templates><template autoinsert\="false" context\="gettercomment_context" deleted\="false" description\="Comment for getter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name\="gettercomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="settercomment_context" deleted\="false" description\="Comment for setter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.settercomment" name\="settercomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="constructorcomment_context" deleted\="false" description\="Comment for created constructors" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name\="constructorcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="true" context\="filecomment_context" deleted\="false" description\="Comment for created Java file
s" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.filecomment" name\="filecomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="typecomment_context" deleted\="false" description\="Comment for created types" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.typecomment" name\="typecomment">/**\r\n *\r\n */</template><template autoinsert\="true" context\="fieldcomment_context" deleted\="false" description\="Comment for fields" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name\="fieldcomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="methodcomment_context" deleted\="false" description\="Comment for non-overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name\="methodcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="false" context\="overridecomment_context" deleted\="false" description\="Comment for overriding methods" enabled\="true
" id\="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name\="overridecomment">/* \r\n *\r\n */</template><template autoinsert\="true" context\="delegatecomment_context" deleted\="false" description\="Comment for delegate methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name\="delegatecomment">/**\r\n * ${tags}\r\n * ${see_to_target}\r\n */</template><template autoinsert\="false" context\="newtype_context" deleted\="false" description\="Newly created files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.newtype" name\="newtype">${filecomment}\r\n${package_declaration}\r\n\r\n/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements. See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the "License"); you may not use this file except
in compliance with\r\n * the License. You may obtain a copy of the License at\r\n *\r\n * http\://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an "AS IS" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\n\r\n${typecomment}\r\n${type_declaration}</template><template autoinsert\="true" context\="classbody_context" deleted\="false" description\="Code in new class type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.classbody" name\="classbody">\r\n</template><template autoinsert\="true" context\="interfacebody_context" deleted\="false" description\="Code in new interface type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name\="interfacebody">\r\n</t
emplate><template autoinsert\="true" context\="enumbody_context" deleted\="false" description\="Code in new enum type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.enumbody" name\="enumbody">\r\n</template><template autoinsert\="true" context\="annotationbody_context" deleted\="false" description\="Code in new annotation type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name\="annotationbody">\r\n</template><template autoinsert\="false" context\="catchblock_context" deleted\="false" description\="Code in new catch blocks" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.catchblock" name\="catchblock">throw new RuntimeException(${exception_var});</template><template autoinsert\="false" context\="methodbody_context" deleted\="false" description\="Code in created method stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodbody" name\="methodbody">${body_statement}</template><template autoinsert\="fa
lse" context\="constructorbody_context" deleted\="false" description\="Code in created constructor stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name\="constructorbody">${body_statement}</template><template autoinsert\="true" context\="getterbody_context" deleted\="false" description\="Code in created getters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.getterbody" name\="getterbody">return ${field};</template><template autoinsert\="true" context\="setterbody_context" deleted\="false" description\="Code in created setters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.setterbody" name\="setterbody">${field} \= ${param};</template></templates>
sp_cleanup.on_save_use_additional_actions=false
Modified: lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml (original)
+++ lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml Mon Dec 21 11:39:57 2015
@@ -2,7 +2,7 @@
<library name="JUnit">
<CLASSES>
<root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/junit-4.10.jar!/" />
- <root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/randomizedtesting-runner-2.1.17.jar!/" />
+ <root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/randomizedtesting-runner-2.3.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
Modified: lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template (original)
+++ lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template Mon Dec 21 11:39:57 2015
@@ -70,6 +70,9 @@
<exclude>**/*.java</exclude>
</excludes>
</testResource>
+ <testResource>
+ <directory>${module-path}/src/test-files</directory>
+ </testResource>
</testResources>
</build>
</project>
Modified: lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template Mon Dec 21 11:39:57 2015
@@ -248,7 +248,7 @@
<reportFormat>plain</reportFormat>
<workingDirectory>${project.build.directory}/test</workingDirectory>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
- <argLine>-Xmx512M -XX:MaxPermSize=256M</argLine>
+ <argLine>-Xmx512M</argLine>
<systemPropertyVariables>
<tempDir>.</tempDir>
<java.awt.headless>true</java.awt.headless>
Modified: lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py (original)
+++ lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py Mon Dec 21 11:39:57 2015
@@ -126,13 +126,14 @@ def update_example_solrconfigs(new_versi
print(' updating example solrconfig.xml files')
matcher = re.compile('<luceneMatchVersion>')
- configset_dir = 'solr/server/solr/configsets'
- if not os.path.isdir(configset_dir):
- raise RuntimeError("Can't locate configset dir (layout change?) : " + configset_dir)
- for root,dirs,files in os.walk(configset_dir, onerror=onerror):
- for f in files:
- if f == 'solrconfig.xml':
- update_solrconfig(os.path.join(root, f), matcher, new_version)
+ paths = ['solr/server/solr/configsets', 'solr/example']
+ for path in paths:
+ if not os.path.isdir(path):
+ raise RuntimeError("Can't locate configset dir (layout change?) : " + path)
+ for root,dirs,files in os.walk(path, onerror=onerror):
+ for f in files:
+ if f == 'solrconfig.xml':
+ update_solrconfig(os.path.join(root, f), matcher, new_version)
def update_solrconfig(filename, matcher, new_version):
print(' %s...' % filename, end='', flush=True)
Modified: lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py (original)
+++ lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py Mon Dec 21 11:39:57 2015
@@ -84,7 +84,12 @@ def getHREFs(urlString):
# Deref any redirects
while True:
url = urllib.parse.urlparse(urlString)
- h = http.client.HTTPConnection(url.netloc)
+ if url.scheme == "http":
+ h = http.client.HTTPConnection(url.netloc)
+ elif url.scheme == "https":
+ h = http.client.HTTPSConnection(url.netloc)
+ else:
+ raise RuntimeError("Unknown protocol: %s" % url.scheme)
h.request('GET', url.path)
r = h.getresponse()
newLoc = r.getheader('location')
Modified: lucene/dev/branches/lucene6835/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/CHANGES.txt?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6835/lucene/CHANGES.txt Mon Dec 21 11:39:57 2015
@@ -15,6 +15,8 @@ System Requirements
New Features
+* LUCENE-6631: Lucene Document classification (Tommaso Teofili, Alessandro Benedetti)
+
* LUCENE-6747: FingerprintFilter is a TokenFilter that outputs a single
token which is a concatenation of the sorted and de-duplicated set of
input tokens. Useful for normalizing short text in clustering/linking
@@ -35,6 +37,24 @@ New Features
that can return false to exit from ExitableDirectoryReader wrapping at
the point fields() is called. (yonik)
+* LUCENE-6825: Add low-level support for block-KD trees (Mike McCandless)
+
+* LUCENE-6852: Add support for dimensionally indexed values to index,
+ document and codec APIs, including a simple text implementation.
+ (Mike McCandless)
+
+* LUCENE-6861: Create Lucene60Codec, supporting dimensional values.
+ (Mike McCandless)
+
+* LUCENE-6879: Allow to define custom CharTokenizer instances without
+ subclassing using Java 8 lambdas or method references. (Uwe Schindler)
+
+* LUCENE-6881: Cutover all BKD implementations to dimensional values
+ (Mike McCandless)
+
+* LUCENE-6837: Add N-best output support to JapaneseTokenizer.
+ (Hiroharu Konno via Christian Moen)
+
API Changes
* LUCENE-3312: The API of oal.document was restructured to
@@ -66,15 +86,110 @@ API Changes
IndexOutput.getName returns its name (Dawid Weiss, Robert Muir, Mike
McCandless)
+* LUCENE-6917: Deprecate and rename NumericXXX classes to
+ LegacyNumericXXX in favor of dimensional values (Mike McCandless)
+
+Optimizations
+
+* LUCENE-6891: Use prefix coding when writing dimensional values in
+ each leaf block in the default codec, to reduce the index
+ size (Mike McCandless)
+
+* LUCENE-6901: Optimize dimensional values indexing: use faster
+ IntroSorter instead of InPlaceMergeSorter, and specialize 1D
+ merging to merge sort the already sorted segments instead of
+ re-indexing (Mike McCandless)
+
Changes in Runtime Behavior
* LUCENE-6789: IndexSearcher's default Similarity is changed to BM25Similarity.
Use ClassicSimilarity to get the old vector space DefaultSimilarity. (Robert Muir)
+* LUCENE-6886: Reserve the .tmp file name extension for temp files,
+ and codec components are no longer allowed to use this extension
+ (Robert Muir, Mike McCandless)
+
+======================= Lucene 5.5.0 =======================
+
+New Features
+
+* LUCENE-5868: JoinUtil.createJoinQuery(..,NumericType,..) query-time join
+ for LONG and INT fields with NUMERIC and SORTED_NUMERIC doc values.
+ (Alexey Zelin via Mikhail Khludnev)
+
+API Changes
+
+* LUCENE-6908: GeoUtils static relational methods have been refactored to new
+ GeoRelationUtils and now correctly handle large irregular rectangles, and
+ pole crossing distance queries. (Nick Knize)
+
+* LUCENE-6900: Grouping sortWithinGroup variables used to allow null to mean
+ Sort.RELEVANCE. Null is no longer permitted. (David Smiley)
+
+* LUCENE-6919: The Scorer class has been refactored to expose an iterator
+ instead of extending DocIdSetIterator. asTwoPhaseIterator() has been renamed
+ to twoPhaseIterator() for consistency. (Adrien Grand)
+
+Optimizations
+
+* LUCENE-6889: BooleanQuery.rewrite now performs some query optimization, in
+ particular to rewrite queries that look like: "+*:* #filter" to a
+ "ConstantScore(filter)". (Adrien Grand)
+
+* LUCENE-6912: Grouping's Collectors now calculate a response to needsScores()
+ instead of always 'true'. (David Smiley)
+
+* LUCENE-6815: DisjunctionScorer now advances two-phased iterators lazily,
+ stopping to evaluate them as soon as a single one matches. The other iterators
+ will be confirmed lazily when computing score() or freq(). (Adrien Grand)
+
+* LUCENE-6926: MUST_NOT clauses now use the match cost API to run the slow bits
+ last whenever possible. (Adrien Grand)
+
+Bug Fixes
+
+* LUCENE-6918: LRUQueryCache.onDocIdSetEviction is only called when at least
+ one DocIdSet is being evicted. (Adrien Grand)
+
+* LUCENE-6910: fix 'if ... > Integer.MAX_VALUE' check in
+ (Binary|Numeric)DocValuesFieldUpdates.merge
+ (https://scan.coverity.com/projects/5620 CID 119973 and CID 120081)
+ (Christine Poerschke, Coverity Scan (via Rishabh Patel))
+
+* LUCENE-6929: Fix SpanNotQuery rewriting to not drop the pre/post parameters.
+ (Tim Allison via Adrien Grand)
+
+* SOLR-7865: BlendedInfixSuggester was returning too many results
+ (Arcadius Ahouansou via Mike McCandless)
+
+Other
+
+* LUCENE-6924: Upgrade randomizedtesting to 2.3.2. (Dawid Weiss)
+
+* LUCENE-6920: Improve custom function checks in expressions module
+ to use MethodHandles and work without extra security privileges.
+ (Uwe Schindler, Robert Muir)
+
+* LUCENE-6921: Fix SPIClassIterator#isParentClassLoader to don't
+ require extra permissions. (Uwe Schindler)
+
+* LUCENE-6923: Fix RamUsageEstimator to access private fields inside
+ AccessController block for computing size. (Robert Muir)
+
+* LUCENE-6907: make TestParser extendable, rename test/.../xml/
+ NumericRangeQueryQuery.xml to NumericRangeQuery.xml
+ (Christine Poerschke)
+
+* LUCENE-6925: add ForceMergePolicy class in test-framework
+ (Christine Poerschke)
+
======================= Lucene 5.4.0 =======================
New Features
+* LUCENE-6875: New Serbian Filter. (Nikola Smolenski via Robert Muir,
+ Dawid Weiss)
+
* LUCENE-6720: New FunctionRangeQuery wrapper around ValueSourceScorer
(returned from ValueSource/FunctionValues.getRangeScorer()). (David Smiley)
@@ -94,6 +209,15 @@ New Features
* LUCENE-6844: PayloadScoreQuery can include or exclude underlying span scores
from its score calculations (Bill Bell, Alan Woodward)
+* LUCENE-6778: Add GeoPointDistanceRangeQuery, to search for points
+ within a "ring" (beyond a minimum distance and below a maximum
+ distance) (Nick Knize via Mike McCandless)
+
+* LUCENE-6874: Add a new UnicodeWhitespaceTokenizer to analysis/common
+ that uses Unicode character properties extracted from ICU4J to tokenize
+ text on whitespace. This tokenizer will split on non-breaking
+ space (NBSP), too. (David Smiley, Uwe Schindler, Steve Rowe)
+
API Changes
* LUCENE-6590: Query.setBoost(), Query.getBoost() and Query.clone() are gone.
@@ -120,6 +244,23 @@ API Changes
* LUCENE-6939: SpanOrQuery.addClause is now deprecated, clauses should all be
provided at construction time. (Paul Elschot via Adrien Grand)
+* LUCENE-6855: CachingWrapperQuery is deprecated and will be removed in 6.0.
+ (Adrien Grand)
+
+* LUCENE-6870: DisjunctionMaxQuery#add is now deprecated, clauses should all be
+ provided at construction time. (Adrien Grand)
+
+* LUCENE-6884: Analyzer.tokenStream() and Tokenizer.setReader() are no longer
+ declared as throwing IOException. (Alan Woodward)
+
+* LUCENE-6849: Expose IndexWriter.flush() method, to move all
+ in-memory segments to disk without opening a near-real-time reader
+ nor calling fsync (Robert Muir, Simon Willnauer, Mike McCandless)
+
+* LUCENE-6911: Add correct StandardQueryParser.getMultiFields() method,
+ deprecate no-op StandardQueryParser.getMultiFields(CharSequence[]) method.
+ (Christine Poerschke, Mikhail Khludnev, Coverity Scan (via Rishabh Patel))
+
Optimizations
* LUCENE-6708: TopFieldCollector does not compute the score several times on the
@@ -148,8 +289,43 @@ Optimizations
strings larger than 64kb by an amount equal to string's utf8 size.
(Dawid Weiss, Robert Muir, shalin)
+* LUCENE-6850: Optimize BooleanScorer for sparse clauses. (Adrien Grand)
+
+* LUCENE-6840: Ordinal indexes for SORTED_SET/SORTED_NUMERIC fields and
+ addresses for BINARY fields are now stored on disk instead of in memory.
+ (Adrien Grand)
+
+* LUCENE-6878: Speed up TopDocs.merge. (Daniel Jelinski via Adrien Grand)
+
+* LUCENE-6885: StandardDirectoryReader (initialCapacity) tweaks
+ (Christine Poerschke)
+
+* LUCENE-6863: Optimized storage requirements of doc values fields when less
+ than 1% of documents have a value. (Adrien Grand)
+
+* LUCENE-6892: various lucene.index initialCapacity tweaks
+ (Christine Poerschke)
+
+* LUCENE-6276: Added TwoPhaseIterator.matchCost() which allows to confirm the
+ least costly TwoPhaseIterators first. (Paul Elschot via Adrien Grand)
+
+* LUCENE-6898: In the default codec, the last stored field value will not
+ be fully read from disk if the supplied StoredFieldVisitor doesn't want it.
+ So put your largest text field value last to benefit. (David Smiley)
+
+* LUCENE-6909: Remove unnecessary synchronized from
+ FacetsConfig.getDimConfig for better concurrency (Sanne Grinovero
+ via Mike McCandless)
+
+* SOLR-7730: Speed up SlowCompositeReaderWrapper.getSortedSetDocValues() by
+ avoiding merging FieldInfos just to check doc value type.
+ (Paul Vasilyev, Yuriy Pakhomov, Mikhail Khludnev, yonik)
+
Bug Fixes
+* LUCENE-6905: Unwrap center longitude for dateline crossing
+ GeoPointDistanceQuery. (Nick Knize)
+
* LUCENE-6817: ComplexPhraseQueryParser.ComplexPhraseQuery does not display
slop in toString(). (Ahmet Arslan via Dawid Weiss)
@@ -204,8 +380,33 @@ Bug Fixes
source for checking for expiration (Ishan Chattopadhyaya via Mike
McCandless)
+* LUCENE-6856: The Weight wrapper used by LRUQueryCache now delegates to the
+ original Weight's BulkScorer when applicable. (Adrien Grand)
+
+* LUCENE-6858: Fix ContextSuggestField to correctly wrap token stream
+ when using CompletionAnalyzer. (Areek Zillur)
+
+* LUCENE-6872: IndexWriter handles any VirtualMachineError, not just OOM,
+ as tragic. (Robert Muir)
+
+* LUCENE-6814: PatternTokenizer no longer hangs onto heap sized to the
+ maximum input string it's ever seen, which can be a large memory
+ "leak" if you tokenize large strings with many threads across many
+ indices (Alex Chow via Mike McCandless)
+
+* LUCENE-6888: Explain output of map() function now also prints default value (janhoy)
+
Other
+* LUCENE-6899: Upgrade randomizedtesting to 2.3.1. (Dawid Weiss)
+
+* LUCENE-6478: Test execution can hang with java.security.debug. (Dawid Weiss)
+
+* LUCENE-6862: Upgrade of RandomizedRunner to version 2.2.0. (Dawid Weiss)
+
+* LUCENE-6857: Validate StandardQueryParser with NOT operator
+ with-in parantheses. (Jigar Shah via Dawid Weiss)
+
* LUCENE-6827: Use explicit capacity ArrayList instead of a LinkedList
in MultiFieldQueryNodeProcessor. (Dawid Weiss).
@@ -263,6 +464,21 @@ Other
* LUCENE-6467: Simplify Query.equals. (Paul Elschot via Adrien Grand)
+* LUCENE-6845: SpanScorer is now merged into Spans (Alan Woodward, David Smiley)
+
+* LUCENE-6887: DefaultSimilarity is deprecated, use ClassicSimilarity for equivilent behavior,
+ or consider switching to BM25Similarity which will become the new default in Lucene 6.0 (hossman)
+
+* LUCENE-6893: factor out CorePlusQueriesParser from CorePlusExtensionsParser
+ (Christine Poerschke)
+
+* LUCENE-6902: Don't retry to fsync files / directories; fail
+ immediately. (Daniel Mitterdorfer, Uwe Schindler)
+
+* LUCENE-6801: Clarify JavaDocs of PhraseQuery that it in fact supports terms
+ at the same position (as does MultiPhraseQuery), treated like a conjunction.
+ Added test. (David Smiley, Adrien Grand)
+
Build
* LUCENE-6732: Improve checker for invalid source patterns to also
@@ -709,11 +925,7 @@ Optimizations
* LUCENE-6621: Removed two unused variables in analysis/stempel/src/java/org/
egothor/stemmer/Compile.java
(Rishabh Patel via Christine Poerschke)
-
-* SOLR-7730: Speed up SlowCompositeReaderWrapper.getSortedSetDocValues() by
- avoiding merging FieldInfos just to check doc value type.
- (Paul Vasilyev, Yuriy Pakhomov, Mikhail Khludnev, yonik)
-
+
Build
* LUCENE-6518: Don't report false thread leaks from IBM J9
Modified: lucene/dev/branches/lucene6835/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/MIGRATE.txt?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/MIGRATE.txt (original)
+++ lucene/dev/branches/lucene6835/lucene/MIGRATE.txt Mon Dec 21 11:39:57 2015
@@ -80,3 +80,11 @@ would be equivalent to the following cod
Query q = ...;
float boost = ...;
q.setBoost(q.getBoost() * boost);
+
+## DimensionalValues replaces NumericField (LUCENE-6917)
+
+DimensionalValues provides faster indexing and searching, a smaller
+index size, and less heap used at search time. The numeric fields
+(IntField, FloatField, LongField, DoubleField) and NumericRangeQuery
+have been moved to the backward-codecs module and prefixed with
+Legacy.
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml Mon Dec 21 11:39:57 2015
@@ -31,6 +31,8 @@
<property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>
+ <property name="unicode-props-file" location="src/java/org/apache/lucene/analysis/util/UnicodeProps.java"/>
+
<target name="jflex" depends="-install-jflex,clean-jflex,-jflex-StandardAnalyzer,-jflex-UAX29URLEmailTokenizer,
-jflex-wiki-tokenizer,-jflex-HTMLStripCharFilter"/>
@@ -114,6 +116,18 @@
</delete>
</target>
+ <target xmlns:ivy="antlib:org.apache.ivy.ant" name="-resolve-icu4j" unless="icu4j.resolved" depends="ivy-availability-check,ivy-configure">
+ <loadproperties prefix="ivyversions" srcFile="${common.dir}/ivy-versions.properties"/>
+ <ivy:cachepath organisation="com.ibm.icu" module="icu4j" revision="${ivyversions./com.ibm.icu/icu4j}"
+ inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
+ <property name="icu4j.resolved" value="true"/>
+ </target>
+
+ <target name="unicode-data" depends="-resolve-icu4j,resolve-groovy">
+ <groovy classpathref="icu4j.classpath" src="src/tools/groovy/generate-unicode-data.groovy"/>
+ <fixcrlf file="${unicode-props-file}" encoding="UTF-8"/>
+ </target>
+
<property name="tld.zones" value="http://www.internic.net/zones/root.zone"/>
<property name="tld.output" location="src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro"/>
@@ -141,7 +155,7 @@
<target name="javadocs" depends="module-build.javadocs"/>
- <target name="regenerate" depends="jflex"/>
+ <target name="regenerate" depends="jflex,unicode-data"/>
<target name="patch-snowball" description="Patches all snowball programs in '${snowball.programs.dir}' to make them work with MethodHandles">
<fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java Mon Dec 21 11:39:57 2015
@@ -22,8 +22,11 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.util.AttributeFactory;
/**
- * A WhitespaceTokenizer is a tokenizer that divides text at whitespace.
+ * A tokenizer that divides text at whitespace characters as defined by
+ * {@link Character#isWhitespace(int)}. Note: That definition explicitly excludes the non-breaking space.
* Adjacent sequences of non-Whitespace characters form tokens.
+ *
+ * @see UnicodeWhitespaceTokenizer
*/
public final class WhitespaceTokenizer extends CharTokenizer {
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java Mon Dec 21 11:39:57 2015
@@ -17,32 +17,56 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
-import java.util.Map;
-
/**
* Factory for {@link WhitespaceTokenizer}.
* <pre class="prettyprint">
* <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
- * <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ * <tokenizer class="solr.WhitespaceTokenizerFactory" rule="unicode"/>
* </analyzer>
* </fieldType></pre>
+ *
+ * Options:
+ * <ul>
+ * <li>rule: either "java" for {@link WhitespaceTokenizer}
+ * or "unicode" for {@link UnicodeWhitespaceTokenizer}</li>
+ * </ul>
*/
public class WhitespaceTokenizerFactory extends TokenizerFactory {
+ public static final String RULE_JAVA = "java";
+ public static final String RULE_UNICODE = "unicode";
+ private static final Collection<String> RULE_NAMES = Arrays.asList(RULE_JAVA, RULE_UNICODE);
+
+ private final String rule;
/** Creates a new WhitespaceTokenizerFactory */
public WhitespaceTokenizerFactory(Map<String,String> args) {
super(args);
+
+ rule = get(args, "rule", RULE_NAMES, RULE_JAVA);
+
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
@Override
- public WhitespaceTokenizer create(AttributeFactory factory) {
- return new WhitespaceTokenizer(factory);
+ public Tokenizer create(AttributeFactory factory) {
+ switch (rule) {
+ case RULE_JAVA:
+ return new WhitespaceTokenizer(factory);
+ case RULE_UNICODE:
+ return new UnicodeWhitespaceTokenizer(factory);
+ default:
+ throw new AssertionError();
+ }
}
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java Mon Dec 21 11:39:57 2015
@@ -137,9 +137,19 @@ public final class PatternTokenizer exte
}
@Override
+ public void close() throws IOException {
+ try {
+ super.close();
+ } finally {
+ str.setLength(0);
+ str.trimToSize();
+ }
+ }
+
+ @Override
public void reset() throws IOException {
super.reset();
- fillBuffer(str, input);
+ fillBuffer(input);
matcher.reset(str);
index = 0;
}
@@ -147,11 +157,11 @@ public final class PatternTokenizer exte
// TODO: we should see if we can make this tokenizer work without reading
// the entire document into RAM, perhaps with Matcher.hitEnd/requireEnd ?
final char[] buffer = new char[8192];
- private void fillBuffer(StringBuilder sb, Reader input) throws IOException {
+ private void fillBuffer(Reader input) throws IOException {
int len;
- sb.setLength(0);
+ str.setLength(0);
while ((len = input.read(buffer)) > 0) {
- sb.append(buffer, 0, len);
+ str.append(buffer, 0, len);
}
}
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,7 @@ package org.apache.lucene.analysis.sr;
* limitations under the License.
*/
+import java.util.Arrays;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
@@ -31,15 +32,19 @@ import org.apache.lucene.analysis.util.T
* <analyzer>
* <tokenizer class="solr.StandardTokenizerFactory"/>
* <filter class="solr.LowerCaseFilterFactory"/>
- * <filter class="solr.SerbianNormalizationFilterFactory"/>
+ * <filter class="solr.SerbianNormalizationFilterFactory"
+ * haircut="bald"/>
* </analyzer>
* </fieldType></pre>
*/
public class SerbianNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+ final String haircut;
/** Creates a new SerbianNormalizationFilterFactory */
public SerbianNormalizationFilterFactory(Map<String,String> args) {
super(args);
+
+ this.haircut = get(args, "haircut", Arrays.asList( "bald", "regular" ), "bald");
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -47,7 +52,11 @@ public class SerbianNormalizationFilterF
@Override
public TokenStream create(TokenStream input) {
- return new SerbianNormalizationFilter(input);
+ if( this.haircut.equals( "regular" ) ) {
+ return new SerbianNormalizationRegularFilter(input);
+ } else {
+ return new SerbianNormalizationFilter(input);
+ }
}
@Override
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Reader;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
@@ -25,9 +28,6 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
-import java.io.IOException;
-import java.io.Reader;
-
/**
* Filters {@link ClassicTokenizer} with {@link ClassicFilter}, {@link
* LowerCaseFilter} and {@link StopFilter}, using a list of
@@ -94,7 +94,7 @@ public final class ClassicAnalyzer exten
tok = new StopFilter(tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void setReader(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) {
src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
super.setReader(reader);
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Reader;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
@@ -25,9 +28,6 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
-import java.io.IOException;
-import java.io.Reader;
-
/**
* Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
* LowerCaseFilter} and {@link StopFilter}, using a list of
@@ -89,7 +89,7 @@ public final class StandardAnalyzer exte
tok = new StopFilter(tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void setReader(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) {
src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
super.setReader(reader);
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Reader;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
@@ -24,9 +27,6 @@ import org.apache.lucene.analysis.core.S
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
-import java.io.IOException;
-import java.io.Reader;
-
/**
* Filters {@link org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer}
* with {@link org.apache.lucene.analysis.standard.StandardFilter},
@@ -91,7 +91,7 @@ public final class UAX29URLEmailAnalyzer
tok = new StopFilter(tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void setReader(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) {
src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength);
super.setReader(reader);
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java Mon Dec 21 11:39:57 2015
@@ -21,6 +21,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.LinkedHashMap;
import java.util.Set;
import java.util.ServiceConfigurationError;
@@ -54,6 +55,9 @@ final class AnalysisSPILoader<S extends
this.suffixes = suffixes;
// if clazz' classloader is not a parent of the given one, we scan clazz's classloader, too:
final ClassLoader clazzClassloader = clazz.getClassLoader();
+ if (classloader == null) {
+ classloader = clazzClassloader;
+ }
if (clazzClassloader != null && !SPIClassIterator.isParentClassLoader(clazzClassloader, classloader)) {
reload(clazzClassloader);
}
@@ -72,6 +76,7 @@ final class AnalysisSPILoader<S extends
* of new service providers on the given classpath/classloader!</em>
*/
public synchronized void reload(ClassLoader classloader) {
+ Objects.requireNonNull(classloader, "classloader");
final LinkedHashMap<String,Class<? extends S>> services =
new LinkedHashMap<>(this.services);
final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java Mon Dec 21 11:39:57 2015
@@ -18,8 +18,15 @@ package org.apache.lucene.analysis.util;
*/
import java.io.IOException;
+import java.util.Objects;
+import java.util.function.IntPredicate;
+import java.util.function.IntUnaryOperator;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LetterTokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.AttributeFactory;
@@ -27,8 +34,16 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.analysis.util.CharacterUtils.CharacterBuffer;
/**
- * An abstract base class for simple, character-oriented tokenizers.
- **/
+ * An abstract base class for simple, character-oriented tokenizers.
+ * <p>
+ * The base class also provides factories to create instances of
+ * {@code CharTokenizer} using Java 8 lambdas or method references.
+ * It is possible to create an instance which behaves exactly like
+ * {@link LetterTokenizer}:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter);
+ * </pre>
+ */
public abstract class CharTokenizer extends Tokenizer {
/**
@@ -47,6 +62,136 @@ public abstract class CharTokenizer exte
super(factory);
}
+ /**
+ * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+ * The predicate should return {@code true} for all valid token characters.
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as {@link LetterTokenizer} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter);
+ * </pre>
+ */
+ public static CharTokenizer fromTokenCharPredicate(final IntPredicate tokenCharPredicate) {
+ return fromTokenCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, tokenCharPredicate);
+ }
+
+ /**
+ * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
+ * The predicate should return {@code true} for all valid token characters.
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as {@link LetterTokenizer} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(factory, Character::isLetter);
+ * </pre>
+ */
+ public static CharTokenizer fromTokenCharPredicate(AttributeFactory factory, final IntPredicate tokenCharPredicate) {
+ return fromTokenCharPredicate(factory, tokenCharPredicate, IntUnaryOperator.identity());
+ }
+
+ /**
+ * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+ * The predicate should return {@code true} for all valid token characters.
+ * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as {@link LowerCaseTokenizer} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter, Character::toLowerCase);
+ * </pre>
+ */
+ public static CharTokenizer fromTokenCharPredicate(final IntPredicate tokenCharPredicate, final IntUnaryOperator normalizer) {
+ return fromTokenCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, tokenCharPredicate, normalizer);
+ }
+
+ /**
+ * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
+ * The predicate should return {@code true} for all valid token characters.
+ * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as {@link LowerCaseTokenizer} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(factory, Character::isLetter, Character::toLowerCase);
+ * </pre>
+ */
+ public static CharTokenizer fromTokenCharPredicate(AttributeFactory factory, final IntPredicate tokenCharPredicate, final IntUnaryOperator normalizer) {
+ Objects.requireNonNull(tokenCharPredicate, "predicate must not be null.");
+ Objects.requireNonNull(normalizer, "normalizer must not be null");
+ return new CharTokenizer(factory) {
+ @Override
+ protected boolean isTokenChar(int c) {
+ return tokenCharPredicate.test(c);
+ }
+
+ @Override
+ protected int normalize(int c) {
+ return normalizer.applyAsInt(c);
+ }
+ };
+ }
+
+ /**
+ * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+ * The predicate should return {@code true} for all valid token separator characters.
+ * This method is provided for convenience to easily use predicates that are negated
+ * (they match the separator characters, not the token characters).
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as {@link WhitespaceTokenizer} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace);
+ * </pre>
+ */
+ public static CharTokenizer fromSeparatorCharPredicate(final IntPredicate separatorCharPredicate) {
+ return fromSeparatorCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, separatorCharPredicate);
+ }
+
+ /**
+ * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
+ * The predicate should return {@code true} for all valid token separator characters.
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as {@link WhitespaceTokenizer} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(factory, Character::isWhitespace);
+ * </pre>
+ */
+ public static CharTokenizer fromSeparatorCharPredicate(AttributeFactory factory, final IntPredicate separatorCharPredicate) {
+ return fromSeparatorCharPredicate(factory, separatorCharPredicate, IntUnaryOperator.identity());
+ }
+
+ /**
+ * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+ * The predicate should return {@code true} for all valid token separator characters.
+ * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as the combination {@link WhitespaceTokenizer} and {@link LowerCaseFilter} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace, Character::toLowerCase);
+ * </pre>
+ */
+ public static CharTokenizer fromSeparatorCharPredicate(final IntPredicate separatorCharPredicate, final IntUnaryOperator normalizer) {
+ return fromSeparatorCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, separatorCharPredicate, normalizer);
+ }
+
+ /**
+ * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate.
+ * The predicate should return {@code true} for all valid token separator characters.
+ * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+ * <p>
+ * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+ * to create an instance which behaves exactly as {@link WhitespaceTokenizer} and {@link LowerCaseFilter} is:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(factory, Character::isWhitespace, Character::toLowerCase);
+ * </pre>
+ */
+ public static CharTokenizer fromSeparatorCharPredicate(AttributeFactory factory, final IntPredicate separatorCharPredicate, final IntUnaryOperator normalizer) {
+ return fromTokenCharPredicate(factory, separatorCharPredicate.negate(), normalizer);
+ }
+
private int offset = 0, bufferIndex = 0, dataLen = 0, finalOffset = 0;
private static final int MAX_WORD_LEN = 255;
private static final int IO_BUFFER_SIZE = 4096;
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java Mon Dec 21 11:39:57 2015
@@ -22,6 +22,7 @@ import java.io.StringReader;
import java.lang.reflect.Modifier;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
@@ -48,6 +49,7 @@ import org.apache.lucene.analysis.miscel
import org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer;
import org.apache.lucene.analysis.sinks.TeeSinkTokenFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
+import org.apache.lucene.analysis.sr.SerbianNormalizationRegularFilter;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
@@ -99,10 +101,17 @@ public class TestAllAnalyzersHaveFactori
ReversePathHierarchyTokenizer.class, // this is supported via an option to PathHierarchyTokenizer's factory
SnowballFilter.class, // this is called SnowballPorterFilterFactory
PatternKeywordMarkerFilter.class,
- SetKeywordMarkerFilter.class
+ SetKeywordMarkerFilter.class,
+ UnicodeWhitespaceTokenizer.class // a supported option via WhitespaceTokenizerFactory
);
}
-
+
+ // The following token filters are excused from having their factory.
+ private static final Set<Class<?>> tokenFiltersWithoutFactory = new HashSet<>();
+ static {
+ tokenFiltersWithoutFactory.add(SerbianNormalizationRegularFilter.class);
+ }
+
private static final ResourceLoader loader = new StringMockResourceLoader("");
public void test() throws Exception {
@@ -117,12 +126,13 @@ public class TestAllAnalyzersHaveFactori
|| testComponents.contains(c)
|| crazyComponents.contains(c)
|| oddlyNamedComponents.contains(c)
+ || tokenFiltersWithoutFactory.contains(c)
|| c.isAnnotationPresent(Deprecated.class) // deprecated ones are typically back compat hacks
|| !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharFilter.class.isAssignableFrom(c))
) {
continue;
}
-
+
Map<String,String> args = new HashMap<>();
args.put("luceneMatchVersion", Version.LATEST.toString());
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java Mon Dec 21 11:39:57 2015
@@ -130,7 +130,7 @@ public class TestAnalyzers extends BaseT
@Override
public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new WhitespaceTokenizer();
+ Tokenizer tokenizer = random().nextBoolean() ? new WhitespaceTokenizer() : new UnicodeWhitespaceTokenizer();
return new TokenStreamComponents(tokenizer, new LowerCaseFilter(tokenizer));
}
@@ -140,7 +140,7 @@ public class TestAnalyzers extends BaseT
@Override
public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new WhitespaceTokenizer();
+ Tokenizer tokenizer = random().nextBoolean() ? new WhitespaceTokenizer() : new UnicodeWhitespaceTokenizer();
return new TokenStreamComponents(tokenizer, new UpperCaseFilter(tokenizer));
}
@@ -230,7 +230,7 @@ public class TestAnalyzers extends BaseT
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() };
+ Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer(), new UnicodeWhitespaceAnalyzer() };
for (Analyzer analyzer : analyzers) {
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
@@ -239,7 +239,7 @@ public class TestAnalyzers extends BaseT
/** blast some random large strings through the analyzer */
public void testRandomHugeStrings() throws Exception {
- Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() };
+ Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer(), new UnicodeWhitespaceAnalyzer() };
for (Analyzer analyzer : analyzers) {
checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192);
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java Mon Dec 21 11:39:57 2015
@@ -17,12 +17,12 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import org.apache.lucene.analysis.NumericTokenStream;
+import java.util.Set;
+
+import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import java.util.Set;
-
/**
* Testcase for {@link TypeTokenFilterFactory}
*/
@@ -46,8 +46,7 @@ public class TestTypeTokenFilterFactory
public void testCreationWithBlackList() throws Exception {
TokenFilterFactory factory = tokenFilterFactory("Type",
"types", "stoptypes-1.txt, stoptypes-2.txt");
- NumericTokenStream input = new NumericTokenStream();
- input.setIntValue(123);
+ CannedTokenStream input = new CannedTokenStream();
factory.create(input);
}
@@ -55,8 +54,7 @@ public class TestTypeTokenFilterFactory
TokenFilterFactory factory = tokenFilterFactory("Type",
"types", "stoptypes-1.txt, stoptypes-2.txt",
"useWhitelist", "true");
- NumericTokenStream input = new NumericTokenStream();
- input.setIntValue(123);
+ CannedTokenStream input = new CannedTokenStream();
factory.create(input);
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java Mon Dec 21 11:39:57 2015
@@ -1,6 +1,5 @@
package org.apache.lucene.analysis.miscellaneous;
-import java.io.IOException;
import java.io.Reader;
import java.util.Collections;
import java.util.Map;
@@ -16,7 +15,6 @@ import org.apache.lucene.analysis.core.S
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.Rethrow;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -105,11 +103,7 @@ public class TestPerFieldAnalyzerWrapper
@Override
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
- try {
- assertNotSame(specialAnalyzer.tokenStream("special", text), components.getTokenStream());
- } catch (IOException e) {
- Rethrow.rethrow(e);
- }
+ assertNotSame(specialAnalyzer.tokenStream("special", text), components.getTokenStream());
TokenFilter filter = new ASCIIFoldingFilter(components.getTokenStream());
return new TokenStreamComponents(components.getTokenizer(), filter);
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java Mon Dec 21 11:39:57 2015
@@ -146,4 +146,37 @@ public class TestPatternTokenizer extend
checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER);
b.close();
}
+
+ // LUCENE-6814
+ public void testHeapFreedAfterClose() throws Exception {
+ // TODO: can we move this to BaseTSTC to catch other "hangs onto heap"ers?
+
+ // Build a 1MB string:
+ StringBuilder b = new StringBuilder();
+ for(int i=0;i<1024;i++) {
+ // 1023 spaces, then an x
+ for(int j=0;j<1023;j++) {
+ b.append(' ');
+ }
+ b.append('x');
+ }
+
+ String big = b.toString();
+
+ Pattern x = Pattern.compile("x");
+
+ List<Tokenizer> tokenizers = new ArrayList<>();
+ for(int i=0;i<512;i++) {
+ Tokenizer stream = new PatternTokenizer(x, -1);
+ tokenizers.add(stream);
+ stream.setReader(new StringReader(big));
+ stream.reset();
+ for(int j=0;j<1024;j++) {
+ assertTrue(stream.incrementToken());
+ }
+ assertFalse(stream.incrementToken());
+ stream.end();
+ stream.close();
+ }
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java Mon Dec 21 11:39:57 2015
@@ -36,7 +36,15 @@ public class TestSerbianNormalizationFil
stream = tokenFilterFactory("SerbianNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "djura" });
}
-
+
+ public void testRegularStemming() throws Exception {
+ Reader reader = new StringReader("ÑÑÑа");
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
+ stream = tokenFilterFactory("SerbianNormalization", "haircut", "regular").create(stream);
+ assertTokenStreamContents(stream, new String[] { "Äura" });
+ }
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
try {
@@ -46,4 +54,5 @@ public class TestSerbianNormalizationFil
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
+
}
Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Mon Dec 21 11:39:57 2015
@@ -182,4 +182,26 @@ public class TestCharTokenizers extends
checkRandomData(random(), analyzer, num);
analyzer.close();
}
+
+ public void testDefinitionUsingMethodReference1() throws Exception {
+ final StringReader reader = new StringReader("Tokenizer Test");
+ final Tokenizer tokenizer = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace);
+ tokenizer.setReader(reader);
+ assertTokenStreamContents(tokenizer, new String[] { "Tokenizer", "Test" });
+ }
+
+ public void testDefinitionUsingMethodReference2() throws Exception {
+ final StringReader reader = new StringReader("Tokenizer(Test)");
+ final Tokenizer tokenizer = CharTokenizer.fromTokenCharPredicate(Character::isLetter, Character::toUpperCase);
+ tokenizer.setReader(reader);
+ assertTokenStreamContents(tokenizer, new String[] { "TOKENIZER", "TEST" });
+ }
+
+ public void testDefinitionUsingLambda() throws Exception {
+ final StringReader reader = new StringReader("Tokenizer\u00A0Test Foo");
+ final Tokenizer tokenizer = CharTokenizer.fromSeparatorCharPredicate(c -> c == '\u00A0' || Character.isWhitespace(c), Character::toLowerCase);
+ tokenizer.setReader(reader);
+ assertTokenStreamContents(tokenizer, new String[] { "tokenizer", "test", "foo" });
+ }
+
}