You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/12/21 12:40:14 UTC

svn commit: r1721134 [2/48] - in /lucene/dev/branches/lucene6835: ./ dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/libraries/ dev-tools/maven/ dev-tools/maven/lucene/analysis/morfologik/ dev-tools/scripts/ lucene/ lucene/analysis/ luc...

Modified: lucene/dev/branches/lucene6835/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/build.xml?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/build.xml (original)
+++ lucene/dev/branches/lucene6835/build.xml Mon Dec 21 11:39:57 2015
@@ -129,6 +129,7 @@
     <property name="validate.baseDir" location="."/>
     <groovy taskname="source-patterns" classpathref="rat.classpath"><![CDATA[
       import org.apache.tools.ant.BuildException;
+      import org.apache.tools.ant.Project;
       import org.apache.rat.Defaults;
       import org.apache.rat.document.impl.FileDocument;
       import org.apache.rat.api.MetaData;
@@ -158,7 +159,7 @@
       def found = 0;
       def violations = new TreeSet();
       def reportViolation = { f, name ->
-        task.log(name + ': ' + f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/'));
+        task.log(name + ': ' + f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/'), Project.MSG_ERR);
         violations.add(name);
         found++;
       }
@@ -166,6 +167,7 @@
       def javadocsPattern = ~$/(?sm)^\Q/**\E(.*?)\Q*/\E/$;
       def lineSplitter = ~$/[\r\n]+/$;
       def licenseMatcher = Defaults.createDefaultMatcher();
+      def validLoggerPattern = ~$/(?s)\b(private\s|static\s|final\s){3}+\s*Logger\s+\p{javaJavaIdentifierStart}+\s+=\s+\QLoggerFactory.getLogger(MethodHandles.lookup().lookupClass());\E/$;
       
       ant.fileScanner{
         fileset(dir: baseDir){
@@ -182,9 +184,11 @@
           // excludes:
           exclude(name: '**/build/**')
           exclude(name: '**/dist/**')
+          exclude(name: '**/CheckLoggingConfiguration.java')
           exclude(name: 'build.xml') // ourselves :-)
         }
       }.each{ f ->
+        task.log('Scanning file: ' + f, Project.MSG_VERBOSE);
         def text = f.getText('UTF-8');
         invalidPatterns.each{ pattern,name ->
           if (pattern.matcher(text).find()) {
@@ -200,6 +204,11 @@
               ratDocument.getMetaData().value(MetaData.RAT_URL_LICENSE_FAMILY_NAME)));
           }
         }
+        if (f.toString().endsWith('.java') && text.contains('org.slf4j.LoggerFactory')) {
+          if (!validLoggerPattern.matcher(text).find()) {
+            reportViolation(f, 'invalid logging pattern [not private static final, uses static class name]');
+          }
+        }
       };
       
       if (found) {
@@ -566,7 +575,7 @@ File | Project Structure | Platform Sett
         int baseLen = basedir.toString().length();
 
         // do some fake check, to verify if this is valid SVN working copy. If this fails ignore checks but log some useful message.
-        task.log('Initializing working copy...');
+        task.log('Initializing working copy...', Project.MSG_INFO);
         try {
           wcClient.doInfo(basedir, SVNRevision.WORKING);
         } catch (SVNException ex) {
@@ -590,7 +599,7 @@ File | Project Structure | Platform Sett
 
         Set missingProps = new TreeSet(), withKeywords = new TreeSet(), unversioned = new TreeSet(), changed = new TreeSet();
 
-        task.log('Getting all versioned and unversioned files...');
+        task.log('Getting all versioned and unversioned files...', Project.MSG_INFO);
         statusClient.doStatus(basedir, SVNRevision.WORKING, SVNDepth.fromRecurse(true), false, true, false, false, {
           status ->
           SVNStatusType nodeStatus = status.getNodeStatus();
@@ -605,12 +614,12 @@ File | Project Structure | Platform Sett
           }
         } as ISVNStatusHandler, null);
 
-        task.log('Filtering files with existing svn:eol-style...');
+        task.log('Filtering files with existing svn:eol-style...', Project.MSG_INFO);
         wcClient.doGetProperty(basedir, 'svn:eol-style', SVNRevision.WORKING, SVNRevision.WORKING, true, {
           file, prop -> missingProps.remove(convertRelative(file));
         } as ISVNPropertyHandler);
         
-        task.log('Filtering files with binary svn:mime-type...');
+        task.log('Filtering files with binary svn:mime-type...', Project.MSG_INFO);
         wcClient.doGetProperty(basedir, 'svn:mime-type', SVNRevision.WORKING, SVNRevision.WORKING, true, {
           file, prop ->
           prop = SVNPropertyValue.getPropertyAsString(prop.getValue());
@@ -619,7 +628,7 @@ File | Project Structure | Platform Sett
           }
         } as ISVNPropertyHandler);
         
-        task.log('Scanning for files with svn:keywords property...');
+        task.log('Scanning for files with svn:keywords property...', Project.MSG_INFO);
         wcClient.doGetProperty(basedir, 'svn:keywords', SVNRevision.WORKING, SVNRevision.WORKING, true, {
           file, prop -> withKeywords.add(convertRelative(file));
         } as ISVNPropertyHandler);

Modified: lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs (original)
+++ lucene/dev/branches/lucene6835/dev-tools/eclipse/dot.settings/org.eclipse.jdt.ui.prefs Mon Dec 21 11:39:57 2015
@@ -3,5 +3,5 @@ eclipse.preferences.version=1
 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=false
 formatter_profile=org.eclipse.jdt.ui.default.eclipse_profile
 formatter_settings_version=12
-org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?><templates><template autoinsert\="false" context\="gettercomment_context" deleted\="false" description\="Comment for getter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name\="gettercomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="settercomment_context" deleted\="false" description\="Comment for setter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.settercomment" name\="settercomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="constructorcomment_context" deleted\="false" description\="Comment for created constructors" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name\="constructorcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="true" context\="filecomment_context" deleted\="false" description\="Comment for created Java file
 s" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.filecomment" name\="filecomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="typecomment_context" deleted\="false" description\="Comment for created types" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.typecomment" name\="typecomment">/**\r\n *\r\n */</template><template autoinsert\="true" context\="fieldcomment_context" deleted\="false" description\="Comment for fields" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name\="fieldcomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="methodcomment_context" deleted\="false" description\="Comment for non-overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name\="methodcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="false" context\="overridecomment_context" deleted\="false" description\="Comment for overriding methods" enabled\="true
 " id\="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name\="overridecomment">/* \r\n *\r\n */</template><template autoinsert\="true" context\="delegatecomment_context" deleted\="false" description\="Comment for delegate methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name\="delegatecomment">/**\r\n * ${tags}\r\n * ${see_to_target}\r\n */</template><template autoinsert\="false" context\="newtype_context" deleted\="false" description\="Newly created files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.newtype" name\="newtype">${filecomment}\r\n${package_declaration}\r\n\r\n/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements.  See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the "License"); you may not use this file except 
 in compliance with\r\n * the License.  You may obtain a copy of the License at\r\n *\r\n *     http\://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an "AS IS" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\n\r\n${typecomment}\r\n${type_declaration}</template><template autoinsert\="true" context\="classbody_context" deleted\="false" description\="Code in new class type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.classbody" name\="classbody">\r\n</template><template autoinsert\="true" context\="interfacebody_context" deleted\="false" description\="Code in new interface type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name\="interfacebody">\r\n</t
 emplate><template autoinsert\="true" context\="enumbody_context" deleted\="false" description\="Code in new enum type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.enumbody" name\="enumbody">\r\n</template><template autoinsert\="true" context\="annotationbody_context" deleted\="false" description\="Code in new annotation type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name\="annotationbody">\r\n</template><template autoinsert\="false" context\="catchblock_context" deleted\="false" description\="Code in new catch blocks" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.catchblock" name\="catchblock">throw new RuntimeException();</template><template autoinsert\="false" context\="methodbody_context" deleted\="false" description\="Code in created method stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodbody" name\="methodbody">${body_statement}</template><template autoinsert\="false" context\="c
 onstructorbody_context" deleted\="false" description\="Code in created constructor stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name\="constructorbody">${body_statement}</template><template autoinsert\="true" context\="getterbody_context" deleted\="false" description\="Code in created getters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.getterbody" name\="getterbody">return ${field};</template><template autoinsert\="true" context\="setterbody_context" deleted\="false" description\="Code in created setters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.setterbody" name\="setterbody">${field} \= ${param};</template></templates>
+org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?><templates><template autoinsert\="false" context\="gettercomment_context" deleted\="false" description\="Comment for getter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name\="gettercomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="settercomment_context" deleted\="false" description\="Comment for setter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.settercomment" name\="settercomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="constructorcomment_context" deleted\="false" description\="Comment for created constructors" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name\="constructorcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="true" context\="filecomment_context" deleted\="false" description\="Comment for created Java file
 s" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.filecomment" name\="filecomment">/**\r\n * \r\n */</template><template autoinsert\="false" context\="typecomment_context" deleted\="false" description\="Comment for created types" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.typecomment" name\="typecomment">/**\r\n *\r\n */</template><template autoinsert\="true" context\="fieldcomment_context" deleted\="false" description\="Comment for fields" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name\="fieldcomment">/**\r\n * \r\n */</template><template autoinsert\="true" context\="methodcomment_context" deleted\="false" description\="Comment for non-overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name\="methodcomment">/**\r\n * ${tags}\r\n */</template><template autoinsert\="false" context\="overridecomment_context" deleted\="false" description\="Comment for overriding methods" enabled\="true
 " id\="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name\="overridecomment">/* \r\n *\r\n */</template><template autoinsert\="true" context\="delegatecomment_context" deleted\="false" description\="Comment for delegate methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name\="delegatecomment">/**\r\n * ${tags}\r\n * ${see_to_target}\r\n */</template><template autoinsert\="false" context\="newtype_context" deleted\="false" description\="Newly created files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.newtype" name\="newtype">${filecomment}\r\n${package_declaration}\r\n\r\n/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements.  See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the "License"); you may not use this file except 
 in compliance with\r\n * the License.  You may obtain a copy of the License at\r\n *\r\n *     http\://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an "AS IS" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\n\r\n${typecomment}\r\n${type_declaration}</template><template autoinsert\="true" context\="classbody_context" deleted\="false" description\="Code in new class type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.classbody" name\="classbody">\r\n</template><template autoinsert\="true" context\="interfacebody_context" deleted\="false" description\="Code in new interface type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name\="interfacebody">\r\n</t
 emplate><template autoinsert\="true" context\="enumbody_context" deleted\="false" description\="Code in new enum type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.enumbody" name\="enumbody">\r\n</template><template autoinsert\="true" context\="annotationbody_context" deleted\="false" description\="Code in new annotation type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name\="annotationbody">\r\n</template><template autoinsert\="false" context\="catchblock_context" deleted\="false" description\="Code in new catch blocks" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.catchblock" name\="catchblock">throw new RuntimeException(${exception_var});</template><template autoinsert\="false" context\="methodbody_context" deleted\="false" description\="Code in created method stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodbody" name\="methodbody">${body_statement}</template><template autoinsert\="fa
 lse" context\="constructorbody_context" deleted\="false" description\="Code in created constructor stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name\="constructorbody">${body_statement}</template><template autoinsert\="true" context\="getterbody_context" deleted\="false" description\="Code in created getters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.getterbody" name\="getterbody">return ${field};</template><template autoinsert\="true" context\="setterbody_context" deleted\="false" description\="Code in created setters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.setterbody" name\="setterbody">${field} \= ${param};</template></templates>
 sp_cleanup.on_save_use_additional_actions=false

Modified: lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml (original)
+++ lucene/dev/branches/lucene6835/dev-tools/idea/.idea/libraries/JUnit.xml Mon Dec 21 11:39:57 2015
@@ -2,7 +2,7 @@
   <library name="JUnit">
     <CLASSES>
       <root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/junit-4.10.jar!/" />
-      <root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/randomizedtesting-runner-2.1.17.jar!/" />
+      <root url="jar://$PROJECT_DIR$/lucene/test-framework/lib/randomizedtesting-runner-2.3.2.jar!/" />
     </CLASSES>
     <JAVADOC />
     <SOURCES />

Modified: lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template (original)
+++ lucene/dev/branches/lucene6835/dev-tools/maven/lucene/analysis/morfologik/pom.xml.template Mon Dec 21 11:39:57 2015
@@ -70,6 +70,9 @@
           <exclude>**/*.java</exclude>
         </excludes>
       </testResource>
+      <testResource>
+        <directory>${module-path}/src/test-files</directory>
+      </testResource>
     </testResources>
   </build>
 </project>

Modified: lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/lucene6835/dev-tools/maven/pom.xml.template Mon Dec 21 11:39:57 2015
@@ -248,7 +248,7 @@
             <reportFormat>plain</reportFormat>
             <workingDirectory>${project.build.directory}/test</workingDirectory>
             <redirectTestOutputToFile>true</redirectTestOutputToFile>
-            <argLine>-Xmx512M -XX:MaxPermSize=256M</argLine>
+            <argLine>-Xmx512M</argLine>
             <systemPropertyVariables>
               <tempDir>.</tempDir>
               <java.awt.headless>true</java.awt.headless>

Modified: lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py (original)
+++ lucene/dev/branches/lucene6835/dev-tools/scripts/addVersion.py Mon Dec 21 11:39:57 2015
@@ -126,13 +126,14 @@ def update_example_solrconfigs(new_versi
   print('  updating example solrconfig.xml files')
   matcher = re.compile('<luceneMatchVersion>')
 
-  configset_dir = 'solr/server/solr/configsets'
-  if not os.path.isdir(configset_dir):
-    raise RuntimeError("Can't locate configset dir (layout change?) : " + configset_dir)
-  for root,dirs,files in os.walk(configset_dir, onerror=onerror):
-    for f in files:
-      if f == 'solrconfig.xml':
-        update_solrconfig(os.path.join(root, f), matcher, new_version) 
+  paths = ['solr/server/solr/configsets', 'solr/example']
+  for path in paths:
+    if not os.path.isdir(path):
+      raise RuntimeError("Can't locate configset dir (layout change?) : " + path)
+    for root,dirs,files in os.walk(path, onerror=onerror):
+      for f in files:
+        if f == 'solrconfig.xml':
+          update_solrconfig(os.path.join(root, f), matcher, new_version)
 
 def update_solrconfig(filename, matcher, new_version):
   print('    %s...' % filename, end='', flush=True)

Modified: lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py (original)
+++ lucene/dev/branches/lucene6835/dev-tools/scripts/smokeTestRelease.py Mon Dec 21 11:39:57 2015
@@ -84,7 +84,12 @@ def getHREFs(urlString):
   # Deref any redirects
   while True:
     url = urllib.parse.urlparse(urlString)
-    h = http.client.HTTPConnection(url.netloc)
+    if url.scheme == "http":
+      h = http.client.HTTPConnection(url.netloc)
+    elif url.scheme == "https":
+      h = http.client.HTTPSConnection(url.netloc)
+    else:
+      raise RuntimeError("Unknown protocol: %s" % url.scheme)
     h.request('GET', url.path)
     r = h.getresponse()
     newLoc = r.getheader('location')

Modified: lucene/dev/branches/lucene6835/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/CHANGES.txt?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6835/lucene/CHANGES.txt Mon Dec 21 11:39:57 2015
@@ -15,6 +15,8 @@ System Requirements
 
 New Features
 
+* LUCENE-6631: Lucene Document classification (Tommaso Teofili, Alessandro Benedetti)
+
 * LUCENE-6747: FingerprintFilter is a TokenFilter that outputs a single
   token which is a concatenation of the sorted and de-duplicated set of 
   input tokens. Useful for normalizing short text in clustering/linking 
@@ -35,6 +37,24 @@ New Features
   that can return false to exit from ExitableDirectoryReader wrapping at
   the point fields() is called. (yonik)
 
+* LUCENE-6825: Add low-level support for block-KD trees (Mike McCandless)
+
+* LUCENE-6852: Add support for dimensionally indexed values to index,
+  document and codec APIs, including a simple text implementation.
+  (Mike McCandless)
+
+* LUCENE-6861: Create Lucene60Codec, supporting dimensional values.
+  (Mike McCandless)
+
+* LUCENE-6879: Allow to define custom CharTokenizer instances without
+  subclassing using Java 8 lambdas or method references. (Uwe Schindler)
+
+* LUCENE-6881: Cutover all BKD implementations to dimensional values
+  (Mike McCandless)
+
+* LUCENE-6837: Add N-best output support to JapaneseTokenizer.
+  (Hiroharu Konno via Christian Moen)
+
 API Changes
 
 * LUCENE-3312: The API of oal.document was restructured to
@@ -66,15 +86,110 @@ API Changes
   IndexOutput.getName returns its name (Dawid Weiss, Robert Muir, Mike
   McCandless)
 
+* LUCENE-6917: Deprecate and rename NumericXXX classes to
+  LegacyNumericXXX in favor of dimensional values (Mike McCandless)
+
+Optimizations
+
+* LUCENE-6891: Use prefix coding when writing dimensional values in 
+  each leaf block in the default codec, to reduce the index
+  size (Mike McCandless)
+
+* LUCENE-6901: Optimize dimensional values indexing: use faster
+  IntroSorter instead of InPlaceMergeSorter, and specialize 1D
+  merging to merge sort the already sorted segments instead of
+  re-indexing (Mike McCandless)
+
 Changes in Runtime Behavior
 
 * LUCENE-6789: IndexSearcher's default Similarity is changed to BM25Similarity.
   Use ClassicSimilarity to get the old vector space DefaultSimilarity. (Robert Muir)
 
+* LUCENE-6886: Reserve the .tmp file name extension for temp files,
+  and codec components are no longer allowed to use this extension
+  (Robert Muir, Mike McCandless)
+
+======================= Lucene 5.5.0 =======================
+
+New Features
+
+* LUCENE-5868: JoinUtil.createJoinQuery(..,NumericType,..) query-time join 
+  for LONG and INT fields with NUMERIC and SORTED_NUMERIC doc values.
+  (Alexey Zelin via Mikhail Khludnev) 
+
+API Changes
+
+* LUCENE-6908: GeoUtils static relational methods have been refactored to new 
+  GeoRelationUtils and now correctly handle large irregular rectangles, and
+  pole crossing distance queries. (Nick Knize)
+
+* LUCENE-6900: Grouping sortWithinGroup variables used to allow null to mean
+  Sort.RELEVANCE.  Null is no longer permitted.  (David Smiley)
+
+* LUCENE-6919: The Scorer class has been refactored to expose an iterator
+  instead of extending DocIdSetIterator. asTwoPhaseIterator() has been renamed
+  to twoPhaseIterator() for consistency. (Adrien Grand)
+
+Optimizations
+
+* LUCENE-6889: BooleanQuery.rewrite now performs some query optimization, in
+  particular to rewrite queries that look like: "+*:* #filter" to a
+  "ConstantScore(filter)". (Adrien Grand)
+
+* LUCENE-6912: Grouping's Collectors now calculate a response to needsScores()
+  instead of always 'true'. (David Smiley)
+
+* LUCENE-6815: DisjunctionScorer now advances two-phased iterators lazily,
+  stopping to evaluate them as soon as a single one matches. The other iterators
+  will be confirmed lazily when computing score() or freq(). (Adrien Grand)
+
+* LUCENE-6926: MUST_NOT clauses now use the match cost API to run the slow bits
+  last whenever possible. (Adrien Grand)
+
+Bug Fixes
+
+* LUCENE-6918: LRUQueryCache.onDocIdSetEviction is only called when at least
+  one DocIdSet is being evicted. (Adrien Grand)
+
+* LUCENE-6910: fix 'if ... > Integer.MAX_VALUE' check in
+  (Binary|Numeric)DocValuesFieldUpdates.merge
+  (https://scan.coverity.com/projects/5620 CID 119973 and CID 120081)
+  (Christine Poerschke, Coverity Scan (via Rishabh Patel))
+
+* LUCENE-6929: Fix SpanNotQuery rewriting to not drop the pre/post parameters.
+  (Tim Allison via Adrien Grand)
+
+* SOLR-7865: BlendedInfixSuggester was returning too many results
+  (Arcadius Ahouansou via Mike McCandless)
+
+Other
+
+* LUCENE-6924: Upgrade randomizedtesting to 2.3.2. (Dawid Weiss)
+
+* LUCENE-6920: Improve custom function checks in expressions module
+  to use MethodHandles and work without extra security privileges.
+  (Uwe Schindler, Robert Muir)
+
+* LUCENE-6921: Fix SPIClassIterator#isParentClassLoader to don't
+  require extra permissions.  (Uwe Schindler)
+
+* LUCENE-6923: Fix RamUsageEstimator to access private fields inside
+  AccessController block for computing size. (Robert Muir)
+
+* LUCENE-6907: make TestParser extendable, rename test/.../xml/
+  NumericRangeQueryQuery.xml to NumericRangeQuery.xml
+  (Christine Poerschke)
+
+* LUCENE-6925: add ForceMergePolicy class in test-framework
+  (Christine Poerschke)
+
 ======================= Lucene 5.4.0 =======================
 
 New Features
 
+* LUCENE-6875: New Serbian Filter. (Nikola Smolenski via Robert Muir, 
+  Dawid Weiss)
+
 * LUCENE-6720: New FunctionRangeQuery wrapper around ValueSourceScorer
   (returned from ValueSource/FunctionValues.getRangeScorer()). (David Smiley)
 
@@ -94,6 +209,15 @@ New Features
 * LUCENE-6844: PayloadScoreQuery can include or exclude underlying span scores
   from its score calculations (Bill Bell, Alan Woodward)
 
+* LUCENE-6778: Add GeoPointDistanceRangeQuery, to search for points
+  within a "ring" (beyond a minimum distance and below a maximum
+  distance) (Nick Knize via Mike McCandless)
+
+* LUCENE-6874: Add a new UnicodeWhitespaceTokenizer to analysis/common
+  that uses Unicode character properties extracted from ICU4J to tokenize
+  text on whitespace. This tokenizer will split on non-breaking
+  space (NBSP), too.  (David Smiley, Uwe Schindler, Steve Rowe)
+
 API Changes
 
 * LUCENE-6590: Query.setBoost(), Query.getBoost() and Query.clone() are gone.
@@ -120,6 +244,23 @@ API Changes
 * LUCENE-6939: SpanOrQuery.addClause is now deprecated, clauses should all be
   provided at construction time. (Paul Elschot via Adrien Grand)
 
+* LUCENE-6855: CachingWrapperQuery is deprecated and will be removed in 6.0.
+  (Adrien Grand)
+
+* LUCENE-6870: DisjunctionMaxQuery#add is now deprecated, clauses should all be
+  provided at construction time. (Adrien Grand) 
+
+* LUCENE-6884: Analyzer.tokenStream() and Tokenizer.setReader() are no longer
+  declared as throwing IOException. (Alan Woodward)
+
+* LUCENE-6849: Expose IndexWriter.flush() method, to move all
+  in-memory segments to disk without opening a near-real-time reader
+  nor calling fsync (Robert Muir, Simon Willnauer, Mike McCandless)
+
+* LUCENE-6911: Add correct StandardQueryParser.getMultiFields() method,
+  deprecate no-op StandardQueryParser.getMultiFields(CharSequence[]) method.
+  (Christine Poerschke, Mikhail Khludnev, Coverity Scan (via Rishabh Patel))
+
 Optimizations
 
 * LUCENE-6708: TopFieldCollector does not compute the score several times on the
@@ -148,8 +289,43 @@ Optimizations
   strings larger than 64kb by an amount equal to string's utf8 size.
   (Dawid Weiss, Robert Muir, shalin)
 
+* LUCENE-6850: Optimize BooleanScorer for sparse clauses. (Adrien Grand)
+
+* LUCENE-6840: Ordinal indexes for SORTED_SET/SORTED_NUMERIC fields and
+  addresses for BINARY fields are now stored on disk instead of in memory.
+  (Adrien Grand)
+
+* LUCENE-6878: Speed up TopDocs.merge. (Daniel Jelinski via Adrien Grand)
+
+* LUCENE-6885: StandardDirectoryReader (initialCapacity) tweaks
+  (Christine Poerschke)
+
+* LUCENE-6863: Optimized storage requirements of doc values fields when less
+  than 1% of documents have a value. (Adrien Grand)
+
+* LUCENE-6892: various lucene.index initialCapacity tweaks
+  (Christine Poerschke)
+
+* LUCENE-6276: Added TwoPhaseIterator.matchCost() which allows to confirm the
+  least costly TwoPhaseIterators first. (Paul Elschot via Adrien Grand)
+
+* LUCENE-6898: In the default codec, the last stored field value will not
+  be fully read from disk if the supplied StoredFieldVisitor doesn't want it.
+  So put your largest text field value last to benefit. (David Smiley)
+
+* LUCENE-6909: Remove unnecessary synchronized from
+  FacetsConfig.getDimConfig for better concurrency (Sanne Grinovero
+  via Mike McCandless)
+  
+* SOLR-7730: Speed up SlowCompositeReaderWrapper.getSortedSetDocValues() by
+  avoiding merging FieldInfos just to check doc value type.
+  (Paul Vasilyev, Yuriy Pakhomov, Mikhail Khludnev, yonik)
+
 Bug Fixes
 
+* LUCENE-6905: Unwrap center longitude for dateline crossing 
+  GeoPointDistanceQuery. (Nick Knize)
+
 * LUCENE-6817: ComplexPhraseQueryParser.ComplexPhraseQuery does not display 
   slop in toString(). (Ahmet Arslan via Dawid Weiss)
 
@@ -204,8 +380,33 @@ Bug Fixes
   source for checking for expiration (Ishan Chattopadhyaya via Mike
   McCandless)
 
+* LUCENE-6856: The Weight wrapper used by LRUQueryCache now delegates to the
+  original Weight's BulkScorer when applicable. (Adrien Grand)
+
+* LUCENE-6858: Fix ContextSuggestField to correctly wrap token stream
+  when using CompletionAnalyzer. (Areek Zillur)
+
+* LUCENE-6872: IndexWriter handles any VirtualMachineError, not just OOM,
+  as tragic. (Robert Muir)
+
+* LUCENE-6814: PatternTokenizer no longer hangs onto heap sized to the
+  maximum input string it's ever seen, which can be a large memory
+  "leak" if you tokenize large strings with many threads across many
+  indices (Alex Chow via Mike McCandless)
+
+* LUCENE-6888: Explain output of map() function now also prints default value (janhoy)
+
 Other
 
+* LUCENE-6899: Upgrade randomizedtesting to 2.3.1. (Dawid Weiss)
+
+* LUCENE-6478: Test execution can hang with java.security.debug. (Dawid Weiss)
+
+* LUCENE-6862: Upgrade of RandomizedRunner to version 2.2.0. (Dawid Weiss)
+
+* LUCENE-6857: Validate StandardQueryParser with NOT operator 
+  with-in parantheses. (Jigar Shah via Dawid Weiss)
+
 * LUCENE-6827: Use explicit capacity ArrayList instead of a LinkedList 
   in MultiFieldQueryNodeProcessor. (Dawid Weiss).
 
@@ -263,6 +464,21 @@ Other
 
 * LUCENE-6467: Simplify Query.equals. (Paul Elschot via Adrien Grand)
 
+* LUCENE-6845: SpanScorer is now merged into Spans (Alan Woodward, David Smiley)
+
+* LUCENE-6887: DefaultSimilarity is deprecated, use ClassicSimilarity for equivilent behavior,
+  or consider switching to BM25Similarity which will become the new default in Lucene 6.0 (hossman)
+
+* LUCENE-6893: factor out CorePlusQueriesParser from CorePlusExtensionsParser
+  (Christine Poerschke)
+
+* LUCENE-6902: Don't retry to fsync files / directories; fail
+  immediately. (Daniel Mitterdorfer, Uwe Schindler)
+
+* LUCENE-6801: Clarify JavaDocs of PhraseQuery that it in fact supports terms
+  at the same position (as does MultiPhraseQuery), treated like a conjunction.
+  Added test. (David Smiley, Adrien Grand)
+
 Build
 
 * LUCENE-6732: Improve checker for invalid source patterns to also
@@ -709,11 +925,7 @@ Optimizations
 * LUCENE-6621: Removed two unused variables in analysis/stempel/src/java/org/
   egothor/stemmer/Compile.java
   (Rishabh Patel via Christine Poerschke)
-
-* SOLR-7730: Speed up SlowCompositeReaderWrapper.getSortedSetDocValues() by
-  avoiding merging FieldInfos just to check doc value type.
-  (Paul Vasilyev, Yuriy Pakhomov, Mikhail Khludnev, yonik)
-
+    
 Build
 
 * LUCENE-6518: Don't report false thread leaks from IBM J9

Modified: lucene/dev/branches/lucene6835/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/MIGRATE.txt?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/MIGRATE.txt (original)
+++ lucene/dev/branches/lucene6835/lucene/MIGRATE.txt Mon Dec 21 11:39:57 2015
@@ -80,3 +80,11 @@ would be equivalent to the following cod
   Query q = ...;
   float boost = ...;
   q.setBoost(q.getBoost() * boost);
+
+## DimensionalValues replaces NumericField (LUCENE-6917)
+
+DimensionalValues provides faster indexing and searching, a smaller
+index size, and less heap used at search time.  The numeric fields
+(IntField, FloatField, LongField, DoubleField) and NumericRangeQuery
+have been moved to the backward-codecs module and prefixed with
+Legacy.

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/build.xml Mon Dec 21 11:39:57 2015
@@ -31,6 +31,8 @@
   
   <property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>  
   
+  <property name="unicode-props-file" location="src/java/org/apache/lucene/analysis/util/UnicodeProps.java"/>
+
   <target name="jflex" depends="-install-jflex,clean-jflex,-jflex-StandardAnalyzer,-jflex-UAX29URLEmailTokenizer,
                                 -jflex-wiki-tokenizer,-jflex-HTMLStripCharFilter"/>
 
@@ -114,6 +116,18 @@
     </delete>
   </target>
   
+  <target xmlns:ivy="antlib:org.apache.ivy.ant" name="-resolve-icu4j" unless="icu4j.resolved" depends="ivy-availability-check,ivy-configure">
+    <loadproperties prefix="ivyversions" srcFile="${common.dir}/ivy-versions.properties"/>
+    <ivy:cachepath organisation="com.ibm.icu" module="icu4j" revision="${ivyversions./com.ibm.icu/icu4j}"
+      inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
+    <property name="icu4j.resolved" value="true"/>
+  </target>
+  
+  <target name="unicode-data" depends="-resolve-icu4j,resolve-groovy">
+    <groovy classpathref="icu4j.classpath" src="src/tools/groovy/generate-unicode-data.groovy"/>
+    <fixcrlf file="${unicode-props-file}" encoding="UTF-8"/>
+  </target>
+  
   <property name="tld.zones" value="http://www.internic.net/zones/root.zone"/>
   <property name="tld.output" location="src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro"/>
 
@@ -141,7 +155,7 @@
 
   <target name="javadocs" depends="module-build.javadocs"/>
 
-  <target name="regenerate" depends="jflex"/>
+  <target name="regenerate" depends="jflex,unicode-data"/>
   
   <target name="patch-snowball" description="Patches all snowball programs in '${snowball.programs.dir}' to make them work with MethodHandles">
       <fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java Mon Dec 21 11:39:57 2015
@@ -22,8 +22,11 @@ import org.apache.lucene.analysis.util.C
 import org.apache.lucene.util.AttributeFactory;
 
 /**
- * A WhitespaceTokenizer is a tokenizer that divides text at whitespace.
+ * A tokenizer that divides text at whitespace characters as defined by
+ * {@link Character#isWhitespace(int)}.  Note: That definition explicitly excludes the non-breaking space.
  * Adjacent sequences of non-Whitespace characters form tokens.
+ *
+ * @see UnicodeWhitespaceTokenizer
  */
 public final class WhitespaceTokenizer extends CharTokenizer {
   

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java Mon Dec 21 11:39:57 2015
@@ -17,32 +17,56 @@ package org.apache.lucene.analysis.core;
  * limitations under the License.
  */
 
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
 import org.apache.lucene.util.AttributeFactory;
 
-import java.util.Map;
-
 /**
  * Factory for {@link WhitespaceTokenizer}. 
  * <pre class="prettyprint">
  * &lt;fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"&gt;
  *   &lt;analyzer&gt;
- *     &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt;
+ *     &lt;tokenizer class="solr.WhitespaceTokenizerFactory" rule="unicode"/&gt;
  *   &lt;/analyzer&gt;
  * &lt;/fieldType&gt;</pre>
+ *
+ * Options:
+ * <ul>
+ *   <li>rule: either "java" for {@link WhitespaceTokenizer}
+ *      or "unicode" for {@link UnicodeWhitespaceTokenizer}</li>
+ * </ul>
  */
 public class WhitespaceTokenizerFactory extends TokenizerFactory {
+  public static final String RULE_JAVA = "java";
+  public static final String RULE_UNICODE = "unicode";
+  private static final Collection<String> RULE_NAMES = Arrays.asList(RULE_JAVA, RULE_UNICODE);
+
+  private final String rule;
 
   /** Creates a new WhitespaceTokenizerFactory */
   public WhitespaceTokenizerFactory(Map<String,String> args) {
     super(args);
+
+    rule = get(args, "rule", RULE_NAMES, RULE_JAVA);
+
     if (!args.isEmpty()) {
       throw new IllegalArgumentException("Unknown parameters: " + args);
     }
   }
 
   @Override
-  public WhitespaceTokenizer create(AttributeFactory factory) {
-    return new WhitespaceTokenizer(factory);
+  public Tokenizer create(AttributeFactory factory) {
+    switch (rule) {
+      case RULE_JAVA:
+        return new WhitespaceTokenizer(factory);
+      case RULE_UNICODE:
+        return new UnicodeWhitespaceTokenizer(factory);
+      default:
+        throw new AssertionError();
+    }
   }
 }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java Mon Dec 21 11:39:57 2015
@@ -137,9 +137,19 @@ public final class PatternTokenizer exte
   }
 
   @Override
+  public void close() throws IOException {
+    try {
+      super.close();
+    } finally {
+      str.setLength(0);
+      str.trimToSize();
+    }
+  }
+
+  @Override
   public void reset() throws IOException {
     super.reset();
-    fillBuffer(str, input);
+    fillBuffer(input);
     matcher.reset(str);
     index = 0;
   }
@@ -147,11 +157,11 @@ public final class PatternTokenizer exte
   // TODO: we should see if we can make this tokenizer work without reading
   // the entire document into RAM, perhaps with Matcher.hitEnd/requireEnd ?
   final char[] buffer = new char[8192];
-  private void fillBuffer(StringBuilder sb, Reader input) throws IOException {
+  private void fillBuffer(Reader input) throws IOException {
     int len;
-    sb.setLength(0);
+    str.setLength(0);
     while ((len = input.read(buffer)) > 0) {
-      sb.append(buffer, 0, len);
+      str.append(buffer, 0, len);
     }
   }
 }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,7 @@ package org.apache.lucene.analysis.sr;
  * limitations under the License.
  */
 
+import java.util.Arrays;
 import java.util.Map;
 
 import org.apache.lucene.analysis.TokenStream;
@@ -31,15 +32,19 @@ import org.apache.lucene.analysis.util.T
  *   &lt;analyzer&gt;
  *     &lt;tokenizer class="solr.StandardTokenizerFactory"/&gt;
  *     &lt;filter class="solr.LowerCaseFilterFactory"/&gt;
- *     &lt;filter class="solr.SerbianNormalizationFilterFactory"/&gt;
+ *     &lt;filter class="solr.SerbianNormalizationFilterFactory"
+ *       haircut="bald"/&gt; 
  *   &lt;/analyzer&gt;
  * &lt;/fieldType&gt;</pre> 
  */
 public class SerbianNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+  final String haircut;
 
   /** Creates a new SerbianNormalizationFilterFactory */
   public SerbianNormalizationFilterFactory(Map<String,String> args) {
     super(args);
+
+  this.haircut = get(args, "haircut", Arrays.asList( "bald", "regular" ), "bald");
     if (!args.isEmpty()) {
       throw new IllegalArgumentException("Unknown parameters: " + args);
     }
@@ -47,7 +52,11 @@ public class SerbianNormalizationFilterF
 
   @Override
   public TokenStream create(TokenStream input) {
-    return new SerbianNormalizationFilter(input);
+    if( this.haircut.equals( "regular" ) ) {
+      return new SerbianNormalizationRegularFilter(input);
+    } else {
+      return new SerbianNormalizationFilter(input);
+    }
   }
 
   @Override

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis.stand
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.io.Reader;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.StopAnalyzer;
@@ -25,9 +28,6 @@ import org.apache.lucene.analysis.util.C
 import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
 import org.apache.lucene.analysis.util.WordlistLoader;
 
-import java.io.IOException;
-import java.io.Reader;
-
 /**
  * Filters {@link ClassicTokenizer} with {@link ClassicFilter}, {@link
  * LowerCaseFilter} and {@link StopFilter}, using a list of
@@ -94,7 +94,7 @@ public final class ClassicAnalyzer exten
     tok = new StopFilter(tok, stopwords);
     return new TokenStreamComponents(src, tok) {
       @Override
-      protected void setReader(final Reader reader) throws IOException {
+      protected void setReader(final Reader reader) {
         src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
         super.setReader(reader);
       }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis.stand
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.io.Reader;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.StopAnalyzer;
@@ -25,9 +28,6 @@ import org.apache.lucene.analysis.util.C
 import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
 import org.apache.lucene.analysis.util.WordlistLoader;
 
-import java.io.IOException;
-import java.io.Reader;
-
 /**
  * Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
  * LowerCaseFilter} and {@link StopFilter}, using a list of
@@ -89,7 +89,7 @@ public final class StandardAnalyzer exte
     tok = new StopFilter(tok, stopwords);
     return new TokenStreamComponents(src, tok) {
       @Override
-      protected void setReader(final Reader reader) throws IOException {
+      protected void setReader(final Reader reader) {
         src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
         super.setReader(reader);
       }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis.stand
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.io.Reader;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.StopAnalyzer;
@@ -24,9 +27,6 @@ import org.apache.lucene.analysis.core.S
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
 
-import java.io.IOException;
-import java.io.Reader;
-
 /**
  * Filters {@link org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer}
  * with {@link org.apache.lucene.analysis.standard.StandardFilter},
@@ -91,7 +91,7 @@ public final class UAX29URLEmailAnalyzer
     tok = new StopFilter(tok, stopwords);
     return new TokenStreamComponents(src, tok) {
       @Override
-      protected void setReader(final Reader reader) throws IOException {
+      protected void setReader(final Reader reader) {
         src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength);
         super.setReader(reader);
       }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java Mon Dec 21 11:39:57 2015
@@ -21,6 +21,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.LinkedHashMap;
 import java.util.Set;
 import java.util.ServiceConfigurationError;
@@ -54,6 +55,9 @@ final class AnalysisSPILoader<S extends
     this.suffixes = suffixes;
     // if clazz' classloader is not a parent of the given one, we scan clazz's classloader, too:
     final ClassLoader clazzClassloader = clazz.getClassLoader();
+    if (classloader == null) {
+      classloader = clazzClassloader;
+    }
     if (clazzClassloader != null && !SPIClassIterator.isParentClassLoader(clazzClassloader, classloader)) {
       reload(clazzClassloader);
     }
@@ -72,6 +76,7 @@ final class AnalysisSPILoader<S extends
    * of new service providers on the given classpath/classloader!</em>
    */
   public synchronized void reload(ClassLoader classloader) {
+    Objects.requireNonNull(classloader, "classloader");
     final LinkedHashMap<String,Class<? extends S>> services =
       new LinkedHashMap<>(this.services);
     final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java Mon Dec 21 11:39:57 2015
@@ -18,8 +18,15 @@ package org.apache.lucene.analysis.util;
  */
 
 import java.io.IOException;
+import java.util.Objects;
+import java.util.function.IntPredicate;
+import java.util.function.IntUnaryOperator;
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LetterTokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.AttributeFactory;
@@ -27,8 +34,16 @@ import org.apache.lucene.analysis.util.C
 import org.apache.lucene.analysis.util.CharacterUtils.CharacterBuffer;
 
 /**
- * An abstract base class for simple, character-oriented tokenizers. 
- **/
+ * An abstract base class for simple, character-oriented tokenizers.
+ * <p>
+ * The base class also provides factories to create instances of
+ * {@code CharTokenizer} using Java 8 lambdas or method references.
+ * It is possible to create an instance which behaves exactly like
+ * {@link LetterTokenizer}:
+ * <pre class="prettyprint lang-java">
+ * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter);
+ * </pre>
+ */
 public abstract class CharTokenizer extends Tokenizer {
   
   /**
@@ -47,6 +62,136 @@ public abstract class CharTokenizer exte
     super(factory);
   }
   
+  /**
+   * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+   * The predicate should return {@code true} for all valid token characters.
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as {@link LetterTokenizer} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter);
+   * </pre>
+   */
+  public static CharTokenizer fromTokenCharPredicate(final IntPredicate tokenCharPredicate) {
+    return fromTokenCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, tokenCharPredicate);
+  }
+  
+  /**
+   * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
+   * The predicate should return {@code true} for all valid token characters.
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as {@link LetterTokenizer} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(factory, Character::isLetter);
+   * </pre>
+   */
+  public static CharTokenizer fromTokenCharPredicate(AttributeFactory factory, final IntPredicate tokenCharPredicate) {
+    return fromTokenCharPredicate(factory, tokenCharPredicate, IntUnaryOperator.identity());
+  }
+  
+  /**
+   * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+   * The predicate should return {@code true} for all valid token characters.
+   * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as {@link LowerCaseTokenizer} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter, Character::toLowerCase);
+   * </pre>
+   */
+  public static CharTokenizer fromTokenCharPredicate(final IntPredicate tokenCharPredicate, final IntUnaryOperator normalizer) {
+    return fromTokenCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, tokenCharPredicate, normalizer);
+  }
+  
+  /**
+   * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
+   * The predicate should return {@code true} for all valid token characters.
+   * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as {@link LowerCaseTokenizer} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(factory, Character::isLetter, Character::toLowerCase);
+   * </pre>
+   */
+  public static CharTokenizer fromTokenCharPredicate(AttributeFactory factory, final IntPredicate tokenCharPredicate, final IntUnaryOperator normalizer) {
+    Objects.requireNonNull(tokenCharPredicate, "predicate must not be null.");
+    Objects.requireNonNull(normalizer, "normalizer must not be null");
+    return new CharTokenizer(factory) {
+      @Override
+      protected boolean isTokenChar(int c) {
+        return tokenCharPredicate.test(c);
+      }
+
+      @Override
+      protected int normalize(int c) {
+        return normalizer.applyAsInt(c);
+      }
+    };
+  }
+  
+  /**
+   * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+   * The predicate should return {@code true} for all valid token separator characters.
+   * This method is provided for convenience to easily use predicates that are negated
+   * (they match the separator characters, not the token characters).
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as {@link WhitespaceTokenizer} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace);
+   * </pre>
+   */
+  public static CharTokenizer fromSeparatorCharPredicate(final IntPredicate separatorCharPredicate) {
+    return fromSeparatorCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, separatorCharPredicate);
+  }
+  
+  /**
+   * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
+   * The predicate should return {@code true} for all valid token separator characters.
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as {@link WhitespaceTokenizer} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(factory, Character::isWhitespace);
+   * </pre>
+   */
+  public static CharTokenizer fromSeparatorCharPredicate(AttributeFactory factory, final IntPredicate separatorCharPredicate) {
+    return fromSeparatorCharPredicate(factory, separatorCharPredicate, IntUnaryOperator.identity());
+  }
+  
+  /**
+   * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
+   * The predicate should return {@code true} for all valid token separator characters.
+   * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as the combination {@link WhitespaceTokenizer} and {@link LowerCaseFilter} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace, Character::toLowerCase);
+   * </pre>
+   */
+  public static CharTokenizer fromSeparatorCharPredicate(final IntPredicate separatorCharPredicate, final IntUnaryOperator normalizer) {
+    return fromSeparatorCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, separatorCharPredicate, normalizer);
+  }
+  
+  /**
+   * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate.
+   * The predicate should return {@code true} for all valid token separator characters.
+   * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
+   * <p>
+   * This factory is intended to be used with lambdas or method references. E.g., an elegant way
+   * to create an instance which behaves exactly as {@link WhitespaceTokenizer} and {@link LowerCaseFilter} is:
+   * <pre class="prettyprint lang-java">
+   * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(factory, Character::isWhitespace, Character::toLowerCase);
+   * </pre>
+   */
+  public static CharTokenizer fromSeparatorCharPredicate(AttributeFactory factory, final IntPredicate separatorCharPredicate, final IntUnaryOperator normalizer) {
+    return fromTokenCharPredicate(factory, separatorCharPredicate.negate(), normalizer);
+  }
+  
   private int offset = 0, bufferIndex = 0, dataLen = 0, finalOffset = 0;
   private static final int MAX_WORD_LEN = 255;
   private static final int IO_BUFFER_SIZE = 4096;

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java Mon Dec 21 11:39:57 2015
@@ -22,6 +22,7 @@ import java.io.StringReader;
 import java.lang.reflect.Modifier;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
@@ -48,6 +49,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer;
 import org.apache.lucene.analysis.sinks.TeeSinkTokenFilter;
 import org.apache.lucene.analysis.snowball.SnowballFilter;
+import org.apache.lucene.analysis.sr.SerbianNormalizationRegularFilter;
 import org.apache.lucene.analysis.util.CharFilterFactory;
 import org.apache.lucene.analysis.util.ResourceLoader;
 import org.apache.lucene.analysis.util.ResourceLoaderAware;
@@ -99,10 +101,17 @@ public class TestAllAnalyzersHaveFactori
       ReversePathHierarchyTokenizer.class, // this is supported via an option to PathHierarchyTokenizer's factory
       SnowballFilter.class, // this is called SnowballPorterFilterFactory
       PatternKeywordMarkerFilter.class,
-      SetKeywordMarkerFilter.class
+      SetKeywordMarkerFilter.class,
+      UnicodeWhitespaceTokenizer.class // a supported option via WhitespaceTokenizerFactory
     );
   }
-  
+
+  // The following token filters are excused from having their factory.
+  private static final Set<Class<?>> tokenFiltersWithoutFactory = new HashSet<>();
+  static {
+    tokenFiltersWithoutFactory.add(SerbianNormalizationRegularFilter.class);
+  }
+
   private static final ResourceLoader loader = new StringMockResourceLoader("");
   
   public void test() throws Exception {
@@ -117,12 +126,13 @@ public class TestAllAnalyzersHaveFactori
         || testComponents.contains(c)
         || crazyComponents.contains(c)
         || oddlyNamedComponents.contains(c)
+        || tokenFiltersWithoutFactory.contains(c)
         || c.isAnnotationPresent(Deprecated.class) // deprecated ones are typically back compat hacks
         || !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharFilter.class.isAssignableFrom(c))
       ) {
         continue;
       }
-      
+
       Map<String,String> args = new HashMap<>();
       args.put("luceneMatchVersion", Version.LATEST.toString());
       

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java Mon Dec 21 11:39:57 2015
@@ -130,7 +130,7 @@ public class TestAnalyzers extends BaseT
 
     @Override
     public TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new WhitespaceTokenizer();
+      Tokenizer tokenizer = random().nextBoolean() ? new WhitespaceTokenizer() : new UnicodeWhitespaceTokenizer();
       return new TokenStreamComponents(tokenizer, new LowerCaseFilter(tokenizer));
     }
     
@@ -140,7 +140,7 @@ public class TestAnalyzers extends BaseT
 
     @Override
     public TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new WhitespaceTokenizer();
+      Tokenizer tokenizer = random().nextBoolean() ? new WhitespaceTokenizer() : new UnicodeWhitespaceTokenizer();
       return new TokenStreamComponents(tokenizer, new UpperCaseFilter(tokenizer));
     }
     
@@ -230,7 +230,7 @@ public class TestAnalyzers extends BaseT
   
   /** blast some random strings through the analyzer */
   public void testRandomStrings() throws Exception {
-    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() };
+    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer(), new UnicodeWhitespaceAnalyzer() };
     for (Analyzer analyzer : analyzers) {
       checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
     }
@@ -239,7 +239,7 @@ public class TestAnalyzers extends BaseT
   
   /** blast some random large strings through the analyzer */
   public void testRandomHugeStrings() throws Exception {
-    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() };
+    Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer(), new UnicodeWhitespaceAnalyzer() };
     for (Analyzer analyzer : analyzers) {
       checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192);
     }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java Mon Dec 21 11:39:57 2015
@@ -17,12 +17,12 @@ package org.apache.lucene.analysis.core;
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.NumericTokenStream;
+import java.util.Set;
+
+import org.apache.lucene.analysis.CannedTokenStream;
 import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
 import org.apache.lucene.analysis.util.TokenFilterFactory;
 
-import java.util.Set;
-
 /**
  * Testcase for {@link TypeTokenFilterFactory}
  */
@@ -46,8 +46,7 @@ public class TestTypeTokenFilterFactory
   public void testCreationWithBlackList() throws Exception {
     TokenFilterFactory factory = tokenFilterFactory("Type",
         "types", "stoptypes-1.txt, stoptypes-2.txt");
-    NumericTokenStream input = new NumericTokenStream();
-    input.setIntValue(123);
+    CannedTokenStream input = new CannedTokenStream();
     factory.create(input);
   }
   
@@ -55,8 +54,7 @@ public class TestTypeTokenFilterFactory
     TokenFilterFactory factory = tokenFilterFactory("Type",
         "types", "stoptypes-1.txt, stoptypes-2.txt",
         "useWhitelist", "true");
-    NumericTokenStream input = new NumericTokenStream();
-    input.setIntValue(123);
+    CannedTokenStream input = new CannedTokenStream();
     factory.create(input);
   }
 

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java Mon Dec 21 11:39:57 2015
@@ -1,6 +1,5 @@
 package org.apache.lucene.analysis.miscellaneous;
 
-import java.io.IOException;
 import java.io.Reader;
 import java.util.Collections;
 import java.util.Map;
@@ -16,7 +15,6 @@ import org.apache.lucene.analysis.core.S
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.Rethrow;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -105,11 +103,7 @@ public class TestPerFieldAnalyzerWrapper
 
       @Override
       protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
-        try {
-          assertNotSame(specialAnalyzer.tokenStream("special", text), components.getTokenStream());
-        } catch (IOException e) {
-          Rethrow.rethrow(e);
-        }
+        assertNotSame(specialAnalyzer.tokenStream("special", text), components.getTokenStream());
         TokenFilter filter = new ASCIIFoldingFilter(components.getTokenStream());
         return new TokenStreamComponents(components.getTokenizer(), filter);
       }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java Mon Dec 21 11:39:57 2015
@@ -146,4 +146,37 @@ public class TestPatternTokenizer extend
     checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER);
     b.close();
   }
+
+  // LUCENE-6814
+  public void testHeapFreedAfterClose() throws Exception {
+    // TODO: can we move this to BaseTSTC to catch other "hangs onto heap"ers?
+
+    // Build a 1MB string:
+    StringBuilder b = new StringBuilder();
+    for(int i=0;i<1024;i++) {
+      // 1023 spaces, then an x
+      for(int j=0;j<1023;j++) {
+        b.append(' ');
+      }
+      b.append('x');
+    }
+
+    String big = b.toString();
+
+    Pattern x = Pattern.compile("x");
+
+    List<Tokenizer> tokenizers = new ArrayList<>();
+    for(int i=0;i<512;i++) {
+      Tokenizer stream = new PatternTokenizer(x, -1);
+      tokenizers.add(stream);
+      stream.setReader(new StringReader(big));
+      stream.reset();
+      for(int j=0;j<1024;j++) {
+        assertTrue(stream.incrementToken());
+      }
+      assertFalse(stream.incrementToken());
+      stream.end();
+      stream.close();
+    }
+  }
 }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/sr/TestSerbianNormalizationFilterFactory.java Mon Dec 21 11:39:57 2015
@@ -36,7 +36,15 @@ public class TestSerbianNormalizationFil
     stream = tokenFilterFactory("SerbianNormalization").create(stream);
     assertTokenStreamContents(stream, new String[] { "djura" });
   }
-  
+
+  public void testRegularStemming() throws Exception {
+    Reader reader = new StringReader("ђура");
+    TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+    ((Tokenizer)stream).setReader(reader);
+    stream = tokenFilterFactory("SerbianNormalization", "haircut", "regular").create(stream);
+    assertTokenStreamContents(stream, new String[] { "đura" });
+  }
+   
   /** Test that bogus arguments result in exception */
   public void testBogusArguments() throws Exception {
     try {
@@ -46,4 +54,5 @@ public class TestSerbianNormalizationFil
       assertTrue(expected.getMessage().contains("Unknown parameters"));
     }
   }
+
 }

Modified: lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/branches/lucene6835/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Mon Dec 21 11:39:57 2015
@@ -182,4 +182,26 @@ public class TestCharTokenizers extends
     checkRandomData(random(), analyzer, num);
     analyzer.close();
   }
+  
+  public void testDefinitionUsingMethodReference1() throws Exception {
+    final StringReader reader = new StringReader("Tokenizer Test");
+    final Tokenizer tokenizer = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace);
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "Tokenizer", "Test" });
+  }
+  
+  public void testDefinitionUsingMethodReference2() throws Exception {
+    final StringReader reader = new StringReader("Tokenizer(Test)");
+    final Tokenizer tokenizer = CharTokenizer.fromTokenCharPredicate(Character::isLetter, Character::toUpperCase);
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "TOKENIZER", "TEST" });
+  }
+  
+  public void testDefinitionUsingLambda() throws Exception {
+    final StringReader reader = new StringReader("Tokenizer\u00A0Test Foo");
+    final Tokenizer tokenizer = CharTokenizer.fromSeparatorCharPredicate(c -> c == '\u00A0' || Character.isWhitespace(c), Character::toLowerCase);
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "tokenizer", "test", "foo" });
+  }
+  
 }