You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/06/07 12:18:19 UTC

[01/18] lucene-solr:jira/solr-12438: SOLR-12444: Updating a cluster policy fails

Repository: lucene-solr
Updated Branches:
  refs/heads/jira/solr-12438 2fa260c14 -> 53d9d081b


SOLR-12444: Updating a cluster policy fails


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/add77d27
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/add77d27
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/add77d27

Branch: refs/heads/jira/solr-12438
Commit: add77d272582e909e9a7cf008d35ea72c3914230
Parents: 6624773
Author: Noble Paul <no...@apache.org>
Authored: Tue Jun 5 22:29:06 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jun 5 22:29:06 2018 +1000

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  2 ++
 .../autoscaling/AutoScalingHandlerTest.java     | 21 ++++++++++++++++++++
 .../client/solrj/cloud/autoscaling/Clause.java  |  2 +-
 3 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/add77d27/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c88de10..1b454a0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -1312,6 +1312,8 @@ Bug Fixes
 * SOLR-11477: Disallow resolving of external entities in the XML query parser (defType=xmlparser).
   (Michael Stepankin, Olga Barinova, Uwe Schindler, Christine Poerschke)
 
+* SOLR-12444: Updating a cluster policy fails (noble)
+
 Optimizations
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/add77d27/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 483b60c..cf11953 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.solr.cloud.autoscaling;
 
+import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.List;
 import java.util.Map;
@@ -26,6 +27,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -1011,6 +1013,25 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
     assertEquals(5L, properties.get(AutoScalingParams.ACTION_THROTTLE_PERIOD_SECONDS));
   }
 
+  public void testUpdatePolicy() throws IOException, SolrServerException {
+    CloudSolrClient solrClient = cluster.getSolrClient();
+    String setPropertiesCommand = "{'set-cluster-policy': [" +
+        "{'cores': '<4','node': '#ANY'}]}";
+    solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
+    SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
+    NamedList<Object> response = solrClient.request(req);
+    assertEquals("<4", Utils.getObjectByPath(response,false,"cluster-policy[0]/cores"));
+    assertEquals("#ANY", Utils.getObjectByPath(response,false,"cluster-policy[0]/node"));
+    setPropertiesCommand = "{'set-cluster-policy': [" +
+        "{'cores': '<3','node': '#ANY'}]}";
+    solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
+    req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
+    response = solrClient.request(req);
+    System.out.println(response);
+
+
+  }
+
   static class AutoScalingRequest extends SolrRequest {
     protected final String message;
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/add77d27/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index c739588..8f198bd 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -140,7 +140,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
     Clause that = (Clause)o;
-    return compareTo(that) == 0;
+    return  Objects.equals(this.original, that.original);
   }
 
   void addTags(Collection<String> params) {


[13/18] lucene-solr:jira/solr-12438: LUCENE-8299: Geo3D wrapper uses new polygon method factory that gives better support for polygons with many points (>100)

Posted by ab...@apache.org.
LUCENE-8299: Geo3D wrapper uses new polygon method factory that gives better support for polygons with many points (>100)


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/af7dfb18
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/af7dfb18
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/af7dfb18

Branch: refs/heads/jira/solr-12438
Commit: af7dfb182ebc175df44b2d0a846a7d11f58bf84b
Parents: 0358fcb
Author: Ignacio Vera <iv...@apache.org>
Authored: Wed Jun 6 11:39:39 2018 +0200
Committer: Ignacio Vera <iv...@apache.org>
Committed: Wed Jun 6 11:39:39 2018 +0200

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  3 +++
 .../spatial/spatial4j/Geo3dShapeFactory.java    | 10 ++++-----
 .../lucene/spatial/spatial4j/Geo3dRptTest.java  | 23 ++++++++++++++++++--
 3 files changed, 28 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af7dfb18/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9eecc42..8f307fe 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -267,6 +267,9 @@ Other
 
 * LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)
 
+* LUCENE-8299: Geo3D wrapper uses new polygon method factory that gives better
+  support for polygons with many points (>100). (Ignacio vera)
+
 * LUCENE-8261: InterpolatedProperties.interpolate and recursive property
   references. (Steve Rowe, Dawid Weiss)
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af7dfb18/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
index a6147df..ccbd6df 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
@@ -311,7 +311,7 @@ public class Geo3dShapeFactory implements S2ShapeFactory {
    */
   private class Geo3dPolygonBuilder extends Geo3dPointBuilder<PolygonBuilder> implements PolygonBuilder {
 
-    List<GeoPolygon> polyHoles;
+    List<GeoPolygonFactory.PolygonDescription> polyHoles = new ArrayList<>();
 
     @Override
     public HoleBuilder hole() {
@@ -321,10 +321,7 @@ public class Geo3dShapeFactory implements S2ShapeFactory {
     class Geo3dHoleBuilder extends Geo3dPointBuilder<PolygonBuilder.HoleBuilder> implements PolygonBuilder.HoleBuilder {
       @Override
       public PolygonBuilder endHole() {
-        if (polyHoles == null) {
-          polyHoles = new ArrayList<>();
-        }
-        polyHoles.add(GeoPolygonFactory.makeGeoPolygon(planetModel, points));
+        polyHoles.add(new GeoPolygonFactory.PolygonDescription(points));
         return Geo3dPolygonBuilder.this;
       }
     }
@@ -332,7 +329,8 @@ public class Geo3dShapeFactory implements S2ShapeFactory {
     @SuppressWarnings("unchecked")
     @Override
     public Shape build() {
-      GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, points, polyHoles);
+      GeoPolygonFactory.PolygonDescription description = new GeoPolygonFactory.PolygonDescription(points, polyHoles);
+      GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, description);
       return new Geo3dShape<>(polygon, context);
     }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af7dfb18/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
index d3b144f..eb6ed5b 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
@@ -18,9 +18,11 @@ package org.apache.lucene.spatial.spatial4j;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import org.apache.lucene.spatial.SpatialTestData;
 import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
 import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
@@ -96,7 +98,7 @@ public class Geo3dRptTest extends RandomSpatialOpStrategyTestCase {
     points.add(new GeoPoint(planetModel, 14 * DEGREES_TO_RADIANS, -180 * DEGREES_TO_RADIANS));
     points.add(new GeoPoint(planetModel, -15 * DEGREES_TO_RADIANS, 153 * DEGREES_TO_RADIANS));
 
-    final Shape triangle = new Geo3dShape(GeoPolygonFactory.makeGeoPolygon(planetModel, points),ctx);
+    final Shape triangle = new Geo3dShape<>(GeoPolygonFactory.makeGeoPolygon(planetModel, points),ctx);
     final Rectangle rect = ctx.makeRectangle(-49, -45, 73, 86);
     testOperation(rect,SpatialOperation.Intersects,triangle, false);
   }
@@ -116,7 +118,7 @@ public class Geo3dRptTest extends RandomSpatialOpStrategyTestCase {
         new GeoPoint(planetModel, 54.0 * DEGREES_TO_RADIANS, 165.0 * DEGREES_TO_RADIANS),
         new GeoPoint(planetModel, -90.0 * DEGREES_TO_RADIANS, 0.0)};
     final GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 29 * DEGREES_TO_RADIANS, pathPoints);
-    final Shape shape = new Geo3dShape(path,ctx);
+    final Shape shape = new Geo3dShape<>(path,ctx);
     final Rectangle rect = ctx.makeRectangle(131, 143, 39, 54);
     testOperation(rect,SpatialOperation.Intersects,shape,true);
   }
@@ -146,6 +148,23 @@ public class Geo3dRptTest extends RandomSpatialOpStrategyTestCase {
     return new Geo3dShape<>(areaShape, ctx);
   }
 
+  @Test
+  public void testOperationsFromFile() throws IOException {
+    setupStrategy();
+    final Iterator<SpatialTestData> indexedSpatialData = getSampleData( "states-poly.txt");
+    final List<Shape> indexedShapes = new ArrayList<>();
+    while(indexedSpatialData.hasNext()) {
+      indexedShapes.add(indexedSpatialData.next().shape);
+    }
+    final Iterator<SpatialTestData> querySpatialData = getSampleData( "states-bbox.txt");
+    final List<Shape> queryShapes = new ArrayList<>();
+    while(querySpatialData.hasNext()) {
+      queryShapes.add(querySpatialData.next().shape);
+      queryShapes.add(randomQueryShape());
+    }
+    testOperation(SpatialOperation.Intersects, indexedShapes, queryShapes, random().nextBoolean());
+  }
+
   //TODO move to a new test class?
   @Test
   public void testWKT() throws Exception {


[09/18] lucene-solr:jira/solr-12438: SOLR-12444: added more assertions to the test

Posted by ab...@apache.org.
SOLR-12444: added more assertions to the test


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9ff3f5a1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9ff3f5a1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9ff3f5a1

Branch: refs/heads/jira/solr-12438
Commit: 9ff3f5a13614c9aefe50c84b9abc653490f536c4
Parents: 124b5e0
Author: Noble Paul <no...@apache.org>
Authored: Wed Jun 6 13:13:20 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Wed Jun 6 13:13:20 2018 +1000

----------------------------------------------------------------------
 .../apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ff3f5a1/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index cf11953..cf57c48 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -1027,8 +1027,8 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
     solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
     req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
     response = solrClient.request(req);
-    System.out.println(response);
-
+    assertEquals("<3", Utils.getObjectByPath(response,false,"cluster-policy[0]/cores"));
+    assertEquals("#ANY", Utils.getObjectByPath(response,false,"cluster-policy[0]/node"));
 
   }
 


[15/18] lucene-solr:jira/solr-12438: SOLR-12445: Upgrade Dropwizard Metrics to version 3.2.6.

Posted by ab...@apache.org.
SOLR-12445: Upgrade Dropwizard Metrics to version 3.2.6.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3022bbce
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3022bbce
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3022bbce

Branch: refs/heads/jira/solr-12438
Commit: 3022bbce2ec493c759eceb4fac7eeab0fb908b59
Parents: c1f4290
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 15:14:22 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 15:14:22 2018 +0200

----------------------------------------------------------------------
 lucene/ivy-versions.properties                | 2 +-
 solr/licenses/metrics-core-3.2.2.jar.sha1     | 1 -
 solr/licenses/metrics-core-3.2.6.jar.sha1     | 1 +
 solr/licenses/metrics-ganglia-3.2.2.jar.sha1  | 1 -
 solr/licenses/metrics-ganglia-3.2.6.jar.sha1  | 1 +
 solr/licenses/metrics-graphite-3.2.2.jar.sha1 | 1 -
 solr/licenses/metrics-graphite-3.2.6.jar.sha1 | 1 +
 solr/licenses/metrics-jetty9-3.2.2.jar.sha1   | 1 -
 solr/licenses/metrics-jetty9-3.2.6.jar.sha1   | 1 +
 solr/licenses/metrics-jvm-3.2.2.jar.sha1      | 1 -
 solr/licenses/metrics-jvm-3.2.6.jar.sha1      | 1 +
 11 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 981bbe6..49a1190 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -60,7 +60,7 @@ com.sun.jersey.version = 1.9
 /dom4j/dom4j = 1.6.1
 /info.ganglia.gmetric4j/gmetric4j = 1.0.7
 
-io.dropwizard.metrics.version = 3.2.2
+io.dropwizard.metrics.version = 3.2.6
 /io.dropwizard.metrics/metrics-core = ${io.dropwizard.metrics.version}
 /io.dropwizard.metrics/metrics-ganglia = ${io.dropwizard.metrics.version}
 /io.dropwizard.metrics/metrics-graphite = ${io.dropwizard.metrics.version}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-core-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-core-3.2.2.jar.sha1 b/solr/licenses/metrics-core-3.2.2.jar.sha1
deleted file mode 100644
index d14a04e..0000000
--- a/solr/licenses/metrics-core-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cd9886f498ee2ab2d994f0c779e5553b2c450416

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-core-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-core-3.2.6.jar.sha1 b/solr/licenses/metrics-core-3.2.6.jar.sha1
new file mode 100644
index 0000000..13fae6d
--- /dev/null
+++ b/solr/licenses/metrics-core-3.2.6.jar.sha1
@@ -0,0 +1 @@
+62fe170cffeded1cef60e9e3402a93b45ce14327

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-ganglia-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-ganglia-3.2.2.jar.sha1 b/solr/licenses/metrics-ganglia-3.2.2.jar.sha1
deleted file mode 100644
index e5d8496..0000000
--- a/solr/licenses/metrics-ganglia-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d5bb1883e9b0daf0e4187e558746f5058f4585c1

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-ganglia-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-ganglia-3.2.6.jar.sha1 b/solr/licenses/metrics-ganglia-3.2.6.jar.sha1
new file mode 100644
index 0000000..32c9d30
--- /dev/null
+++ b/solr/licenses/metrics-ganglia-3.2.6.jar.sha1
@@ -0,0 +1 @@
+a44039835eafd2dad8842a9ed16a60c088c5b7ef

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-graphite-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-graphite-3.2.2.jar.sha1 b/solr/licenses/metrics-graphite-3.2.2.jar.sha1
deleted file mode 100644
index 5d11db4..0000000
--- a/solr/licenses/metrics-graphite-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-908e8cbec1bbdb2f4023334e424c7de2832a95af

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-graphite-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-graphite-3.2.6.jar.sha1 b/solr/licenses/metrics-graphite-3.2.6.jar.sha1
new file mode 100644
index 0000000..26a1bbc
--- /dev/null
+++ b/solr/licenses/metrics-graphite-3.2.6.jar.sha1
@@ -0,0 +1 @@
+ecbc470e9097bb3d7ff0232cca47f3badde2e20b

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jetty9-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jetty9-3.2.2.jar.sha1 b/solr/licenses/metrics-jetty9-3.2.2.jar.sha1
deleted file mode 100644
index 92d3508..0000000
--- a/solr/licenses/metrics-jetty9-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3fc94d99f41dc3f5be5483c81828138104df4449

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jetty9-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jetty9-3.2.6.jar.sha1 b/solr/licenses/metrics-jetty9-3.2.6.jar.sha1
new file mode 100644
index 0000000..5d7bfa4
--- /dev/null
+++ b/solr/licenses/metrics-jetty9-3.2.6.jar.sha1
@@ -0,0 +1 @@
+5dae1c13d8607663fbc7b22cf8c05aacd22f802e

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jvm-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jvm-3.2.2.jar.sha1 b/solr/licenses/metrics-jvm-3.2.2.jar.sha1
deleted file mode 100644
index 0c02f93..0000000
--- a/solr/licenses/metrics-jvm-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9cbf2030242f7ffb97fae23f8a81421eb8d4ad45

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jvm-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jvm-3.2.6.jar.sha1 b/solr/licenses/metrics-jvm-3.2.6.jar.sha1
new file mode 100644
index 0000000..219d02b
--- /dev/null
+++ b/solr/licenses/metrics-jvm-3.2.6.jar.sha1
@@ -0,0 +1 @@
+a7a475393fe47dfee2042415430da3f01d4fe94e


[11/18] lucene-solr:jira/solr-12438: SOLR-12250: Create the temporary tlog file properly

Posted by ab...@apache.org.
SOLR-12250: Create the temporary tlog file properly


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2f19ae19
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2f19ae19
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2f19ae19

Branch: refs/heads/jira/solr-12438
Commit: 2f19ae1907e63864e19eda5b6325083f61f9cf66
Parents: 2b406a5
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed Jun 6 15:50:19 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed Jun 6 15:50:19 2018 +0700

----------------------------------------------------------------------
 .../test/org/apache/solr/update/TransactionLogTest.java   | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f19ae19/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
index d2b4b26..100d4d9 100644
--- a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
+++ b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
@@ -17,8 +17,10 @@
 
 package org.apache.solr.update;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Locale;
 
@@ -29,9 +31,11 @@ import org.junit.Test;
 public class TransactionLogTest extends LuceneTestCase {
 
   @Test
-  public void testBigLastAddSize() throws IOException {
-    String tlogFileName = String.format(Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, 0);
-    try (TransactionLog transactionLog = new TransactionLog(Files.createTempFile(tlogFileName, "").toFile(), new ArrayList<>())) {
+  public void testBigLastAddSize() {
+    String tlogFileName = String.format(Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, Long.MAX_VALUE);
+    Path path = createTempDir();
+    File logFile = new File(path.toFile(), tlogFileName);
+    try (TransactionLog transactionLog = new TransactionLog(logFile, new ArrayList<>())) {
       transactionLog.lastAddSize = 2000000000;
       AddUpdateCommand updateCommand = new AddUpdateCommand(null);
       updateCommand.solrDoc = new SolrInputDocument();


[08/18] lucene-solr:jira/solr-12438: SOLR-11453: fix typos in the CHANGES entry

Posted by ab...@apache.org.
SOLR-11453: fix typos in the CHANGES entry


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/124b5e04
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/124b5e04
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/124b5e04

Branch: refs/heads/jira/solr-12438
Commit: 124b5e047df697bfd61f1d14f87a0ee5d24f68d8
Parents: 0c6f38a
Author: Varun Thacker <va...@apache.org>
Authored: Tue Jun 5 15:16:12 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Tue Jun 5 15:16:12 2018 -0700

----------------------------------------------------------------------
 solr/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/124b5e04/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5adc677..6dd4889 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -70,8 +70,8 @@ Upgrade Notes
   To return the previous behavior pass false to skipCommitOnMasterVersionZero in slave section of replication
   handler configuration, or pass it to the fetchindex command.
 
-* SOLR-11453: Configuring slowQueryThresholdMillis now logs slow requests to a separate file - solr_slow_requests.log .
-  Previously they would get logged in the solr.xml file
+* SOLR-11453: Configuring slowQueryThresholdMillis now logs slow requests to a separate file - solr_slow_requests.log.
+  Previously they would get logged in the solr.log file.
 
 New Features
 ----------------------
@@ -140,7 +140,7 @@ New Features
 * SOLR-12328: JSON Facet API: Domain change with graph query.
   (Daniel Meehl, Kevin Watters, yonik)
 
-* SOLR-11453: Configuring slowQueryThresholdMillis logs slow requests to a separate file - solr_slow_requests.log .
+* SOLR-11453: Configuring slowQueryThresholdMillis logs slow requests to a separate file - solr_slow_requests.log.
   (Shawn Heisey, Remko Popma, Varun Thacker)
 
 * SOLR-12401: Add getValue() and setValue() Stream Evaluators (Joel Bernstein, janhoy)


[02/18] lucene-solr:jira/solr-12438: SOLR-12387: fixing a test failure

Posted by ab...@apache.org.
SOLR-12387: fixing a test failure


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f9d807af
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f9d807af
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f9d807af

Branch: refs/heads/jira/solr-12438
Commit: f9d807af755f33dea1384cb8b9c591875076dd1a
Parents: add77d2
Author: Noble Paul <no...@apache.org>
Authored: Tue Jun 5 23:14:11 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jun 5 23:14:11 2018 +1000

----------------------------------------------------------------------
 .../solr/handler/admin/CollectionsHandler.java    |  5 +++--
 .../solr/common/cloud/ClusterProperties.java      | 18 ++++++++++++++++++
 .../apache/solr/common/cloud/ZkStateReader.java   |  6 ++++++
 3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f9d807af/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 01d2fe8..269bb50 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -206,9 +206,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
     return this.coreContainer;
   }
 
-  protected void copyFromClusterProp(Map<String, Object> props, String prop) {
+  protected void copyFromClusterProp(Map<String, Object> props, String prop) throws IOException {
     if (props.get(prop) != null) return;//if it's already specified , return
-    Object defVal = coreContainer.getZkController().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null);
+    Object defVal = new ClusterProperties(coreContainer.getZkController().getZkStateReader().getZkClient())
+        .getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null);
     if (defVal != null) props.put(prop, String.valueOf(defVal));
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f9d807af/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
index 87896da..446923b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.solr.common.SolrException;
@@ -68,6 +69,23 @@ public class ClusterProperties {
   }
 
   /**
+   * Read the value of a cluster property, returning a default if it is not set
+   *
+   * @param key          the property name or the full path to the property as a list of parts.
+   * @param defaultValue the default value
+   * @param <T>          the type of the property
+   * @return the property value
+   * @throws IOException if there is an error reading the value from the cluster
+   */
+  @SuppressWarnings("unchecked")
+  public <T> T getClusterProperty(List<String> key, T defaultValue) throws IOException {
+    T value = (T) Utils.getObjectByPath(getClusterProperties(), false, key);
+    if (value == null)
+      return defaultValue;
+    return value;
+  }
+
+  /**
    * Return the cluster properties
    * @throws IOException if there is an error reading properties from the cluster
    */

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f9d807af/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 6b65c34..a86c5e2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -961,6 +961,12 @@ public class ZkStateReader implements Closeable {
     return value;
   }
 
+  /**Same as the above but allows a full json path as a list of parts
+   *
+   * @param keyPath path to the property example ["collectionDefauls", "numShards"]
+   * @param defaultValue a default value to use if no such property exists
+   * @return the cluster property, or a default if the property is not set
+   */
   public <T> T getClusterProperty(List<String> keyPath, T defaultValue) {
     T value = (T) Utils.getObjectByPath( clusterProperties, false, keyPath);
     if (value == null)


[10/18] lucene-solr:jira/solr-12438: LUCENE-8346: Remove final modifier on QueryBuilder#createSpanQuery to allow override

Posted by ab...@apache.org.
LUCENE-8346: Remove final modifier on QueryBuilder#createSpanQuery to allow override


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2b406a57
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2b406a57
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2b406a57

Branch: refs/heads/jira/solr-12438
Commit: 2b406a57c41c28a95db1deefc17527e600fa244d
Parents: 9ff3f5a
Author: Jim Ferenczi <ji...@apache.org>
Authored: Wed Jun 6 09:18:25 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed Jun 6 09:18:25 2018 +0200

----------------------------------------------------------------------
 lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2b406a57/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java b/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
index 2cb066b..697e3bb 100644
--- a/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
@@ -346,9 +346,9 @@ public class QueryBuilder {
 
   /**
    * Creates a span query from the tokenstream.  In the case of a single token, a simple <code>SpanTermQuery</code> is
-   * returned.  When multiple tokens, an ordered <code>SpanNearQuery</code> with slop of 0 is returned.
+   * returned.  When multiple tokens, an ordered <code>SpanNearQuery</code> with slop 0 is returned.
    */
-  protected final SpanQuery createSpanQuery(TokenStream in, String field) throws IOException {
+  protected SpanQuery createSpanQuery(TokenStream in, String field) throws IOException {
     TermToBytesRefAttribute termAtt = in.getAttribute(TermToBytesRefAttribute.class);
     if (termAtt == null) {
       return null;


[17/18] lucene-solr:jira/solr-12438: Merge branch 'master' into jira/solr-12438

Posted by ab...@apache.org.
Merge branch 'master' into jira/solr-12438


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3e724818
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3e724818
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3e724818

Branch: refs/heads/jira/solr-12438
Commit: 3e724818c39623d40024025c7dfe5c1be694c547
Parents: 2fa260c d163159
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 16:48:27 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 16:48:27 2018 +0200

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |   3 +
 .../org/apache/lucene/util/QueryBuilder.java    |   4 +-
 lucene/ivy-versions.properties                  |   2 +-
 .../spatial/spatial4j/Geo3dShapeFactory.java    |  10 +-
 .../lucene/spatial/spatial4j/Geo3dRptTest.java  |  23 +-
 solr/CHANGES.txt                                |  19 +-
 solr/NOTICE.txt                                 |  14 +
 .../src/java/org/apache/solr/core/SolrCore.java |   2 +-
 .../apache/solr/core/SolrResourceLoader.java    |  12 +-
 .../solr/handler/admin/CollectionsHandler.java  |   5 +-
 .../solr/handler/tagger/OffsetCorrector.java    | 178 +++++++++
 .../solr/handler/tagger/TagClusterReducer.java  | 103 +++++
 .../org/apache/solr/handler/tagger/TagLL.java   | 176 ++++++++
 .../org/apache/solr/handler/tagger/Tagger.java  | 230 +++++++++++
 .../handler/tagger/TaggerRequestHandler.java    | 397 +++++++++++++++++++
 .../solr/handler/tagger/TaggingAttribute.java   |  65 +++
 .../handler/tagger/TaggingAttributeImpl.java    |  79 ++++
 .../solr/handler/tagger/TermPrefixCursor.java   | 189 +++++++++
 .../solr/handler/tagger/XmlOffsetCorrector.java | 113 ++++++
 .../solr/handler/tagger/package-info.java       |  27 ++
 .../org/apache/solr/search/QParserPlugin.java   |  80 ++--
 .../solr/collection1/conf/schema-tagger.xml     | 187 +++++++++
 .../solr/collection1/conf/solrconfig-tagger.xml |  59 +++
 .../autoscaling/AutoScalingHandlerTest.java     |  21 +
 .../autoscaling/sim/SimSolrCloudTestCase.java   |  56 ++-
 .../apache/solr/core/ResourceLoaderTest.java    |   6 +-
 .../tagger/EmbeddedSolrNoSerializeTest.java     | 153 +++++++
 .../handler/tagger/RandomizedTaggerTest.java    | 150 +++++++
 .../apache/solr/handler/tagger/Tagger2Test.java | 175 ++++++++
 .../apache/solr/handler/tagger/TaggerTest.java  | 296 ++++++++++++++
 .../solr/handler/tagger/TaggerTestCase.java     | 251 ++++++++++++
 .../handler/tagger/TaggingAttributeTest.java    |  73 ++++
 .../handler/tagger/WordLengthTaggingFilter.java | 110 +++++
 .../tagger/WordLengthTaggingFilterFactory.java  |  67 ++++
 .../handler/tagger/XmlInterpolationTest.java    | 224 +++++++++++
 .../solr/search/TestStandardQParsers.java       |  10 +-
 .../apache/solr/update/TransactionLogTest.java  |  12 +-
 solr/licenses/metrics-core-3.2.2.jar.sha1       |   1 -
 solr/licenses/metrics-core-3.2.6.jar.sha1       |   1 +
 solr/licenses/metrics-ganglia-3.2.2.jar.sha1    |   1 -
 solr/licenses/metrics-ganglia-3.2.6.jar.sha1    |   1 +
 solr/licenses/metrics-graphite-3.2.2.jar.sha1   |   1 -
 solr/licenses/metrics-graphite-3.2.6.jar.sha1   |   1 +
 solr/licenses/metrics-jetty9-3.2.2.jar.sha1     |   1 -
 solr/licenses/metrics-jetty9-3.2.6.jar.sha1     |   1 +
 solr/licenses/metrics-jvm-3.2.2.jar.sha1        |   1 -
 solr/licenses/metrics-jvm-3.2.6.jar.sha1        |   1 +
 solr/solr-ref-guide/src/searching.adoc          |  33 +-
 solr/solr-ref-guide/src/the-tagger-handler.adoc | 265 +++++++++++++
 .../client/solrj/cloud/autoscaling/Clause.java  |   2 +-
 .../solr/common/cloud/ClusterProperties.java    |  18 +
 .../apache/solr/common/cloud/ZkStateReader.java |   6 +
 52 files changed, 3803 insertions(+), 112 deletions(-)
----------------------------------------------------------------------



[07/18] lucene-solr:jira/solr-12438: SOLR-12233: QParserPlugin's static registry of builtins can be optimized to avoid needless ClassLoader activity on SolrCore load.

Posted by ab...@apache.org.
SOLR-12233: QParserPlugin's static registry of builtins can be optimized
 to avoid needless ClassLoader activity on SolrCore load.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0c6f38a3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0c6f38a3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0c6f38a3

Branch: refs/heads/jira/solr-12438
Commit: 0c6f38a315d0df5abd01e7d4efe481bc53444a49
Parents: cf63392
Author: Jeff <je...@salesforce.com>
Authored: Tue Jun 5 15:11:32 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Jun 5 15:11:32 2018 -0400

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  6 +-
 .../src/java/org/apache/solr/core/SolrCore.java |  2 +-
 .../org/apache/solr/search/QParserPlugin.java   | 80 ++++++++++----------
 .../solr/search/TestStandardQParsers.java       | 10 +--
 4 files changed, 51 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 479406f..5adc677 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -155,7 +155,7 @@ New Features
 * SOLR-12389: support deeply nested json objects in clusterprops.json (noble)
 
 * SOLR-12376: Added the TaggerRequestHandler (AKA SolrTextTagger) for tagging text.  It's used as a component of
-  NER/ERD systems including query-understanding.  See the ref guide for more info.  (David Smiley
+  NER/ERD systems including query-understanding.  See the ref guide for more info.  (David Smiley)
 
 Bug Fixes
 ----------------------
@@ -332,6 +332,10 @@ Optimizations
 
 * SOLR-9922: Write buffering updates to another tlog. (Cao Manh Dat)
 
+* SOLR-12233: QParserPlugin's built-in static registry now holds actual QParserPlugin instances instead of class
+  references.  This is consistent with other plugin registries and allows a SolrCore to load faster.
+  (Jeff Miller, David Smiley)
+
 Other Changes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 99c0cca..feab22d 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -958,7 +958,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
       initIndex(prev != null, reload);
 
       initWriters();
-      qParserPlugins.init(createInstances(QParserPlugin.standardPlugins), this);
+      qParserPlugins.init(QParserPlugin.standardPlugins, this);
       valueSourceParsers.init(ValueSourceParser.standardValueSourceParsers, this);
       transformerFactories.init(TransformerFactory.defaultFactories, this);
       loadSearchComponents();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
index f80bc9c..b20c3c8 100644
--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
@@ -37,53 +37,53 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
   public static final String DEFAULT_QTYPE = LuceneQParserPlugin.NAME;
 
   /**
-   * Internal use - name to class mappings of builtin parsers.
+   * Internal use - name to parser for the builtin parsers.
    * Each query parser plugin extending {@link QParserPlugin} has own instance of standardPlugins.
    * This leads to cyclic dependencies of static fields and to case when NAME field is not yet initialized.
    * This result to NPE during initialization.
    * For every plugin, listed here, NAME field has to be final and static.
    */
-  public static final Map<String, Class<? extends QParserPlugin>> standardPlugins;
+  public static final Map<String, QParserPlugin> standardPlugins;
 
   static {
-    HashMap<String, Class<? extends QParserPlugin>> map = new HashMap<>(30, 1);
-    map.put(LuceneQParserPlugin.NAME, LuceneQParserPlugin.class);
-    map.put(FunctionQParserPlugin.NAME, FunctionQParserPlugin.class);
-    map.put(PrefixQParserPlugin.NAME, PrefixQParserPlugin.class);
-    map.put(BoostQParserPlugin.NAME, BoostQParserPlugin.class);
-    map.put(DisMaxQParserPlugin.NAME, DisMaxQParserPlugin.class);
-    map.put(ExtendedDismaxQParserPlugin.NAME, ExtendedDismaxQParserPlugin.class);
-    map.put(FieldQParserPlugin.NAME, FieldQParserPlugin.class);
-    map.put(RawQParserPlugin.NAME, RawQParserPlugin.class);
-    map.put(TermQParserPlugin.NAME, TermQParserPlugin.class);
-    map.put(TermsQParserPlugin.NAME, TermsQParserPlugin.class);
-    map.put(NestedQParserPlugin.NAME, NestedQParserPlugin.class);
-    map.put(FunctionRangeQParserPlugin.NAME, FunctionRangeQParserPlugin.class);
-    map.put(SpatialFilterQParserPlugin.NAME, SpatialFilterQParserPlugin.class);
-    map.put(SpatialBoxQParserPlugin.NAME, SpatialBoxQParserPlugin.class);
-    map.put(JoinQParserPlugin.NAME, JoinQParserPlugin.class);
-    map.put(SurroundQParserPlugin.NAME, SurroundQParserPlugin.class);
-    map.put(SwitchQParserPlugin.NAME, SwitchQParserPlugin.class);
-    map.put(MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class);
-    map.put(BlockJoinParentQParserPlugin.NAME, BlockJoinParentQParserPlugin.class);
-    map.put(BlockJoinChildQParserPlugin.NAME, BlockJoinChildQParserPlugin.class);
-    map.put(FiltersQParserPlugin.NAME, FiltersQParserPlugin.class);
-    map.put(CollapsingQParserPlugin.NAME, CollapsingQParserPlugin.class);
-    map.put(SimpleQParserPlugin.NAME, SimpleQParserPlugin.class);
-    map.put(ComplexPhraseQParserPlugin.NAME, ComplexPhraseQParserPlugin.class);
-    map.put(ReRankQParserPlugin.NAME, ReRankQParserPlugin.class);
-    map.put(ExportQParserPlugin.NAME, ExportQParserPlugin.class);
-    map.put(MLTQParserPlugin.NAME, MLTQParserPlugin.class);
-    map.put(HashQParserPlugin.NAME, HashQParserPlugin.class);
-    map.put(GraphQParserPlugin.NAME, GraphQParserPlugin.class);
-    map.put(XmlQParserPlugin.NAME, XmlQParserPlugin.class);
-    map.put(GraphTermsQParserPlugin.NAME, GraphTermsQParserPlugin.class);
-    map.put(IGainTermsQParserPlugin.NAME, IGainTermsQParserPlugin.class);
-    map.put(TextLogisticRegressionQParserPlugin.NAME, TextLogisticRegressionQParserPlugin.class);
-    map.put(SignificantTermsQParserPlugin.NAME, SignificantTermsQParserPlugin.class);
-    map.put(PayloadScoreQParserPlugin.NAME, PayloadScoreQParserPlugin.class);
-    map.put(PayloadCheckQParserPlugin.NAME, PayloadCheckQParserPlugin.class);
-    map.put(BoolQParserPlugin.NAME, BoolQParserPlugin.class);
+    HashMap<String, QParserPlugin> map = new HashMap<>(30, 1);
+    map.put(LuceneQParserPlugin.NAME, new LuceneQParserPlugin());
+    map.put(FunctionQParserPlugin.NAME, new FunctionQParserPlugin());
+    map.put(PrefixQParserPlugin.NAME, new PrefixQParserPlugin());
+    map.put(BoostQParserPlugin.NAME, new BoostQParserPlugin());
+    map.put(DisMaxQParserPlugin.NAME, new DisMaxQParserPlugin());
+    map.put(ExtendedDismaxQParserPlugin.NAME, new ExtendedDismaxQParserPlugin());
+    map.put(FieldQParserPlugin.NAME, new FieldQParserPlugin());
+    map.put(RawQParserPlugin.NAME, new RawQParserPlugin());
+    map.put(TermQParserPlugin.NAME, new TermQParserPlugin());
+    map.put(TermsQParserPlugin.NAME, new TermsQParserPlugin());
+    map.put(NestedQParserPlugin.NAME, new NestedQParserPlugin());
+    map.put(FunctionRangeQParserPlugin.NAME, new FunctionRangeQParserPlugin());
+    map.put(SpatialFilterQParserPlugin.NAME, new SpatialFilterQParserPlugin());
+    map.put(SpatialBoxQParserPlugin.NAME, new SpatialBoxQParserPlugin());
+    map.put(JoinQParserPlugin.NAME, new JoinQParserPlugin());
+    map.put(SurroundQParserPlugin.NAME, new SurroundQParserPlugin());
+    map.put(SwitchQParserPlugin.NAME, new SwitchQParserPlugin());
+    map.put(MaxScoreQParserPlugin.NAME, new MaxScoreQParserPlugin());
+    map.put(BlockJoinParentQParserPlugin.NAME, new BlockJoinParentQParserPlugin());
+    map.put(BlockJoinChildQParserPlugin.NAME, new BlockJoinChildQParserPlugin());
+    map.put(FiltersQParserPlugin.NAME, new FiltersQParserPlugin());
+    map.put(CollapsingQParserPlugin.NAME, new CollapsingQParserPlugin());
+    map.put(SimpleQParserPlugin.NAME, new SimpleQParserPlugin());
+    map.put(ComplexPhraseQParserPlugin.NAME, new ComplexPhraseQParserPlugin());
+    map.put(ReRankQParserPlugin.NAME, new ReRankQParserPlugin());
+    map.put(ExportQParserPlugin.NAME, new ExportQParserPlugin());
+    map.put(MLTQParserPlugin.NAME, new MLTQParserPlugin());
+    map.put(HashQParserPlugin.NAME, new HashQParserPlugin());
+    map.put(GraphQParserPlugin.NAME, new GraphQParserPlugin());
+    map.put(XmlQParserPlugin.NAME, new XmlQParserPlugin());
+    map.put(GraphTermsQParserPlugin.NAME, new GraphTermsQParserPlugin());
+    map.put(IGainTermsQParserPlugin.NAME, new IGainTermsQParserPlugin());
+    map.put(TextLogisticRegressionQParserPlugin.NAME, new TextLogisticRegressionQParserPlugin());
+    map.put(SignificantTermsQParserPlugin.NAME, new SignificantTermsQParserPlugin());
+    map.put(PayloadScoreQParserPlugin.NAME, new PayloadScoreQParserPlugin());
+    map.put(PayloadCheckQParserPlugin.NAME, new PayloadCheckQParserPlugin());
+    map.put(BoolQParserPlugin.NAME, new BoolQParserPlugin());
 
     standardPlugins = Collections.unmodifiableMap(map);
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java b/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
index ff9ffff..cab9026 100644
--- a/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
+++ b/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
@@ -16,15 +16,15 @@
  */
 package org.apache.solr.search;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.junit.Test;
-
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
 /**
  * Check standard query parsers for class loading problems during initialization (NAME field is final and static).
  * Because every query plugin extend {@link org.apache.solr.search.QParserPlugin} and contains own instance of {@link org.apache.solr.search.QParserPlugin#standardPlugins},
@@ -50,9 +50,9 @@ public class TestStandardQParsers extends LuceneTestCase {
     List<String> notFinal = new ArrayList<>(QParserPlugin.standardPlugins.size());
     List<String> mismatch = new ArrayList<>(QParserPlugin.standardPlugins.size());
 
-    for (Map.Entry<String, Class<? extends QParserPlugin>> pair : QParserPlugin.standardPlugins.entrySet()) {
+    for (Map.Entry<String, QParserPlugin> pair : QParserPlugin.standardPlugins.entrySet()) {
       String regName = pair.getKey();
-      Class<? extends QParserPlugin> clazz = pair.getValue();
+      Class<? extends QParserPlugin> clazz = pair.getValue().getClass();;
 
       Field nameField = clazz.getField(FIELD_NAME);
       int modifiers = nameField.getModifiers();


[14/18] lucene-solr:jira/solr-12438: SOLR-11911: Move simulator reset to @Before method.

Posted by ab...@apache.org.
SOLR-11911: Move simulator reset to @Before method.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c1f42906
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c1f42906
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c1f42906

Branch: refs/heads/jira/solr-12438
Commit: c1f42906ef1e0929f3887736dd9bb5bd0f9975c8
Parents: af7dfb1
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 11:06:50 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 11:48:42 2018 +0200

----------------------------------------------------------------------
 .../autoscaling/sim/SimSolrCloudTestCase.java   | 56 ++++++++++----------
 1 file changed, 27 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1f42906/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
index e83f72f..270e7e7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
@@ -80,40 +80,38 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 {
   @Override
   public void setUp() throws Exception {
     super.setUp();
-    if (cluster != null) {
-      // clear any persisted configuration
-      cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
-      cluster.getDistribStateManager().setData(ZkStateReader.ROLES, Utils.toJSON(new HashMap<>()), -1);
-      cluster.getSimClusterStateProvider().simDeleteAllCollections();
-      cluster.simClearSystemCollection();
-      cluster.getSimNodeStateProvider().simRemoveDeadNodes();
-      cluster.getSimClusterStateProvider().simRemoveDeadNodes();
-      // restore the expected number of nodes
-      int currentSize = cluster.getLiveNodesSet().size();
-      if (currentSize < clusterNodeCount) {
-        int addCnt = clusterNodeCount - currentSize;
-        while (addCnt-- > 0) {
-          cluster.simAddNode();
-        }
-      } else if (currentSize > clusterNodeCount) {
-        cluster.simRemoveRandomNodes(currentSize - clusterNodeCount, true, random());
-      }
-      // clean any persisted trigger state or events
-      removeChildren(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
-      removeChildren(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
-      removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
-      removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
-      cluster.getSimClusterStateProvider().simResetLeaderThrottles();
-      cluster.simRestartOverseer(null);
-      cluster.getTimeSource().sleep(5000);
-      cluster.simResetOpCounts();
-    }
   }
 
   @Before
-  public void checkClusterConfiguration() {
+  public void checkClusterConfiguration() throws Exception {
     if (cluster == null)
       throw new RuntimeException("SimCloudManager not configured - have you called configureCluster()?");
+    // clear any persisted configuration
+    cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
+    cluster.getDistribStateManager().setData(ZkStateReader.ROLES, Utils.toJSON(new HashMap<>()), -1);
+    cluster.getSimClusterStateProvider().simDeleteAllCollections();
+    cluster.simClearSystemCollection();
+    cluster.getSimNodeStateProvider().simRemoveDeadNodes();
+    cluster.getSimClusterStateProvider().simRemoveDeadNodes();
+    // restore the expected number of nodes
+    int currentSize = cluster.getLiveNodesSet().size();
+    if (currentSize < clusterNodeCount) {
+      int addCnt = clusterNodeCount - currentSize;
+      while (addCnt-- > 0) {
+        cluster.simAddNode();
+      }
+    } else if (currentSize > clusterNodeCount) {
+      cluster.simRemoveRandomNodes(currentSize - clusterNodeCount, true, random());
+    }
+    // clean any persisted trigger state or events
+    removeChildren(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
+    removeChildren(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
+    removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
+    removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
+    cluster.getSimClusterStateProvider().simResetLeaderThrottles();
+    cluster.simRestartOverseer(null);
+    cluster.getTimeSource().sleep(5000);
+    cluster.simResetOpCounts();
   }
 
   protected void removeChildren(String path) throws Exception {


[16/18] lucene-solr:jira/solr-12438: SOLR-12445: Update CHANGES.

Posted by ab...@apache.org.
SOLR-12445: Update CHANGES.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d1631593
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d1631593
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d1631593

Branch: refs/heads/jira/solr-12438
Commit: d1631593f3a0ff9f58468ff0f32d7c1e8a404567
Parents: 3022bbc
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 15:30:32 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 15:30:32 2018 +0200

----------------------------------------------------------------------
 solr/CHANGES.txt | 2 ++
 1 file changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d1631593/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6dd4889..c93ccd8 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -295,6 +295,8 @@ Bug Fixes
 
 * SOLR-12354: Register the /admin/info/key end-point at the startup time to avoid 404 (noble)
 
+* SOLR-12445: Upgrade Dropwizard Metrics to version 3.2.6. (ab)
+
 Optimizations
 ----------------------
 


[03/18] lucene-solr:jira/solr-12438: SOLR-12417: doc: fix CHANGES credit

Posted by ab...@apache.org.
SOLR-12417: doc: fix CHANGES credit


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7d0b64f9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7d0b64f9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7d0b64f9

Branch: refs/heads/jira/solr-12438
Commit: 7d0b64f9d5484c1f03ccf422bdee50c31443344c
Parents: f9d807a
Author: yonik <yo...@apache.org>
Authored: Tue Jun 5 10:55:53 2018 -0400
Committer: yonik <yo...@apache.org>
Committed: Tue Jun 5 10:55:53 2018 -0400

----------------------------------------------------------------------
 solr/CHANGES.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d0b64f9/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 1b454a0..66d2026 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -283,7 +283,7 @@ Bug Fixes
 * SOLR-12374: SnapShooter.getIndexCommit can forget to decref the searcher; though it's not clear in practice when.
   (David Smiley)
 
-* SOLR-12417: velocity response writer should enforce valid function name for v.json parameter (yonik)
+* SOLR-12417: velocity response writer should enforce valid function name for v.json parameter (Mano Kovacs, yonik)
 
 * SOLR-12271: Fixed bug in how Analytics component reads negative values from float and double fields. (Houston Putman)
 


[12/18] lucene-solr:jira/solr-12438: SOLR-12250: Fix precommit

Posted by ab...@apache.org.
SOLR-12250: Fix precommit


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0358fcb1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0358fcb1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0358fcb1

Branch: refs/heads/jira/solr-12438
Commit: 0358fcb1759038f76d91c124703c5d3244125f75
Parents: 2f19ae1
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed Jun 6 15:52:15 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed Jun 6 15:52:15 2018 +0700

----------------------------------------------------------------------
 solr/core/src/test/org/apache/solr/update/TransactionLogTest.java | 2 --
 1 file changed, 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0358fcb1/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
index 100d4d9..66ecbc6 100644
--- a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
+++ b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
@@ -18,8 +18,6 @@
 package org.apache.solr.update;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Locale;


[05/18] lucene-solr:jira/solr-12438: SOLR-12376: New TaggerRequestHandler (SolrTextTagger).

Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java
new file mode 100644
index 0000000..cb742a8
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java
@@ -0,0 +1,150 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Random;
+import java.util.Set;
+
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Randomly generate taggable text and verify via simple tag algorithm.
+ */
+@Repeat(iterations = 10)
+public class RandomizedTaggerTest extends TaggerTestCase {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+  }
+
+  @Test
+  public void test() throws Exception {
+    final Random R = random();
+
+    Set<String> names = new HashSet<>();
+    //random list of single-word names
+    final int NUM_SINGLES = 4;//RandomInts.randomIntBetween(R, 1, 5);
+    for (int i = 0; i < NUM_SINGLES; i++) {
+      if (i == 0)//first is a big string (perhaps triggers bugs related to growing buffers)
+        names.add(randomStringOfLength(16, 32));
+      else
+        names.add(randomString());
+    }
+
+    //add random list of multi-word names, partially including existing names
+    final int NUM_MULTI = 10;
+    for (int i = 0; i < NUM_MULTI; i++) {
+      final int numWords = RandomNumbers.randomIntBetween(R, 2, 4);
+      StringBuilder buf = new StringBuilder();
+      for (int j = 0; j < numWords; j++) {
+        if (j != 0)
+          buf.append(' ');
+        if (R.nextBoolean()) {//new likely non-existent word
+          buf.append(randomString());
+        } else {//existing word (possible multi-word from prev iteration)
+          buf.append(RandomPicks.randomFrom(R, names));
+        }
+      }
+      names.add(buf.toString());
+    }
+
+    // BUILD NAMES
+    buildNames(names.toArray(new String[names.size()]));
+
+    // QUERY LOOP
+    for (int tTries = 0; tTries < 10 * RANDOM_MULTIPLIER; tTries++) {
+      // Build up random input, similar to multi-word random names above
+      StringBuilder input = new StringBuilder();
+      final int INPUT_WORD_LEN = 20;
+      input.append(' ');//must start with space based on assertBruteForce logic
+      for (int i = 0; i < INPUT_WORD_LEN; i++) {
+        if (R.nextBoolean()) {//new likely non-existent word
+          input.append(randomString());
+        } else {//existing word (possible multi-word from prev iteration)
+          input.append(RandomPicks.randomFrom(R, NAMES));
+        }
+        input.append(' ');//must end with a space
+      }
+
+      boolean madeIt = false;
+      try {
+        assertBruteForce(input.toString());
+        madeIt = true;
+      } finally {
+        if (!madeIt) {
+          System.out.println("Reproduce with:");
+          System.out.print(" buildNames(");
+          for (int i = 0; i < NAMES.size(); i++) {
+            if (i != 0)
+              System.out.print(',');
+            System.out.print('"');
+            System.out.print(NAMES.get(i));
+            System.out.print('"');
+          }
+          System.out.println(");");
+          System.out.println(" assertBruteForce(\"" + input+"\");");
+        }
+      }
+    }
+
+  }
+
+  private void assertBruteForce(String input) throws Exception {
+    assert input.matches(" .* ");
+    baseParams.set("overlaps", "ALL");
+
+    //loop through NAMES and find all tag offsets
+    List<TestTag> testTags = new ArrayList<>();
+    for (String name : NAMES) {
+      String spaceName = " "+name+" ";
+      int off = 0;
+      while (true) {
+        int idx = input.indexOf(spaceName, off);
+        if (idx < 0)
+          break;
+        testTags.add(new TestTag(idx + 1, idx + 1 + name.length(), name, name));
+        off = idx + 1;
+      }
+    }
+
+    //assert
+    assertTags(reqDoc(input), testTags.toArray(new TestTag[testTags.size()]));
+  }
+
+  private String randomString() { return randomStringOfLength(1, 1); }
+
+  private String randomStringOfLength(int min, int max) {
+    return RandomStrings.randomAsciiLettersOfLengthBetween(random(), min, max).toLowerCase(Locale.ROOT);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java b/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java
new file mode 100644
index 0000000..c7580e1
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java
@@ -0,0 +1,175 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.nio.charset.StandardCharsets;
+
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Test the {@link TaggerRequestHandler}.
+ */
+public class Tagger2Test extends TaggerTestCase {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+  }
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    baseParams.set("overlaps", "LONGEST_DOMINANT_RIGHT");
+  }
+
+  /** whole matching, no sub-tags */
+  @Test
+  public void testLongestDominantRight() throws Exception {
+    buildNames("in", "San", "in San", "Francisco", "San Francisco",
+        "San Francisco State College", "College of California",
+        "Clayton", "Clayton North", "North Carolina");
+
+    assertTags("He lived in San Francisco.",
+        "in", "San Francisco");
+
+    assertTags("He enrolled in San Francisco State College of California",
+        "in", "San Francisco State College");
+
+    assertTags("He lived in Clayton North Carolina",
+        "in", "Clayton", "North Carolina");
+
+  }
+
+  // As of Lucene/Solr 4.9, StandardTokenizer never does this anymore (reported to Lucene dev-list,
+  // Jan 26th 2015.  Honestly it's not particularly important to us but it renders this test
+  // pointless.
+  /** Orig issue https://github.com/OpenSextant/SolrTextTagger/issues/2  related: #13 */
+  @Test
+  @Ignore
+  public void testVeryLongWord() throws Exception {
+    String SANFRAN = "San Francisco";
+    buildNames(SANFRAN);
+
+    // exceeds default 255 max token length which means it in-effect becomes a stop-word
+    StringBuilder STOP = new StringBuilder(260);//>255
+    for (int i = 0; i < STOP.capacity(); i++) {
+      STOP.append((char) ('0' + (i % 10)));
+    }
+
+    String doc = "San " + STOP + " Francisco";
+    assertTags(doc);//no match due to default stop word handling
+    //and we find it when we ignore stop words
+    assertTags(reqDoc(doc, "ignoreStopwords", "true"), new TestTag(0, doc.length(), doc, lookupByName(SANFRAN)));
+  }
+
+  /** Support for stopwords (posInc &gt; 1);
+   * discussion: https://github.com/OpenSextant/SolrTextTagger/issues/13 */
+  @Test
+  public void testStopWords() throws Exception {
+    baseParams.set("field", "name_tagStop");//stop filter (pos inc enabled) index & query
+
+    String SOUTHOFWALES = "South of Wales";//'of' is stop word index time & query
+    String ACITYA = "A City A";
+
+    buildNames(SOUTHOFWALES, ACITYA);
+
+    //round-trip works
+    assertTags(reqDoc(SOUTHOFWALES), new TestTag(0, SOUTHOFWALES.length(), SOUTHOFWALES,
+            lookupByName(SOUTHOFWALES)));
+    //  but offsets doesn't include stopword when leading or trailing...
+    assertTags(reqDoc(ACITYA), new TestTag(2, 6, "City",
+            lookupByName(ACITYA)));
+    //break on stop words
+    assertTags(reqDoc(SOUTHOFWALES, "ignoreStopwords", "false"));//match nothing
+  }
+
+  /** Tests WordDelimiterGraphFilter, stacked/synonymous tokens at index time (catenate options) */
+  @Test
+  public void testWDF() throws Exception {
+    baseParams.set("field", "name_tagWDF");
+
+    final String WINSTONSALEM = "City of Winston-Salem";//hyphen
+    final String BOSTONHARBOR = "Boston Harbor";//space
+    buildNames(WINSTONSALEM, BOSTONHARBOR);
+
+    //round-trip works
+    assertTags(reqDoc(WINSTONSALEM), new TestTag(0, WINSTONSALEM.length(), WINSTONSALEM,
+        lookupByName(WINSTONSALEM)));
+
+    // space separated works
+    final String WS_SPACE = WINSTONSALEM.replace('-', ' ');
+    assertTags(reqDoc(WS_SPACE),
+        new TestTag(0, WS_SPACE.length(), WS_SPACE,
+        lookupByName(WINSTONSALEM)));
+
+    //must be full match
+    assertTags(reqDoc("Winston"));//match nothing
+    assertTags(reqDoc("Salem"));//match nothing
+
+    // round-trip works
+    assertTags(reqDoc(BOSTONHARBOR), new TestTag(0, BOSTONHARBOR.length(), BOSTONHARBOR,
+        lookupByName(BOSTONHARBOR)));
+
+    // hyphen separated works
+    final String BH_HYPHEN = BOSTONHARBOR.replace(' ', '-');
+    assertTags(reqDoc(BH_HYPHEN),
+        new TestTag(0, BH_HYPHEN.length(), BH_HYPHEN,
+            lookupByName(BOSTONHARBOR)));
+    //must be full match
+    assertTags(reqDoc("Boston"));//match nothing
+    assertTags(reqDoc("Harbor"));//match nothing
+  }
+
+  /** Ensure character offsets work for multi-byte characters */
+  @Test
+  public void testMultibyteChar() throws Exception {
+    //  https://unicode-table.com/en/2019/
+    //             0         1         2         3         4
+    //             01234567890123456789012345678901234567890
+    String TEXT = "He mentionned ’Obama’ in the White House";
+    assertEquals(40, TEXT.length()); // char length (in Java, UTF16)
+
+    String QUOTE = TEXT.substring(14, 15);
+    assertEquals(8217, QUOTE.codePointAt(0));
+
+    //UTF8
+    assertEquals(3, QUOTE.getBytes(StandardCharsets.UTF_8).length);
+    assertEquals(1, "a".getBytes(StandardCharsets.UTF_8).length);
+    assertEquals(40 + 2*2, TEXT.getBytes(StandardCharsets.UTF_8).length);
+
+    //UTF16 big endian    (by specifying big/little endian, there is no "byte order mark")
+    assertEquals(2, QUOTE.getBytes(StandardCharsets.UTF_16BE).length);
+    assertEquals(2, "a".getBytes(StandardCharsets.UTF_16BE).length);
+    assertEquals(40 * 2, TEXT.getBytes(StandardCharsets.UTF_16BE).length);
+
+
+    buildNames("Obama");
+
+    assertTags(TEXT, "Obama");
+
+    // TODO test surrogate pairs (i.e. code points not in the BMP)
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java
new file mode 100644
index 0000000..93b11b5
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java
@@ -0,0 +1,296 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+/**
+ * The original test for {@link TaggerRequestHandler}.
+ */
+public class TaggerTest extends TaggerTestCase {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+  }
+
+  private void indexAndBuild() throws Exception {
+    N[] names = N.values();
+    String[] namesStrs = new String[names.length];
+    for (int i = 0; i < names.length; i++) {
+      namesStrs[i] = names[i].getName();
+    }
+    buildNames(namesStrs);
+  }
+
+  /** Name corpus */
+  enum N {
+    //keep order to retain ord()
+    London, London_Business_School, Boston, City_of_London,
+    of, the//filtered out of the corpus by a custom query
+    ;
+
+    String getName() { return name().replace('_',' '); }
+    static N lookupByName(String name) { return N.valueOf(name.replace(' ', '_')); }
+    int getId() { return ordinal(); }
+  }
+
+  public void testFormat() throws Exception {
+    baseParams.set("overlaps", "NO_SUB");
+    indexAndBuild();
+
+    String rspStr = _testFormatRequest(false);
+    String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+        "<response>\n" +
+        "\n" +
+        "<int name=\"tagsCount\">1</int>\n" +
+        "<arr name=\"tags\">\n" +
+        "  <lst>\n" +
+        "    <int name=\"startOffset\">0</int>\n" +
+        "    <int name=\"endOffset\">22</int>\n" +
+        "    <arr name=\"ids\">\n" +
+        "      <str>1</str>\n" +
+        "    </arr>\n" +
+        "  </lst>\n" +
+        "</arr>\n" +
+        "<result name=\"response\" numFound=\"1\" start=\"0\">\n" +
+        "  <doc>\n" +
+        "    <str name=\"id\">1</str>\n" +
+        "    <str name=\"name\">London Business School</str></doc>\n" +
+        "</result>\n" +
+        "</response>\n";
+    assertEquals(expected, rspStr);
+  }
+
+  public void testFormatMatchText() throws Exception {
+    baseParams.set("overlaps", "NO_SUB");
+    indexAndBuild();
+
+    String rspStr = _testFormatRequest(true);
+    String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+        "<response>\n" +
+        "\n" +
+        "<int name=\"tagsCount\">1</int>\n" +
+        "<arr name=\"tags\">\n" +
+        "  <lst>\n" +
+        "    <int name=\"startOffset\">0</int>\n" +
+        "    <int name=\"endOffset\">22</int>\n" +
+        "    <str name=\"matchText\">london business school</str>\n" +
+        "    <arr name=\"ids\">\n" +
+        "      <str>1</str>\n" +
+        "    </arr>\n" +
+        "  </lst>\n" +
+        "</arr>\n" +
+        "<result name=\"response\" numFound=\"1\" start=\"0\">\n" +
+        "  <doc>\n" +
+        "    <str name=\"id\">1</str>\n" +
+        "    <str name=\"name\">London Business School</str></doc>\n" +
+        "</result>\n" +
+        "</response>\n";
+    assertEquals(expected, rspStr);
+  }
+
+  private String _testFormatRequest(boolean matchText) throws Exception {
+    String doc = "london business school";//just one tag
+    SolrQueryRequest req = reqDoc(doc, "indent", "on", "omitHeader", "on", "matchText", ""+matchText);
+    String rspStr = h.query(req);
+    req.close();
+    return rspStr;
+  }
+
+  /** Partial matching, no sub-tags */
+  @Ignore //TODO ConcatenateGraphFilter uses a special separator char that we can't put into XML (invalid char)
+  public void testPartialMatching() throws Exception {
+    baseParams.set("field", "name_tagPartial");
+    baseParams.set("overlaps", "NO_SUB");
+    baseParams.set("fq", "NOT name:(of the)");//test filtering
+    indexAndBuild();
+
+    //these match nothing
+    assertTags(reqDoc("") );
+    assertTags(reqDoc(" ") );
+    assertTags(reqDoc("the") );
+
+    String doc;
+
+    //just London Business School via "school" substring
+    doc = "school";
+    assertTags(reqDoc(doc), tt(doc,"school", 0, N.London_Business_School));
+
+    doc = "a school";
+    assertTags(reqDoc(doc), tt(doc,"school", 0, N.London_Business_School));
+
+    doc = "school a";
+    assertTags(reqDoc(doc), tt(doc,"school", 0, N.London_Business_School));
+
+    //More interesting
+
+    doc = "school City";
+    assertTags(reqDoc(doc),
+        tt(doc, "school", 0, N.London_Business_School),
+        tt(doc, "City", 0, N.City_of_London) );
+
+    doc = "City of London Business School";
+    assertTags(reqDoc(doc),   //no plain London (sub-tag)
+        tt(doc, "City of London", 0, N.City_of_London),
+        tt(doc, "London Business School", 0, N.London_Business_School));
+  }
+
+  /** whole matching, no sub-tags */
+  public void testWholeMatching() throws Exception {
+    baseParams.set("overlaps", "NO_SUB");
+    baseParams.set("fq", "NOT name:(of the)");//test filtering
+    indexAndBuild();
+
+    //these match nothing
+    assertTags(reqDoc(""));
+    assertTags(reqDoc(" ") );
+    assertTags(reqDoc("the") );
+
+    //partial on N.London_Business_School matches nothing
+    assertTags(reqDoc("school") );
+    assertTags(reqDoc("a school") );
+    assertTags(reqDoc("school a") );
+    assertTags(reqDoc("school City") );
+
+    String doc;
+
+    doc = "school business london";//backwards
+    assertTags(reqDoc(doc), tt(doc,"london", 0, N.London));
+
+    doc = "of London Business School";
+    assertTags(reqDoc(doc),   //no plain London (sub-tag)
+        tt(doc, "London Business School", 0, N.London_Business_School));
+
+    //More interesting
+    doc = "City of London Business School";
+    assertTags(reqDoc(doc),   //no plain London (sub-tag)
+        tt(doc, "City of London", 0, N.City_of_London),
+        tt(doc, "London Business School", 0, N.London_Business_School));
+
+    doc = "City of London Business";
+    assertTags(reqDoc(doc),   //no plain London (sub-tag) no Business (partial-match)
+        tt(doc, "City of London", 0, N.City_of_London));
+
+    doc = "London Business magazine";
+    assertTags(reqDoc(doc),  //Just London; L.B.S. fails
+        tt(doc, "London", 0, N.London));
+  }
+
+  /** whole matching, with sub-tags */
+  public void testSubTags() throws Exception {
+    baseParams.set("overlaps", "ALL");
+    baseParams.set("fq", "NOT name:(of the)");//test filtering
+    indexAndBuild();
+
+    //these match nothing
+    assertTags(reqDoc(""));
+    assertTags(reqDoc(" ") );
+    assertTags(reqDoc("the") );
+
+    //partial on N.London_Business_School matches nothing
+    assertTags(reqDoc("school") );
+    assertTags(reqDoc("a school") );
+    assertTags(reqDoc("school a") );
+    assertTags(reqDoc("school City") );
+
+    String doc;
+
+    doc = "school business london";//backwards
+    assertTags(reqDoc(doc), tt(doc,"london", 0, N.London));
+
+    //More interesting
+    doc = "City of London Business School";
+    assertTags(reqDoc(doc),
+        tt(doc, "City of London", 0, N.City_of_London),
+        tt(doc, "London", 0, N.London),
+        tt(doc, "London Business School", 0, N.London_Business_School));
+
+    doc = "City of London Business";
+    assertTags(reqDoc(doc),
+        tt(doc, "City of London", 0, N.City_of_London),
+        tt(doc, "London", 0, N.London));
+  }
+
+  public void testMultipleFilterQueries() throws Exception {
+    baseParams.set("overlaps", "ALL");
+
+    // build up the corpus with some additional fields for filtering purposes
+    deleteByQueryAndGetVersion("*:*", null);
+
+    int i = 0;
+    assertU(adoc("id", ""+i++, "name", N.London.getName(), "type", "city", "country", "UK"));
+    assertU(adoc("id", ""+i++, "name", N.London_Business_School.getName(), "type", "school", "country", "UK"));
+    assertU(adoc("id", ""+i++, "name", N.Boston.getName(), "type", "city", "country", "US"));
+    assertU(adoc("id", ""+i++, "name", N.City_of_London.getName(), "type", "org", "country", "UK"));
+    assertU(commit());
+
+    // not calling buildNames so that we can bring along extra attributes for filtering
+    NAMES = Arrays.stream(N.values()).map(N::getName).collect(Collectors.toList());
+
+    // phrase that matches everything
+    String doc = "City of London Business School in Boston";
+
+    // first do no filtering
+    ModifiableSolrParams p = new ModifiableSolrParams();
+    p.add(CommonParams.Q, "*:*");
+    assertTags(reqDoc(doc, p),
+        tt(doc, "City of London", 0, N.City_of_London),
+        tt(doc, "London", 0, N.London),
+        tt(doc, "London Business School", 0, N.London_Business_School),
+        tt(doc, "Boston", 0, N.Boston));
+
+    // add a single fq
+    p.add(CommonParams.FQ, "type:city");
+    assertTags(reqDoc(doc, p),
+        tt(doc, "London", 0, N.London),
+        tt(doc, "Boston", 0, N.Boston));
+
+    // add another fq
+    p.add(CommonParams.FQ, "country:US");
+    assertTags(reqDoc(doc, p),
+        tt(doc, "Boston", 0, N.Boston));
+  }
+
+  private TestTag tt(String doc, String substring, int substringIndex, N name) {
+    assert substringIndex == 0;
+
+    //little bit of copy-paste code from super.tt()
+    int startOffset = -1, endOffset;
+    int substringIndex1 = 0;
+    for(int i = 0; i <= substringIndex1; i++) {
+      startOffset = doc.indexOf(substring, ++startOffset);
+      assert startOffset >= 0 : "The test itself is broken";
+    }
+    endOffset = startOffset+ substring.length();//1 greater (exclusive)
+    return new TestTag(startOffset, endOffset, substring, lookupByName(name.getName()));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java
new file mode 100644
index 0000000..e525ce9
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java
@@ -0,0 +1,251 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+import org.apache.commons.lang.builder.CompareToBuilder;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.lucene.document.Document;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.junit.Rule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class TaggerTestCase extends SolrTestCaseJ4 {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  @Rule
+  public TestWatcher watchman = new TestWatcher() {
+    @Override
+    protected void starting(Description description) {
+      log.info("{} being run...", description.getDisplayName());
+    }
+  };
+
+  protected final ModifiableSolrParams baseParams = new ModifiableSolrParams();
+
+  //populated in buildNames; tested in assertTags
+  protected static List<String> NAMES;
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    baseParams.clear();
+    baseParams.set(CommonParams.QT, "/tag");
+    baseParams.set(CommonParams.WT, "xml");
+  }
+
+  protected void assertTags(String doc, String... tags) throws Exception {
+    TestTag[] tts = new TestTag[tags.length];
+    for (int i = 0; i < tags.length; i++) {
+      tts[i] = tt(doc, tags[i]);
+    }
+    assertTags(reqDoc(doc), tts);
+  }
+
+  protected static void buildNames(String... names) throws Exception {
+    deleteByQueryAndGetVersion("*:*", null);
+    NAMES = Arrays.asList(names);
+    //Collections.sort(NAMES);
+    int i = 0;
+    for (String n : NAMES) {
+      assertU(adoc("id", ""+(i++), "name", n));
+    }
+    assertU(commit());
+  }
+
+  protected String lookupByName(String name) {
+    for (String n : NAMES) {
+      if (n.equalsIgnoreCase(name))
+        return n;
+    }
+    return null;
+  }
+
+  protected TestTag tt(String doc, String substring) {
+    int startOffset = -1, endOffset;
+    int substringIndex = 0;
+    for(int i = 0; i <= substringIndex; i++) {
+      startOffset = doc.indexOf(substring,++startOffset);
+      assert startOffset >= 0 : "The test itself is broken";
+    }
+    endOffset = startOffset+substring.length();//1 greater (exclusive)
+    return new TestTag(startOffset, endOffset, substring, lookupByName(substring));
+  }
+
+  /** Asserts the tags.  Will call req.close(). */
+  protected void assertTags(SolrQueryRequest req, TestTag... eTags) throws Exception {
+    try {
+      SolrQueryResponse rsp = h.queryAndResponse(req.getParams().get(CommonParams.QT), req);
+      TestTag[] aTags = pullTagsFromResponse(req, rsp);
+
+      String message;
+      if (aTags.length > 10)
+        message = null;
+      else
+        message = Arrays.asList(aTags).toString();
+      Arrays.sort(eTags);
+      assertSortedArrayEquals(message, eTags, aTags);
+
+    } finally {
+      req.close();
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  protected TestTag[] pullTagsFromResponse(SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException {
+    NamedList rspValues = rsp.getValues();
+    Map<String, String> matchingNames = new HashMap<>();
+    SolrIndexSearcher searcher = req.getSearcher();
+    DocList docList = (DocList) rspValues.get("response");
+    DocIterator iter = docList.iterator();
+    while (iter.hasNext()) {
+      int docId = iter.next();
+      Document doc = searcher.doc(docId);
+      String id = doc.getField("id").stringValue();
+      String name = lookupByName(doc.get("name"));
+      assertEquals("looking for "+name, NAMES.indexOf(name)+"", id);
+      matchingNames.put(id, name);
+    }
+
+    //build TestTag[] aTags from response ('a' is actual)
+    List<NamedList> mTagsList = (List<NamedList>) rspValues.get("tags");
+    List<TestTag> aTags = new ArrayList<>();
+    for (NamedList map : mTagsList) {
+      List<String> foundIds = (List<String>) map.get("ids");
+      for (String id  : foundIds) {
+        aTags.add(new TestTag(
+            ((Number)map.get("startOffset")).intValue(),
+            ((Number)map.get("endOffset")).intValue(),
+            null,
+            matchingNames.get(id)));
+      }
+    }
+    return aTags.toArray(new TestTag[0]);
+  }
+
+  /** REMEMBER to close() the result req object. */
+  protected SolrQueryRequest reqDoc(String doc, String... moreParams) {
+    return reqDoc(doc, params(moreParams));
+  }
+
+  /** REMEMBER to close() the result req object. */
+  protected SolrQueryRequest reqDoc(String doc, SolrParams moreParams) {
+    log.debug("Test doc: "+doc);
+    SolrParams params = SolrParams.wrapDefaults(moreParams, baseParams);
+    SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(), params) {};
+    Iterable<ContentStream> stream = Collections.singleton((ContentStream)new ContentStreamBase.StringStream(doc));
+    req.setContentStreams(stream);
+    return req;
+  }
+
+  /** Asserts the sorted arrays are equals, with a helpful error message when not.*/
+  public void assertSortedArrayEquals(String message, Object[] expecteds, Object[] actuals) {
+    AssertionError error = null;
+    try {
+      assertArrayEquals(null, expecteds, actuals);
+    } catch (AssertionError e) {
+      error = e;
+    }
+    if (error == null)
+      return;
+    TreeSet<Object> expectedRemaining = new TreeSet<>(Arrays.asList(expecteds));
+    expectedRemaining.removeAll(Arrays.asList(actuals));
+    if (!expectedRemaining.isEmpty())
+      fail(message+": didn't find expected "+expectedRemaining.first()+" (of "+expectedRemaining.size()+"); "+ error);
+    TreeSet<Object> actualsRemaining = new TreeSet<>(Arrays.asList(actuals));
+    actualsRemaining.removeAll(Arrays.asList(expecteds));
+    fail(message+": didn't expect "+actualsRemaining.first()+" (of "+actualsRemaining.size()+"); "+ error);
+  }
+
+  class TestTag implements Comparable {
+    final int startOffset, endOffset;
+    final String substring;
+    final String docName;
+
+    TestTag(int startOffset, int endOffset, String substring, String docName) {
+      this.startOffset = startOffset;
+      this.endOffset = endOffset;
+      this.substring = substring;
+      this.docName = docName;
+    }
+
+    @Override
+    public String toString() {
+      return "TestTag{" +
+          "[" + startOffset + "-" + endOffset + "]" +
+          " doc=" + NAMES.indexOf(docName) + ":'" + docName + "'" +
+          (docName.equals(substring) || substring == null ? "" : " substr="+substring)+
+          '}';
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      TestTag that = (TestTag) obj;
+      return new EqualsBuilder()
+          .append(this.startOffset, that.startOffset)
+          .append(this.endOffset, that.endOffset)
+          .append(this.docName, that.docName)
+          .isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+      return startOffset;//cheesy but acceptable
+    }
+
+    @Override
+    public int compareTo(Object o) {
+      TestTag that = (TestTag) o;
+      return new CompareToBuilder()
+          .append(this.startOffset, that.startOffset)
+          .append(this.endOffset, that.endOffset)
+          .append(this.docName,that.docName)
+          .toComparison();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java
new file mode 100644
index 0000000..39c7828
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java
@@ -0,0 +1,73 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the {@link TaggerRequestHandler} with
+ * a Analyzer chain that does use the {@link TaggingAttribute}. See the test
+ * configuration under 'taggingattribute'.
+ */
+public class TaggingAttributeTest extends TaggerTestCase {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+  }
+
+  /**
+   * Whole matching, no sub-tags. Links only words with &gt; 3 letters.
+   * Because of that "San" is not used to start tags
+   *
+   */
+  @Test
+  public void testTaggingAttribute() throws Exception {
+    baseParams.set("field", "name_tagAttribute"); // has WordLengthTaggingFilter using the TaggingAttribute
+    // this test is based on the longest dominant right test, so we use the
+    // the same TagClusterReducer setting
+    baseParams.set("overlaps", "LONGEST_DOMINANT_RIGHT");
+
+    buildNames("in", "San", "in San", "Francisco", "San Francisco",
+        "San Francisco State College", "College of California",
+        "Clayton", "Clayton North", "North Carolina");
+
+    assertTags("He lived in San Francisco.",
+        //"in", "San Francisco"); //whis would be expected without taggable
+        "Francisco");// this are the expected results with taggable
+
+    assertTags("He enrolled in San Francisco State College of California",
+        //"in", "San Francisco State College"); //without taggable enabled
+        "Francisco", "College of California");// With taggable
+    //NOTE this also tests that started tags are advanced for non-taggable
+    //     tokens, as otherwise 'College of California' would not be
+    //     suggested.
+
+    assertTags("He lived in Clayton North Carolina",
+        //"in", "Clayton", "North Carolina");
+        "Clayton", "North Carolina");
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java
new file mode 100644
index 0000000..237a8b8
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java
@@ -0,0 +1,110 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+
+/**
+ * Simple TokenFilter that lookup only Tokens with more as the parsed number
+ * of chars.<p>
+ * <b>NOTE:</b>This implementation is only intended to be used as an example
+ * and for unit testing the {@link TaggingAttribute} feature. Typically
+ * implementations will be based on NLP results (e.g. using POS tags or
+ * detected Named Entities).
+ * <p>
+ * <b>Example Usage:</b><p>
+ * Currently the usage requires to modify the Analyzer as defined by the
+ * <code>indexedField</code>. An alternative would be to allow the configuration
+ * of a special FieldType in the schema.xml and use this Analyzer for processing
+ * the text sent to the request.<p>
+ * While the current solution is fine for direct API usage, defining the
+ * Analyzer in the schema.xml would be better suitable for using this feature
+ * with the {@link TaggerRequestHandler}.
+ *
+ * <pre class="prettyprint">
+ *     Analyzer analyzer = req.getSchema().getField(indexedField).getType().getAnalyzer();
+ *     //get the TokenStream from the Analyzer
+ *     TokenStream baseStream = analyzer.tokenStream("", reader);
+ *     //add a FilterStream that sets the LookupAttribute to the end
+ *     TokenStream filterStream = new WordLengthLookupFilter(baseStream);
+ *     //create the Tagger using the modified analyzer chain.
+ *     new Tagger(corpus, filterStream, tagClusterReducer) {
+ *
+ *         protected void tagCallback(int startOffset, int endOffset, long docIdsKey) {
+ *             //implement the callback
+ *         }
+ *
+ *     }.process();
+ * </pre>
+ */
+public class WordLengthTaggingFilter extends TokenFilter {
+
+  /**
+   * The default minimum length is <code>3</code>
+   */
+  public static final int DEFAULT_MIN_LENGTH = 3;
+  private final TaggingAttribute lookupAtt = addAttribute(TaggingAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private int minLength;
+
+  /**
+   * TokenFilter only marks tokens to be looked up with equals or more as
+   * {@link #DEFAULT_MIN_LENGTH} characters
+   */
+  public WordLengthTaggingFilter(TokenStream input) {
+    this(input, null);
+  }
+
+  /**
+   * TokenFilter only marks tokens to be looked up with equals or more characters
+   * as the parsed minimum.
+   *
+   * @param input     the TokenStream to consume tokens from
+   * @param minLength The minimum length to lookup a Token. <code>null</code>
+   *                  or &lt;= 0 to use the #DEFAULT_MIN_LENGTH
+   */
+  public WordLengthTaggingFilter(TokenStream input, Integer minLength) {
+    super(input);
+    if (minLength == null || minLength <= 0) {
+      this.minLength = DEFAULT_MIN_LENGTH;
+    } else {
+      this.minLength = minLength;
+    }
+  }
+
+  @Override
+  public final boolean incrementToken() throws IOException {
+    if (input.incrementToken()) {
+      int size = offsetAtt.endOffset() - offsetAtt.startOffset();
+      lookupAtt.setTaggable(size >= minLength);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java
new file mode 100644
index 0000000..dbfc538
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java
@@ -0,0 +1,67 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class WordLengthTaggingFilterFactory extends TokenFilterFactory {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  public static final String MIN_LENGTH = "minLength";
+
+  private final Integer minLength;
+
+  public WordLengthTaggingFilterFactory(Map<String, String> args) {
+    super(args);
+    int minLength = -1;
+    Object value = args.get(MIN_LENGTH);
+    if (value != null) {
+      try {
+        minLength = Integer.parseInt(value.toString());
+      } catch (NumberFormatException e) {
+        log.warn("Unable to parse minLength from value 'minLength=\"{}\"'", value);
+
+      }
+    }
+    if (minLength <= 0) {
+      log.info("use default minLength={}", WordLengthTaggingFilter.DEFAULT_MIN_LENGTH);
+      this.minLength = null;
+    } else {
+      log.info("set minLength={}", minLength);
+      this.minLength = minLength;
+    }
+  }
+
+  @Override
+  public TokenStream create(TokenStream input) {
+    return new WordLengthTaggingFilter(input, minLength);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java
new file mode 100644
index 0000000..d7dd5df
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java
@@ -0,0 +1,224 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.xml.sax.InputSource;
+
+public class XmlInterpolationTest extends TaggerTestCase {
+
+  private static DocumentBuilder xmlDocBuilder;
+
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    DocumentBuilderFactory xmlDocBuilderFactory = DocumentBuilderFactory.newInstance();
+    xmlDocBuilderFactory.setValidating(true);
+    xmlDocBuilderFactory.setNamespaceAware(true);
+    xmlDocBuilder = xmlDocBuilderFactory.newDocumentBuilder();
+
+    initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+  }
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    baseParams.set("field", "name_tagXml");
+    baseParams.set("overlaps", "LONGEST_DOMINANT_RIGHT");
+    baseParams.set("xmlOffsetAdjust", "true");
+  }
+
+  @Test
+  public void test() throws Exception {
+    buildNames("start end");
+
+    assertXmlTag("<doc>before start <!-- c --> end after</doc>", true);
+    assertXmlTag("<doc>before start <br/> end after</doc>", true);
+    assertXmlTag("<doc>before <em>start</em> <b>end</b> after</doc>", true);
+    assertXmlTag("<doc>before <em>start</em> end after</doc>", true);
+    assertXmlTag("<doc>before start end<em> after</em></doc>", true);
+    assertXmlTag("<doc><em>before </em>start end after</doc>", true);//adjacent tags
+    assertXmlTag("<doc>before <b> <em>start</em> </b> end after</doc>", true);
+    assertXmlTag("<doc>before <b> <em>start</em> </b> <em>  end  </em> after</doc>", true);
+
+    assertXmlTag("<doc><p>before start</p> end after</doc>", false);
+    assertXmlTag("<doc>before start <p>end after</p> </doc>", false);
+
+    assertXmlTag("<doc>before <em a='A' b='B'>start</em> <b a='A' b='B'>end</b> after</doc>", true);
+  }
+
+  @Test(expected = SolrException.class)
+  public void testInvalidXml() throws Exception {
+    assertXmlTag("notXml", false);
+  }
+
+  @Test(expected = Exception.class)
+  public void testValidatingXml() throws Exception {
+    validateXml("foo");
+  }
+
+  protected void assertXmlTag(String docText, boolean expected) throws Exception {
+    final SolrQueryRequest req = reqDoc(docText);
+    try { // 5.4 and beyond we can use try-with-resources
+      final SolrQueryResponse rsp = h.queryAndResponse(req.getParams().get("qt"), req);
+      final TestTag[] testTags = pullTagsFromResponse(req, rsp);
+      if (!expected) {
+        assertEquals(0, testTags.length);
+      } else {
+        assertEquals(1, testTags.length);
+        final TestTag tag = testTags[0];
+        validateXml(insertAnchorAtOffsets(docText, tag.startOffset, tag.endOffset, tag.docName));
+      }
+    } finally {
+      req.close();
+    }
+  }
+
+  protected void validateXml(String xml) throws Exception {
+    // the "parse" method also validates XML, will throw an exception if mis-formatted
+    xmlDocBuilder.parse(new InputSource(new StringReader(xml)));
+  }
+
+
+  @Test
+  public void testLuceneHtmlFilterBehavior() {
+    String docText;
+
+    //Close tag adjacent to start & end results in end offset including the close tag. LUCENE-5734
+    docText = "<doc><a><b>start</b> end</a></doc>";
+    assertArrayEquals(tagExpect(docText, "start", "end</a>"), analyzeTagOne(docText, "start", "end"));
+
+    //Space after "end" means offset doesn't include </a>
+    docText = "<doc><a><b>start</b> end </a></doc>";
+    assertArrayEquals(tagExpect(docText, "start", "end"), analyzeTagOne(docText, "start", "end"));
+
+    //Matches entity at end
+    final String endStr = String.format(Locale.ROOT, "en&#x%02x;", (int) 'd');
+    docText = "<doc>start " + endStr + "</doc>";
+    assertArrayEquals(tagExpect(docText, "start", endStr), analyzeTagOne(docText, "start", "end"));
+    //... and at start
+    final String startStr = String.format(Locale.ROOT, "&#x%02x;tart", (int) 's');
+    docText = "<doc>" + startStr + " end</doc>";
+    assertArrayEquals(tagExpect(docText, startStr, "end"), analyzeTagOne(docText, "start", "end"));
+
+    //Test ignoring proc instructions & comments. Note: doesn't expand the entity to "start".
+    docText = "<!DOCTYPE start [ "
+            + "<!ENTITY start \"start\">"
+            + "]><start><?start start ?><!-- start --><start/>&start;</start>";
+    assertArrayEquals(new int[]{-1, -1}, analyzeTagOne(docText, "start", "start"));
+
+    //Test entity behavior
+    docText =                " &mdash; &ndash; &amp; &foo; &#xA0; a&nbsp;b";
+    assertArrayEquals(new String[]{"—", "–", "&", "&foo;", "\u00A0", "a", "b"},
+            analyzeReturnTokens(docText));
+
+    //Observe offset adjustment of trailing entity to end tag
+    docText = "foo&nbsp;bar";
+    assertArrayEquals(tagExpect(docText, "foo", "foo"), analyzeTagOne(docText, "foo", "foo"));
+  }
+
+  private String insertAnchorAtOffsets(String docText, int startOffset, int endOffset, String id) {
+    String insertStart = "<A id='"+ id +"'>";// (normally we'd escape id)
+    String insertEnd = "</A>";
+    return docText.substring(0, startOffset)
+            + insertStart
+            + docText.substring(startOffset, endOffset)
+            + insertEnd
+            + docText.substring(endOffset);
+  }
+
+  private int[] tagExpect(String docText, String start, String end) {
+    return new int[]{docText.indexOf(start), docText.indexOf(end) + end.length()};
+  }
+
+  private int[] analyzeTagOne(String docText, String start, String end) {
+    int[] result = {-1, -1};
+
+    Reader filter = new HTMLStripCharFilter(new StringReader(docText));
+
+    WhitespaceTokenizer ts = new WhitespaceTokenizer();
+    final CharTermAttribute termAttribute = ts.addAttribute(CharTermAttribute.class);
+    final OffsetAttribute offsetAttribute = ts.addAttribute(OffsetAttribute.class);
+    try {
+      ts.setReader(filter);
+      ts.reset();
+      while (ts.incrementToken()) {
+        final String termString = termAttribute.toString();
+        if (termString.equals(start))
+          result[0] = offsetAttribute.startOffset();
+        if (termString.equals(end)) {
+          result[1] = offsetAttribute.endOffset();
+          return result;
+        }
+      }
+      ts.end();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    } finally {
+      IOUtils.closeQuietly(ts);
+    }
+    return result;
+  }
+
+  private String[] analyzeReturnTokens(String docText) {
+    List<String> result = new ArrayList<>();
+
+    Reader filter = new HTMLStripCharFilter(new StringReader(docText),
+            Collections.singleton("unescaped"));
+    WhitespaceTokenizer ts = new WhitespaceTokenizer();
+    final CharTermAttribute termAttribute = ts.addAttribute(CharTermAttribute.class);
+    try {
+      ts.setReader(filter);
+      ts.reset();
+      while (ts.incrementToken()) {
+        result.add(termAttribute.toString());
+      }
+      ts.end();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    } finally {
+      IOUtils.closeQuietly(ts);
+    }
+    return result.toArray(new String[result.size()]);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/solr-ref-guide/src/searching.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/searching.adoc b/solr/solr-ref-guide/src/searching.adoc
index 145c1a4..753c2d8 100644
--- a/solr/solr-ref-guide/src/searching.adoc
+++ b/solr/solr-ref-guide/src/searching.adoc
@@ -1,5 +1,35 @@
 = Searching
-:page-children: overview-of-searching-in-solr, velocity-search-ui, relevance, query-syntax-and-parsing, json-request-api, json-facet-api, faceting, highlighting, spell-checking, query-re-ranking, transforming-result-documents, suggester, morelikethis, pagination-of-results, collapse-and-expand-results, result-grouping, result-clustering, spatial-search, the-terms-component, the-term-vector-component, the-stats-component, the-query-elevation-component, response-writers, near-real-time-searching, realtime-get, exporting-result-sets, streaming-expressions, parallel-sql-interface, analytics
+:page-children: overview-of-searching-in-solr, +
+  velocity-search-ui, +
+  relevance, +
+  query-syntax-and-parsing, +
+  json-request-api, +
+  json-facet-api, +
+  faceting, +
+  highlighting, +
+  spell-checking, +
+  query-re-ranking, +
+  transforming-result-documents, +
+  suggester, +
+  morelikethis, +
+  pagination-of-results, +
+  collapse-and-expand-results, +
+  result-grouping, +
+  result-clustering, +
+  spatial-search, +
+  the-terms-component, +
+  the-term-vector-component, +
+  the-stats-component, +
+  the-query-elevation-component, +
+  the-tagger-handler, +
+  response-writers, +
+  near-real-time-searching, +
+  realtime-get, +
+  exporting-result-sets, +
+  streaming-expressions, +
+  parallel-sql-interface, +
+  analytics
+
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
@@ -50,6 +80,7 @@ This section describes how Solr works with search requests. It covers the follow
 * <<the-term-vector-component.adoc#the-term-vector-component,The Term Vector Component>>: How to get term information about specific documents.
 * <<the-stats-component.adoc#the-stats-component,The Stats Component>>: How to return information from numeric fields within a document set.
 * <<the-query-elevation-component.adoc#the-query-elevation-component,The Query Elevation Component>>: How to force documents to the top of the results for certain queries.
+* <<the-tagger-handler.adoc#the-tagger-handler,The Tagger Handler>>: The SolrTextTagger, for basic named entity tagging in text.
 * <<response-writers.adoc#response-writers,Response Writers>>: Detailed information about configuring and using Solr's response writers.
 * <<near-real-time-searching.adoc#near-real-time-searching,Near Real Time Searching>>: How to include documents in search results nearly immediately after they are indexed.
 * <<realtime-get.adoc#realtime-get,RealTime Get>>: How to get the latest version of a document without opening a searcher.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/solr-ref-guide/src/the-tagger-handler.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/the-tagger-handler.adoc b/solr/solr-ref-guide/src/the-tagger-handler.adoc
new file mode 100644
index 0000000..14ba8ed
--- /dev/null
+++ b/solr/solr-ref-guide/src/the-tagger-handler.adoc
@@ -0,0 +1,265 @@
+[[the-tagger-handler]]
+= The Tagger Handler
+
+The "Tagger" Request Handler, AKA the "SolrTextTagger" is a "text tagger".
+Given a dictionary (a Solr index) with a name-like field,
+  you post text to this request handler and it will return every occurrence of one of those names with offsets and other document metadata desired.
+It's used for named entity recognition (NER).
+It doesn't do any NLP (outside of Lucene text analysis) so it's said to be a "naive tagger",
+  but it's definitely useful as-is and a more complete NER or ERD (entity recognition and disambiguation)
+  system can be built with this as a key component.
+The SolrTextTagger might be used on queries for query-understanding or large documents as well.
+
+To get a sense of how to use it, jump to the tutorial below.
+
+The tagger does not yet support a sharded index.
+Tens, perhaps hundreds of millions of names (documents) are supported, mostly limited by memory.
+
+[[tagger-configuration]]
+== Configuration
+
+The Solr schema needs 2 things:
+
+* A unique key field (see `<uniqueKey>`).
+  Recommended field settings: set `docValues=true`
+* A tag field, a TextField, with `ConcatenateGraphFilterFactory` at the end of the index chain (not the query chain):
+  Set `preservePositionIncrements=false` on that filter.
+  Recommended field settings: `omitNorms=true`, `omitTermFreqAndPositions=true` and `postingsFormat=FST50`
+
+The text field's _index analysis chain_, aside from needing ConcatenateGraphFilterFactory at the end,
+  can otherwise have whatever tokenizer and filters suit your matching preferences.
+It can have multi-word synonyms and use WordDelimiterGraphFilterFactory for example.
+However, do _not_ use FlattenGraphFilterFactory as it will interfere with ConcatenateGraphFilterFactory.
+Position gaps (e.g. stop words) get ignored; it's not (yet) supported for the gap to be significant.
+
+The text field's _query analysis chain_, on the other hand, is more limited.
+There should not be tokens at the same position, thus no synonym expansion -- do that at index time instead.
+Stop words (or any other filter introducing a position gap) are supported.
+At runtime the tagger can be configured to either treat it as a tag break or to ignore it.
+
+The Solr config needs the `solr.TagRequestHandler` defined, which supports `defaults`, `invariants`, and `appends`
+sections just like the search handler.
+
+[[tagger-parameters]]
+== Tagger Parameters
+
+The tagger's execution is completely configurable with request parameters.  Only `field` is required.
+
+`field`::
+  The tag field that serves as the dictionary.
+  This is required; you'll probably specify it in the request handler.
+
+`fq`::
+  You can specify some number of _filter queries_ to limit the dictionary used for tagging.
+  This parameter is the same as is used by the `solr.SearchHandler`.
+
+`rows`::
+  The maximum number of documents to return, but defaulting to 10000 for a tag request.
+  This parameter is the same as is used by the `solr.SearchHandler`.
+
+`fl`::
+  Solr's standard param for listing the fields to return.
+  This parameter is the same as is used by the `solr.SearchHandler`.
+
+`overlaps`::
+  Choose the algorithm to determine which tags in an overlapping set should be retained, versus being pruned away.
+  Options are:
+
+  * `ALL`: Emit all tags.
+  * `NO_SUB`: Don't emit a tag that is completely within another tag (i.e. no subtag).
+  * `LONGEST_DOMINANT_RIGHT`: Given a cluster of overlapping tags, emit the longest one (by character length).
+     If there is a tie, pick the right-most.
+     Remove any tags overlapping with this tag then repeat the algorithm to potentially find other tags
+     that can be emitted in the cluster.
+
+`matchText`::
+  A boolean indicating whether to return the matched text in the tag response.
+  This will trigger the tagger to fully buffer the input before tagging.
+
+`tagsLimit`::
+  The maximum number of tags to return in the response.
+  Tagging effectively stops after this point.
+  By default this is 1000.
+
+`skipAltTokens`::
+  A boolean flag used to suppress errors that can occur if, for example,
+  you enable synonym expansion at query time in the analyzer, which you normally shouldn't do.
+  Let this default to false unless you know that such tokens can't be avoided.
+
+`ignoreStopwords`::
+  A boolean flag that causes stopwords (or any condition causing positions to skip like >255 char words)
+  to be ignored as if it wasn't there.
+  Otherwise, the behavior is to treat them as breaks in tagging on the presumption your indexed text-analysis
+  configuration doesn't have a StopWordFilter.
+  By default the indexed analysis chain is checked for the presence of a StopWordFilter and if found
+  then ignoreStopWords is true if unspecified.
+  You probably shouldn't have a StopWordFilter configured and probably won't need to set this param either.
+
+`xmlOffsetAdjust`::
+  A boolean indicating that the input is XML and furthermore that the offsets of returned tags should be adjusted as
+  necessary to allow for the client to insert an openening and closing element at the tag offset pair.
+  If it isn't possible to do so then the tag will be omitted.
+  You are expected to configure `HTMLStripCharFilterFactory` in the schema when using this option.
+  This will trigger the tagger to fully buffer the input before tagging.
+
+Solr's parameters for controlling the response format are supported, like:
+  `echoParams`, `wt`, `indent`, etc.
+
+[[tagger-tutorial-with-geonames]]
+== Tutorial with Geonames
+
+This is a tutorial that demonstrates how to configure and use the text
+tagger with the popular Geonames data set. It's more than a tutorial;
+it's a how-to with information that wasn't described above.
+
+[[tagger-create-and-configure-a-solr-collection]]
+=== Create and Configure a Solr Collection
+
+Create a Solr collection named "geonames". For the tutorial, we'll
+assume the default "data-driven" configuration. It's good for
+experimentation and getting going fast but not for production or being
+optimal.
+
+....
+bin/solr create -c geonames
+....
+
+[[tagger-configuring]]
+==== Configuring
+
+We need to configure the schema first. The "data driven" mode we're
+using allows us to keep this step fairly minimal -- we just need to
+declare a field type, 2 fields, and a copy-field. The critical part
+up-front is to define the "tag" field type. There are many many ways to
+configure text analysis; and we're not going to get into those choices
+here. But an important bit is the `ConcatenateGraphFilterFactory` at the
+end of the index analyzer chain. Another important bit for performance
+is postingsFormat=FST50 resulting in a compact FST based in-memory data
+structure that is especially beneficial for the text tagger.
+
+Schema configuration:
+
+....
+curl -X POST -H 'Content-type:application/json'  http://localhost:8983/solr/geonames/schema -d '{
+  "add-field-type":{
+    "name":"tag",
+    "class":"solr.TextField",
+    "postingsFormat":"FST50",
+    "omitNorms":true,
+    "omitTermFreqAndPositions":true,
+    "indexAnalyzer":{
+      "tokenizer":{
+         "class":"solr.StandardTokenizerFactory" },
+      "filters":[
+        {"class":"solr.EnglishPossessiveFilterFactory"},
+        {"class":"solr.ASCIIFoldingFilterFactory"},
+        {"class":"solr.LowerCaseFilterFactory"},
+        {"class":"solr.ConcatenateGraphFilterFactory", "preservePositionIncrements":false }
+      ]},
+    "queryAnalyzer":{
+      "tokenizer":{
+         "class":"solr.StandardTokenizerFactory" },
+      "filters":[
+        {"class":"solr.EnglishPossessiveFilterFactory"},
+        {"class":"solr.ASCIIFoldingFilterFactory"},
+        {"class":"solr.LowerCaseFilterFactory"}
+      ]}
+    },
+
+  "add-field":{ "name":"name",     "type":"text_general"},
+
+  "add-field":{ "name":"name_tag", "type":"tag",          "stored":false },
+
+  "add-copy-field":{ "source":"name", "dest":[ "name_tag" ]}
+}'
+....
+
+Configure a custom Solr Request Handler:
+
+....
+curl -X POST -H 'Content-type:application/json' http://localhost:8983/solr/geonames/config -d '{
+  "add-requesthandler" : {
+    "name": "/tag",
+    "class":"solr.TaggerRequestHandler",
+    "defaults":{ "field":"name_tag" }
+  }
+}'
+....
+
+[[tagger-load-some-sample-data]]
+=== Load Some Sample Data
+
+We'll go with some Geonames.org data in CSV format. Solr is quite
+flexible in loading data in a variety of formats. This
+http://download.geonames.org/export/dump/cities1000.zip[cities1000.zip]
+should be almost 7MB file expanding to a cities1000.txt file around
+22.2MB containing 145k lines, each a city in the world of at least 1000
+population.
+
+Using bin/post:
+....
+bin/post -c geonames -type text/csv \
+  -params 'optimize=true&separator=%09&encapsulator=%00&fieldnames=id,name,,alternative_names,latitude,longitude,,,countrycode,,,,,,population,elevation,,timezone,lastupdate' \
+  /tmp/cities1000.txt
+....
+or using curl:
+....
+curl -X POST --data-binary @/path/to/cities1000.txt -H 'Content-type:application/csv' \
+  'http://localhost:8983/solr/geonames/update?commit=true&optimize=true&separator=%09&encapsulator=%00&fieldnames=id,name,,alternative_names,latitude,longitude,,,countrycode,,,,,,population,elevation,,timezone,lastupdate'
+....
+
+That might take around 35 seconds; it depends. It can be a lot faster if
+the schema were tuned to only have what we truly need (no text search if
+not needed).
+
+In that command we said optimize=true to put the index in a state that
+will make tagging faster. The encapsulator=%00 is a bit of a hack to
+disable the default double-quote.
+
+[[tagger-tag-time]]
+=== Tag Time!
+
+This is a trivial example tagging a small piece of text. For more
+options, see the earlier documentation.
+
+....
+curl -X POST \
+  'http://localhost:8983/solr/geonames/tag?overlaps=NO_SUB&tagsLimit=5000&fl=id,name,countrycode&wt=json&indent=on' \
+  -H 'Content-Type:text/plain' -d 'Hello New York City'
+....
+
+The response should be this (the QTime may vary):
+
+....
+{
+  "responseHeader":{
+    "status":0,
+    "QTime":1},
+  "tagsCount":1,
+  "tags":[[
+      "startOffset",6,
+      "endOffset",19,
+      "ids",["5128581"]]],
+  "response":{"numFound":1,"start":0,"docs":[
+      {
+        "id":"5128581",
+        "name":["New York City"],
+        "countrycode":["US"]}]
+  }}
+....
+
+[[tagger-tips]]
+== Tips
+
+Performance Tips:
+
+* Follow the recommended configuration field settings, especially `postingsFormat=FST50`.
+* "optimize" after loading your dictionary down to 1 Lucene segment, or at least to as few as possible.
+* For bulk tagging lots of documents, there are some strategies, not mutually exclusive:
+** Batch them.
+   The tagger doesn't directly support batching but as a hack you can send a bunch of documents concatenated with
+     a nonsense word that is not in the dictionary like "ZZYYXXAABBCC" between them.
+     You'll need to keep track of the character offsets of these so you can subtract them from the results.
+** For reducing tagging latency even further, consider embedding Solr with `EmbeddedSolrServer`.
+   See `EmbeddedSolrNoSerializeTest`.
+** Use more than one thread -- perhaps as many as there are CPU cores available to Solr.
\ No newline at end of file


[06/18] lucene-solr:jira/solr-12438: SOLR-12376: New TaggerRequestHandler (SolrTextTagger).

Posted by ab...@apache.org.
SOLR-12376: New TaggerRequestHandler (SolrTextTagger).


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cf633921
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cf633921
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cf633921

Branch: refs/heads/jira/solr-12438
Commit: cf63392183ffc96428fc4c52f546fec2cdf766d5
Parents: c587598
Author: David Smiley <ds...@apache.org>
Authored: Tue Jun 5 14:04:55 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Jun 5 14:04:55 2018 -0400

----------------------------------------------------------------------
 solr/CHANGES.txt                                |   3 +
 solr/NOTICE.txt                                 |  14 +
 .../apache/solr/core/SolrResourceLoader.java    |  12 +-
 .../solr/handler/tagger/OffsetCorrector.java    | 178 +++++++++
 .../solr/handler/tagger/TagClusterReducer.java  | 103 +++++
 .../org/apache/solr/handler/tagger/TagLL.java   | 176 ++++++++
 .../org/apache/solr/handler/tagger/Tagger.java  | 230 +++++++++++
 .../handler/tagger/TaggerRequestHandler.java    | 397 +++++++++++++++++++
 .../solr/handler/tagger/TaggingAttribute.java   |  65 +++
 .../handler/tagger/TaggingAttributeImpl.java    |  79 ++++
 .../solr/handler/tagger/TermPrefixCursor.java   | 189 +++++++++
 .../solr/handler/tagger/XmlOffsetCorrector.java | 113 ++++++
 .../solr/handler/tagger/package-info.java       |  27 ++
 .../solr/collection1/conf/schema-tagger.xml     | 187 +++++++++
 .../solr/collection1/conf/solrconfig-tagger.xml |  59 +++
 .../tagger/EmbeddedSolrNoSerializeTest.java     | 153 +++++++
 .../handler/tagger/RandomizedTaggerTest.java    | 150 +++++++
 .../apache/solr/handler/tagger/Tagger2Test.java | 175 ++++++++
 .../apache/solr/handler/tagger/TaggerTest.java  | 296 ++++++++++++++
 .../solr/handler/tagger/TaggerTestCase.java     | 251 ++++++++++++
 .../handler/tagger/TaggingAttributeTest.java    |  73 ++++
 .../handler/tagger/WordLengthTaggingFilter.java | 110 +++++
 .../tagger/WordLengthTaggingFilterFactory.java  |  67 ++++
 .../handler/tagger/XmlInterpolationTest.java    | 224 +++++++++++
 solr/solr-ref-guide/src/searching.adoc          |  33 +-
 solr/solr-ref-guide/src/the-tagger-handler.adoc | 265 +++++++++++++
 26 files changed, 3622 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 66d2026..479406f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -154,6 +154,9 @@ New Features
 
 * SOLR-12389: support deeply nested json objects in clusterprops.json (noble)
 
+* SOLR-12376: Added the TaggerRequestHandler (AKA SolrTextTagger) for tagging text.  It's used as a component of
+  NER/ERD systems including query-understanding.  See the ref guide for more info.  (David Smiley
+
 Bug Fixes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/NOTICE.txt
----------------------------------------------------------------------
diff --git a/solr/NOTICE.txt b/solr/NOTICE.txt
index fd954f4..a5b2070 100644
--- a/solr/NOTICE.txt
+++ b/solr/NOTICE.txt
@@ -537,3 +537,17 @@ See http://www.restlet.org/
 Protocol Buffers - Google's data interchange format
 Copyright 2008 Google Inc.
 http://code.google.com/apis/protocolbuffers/
+
+=========================================================================
+==     SolrTextTagger Notice                                           ==
+=========================================================================
+
+The TaggerRequestHandler and related classes in its package came from the
+OpenSextant Solr Text Tagger,
+Copyright 2013 The MITRE Corporation. All Rights Reserved.
+
+  This software was produced for the U. S. Government
+  under Contract No. W15P7T-11-C-F600, and is
+  subject to the Rights in Noncommercial Computer Software
+  and Noncommercial Computer Software Documentation
+  Clause 252.227-7014 (JUN 1995)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 22753dd..0ff5c7b 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -16,6 +16,10 @@
  */
 package org.apache.solr.core;
 
+import javax.naming.Context;
+import javax.naming.InitialContext;
+import javax.naming.NamingException;
+import javax.naming.NoInitialContextException;
 import java.io.Closeable;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -47,10 +51,6 @@ import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import javax.naming.Context;
-import javax.naming.InitialContext;
-import javax.naming.NamingException;
-import javax.naming.NoInitialContextException;
 
 import org.apache.lucene.analysis.WordlistLoader;
 import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -88,9 +88,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   static final String project = "solr";
   static final String base = "org.apache" + "." + project;
   static final String[] packages = {
-      "", "analysis.", "schema.", "handler.", "search.", "update.", "core.", "response.", "request.",
+      "", "analysis.", "schema.", "handler.", "handler.tagger.", "search.", "update.", "core.", "response.", "request.",
       "update.processor.", "util.", "spelling.", "handler.component.", "handler.dataimport.",
-      "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.","handler.admin.",
+      "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.", "handler.admin.",
       "cloud.autoscaling."
   };
   private static final java.lang.String SOLR_CORE_NAME = "solr.core.name";

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java b/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java
new file mode 100644
index 0000000..1fb4911
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java
@@ -0,0 +1,178 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.util.Arrays;
+
+import com.carrotsearch.hppc.IntArrayList;
+
+public abstract class OffsetCorrector {
+
+  //TODO support a streaming style of consuming input text so that we need not take a
+  // String. Trickier because we need to keep more information as we parse to know when tags
+  // are adjacent with/without whitespace
+
+  //Data structure requirements:
+  // Given a character offset:
+  //   * determine what tagId is it's parent.
+  //   * determine if it is adjacent to the parent open tag, ignoring whitespace
+  //   * determine if it is adjacent to the parent close tag, ignoring whitespace
+  // Given a tagId:
+  //   * What is it's parent tagId
+  //   * What's the char offset of the start and end of the open tag
+  //   * What's the char offset of the start and end of the close tag
+
+  /** Document text. */
+  protected final String docText;
+
+  /** Array of tag info comprised of 5 int fields:
+   *    [int parentTag, int openStartOff, int openEndOff, int closeStartOff, int closeEndOff].
+   * It's size indicates how many tags there are. Tag's are ID'ed sequentially from 0. */
+  protected final IntArrayList tagInfo;
+
+  /** offsets of parent tag id change (ascending order) */
+  protected final IntArrayList parentChangeOffsets;
+  /** tag id; parallel array to parentChangeOffsets */
+  protected final IntArrayList parentChangeIds;
+
+  protected final int[] offsetPair = new int[] { -1, -1};//non-thread-safe state
+
+  /** Disjoint start and end span offsets (inclusive) of non-taggable sections. Null if none. */
+  protected final IntArrayList nonTaggableOffsets;
+
+  /**
+   * Initialize based on the document text.
+   * @param docText non-null structured content.
+   * @param hasNonTaggable if there may be "non-taggable" tags to track
+   */
+  protected OffsetCorrector(String docText, boolean hasNonTaggable) {
+    this.docText = docText;
+    final int guessNumElements = Math.max(docText.length() / 20, 4);
+
+    tagInfo = new IntArrayList(guessNumElements * 5);
+    parentChangeOffsets = new IntArrayList(guessNumElements * 2);
+    parentChangeIds = new IntArrayList(guessNumElements * 2);
+    nonTaggableOffsets = hasNonTaggable ? new IntArrayList(guessNumElements / 5) : null;
+  }
+
+  /** Corrects the start and end offset pair. It will return null if it can't
+   * due to a failure to keep the offsets balance-able, or if it spans "non-taggable" tags.
+   * The start (left) offset is pulled left as needed over whitespace and opening tags. The end
+   * (right) offset is pulled right as needed over whitespace and closing tags. It's returned as
+   * a 2-element array.
+   * <p>Note that the returned array is internally reused; just use it to examine the response.
+   */
+  public int[] correctPair(int leftOffset, int rightOffset) {
+    rightOffset = correctEndOffsetForCloseElement(rightOffset);
+    if (spansNonTaggable(leftOffset, rightOffset))
+      return null;
+
+    int startTag = lookupTag(leftOffset);
+    //offsetPair[0] = Math.max(offsetPair[0], getOpenStartOff(startTag));
+    int endTag = lookupTag(rightOffset-1);
+    //offsetPair[1] = Math.min(offsetPair[1], getCloseStartOff(endTag));
+
+    // Find the ancestor tag enclosing offsetPair.  And bump out left offset along the way.
+    int iTag = startTag;
+    for (; !tagEnclosesOffset(iTag, rightOffset); iTag = getParentTag(iTag)) {
+      //Ensure there is nothing except whitespace thru OpenEndOff
+      int tagOpenEndOff = getOpenEndOff(iTag);
+      if (hasNonWhitespace(tagOpenEndOff, leftOffset))
+        return null;
+      leftOffset = getOpenStartOff(iTag);
+    }
+    final int ancestorTag = iTag;
+    // Bump out rightOffset until we get to ancestorTag.
+    for (iTag = endTag; iTag != ancestorTag; iTag = getParentTag(iTag)) {
+      //Ensure there is nothing except whitespace thru CloseStartOff
+      int tagCloseStartOff = getCloseStartOff(iTag);
+      if (hasNonWhitespace(rightOffset, tagCloseStartOff))
+        return null;
+      rightOffset = getCloseEndOff(iTag);
+    }
+
+    offsetPair[0] = leftOffset;
+    offsetPair[1] = rightOffset;
+    return offsetPair;
+  }
+
+  /** Correct endOffset for adjacent element at the right side.  E.g. offsetPair might point to:
+   * <pre>
+   *   foo&lt;/tag&gt;
+   * </pre>
+   * and this method pulls the end offset left to the '&lt;'. This is necessary for use with
+   * {@link org.apache.lucene.analysis.charfilter.HTMLStripCharFilter}.
+   *
+   * See https://issues.apache.org/jira/browse/LUCENE-5734 */
+  protected int correctEndOffsetForCloseElement(int endOffset) {
+    if (docText.charAt(endOffset-1) == '>') {
+      final int newEndOffset = docText.lastIndexOf('<', endOffset - 2);
+      if (newEndOffset > offsetPair[0])//just to be sure
+        return newEndOffset;
+    }
+    return endOffset;
+  }
+
+  protected boolean hasNonWhitespace(int start, int end) {
+    for (int i = start; i < end; i++) {
+      if (!Character.isWhitespace(docText.charAt(i)))
+        return true;
+    }
+    return false;
+  }
+
+  protected boolean tagEnclosesOffset(int tag, int off) {
+    return off >= getOpenStartOff(tag) && off < getCloseEndOff(tag);
+  }
+
+  protected int getParentTag(int tag) { return tagInfo.get(tag * 5 + 0); }
+  protected int getOpenStartOff(int tag) { return tagInfo.get(tag * 5 + 1); }
+  protected int getOpenEndOff(int tag) { return tagInfo.get(tag * 5 + 2); }
+  protected int getCloseStartOff(int tag) { return tagInfo.get(tag * 5 + 3); }
+  protected int getCloseEndOff(int tag) { return tagInfo.get(tag * 5 + 4); }
+
+  protected int lookupTag(int off) {
+    int idx = Arrays.binarySearch(parentChangeOffsets.buffer, 0, parentChangeOffsets.size(), off);
+    if (idx < 0)
+      idx = (-idx - 1) - 1;//round down
+    return parentChangeIds.get(idx);
+  }
+
+  protected boolean spansNonTaggable(int startOff, int endOff) {
+    if (nonTaggableOffsets == null)
+      return false;
+    int idx = Arrays.binarySearch(nonTaggableOffsets.buffer, 0, nonTaggableOffsets.size(), startOff);
+    //if tag start coincides with first or last char of non-taggable span then result is true.
+    // (probably never happens since those characters are actual element markup)
+    if (idx >= 0)
+      return true;
+    idx = -idx - 1;//modify for where we would insert
+    //if idx is odd then our span intersects a non-taggable span; return true
+    if ((idx & 1) == 1)
+      return true;
+    //it's non-taggable if the next non-taggable start span is before our endOff
+    if (idx == nonTaggableOffsets.size())
+      return false;
+    return nonTaggableOffsets.get(idx) < endOff;
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java b/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java
new file mode 100644
index 0000000..9310a04
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java
@@ -0,0 +1,103 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+public interface TagClusterReducer {
+  /**
+   * Reduces the linked-list to only those tags that should be emitted
+   * @param head not null; 1-element array to head which isn't null either
+   */
+  void reduce(TagLL[] head);
+
+  static final TagClusterReducer ALL = new TagClusterReducer() {
+    @Override
+    public void reduce(TagLL[] head) {
+    }
+  };
+
+  static final TagClusterReducer NO_SUB = new TagClusterReducer() {
+    @Override
+    public void reduce(TagLL[] head) {
+      //loop forward over all tags
+      for (TagLL tag = head[0].nextTag; tag != null; tag = tag.nextTag) {
+        //loop backwards over prev tags from this tag
+        for (TagLL tPrev = tag.prevTag; tPrev != null; tPrev = tPrev.prevTag) {
+          assert tPrev.startOffset <= tag.startOffset;
+          //if a previous tag's endOffset is <= this one's, tForward can be removed
+          if (tPrev.endOffset >= tag.endOffset) {
+            tag.removeLL();
+            break;
+          } else if (tPrev.startOffset == tag.startOffset) {
+            tPrev.removeLL();
+            //continue; 'tag' is still valid
+          }
+        }
+      }
+    }
+  };
+
+  static final TagClusterReducer LONGEST_DOMINANT_RIGHT = new TagClusterReducer() {
+    @Override
+    public void reduce(TagLL[] head) {
+
+      //--Optimize for common single-tag case
+      if (head[0].nextTag == null)
+        return;
+
+      while (true) {
+        //--Find longest not already marked
+        TagLL longest = null;
+        for (TagLL t = head[0]; t != null; t = t.nextTag) {
+          if (!t.mark && (longest == null || t.charLen() >= longest.charLen()))
+            longest = t;
+        }
+        if (longest == null)
+          break;
+        //--Mark longest (so we return it eventually)
+        longest.mark = true;
+        //--Remove tags overlapping this longest
+        for (TagLL t = head[0]; t != null; t = t.nextTag) {
+          if (t.mark)
+            continue;
+
+          if (t.overlaps(longest)) {
+            t.removeLL();
+          } else if (t.startOffset >= longest.endOffset) {
+            break;//no subsequent can possibly overlap
+          }
+        }
+      }//loop
+
+      //all-remaining should be marked
+//      for (TagLL t = head; t != null; t = t.nextTag) {
+//        assert t.mark;
+////        if (!t.mark) {
+////          t.removeLL();
+////          if (head == t)
+////            head = t.nextTag;
+////        }
+//      }
+      assert head[0].mark;
+    }
+  };
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java b/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java
new file mode 100644
index 0000000..e8bb0a3
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java
@@ -0,0 +1,176 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * This is a Tag -- a startOffset, endOffset and value.
+ * <p>
+ * A Tag starts without a value in an
+ * "advancing" state.  {@link #advance(org.apache.lucene.util.BytesRef, int)}
+ * is called with subsequent words and then eventually it won't advance any
+ * more, and value is set (could be null).
+ * <p>
+ * A Tag is also a doubly-linked-list (hence the LL in the name). All tags share
+ * a reference to the head via a 1-element array, which is potentially modified
+ * if any of the linked-list methods are called. Tags in the list should have
+ * equal or increasing start offsets.
+ */
+public class TagLL{
+
+  private final TagLL[] head;//a shared pointer to the head; 1 element
+  TagLL prevTag, nextTag; // linked list
+
+  private TermPrefixCursor cursor;
+
+  final int startOffset;//inclusive
+  int endOffset;//exclusive
+  Object value;//null means unset
+
+  /** optional boolean used by some TagClusterReducer's */
+  boolean mark = false;
+
+  TagLL(TagLL[] head, TermPrefixCursor cursor, int startOffset, int endOffset, Object value) {
+    this.head = head;
+    this.cursor = cursor;
+    this.startOffset = startOffset;
+    this.endOffset = endOffset;
+    this.value = value;
+  }
+
+  /**
+   * Advances this tag with "word" at offset "offset".  If this tag is not in
+   * an advancing state then it does nothing. If it is advancing and prior to
+   * advancing further it sees a value, then a non-advancing tag may be inserted
+   * into the LL as side-effect. If this returns false (it didn't advance) and
+   * if there is no value, then it will also be removed.
+   *
+   *
+   * @param word      The next word or null if at an end
+   * @param offset    The last character in word's offset in the underlying
+   *                  stream. If word is null then it's meaningless.
+   *
+   * @return          Whether it advanced or not.
+   */
+  boolean advance(BytesRef word, int offset) throws IOException {
+    if (!isAdvancing())
+      return false;
+
+    Object iVal = cursor.getDocIds();
+
+    if (word != null && cursor.advance(word)) {
+
+      if (iVal != null) {
+        addBeforeLL(new TagLL(head, null, startOffset, endOffset, iVal));
+      }
+
+      assert offset >= endOffset;
+      endOffset = offset;
+      return true;
+    } else {
+      this.value = iVal;
+      this.cursor = null;
+      if (iVal == null)
+        removeLL();
+      return false;
+    }
+  }
+
+  /** Removes this tag from the chain, connecting prevTag and nextTag. Does not
+   * modify "this" object's pointers, so the caller can refer to nextTag after
+   * removing it. */
+  public void removeLL() {
+    if (head[0] == this)
+      head[0] = nextTag;
+    if (prevTag != null) {
+      prevTag.nextTag = nextTag;
+    }
+    if (nextTag != null) {
+      nextTag.prevTag = prevTag;
+    }
+  }
+
+  void addBeforeLL(TagLL tag) {
+    assert tag.startOffset <= startOffset;
+    if (prevTag != null) {
+      assert prevTag.startOffset <= tag.startOffset;
+      prevTag.nextTag = tag;
+      tag.prevTag = prevTag;
+    } else {
+      assert head[0] == this;
+      head[0] = tag;
+    }
+    prevTag = tag;
+    tag.nextTag = this;
+  }
+
+  void addAfterLL(TagLL tag) {
+    assert tag.startOffset >= startOffset;
+    if (nextTag != null) {
+      assert nextTag.startOffset >= tag.startOffset;
+      nextTag.prevTag = tag;
+      tag.nextTag = nextTag;
+    }
+    nextTag = tag;
+    tag.prevTag = this;
+  }
+
+  public int charLen() {
+    return endOffset - startOffset;
+  }
+
+  public TagLL getNextTag() {
+    return nextTag;
+  }
+
+  public TagLL getPrevTag() {
+    return prevTag;
+  }
+
+  public int getStartOffset() {
+    return startOffset;
+  }
+  public int getEndOffset() {
+    return endOffset;
+  }
+  public boolean overlaps(TagLL other) {
+    //don't use >= or <= because startOffset is inclusive while endOffset is exclusive
+    if (startOffset < other.startOffset)
+      return endOffset > other.startOffset;
+    else
+      return startOffset < other.endOffset;
+  }
+
+  boolean isAdvancing() {
+    return cursor != null;
+  }
+
+  @Override
+  public String toString() {
+    return (prevTag != null ? '*' : '-') + "|" + (nextTag != null ? '*' : '-') +
+        " " + startOffset + " to " + endOffset + (isAdvancing() ? '+' : " #" + value);
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java b/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java
new file mode 100644
index 0000000..12a4cf0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java
@@ -0,0 +1,230 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tags maximum string of words in a corpus.  This is a callback-style API
+ * in which you implement {@link #tagCallback(int, int, Object)}.
+ *
+ * This class should be independently usable outside Solr.
+ */
+public abstract class Tagger {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private final TokenStream tokenStream;
+  private final TermToBytesRefAttribute byteRefAtt;
+  private final PositionIncrementAttribute posIncAtt;
+  private final OffsetAttribute offsetAtt;
+  private final TaggingAttribute taggingAtt;
+
+  private final TagClusterReducer tagClusterReducer;
+  private final Terms terms;
+  private final Bits liveDocs;
+  private final boolean skipAltTokens;
+  private final boolean ignoreStopWords;
+
+  private Map<BytesRef, IntsRef> docIdsCache;
+
+  /** Whether the WARNING about skipped tokens was already logged. */
+  private boolean loggedSkippedAltTokenWarning = false;
+
+  public Tagger(Terms terms, Bits liveDocs, TokenStream tokenStream,
+                TagClusterReducer tagClusterReducer, boolean skipAltTokens,
+                boolean ignoreStopWords) throws IOException {
+    this.terms = terms;
+    this.liveDocs = liveDocs;
+    this.tokenStream = tokenStream;
+    this.skipAltTokens = skipAltTokens;
+    this.ignoreStopWords = ignoreStopWords;
+    byteRefAtt = tokenStream.addAttribute(TermToBytesRefAttribute.class);
+    posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
+    offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
+    taggingAtt = tokenStream.addAttribute(TaggingAttribute.class);
+    tokenStream.reset();
+
+    this.tagClusterReducer = tagClusterReducer;
+  }
+
+  public void enableDocIdsCache(int initSize) {
+    if (initSize > 0)
+      docIdsCache = new HashMap<>(initSize);
+  }
+
+  public void process() throws IOException {
+    if (terms == null)
+      return;
+
+    //a shared pointer to the head used by this method and each Tag instance.
+    final TagLL[] head = new TagLL[1];
+
+    TermPrefixCursor cursor = null;//re-used
+
+    //boolean switch used to log warnings in case tokens where skipped during tagging.
+    boolean skippedTokens = false;
+
+    while (tokenStream.incrementToken()) {
+      if (log.isTraceEnabled()) {
+        log.trace("Token: {}, posInc: {},  offset: [{},{}]",
+                byteRefAtt, posIncAtt.getPositionIncrement(),
+                offsetAtt.startOffset(), offsetAtt.endOffset());
+      }
+      //check for posInc < 1 (alternate Tokens, such as expanded Synonyms)
+      if (posIncAtt.getPositionIncrement() < 1) {
+        //(a) Deal with this as a configuration issue and throw an exception
+        if (!skipAltTokens) {
+          //TODO throw UnsupportedTokenException when PhraseBuilder is ported
+          throw new IllegalStateException("Query Analyzer generates alternate "
+              + "Tokens (posInc == 0). Please adapt your Analyzer configuration or "
+              + "enable '" + TaggerRequestHandler.SKIP_ALT_TOKENS + "' to skip such "
+              + "tokens. NOTE: enabling '" + TaggerRequestHandler.SKIP_ALT_TOKENS
+              + "' might result in wrong tagging results if the index time analyzer "
+              + "is not configured accordingly. For detailed information see "
+              + "https://github.com/OpenSextant/SolrTextTagger/pull/11#issuecomment-24936225");
+        } else {
+          //(b) In case the index time analyser had indexed all variants (users
+          //    need to ensure that) processing of alternate tokens can be skipped
+          //    as anyways all alternatives will be contained in the FST.
+          skippedTokens = true;
+          log.trace("  ... ignored token");
+          continue;
+        }
+      }
+      //-- If PositionIncrement > 1 (stopwords)
+      if (!ignoreStopWords && posIncAtt.getPositionIncrement() > 1) {
+        log.trace("   - posInc > 1 ... mark cluster as done");
+        advanceTagsAndProcessClusterIfDone(head, null);
+      }
+
+      final BytesRef term;
+      //NOTE: we need to lookup tokens if
+      // * the LookupAtt is true OR
+      // * there are still advancing tags (to find the longest possible match)
+      if(taggingAtt.isTaggable() || head[0] != null){
+        //-- Lookup the term id from the next token
+        term = byteRefAtt.getBytesRef();
+        if (term.length == 0) {
+          throw new IllegalArgumentException("term: " + term.utf8ToString() + " analyzed to a zero-length token");
+        }
+      } else { //no current cluster AND lookup == false ...
+        term = null; //skip this token
+      }
+
+      //-- Process tag
+      advanceTagsAndProcessClusterIfDone(head, term);
+
+      //-- only create new Tags for Tokens we need to lookup
+      if (taggingAtt.isTaggable() && term != null) {
+
+        //determine if the terms index has a term starting with the provided term
+        // TODO create a pool of these cursors to reuse them more?  could be trivial impl
+        if (cursor == null)// (else the existing cursor will be re-used)
+          cursor = new TermPrefixCursor(terms.iterator(), liveDocs, docIdsCache);
+        if (cursor.advance(term)) {
+          TagLL newTail = new TagLL(head, cursor, offsetAtt.startOffset(), offsetAtt.endOffset(), null);
+          cursor = null;//because the new tag now "owns" this instance
+          //and add it to the end
+          if (head[0] == null) {
+            head[0] = newTail;
+          } else {
+            for (TagLL t = head[0]; true; t = t.nextTag) {
+              if (t.nextTag == null) {
+                t.addAfterLL(newTail);
+                break;
+              }
+            }
+          }
+        }
+      }//if termId >= 0
+    }//end while(incrementToken())
+
+    //-- Finish all tags
+    advanceTagsAndProcessClusterIfDone(head, null);
+    assert head[0] == null;
+
+    if(!loggedSkippedAltTokenWarning && skippedTokens){
+      loggedSkippedAltTokenWarning = true; //only log once
+      log.warn("The Tagger skipped some alternate tokens (tokens with posInc == 0) "
+          + "while processing text. This may cause problems with some Analyzer "
+          + "configurations (e.g. query time synonym expansion). For details see "
+          + "https://github.com/OpenSextant/SolrTextTagger/pull/11#issuecomment-24936225");
+    }
+
+    tokenStream.end();
+    //tokenStream.close(); caller closes because caller acquired it
+  }
+
+  private void advanceTagsAndProcessClusterIfDone(TagLL[] head, BytesRef term) throws IOException {
+    //-- Advance tags
+    final int endOffset = term != null ? offsetAtt.endOffset() : -1;
+    boolean anyAdvance = false;
+    for (TagLL t = head[0]; t != null; t = t.nextTag) {
+      anyAdvance |= t.advance(term, endOffset);
+    }
+
+    //-- Process cluster if done
+    if (!anyAdvance && head[0] != null) {
+      tagClusterReducer.reduce(head);
+      for (TagLL t = head[0]; t != null; t = t.nextTag) {
+        assert t.value != null;
+        tagCallback(t.startOffset, t.endOffset, t.value);
+      }
+      head[0] = null;
+    }
+  }
+
+  /**
+   * Invoked by {@link #process()} for each tag found.  endOffset is always &gt;= the endOffset
+   * given in the previous call.
+   *
+   * @param startOffset The character offset of the original stream where the tag starts.
+   * @param endOffset One more than the character offset of the original stream where the tag ends.
+   * @param docIdsKey A reference to the matching docIds that can be resolved via {@link #lookupDocIds(Object)}.
+   */
+  protected abstract void tagCallback(int startOffset, int endOffset, Object docIdsKey);
+
+  /**
+   * Returns a sorted array of integer docIds given the corresponding key.
+   * @param docIdsKey The lookup key.
+   * @return Not null
+   */
+  protected IntsRef lookupDocIds(Object docIdsKey) {
+    return (IntsRef) docIdsKey;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java
new file mode 100644
index 0000000..a972e47
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java
@@ -0,0 +1,397 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import javax.xml.stream.XMLStreamException;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import com.google.common.io.CharStreams;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.StopFilterFactory;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IntsRef;
+import org.apache.solr.analysis.TokenizerChain;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.BitDocSet;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.DocSlice;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SolrReturnFields;
+import org.apache.solr.search.SyntaxError;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Scans posted text, looking for matching strings in the Solr index.
+ * The public static final String members are request parameters.
+ * This handler is also called the "SolrTextTagger".
+ *
+ * @since 7.4.0
+ */
+public class TaggerRequestHandler extends RequestHandlerBase {
+
+  /** Request parameter. */
+  public static final String OVERLAPS = "overlaps";
+  /** Request parameter. */
+  public static final String TAGS_LIMIT = "tagsLimit";
+  /** Request parameter. */
+  public static final String MATCH_TEXT = "matchText";
+  /** Request parameter. */
+  public static final String SKIP_ALT_TOKENS = "skipAltTokens";
+  /** Request parameter. */
+  public static final String IGNORE_STOPWORDS = "ignoreStopwords";
+  /** Request parameter. */
+  public static final String XML_OFFSET_ADJUST = "xmlOffsetAdjust";
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  @Override
+  public String getDescription() {
+    return "Processes input text to find matching tokens stored in the index.";
+  }
+
+  @Override
+  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+
+    //--Read params
+    final String indexedField = req.getParams().get("field");
+    if (indexedField == null)
+      throw new RuntimeException("required param 'field'");
+
+    final TagClusterReducer tagClusterReducer =
+            chooseTagClusterReducer(req.getParams().get(OVERLAPS));
+    final int rows = req.getParams().getInt(CommonParams.ROWS, 10000);
+    final int tagsLimit = req.getParams().getInt(TAGS_LIMIT, 1000);
+    final boolean addMatchText = req.getParams().getBool(MATCH_TEXT, false);
+    final SchemaField idSchemaField = req.getSchema().getUniqueKeyField();
+    if (idSchemaField == null) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The tagger requires a" +
+              "uniqueKey in the schema.");//TODO this could be relaxed
+    }
+    final boolean skipAltTokens = req.getParams().getBool(SKIP_ALT_TOKENS, false);
+    final boolean ignoreStopWords = req.getParams().getBool(IGNORE_STOPWORDS,
+            fieldHasIndexedStopFilter(indexedField, req));
+
+    //--Get posted data
+    Reader inputReader = null;
+    Iterable<ContentStream> streams = req.getContentStreams();
+    if (streams != null) {
+      Iterator<ContentStream> iter = streams.iterator();
+      if (iter.hasNext()) {
+        inputReader = iter.next().getReader();
+      }
+      if (iter.hasNext()) {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+            getClass().getSimpleName()+" does not support multiple ContentStreams"); //TODO support bulk tagging?
+      }
+    }
+    if (inputReader == null) {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+          getClass().getSimpleName()+" requires text to be POSTed to it");
+    }
+
+    // We may or may not need to read the input into a string
+    final InputStringLazy inputStringFuture = new InputStringLazy(inputReader);
+
+    final OffsetCorrector offsetCorrector = getOffsetCorrector(req.getParams(), inputStringFuture);
+
+    final String inputString;//only populated if needed
+    if (addMatchText || inputStringFuture.inputString != null) {
+      //Read the input fully into a String buffer that we'll need later,
+      // then replace the input with a reader wrapping the buffer.
+      inputString = inputStringFuture.call();
+      inputReader.close();
+      inputReader = new StringReader(inputString);
+    } else {
+      inputString = null;//not used
+    }
+
+    final SolrIndexSearcher searcher = req.getSearcher();
+    final FixedBitSet matchDocIdsBS = new FixedBitSet(searcher.maxDoc());
+    final List tags = new ArrayList(2000);
+
+    try {
+      Analyzer analyzer = req.getSchema().getField(indexedField).getType().getQueryAnalyzer();
+      try (TokenStream tokenStream = analyzer.tokenStream("", inputReader)) {
+        Terms terms = searcher.getSlowAtomicReader().terms(indexedField);
+        if (terms == null)
+          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+                  "field " + indexedField + " has no indexed data");
+        Tagger tagger = new Tagger(terms, computeDocCorpus(req), tokenStream, tagClusterReducer,
+                skipAltTokens, ignoreStopWords) {
+          @SuppressWarnings("unchecked")
+          @Override
+          protected void tagCallback(int startOffset, int endOffset, Object docIdsKey) {
+            if (tags.size() >= tagsLimit)
+              return;
+            if (offsetCorrector != null) {
+              int[] offsetPair = offsetCorrector.correctPair(startOffset, endOffset);
+              if (offsetPair == null) {
+                log.debug("Discarded offsets [{}, {}] because couldn't balance XML.",
+                        startOffset, endOffset);
+                return;
+              }
+              startOffset = offsetPair[0];
+              endOffset = offsetPair[1];
+            }
+
+            NamedList tag = new NamedList();
+            tag.add("startOffset", startOffset);
+            tag.add("endOffset", endOffset);
+            if (addMatchText)
+              tag.add("matchText", inputString.substring(startOffset, endOffset));
+            //below caches, and also flags matchDocIdsBS
+            tag.add("ids", lookupSchemaDocIds(docIdsKey));
+            tags.add(tag);
+          }
+
+          Map<Object, List> docIdsListCache = new HashMap<>(2000);
+
+          ValueSourceAccessor uniqueKeyCache = new ValueSourceAccessor(searcher,
+                  idSchemaField.getType().getValueSource(idSchemaField, null));
+
+          @SuppressWarnings("unchecked")
+          private List lookupSchemaDocIds(Object docIdsKey) {
+            List schemaDocIds = docIdsListCache.get(docIdsKey);
+            if (schemaDocIds != null)
+              return schemaDocIds;
+            IntsRef docIds = lookupDocIds(docIdsKey);
+            //translate lucene docIds to schema ids
+            schemaDocIds = new ArrayList(docIds.length);
+            for (int i = docIds.offset; i < docIds.offset + docIds.length; i++) {
+              int docId = docIds.ints[i];
+              assert i == docIds.offset || docIds.ints[i - 1] < docId : "not sorted?";
+              matchDocIdsBS.set(docId);//also, flip docid in bitset
+              try {
+                schemaDocIds.add(uniqueKeyCache.objectVal(docId));//translates here
+              } catch (IOException e) {
+                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+              }
+            }
+            assert !schemaDocIds.isEmpty();
+
+            docIdsListCache.put(docIds, schemaDocIds);
+            return schemaDocIds;
+          }
+
+        };
+        tagger.enableDocIdsCache(2000);//TODO configurable
+        tagger.process();
+      }
+    } finally {
+      inputReader.close();
+    }
+    rsp.add("tagsCount",tags.size());
+    rsp.add("tags", tags);
+
+    rsp.setReturnFields(new SolrReturnFields( req ));
+
+    //Solr's standard name for matching docs in response
+    rsp.add("response", getDocList(rows, matchDocIdsBS));
+  }
+
+  private static class InputStringLazy implements Callable<String> {
+    final Reader inputReader;
+    String inputString;
+
+    InputStringLazy(Reader inputReader) {
+      this.inputReader = inputReader;
+    }
+
+    @Override
+    public String call() throws IOException {
+      if (inputString == null) {
+        inputString = CharStreams.toString(inputReader);
+      }
+      return inputString;
+    }
+  }
+
+  protected OffsetCorrector getOffsetCorrector(SolrParams params, Callable<String> inputStringProvider) throws Exception {
+    final boolean xmlOffsetAdjust = params.getBool(XML_OFFSET_ADJUST, false);
+    if (!xmlOffsetAdjust) {
+      return null;
+    }
+    try {
+      return new XmlOffsetCorrector(inputStringProvider.call());
+    } catch (XMLStreamException e) {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+          "Expecting XML but wasn't: " + e, e);
+    }
+  }
+
+  private DocList getDocList(int rows, FixedBitSet matchDocIdsBS) throws IOException {
+    //Now we must supply a Solr DocList and add it to the response.
+    //  Typically this is gotten via a SolrIndexSearcher.search(), but in this case we
+    //  know exactly what documents to return, the order doesn't matter nor does
+    //  scoring.
+    //  Ideally an implementation of DocList could be directly implemented off
+    //  of a BitSet, but there are way too many methods to implement for a minor
+    //  payoff.
+    int matchDocs = matchDocIdsBS.cardinality();
+    int[] docIds = new int[ Math.min(rows, matchDocs) ];
+    DocIdSetIterator docIdIter = new BitSetIterator(matchDocIdsBS, 1);
+    for (int i = 0; i < docIds.length; i++) {
+      docIds[i] = docIdIter.nextDoc();
+    }
+    return new DocSlice(0, docIds.length, docIds, null, matchDocs, 1f);
+  }
+
+  private TagClusterReducer chooseTagClusterReducer(String overlaps) {
+    TagClusterReducer tagClusterReducer;
+    if (overlaps == null || overlaps.equals("NO_SUB")) {
+      tagClusterReducer = TagClusterReducer.NO_SUB;
+    } else if (overlaps.equals("ALL")) {
+      tagClusterReducer = TagClusterReducer.ALL;
+    } else if (overlaps.equals("LONGEST_DOMINANT_RIGHT")) {
+      tagClusterReducer = TagClusterReducer.LONGEST_DOMINANT_RIGHT;
+    } else {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+          "unknown tag overlap mode: "+overlaps);
+    }
+    return tagClusterReducer;
+  }
+
+  /**
+   * The set of documents matching the provided 'fq' (filter query). Don't include deleted docs
+   * either. If null is returned, then all docs are available.
+   */
+  private Bits computeDocCorpus(SolrQueryRequest req) throws SyntaxError, IOException {
+    final String[] corpusFilterQueries = req.getParams().getParams("fq");
+    final SolrIndexSearcher searcher = req.getSearcher();
+    final Bits docBits;
+    if (corpusFilterQueries != null && corpusFilterQueries.length > 0) {
+      List<Query> filterQueries = new ArrayList<Query>(corpusFilterQueries.length);
+      for (String corpusFilterQuery : corpusFilterQueries) {
+        QParser qParser = QParser.getParser(corpusFilterQuery, null, req);
+        try {
+          filterQueries.add(qParser.parse());
+        } catch (SyntaxError e) {
+          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+        }
+      }
+
+      final DocSet docSet = searcher.getDocSet(filterQueries);//hopefully in the cache
+      //note: before Solr 4.7 we could call docSet.getBits() but no longer.
+      if (docSet instanceof BitDocSet) {
+        docBits = ((BitDocSet)docSet).getBits();
+      } else {
+        docBits = new Bits() {
+
+          @Override
+          public boolean get(int index) {
+            return docSet.exists(index);
+          }
+
+          @Override
+          public int length() {
+            return searcher.maxDoc();
+          }
+        };
+      }
+    } else {
+      docBits = searcher.getSlowAtomicReader().getLiveDocs();
+    }
+    return docBits;
+  }
+
+  private boolean fieldHasIndexedStopFilter(String field, SolrQueryRequest req) {
+    FieldType fieldType = req.getSchema().getFieldType(field);
+    Analyzer analyzer = fieldType.getIndexAnalyzer();//index analyzer
+    if (analyzer instanceof TokenizerChain) {
+      TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
+      TokenFilterFactory[] tokenFilterFactories = tokenizerChain.getTokenFilterFactories();
+      for (TokenFilterFactory tokenFilterFactory : tokenFilterFactories) {
+        if (tokenFilterFactory instanceof StopFilterFactory)
+          return true;
+      }
+    }
+    return false;
+  }
+
+  /** See LUCENE-4541 or {@link org.apache.solr.response.transform.ValueSourceAugmenter}. */
+  static class ValueSourceAccessor {
+    private final List<LeafReaderContext> readerContexts;
+    private final ValueSource valueSource;
+    private final Map fContext;
+    private final FunctionValues[] functionValuesPerSeg;
+    private final int[] functionValuesDocIdPerSeg;
+
+    ValueSourceAccessor(IndexSearcher searcher, ValueSource valueSource) {
+      readerContexts = searcher.getIndexReader().leaves();
+      this.valueSource = valueSource;
+      fContext = ValueSource.newContext(searcher);
+      functionValuesPerSeg = new FunctionValues[readerContexts.size()];
+      functionValuesDocIdPerSeg = new int[readerContexts.size()];
+    }
+
+    Object objectVal(int topDocId) throws IOException {
+      // lookup segment level stuff:
+      int segIdx = ReaderUtil.subIndex(topDocId, readerContexts);
+      LeafReaderContext rcontext = readerContexts.get(segIdx);
+      int segDocId = topDocId - rcontext.docBase;
+      // unfortunately Lucene 7.0 requires forward only traversal (with no reset method).
+      //   So we need to track our last docId (per segment) and re-fetch the FunctionValues. :-(
+      FunctionValues functionValues = functionValuesPerSeg[segIdx];
+      if (functionValues == null || segDocId < functionValuesDocIdPerSeg[segIdx]) {
+        functionValues = functionValuesPerSeg[segIdx] = valueSource.getValues(fContext, rcontext);
+      }
+      functionValuesDocIdPerSeg[segIdx] = segDocId;
+
+      // get value:
+      return functionValues.objectVal(segDocId);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java
new file mode 100644
index 0000000..b7803e4
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java
@@ -0,0 +1,65 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.Attribute;
+
+/**
+ * Attribute used by the {@link Tagger} to decide if a token can start a
+ * new {@link TagLL tag}.
+ * <p>
+ * By default this Attribute will return <code>true</code>, but it might be
+ * reset by some {@link TokenFilter} added to the {@link TokenStream} used
+ * to analyze the parsed text. Typically this will be done based on NLP
+ * processing results (e.g. to only lookup Named Entities).
+ * <p>
+ * NOTE: that all Tokens are used to advance existing {@link TagLL tags}.
+ */
+public interface TaggingAttribute extends Attribute {
+
+  /**
+   * By default this Attribute will be initialised with <code>true</code>.
+   * This ensures that all tokens are taggable by default (especially if
+   * the {@link TaggingAttribute} is not set by any component in the configured
+   * {@link TokenStream}
+   */
+  public static final boolean DEFAULT_TAGGABLE = true;
+
+  /**
+   * Getter for the taggable state of the current Token
+   *
+   * @return the state
+   */
+  public boolean isTaggable();
+
+  /**
+   * Setter for the taggable state. Typically called by code within
+   * {@link TokenFilter#incrementToken()}.
+   *
+   * @param lookup the state
+   */
+  public void setTaggable(boolean lookup);
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java
new file mode 100644
index 0000000..55ecfbc6
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java
@@ -0,0 +1,79 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+
+/**
+ * Implementation of the {@link TaggingAttribute}
+ */
+public class TaggingAttributeImpl extends AttributeImpl implements TaggingAttribute {
+
+  /**
+   * the private field initialised with {@link TaggingAttribute#DEFAULT_TAGGABLE}
+   */
+  private boolean taggable = TaggingAttribute.DEFAULT_TAGGABLE;
+
+  /*
+   * (non-Javadoc)
+   * @see org.opensextant.solrtexttagger.LookupAttribute#isLookup()
+   */
+  @Override
+  public boolean isTaggable() {
+    return taggable;
+  }
+
+  /*
+   * (non-Javadoc)
+   * @see org.opensextant.solrtexttagger.LookupAttribute#setLookup(boolean)
+   */
+  @Override
+  public void setTaggable(boolean lookup) {
+    this.taggable = lookup;
+  }
+
+  /*
+   * (non-Javadoc)
+   * @see org.apache.lucene.util.AttributeImpl#clear()
+   */
+  @Override
+  public void clear() {
+    taggable = DEFAULT_TAGGABLE;
+  }
+
+  /*
+   * (non-Javadoc)
+   * @see org.apache.lucene.util.AttributeImpl#copyTo(org.apache.lucene.util.AttributeImpl)
+   */
+  @Override
+  public void copyTo(AttributeImpl target) {
+    ((TaggingAttribute) target).setTaggable(taggable);
+  }
+
+  @Override
+  public void reflectWith(AttributeReflector reflector) {
+    reflector.reflect(TaggingAttribute.class, "taggable", isTaggable());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java b/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java
new file mode 100644
index 0000000..1e82dbe
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java
@@ -0,0 +1,189 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.analysis.miscellaneous.ConcatenateGraphFilter;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.IntsRef;
+
+/**
+ * Cursor into the terms that advances by prefix.
+ */
+class TermPrefixCursor {
+
+  //Note: this could be a lot more efficient if MemoryPostingsFormat supported ordinal lookup.
+  // Maybe that could be added to Lucene.
+
+  // TODO add bloom filter of hashcode of first ~ 6 bytes to avoid lookup into terms dict?
+
+  private static final byte SEPARATOR_CHAR = ConcatenateGraphFilter.SEP_LABEL; // used to be ' '; TODO configurable?
+  private static final IntsRef EMPTY_INTSREF = new IntsRef();
+
+  private final TermsEnum termsEnum;
+  private final Bits liveDocs;
+  private final Map<BytesRef, IntsRef> docIdsCache;
+
+  private BytesRef prefixBuf;//we append to this
+  private BytesRefBuilder prefixBufBuilder = new BytesRefBuilder();
+  private boolean prefixBufOnLoan;//if true, PB is loaned; needs to be copied
+  private PostingsEnum postingsEnum;
+  private IntsRef docIds;
+
+  TermPrefixCursor(TermsEnum termsEnum, Bits liveDocs, Map<BytesRef, IntsRef> docIdsCache) {
+    this.termsEnum = termsEnum;
+    this.liveDocs = liveDocs;
+    this.docIdsCache = docIdsCache;
+  }
+
+  /** Appends the separator char (if not the first) plus the given word to the prefix buffer,
+   * then seeks to it. If the seek fails, false is returned and this cursor
+   * can be re-used as if in a new state.  The {@code word} BytesRef is considered temporary,
+   * and is not saved within this class. */
+  boolean advance(BytesRef word) throws IOException {
+    if (prefixBuf == null) { // first advance
+      //set prefixBuf to word temporary. When advance() completes, we either null out or copy.
+      prefixBuf = word;
+      prefixBufOnLoan = true;
+      if (seekPrefix()) {//... and we have to
+        ensureBufIsACopy();
+        return true;
+      } else {
+        prefixBuf = null;//just to be darned sure 'word' isn't referenced here
+        return false;
+      }
+
+    } else { // subsequent advance
+      //append to existing
+      assert !prefixBufOnLoan;
+
+      prefixBufBuilder.append(SEPARATOR_CHAR);
+      prefixBufBuilder.append(word);
+      prefixBuf = prefixBufBuilder.get();
+      if (seekPrefix()) {
+        return true;
+      } else {
+        prefixBuf = null;
+        return false;
+      }
+    }
+  }
+
+  private void ensureBufIsACopy() {
+    if (!prefixBufOnLoan)
+      return;
+
+    prefixBufBuilder.clear();
+    prefixBufBuilder.copyBytes(prefixBuf);
+    prefixBuf = prefixBufBuilder.get();
+    prefixBufOnLoan = false;
+  }
+
+  /** Seeks to prefixBuf or the next term that is prefixed by prefixBuf plus the separator char.
+   * Sets docIds. **/
+  private boolean seekPrefix() throws IOException {
+    TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(prefixBuf);
+
+    docIds = null;//invalidate
+    switch (seekStatus) {
+      case END:
+        return false;
+
+      case FOUND:
+        postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
+        docIds = postingsEnumToIntsRef(postingsEnum, liveDocs);
+        if (docIds.length > 0) {
+          return true;
+        }
+
+        //Pretend we didn't find it; go to next term
+        docIds = null;
+        if (termsEnum.next() == null) { // case END
+          return false;
+        }
+        //fall through to NOT_FOUND
+
+      case NOT_FOUND:
+        //termsEnum must start with prefixBuf to continue
+        BytesRef teTerm = termsEnum.term();
+
+        if (teTerm.length > prefixBuf.length) {
+          for (int i = 0; i < prefixBuf.length; i++) {
+            if (prefixBuf.bytes[prefixBuf.offset + i] != teTerm.bytes[teTerm.offset + i])
+              return false;
+          }
+          if (teTerm.bytes[teTerm.offset + prefixBuf.length] != SEPARATOR_CHAR)
+            return false;
+          return true;
+        }
+        return false;
+    }
+    throw new IllegalStateException(seekStatus.toString());
+  }
+
+  /** Returns an IntsRef either cached or reading postingsEnum. Not null. */
+  private IntsRef postingsEnumToIntsRef(PostingsEnum postingsEnum, Bits liveDocs) throws IOException {
+    // (The cache can have empty IntsRefs)
+
+    //lookup prefixBuf in a cache
+    if (docIdsCache != null) {
+      docIds = docIdsCache.get(prefixBuf);
+      if (docIds != null) {
+        return docIds;
+      }
+    }
+
+    //read postingsEnum
+    docIds = new IntsRef(termsEnum.docFreq());
+    int docId;
+    while ((docId = postingsEnum.nextDoc()) != PostingsEnum.NO_MORE_DOCS) {
+      if (liveDocs != null && !liveDocs.get(postingsEnum.docID())) {
+        continue;
+      }
+      docIds.ints[docIds.length++] = docId;
+    }
+    if (docIds.length == 0)
+      docIds = EMPTY_INTSREF;
+
+    //cache
+    if (docIdsCache != null) {
+      ensureBufIsACopy();
+      //clone is shallow; that's okay as the prefix isn't overwritten; it's just appended to
+      docIdsCache.put(prefixBuf.clone(), docIds);
+    }
+    return docIds;
+  }
+
+  /** The docIds of the last call to advance, if it returned true. It might be null, but
+   * its length won't be 0. Treat as immutable. */
+  IntsRef getDocIds() {
+    assert docIds == null || docIds.length != 0;
+    return docIds;
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java b/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java
new file mode 100644
index 0000000..576328f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java
@@ -0,0 +1,113 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import javax.xml.stream.XMLResolver;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.XMLEvent;
+import java.io.InputStream;
+import java.io.StringReader;
+
+import com.ctc.wstx.stax.WstxInputFactory;
+import org.apache.commons.io.input.ClosedInputStream;
+import org.codehaus.stax2.LocationInfo;
+import org.codehaus.stax2.XMLInputFactory2;
+import org.codehaus.stax2.XMLStreamReader2;
+
+/**
+ * Corrects offsets to adjust for XML formatted data. The goal is such that the caller should be
+ * able to insert a start XML tag at the start offset and a corresponding end XML tag at the end
+ * offset of the tagger, and have it be valid XML.  See {@link #correctPair(int, int)}.
+ *
+ * This will not work on invalid XML.
+ *
+ * Not thread-safe.
+ */
+public class XmlOffsetCorrector extends OffsetCorrector {
+
+  //TODO use StAX without hard requirement on woodstox.   xmlStreamReader.getLocation().getCharacterOffset()
+
+  private static final XMLInputFactory2 XML_INPUT_FACTORY;
+  static {
+    // note: similar code in Solr's EmptyEntityResolver
+    XML_INPUT_FACTORY = new WstxInputFactory();
+    XML_INPUT_FACTORY.setXMLResolver(new XMLResolver() {
+      @Override
+      public InputStream resolveEntity(String publicId, String systemId, String baseURI, String namespace) {
+        return ClosedInputStream.CLOSED_INPUT_STREAM;
+      }
+    });
+    // TODO disable DTD?
+    // XML_INPUT_FACTORY.setProperty(XMLInputFactory.IS_VALIDATING, Boolean.FALSE)
+    XML_INPUT_FACTORY.configureForSpeed();
+  }
+
+  /**
+   * Initialize based on the document text.
+   * @param docText non-null XML content.
+   * @throws XMLStreamException If there's a problem parsing the XML.
+   */
+  public XmlOffsetCorrector(String docText) throws XMLStreamException {
+    super(docText, false);
+
+    int tagCounter = 0;
+    int thisTag = -1;
+
+    //note: we *could* add a virtual outer tag to guarantee all text is in the context of a tag,
+    // but we shouldn't need to because there is no findable text outside the top element.
+
+    final XMLStreamReader2 xmlStreamReader =
+            (XMLStreamReader2) XML_INPUT_FACTORY.createXMLStreamReader(new StringReader(docText));
+
+    while (xmlStreamReader.hasNext()) {
+      int eventType = xmlStreamReader.next();
+      switch (eventType) {
+        case XMLEvent.START_ELEMENT: {
+          tagInfo.ensureCapacity(tagInfo.size() + 5);
+          final int parentTag = thisTag;
+          final LocationInfo info = xmlStreamReader.getLocationInfo();
+          tagInfo.add(parentTag);
+          tagInfo.add((int) info.getStartingCharOffset(), (int) info.getEndingCharOffset());
+          tagInfo.add(-1, -1);//these 2 will be populated when we get to the close tag
+          thisTag = tagCounter++;
+
+          parentChangeOffsets.add((int) info.getStartingCharOffset());
+          parentChangeIds.add(thisTag);
+          break;
+        }
+        case XMLEvent.END_ELEMENT: {
+          final LocationInfo info = xmlStreamReader.getLocationInfo();
+          tagInfo.set(5 * thisTag + 3, (int) info.getStartingCharOffset());
+          tagInfo.set(5 * thisTag + 4, (int) info.getEndingCharOffset());
+          thisTag = getParentTag(thisTag);
+
+          parentChangeOffsets.add((int) info.getEndingCharOffset());
+          parentChangeIds.add(thisTag);
+          break;
+        }
+        default: //do nothing
+      }
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java b/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java
new file mode 100644
index 0000000..c2055b3
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The {@link org.apache.solr.handler.tagger.TaggerRequestHandler} and supporting classes.
+ * This was formerly known as OpenSextant's SolrTextTagger.
+ */
+package org.apache.solr.handler.tagger;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml
new file mode 100644
index 0000000..051cd10
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml
@@ -0,0 +1,187 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  This software was produced for the U. S. Government
+  under Contract No. W15P7T-11-C-F600, and is
+  subject to the Rights in Noncommercial Computer Software
+  and Noncommercial Computer Software Documentation
+  Clause 252.227-7014 (JUN 1995)
+
+  Copyright 2013 The MITRE Corporation. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<schema name="minimal" version="1.6">
+
+  <fields>
+    <field name="id" type="string" docValues="true" required="true"/>
+    <field name="name" type="string"/>
+    <!-- freq, positions, and norms are not needed by the tagger. However if you
+    intend to have this field be used for general search, you should not exclude
+    these stats. -->
+    <field name="name_tag" type="tag" stored="false"/>
+    <field name="name_tagStop" type="tagStop" stored="false"/>
+    <field name="name_tagWDF" type="tagWDF" stored="false"/>
+    <!--<field name="name_tagPartial" type="tagPartial" stored="false"/>-->
+    <field name="name_tagXml" type="tagXml" stored="false"/>
+    <field name="name_tagAttribute" type="tagAttribute" stored="false"/>
+
+    <copyField source="name" dest="name_tag"/>
+    <copyField source="name" dest="name_tagStop"/>
+    <copyField source="name" dest="name_tagWDF"/>
+    <!--<copyField source="name" dest="name_tagPartial"/>-->
+    <copyField source="name" dest="name_tagXml"/>
+    <copyField source="name" dest="name_tagAttribute"/>
+
+    <dynamicField name="*" type="string" indexed="true" stored="true"/>
+  </fields>
+
+  <uniqueKey>id</uniqueKey>
+
+  <types>
+    <fieldType name="string" class="solr.StrField"/>
+
+    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+
+    <fieldType name="tag" class="solr.TextField" positionIncrementGap="100"
+               postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <!--Usually good but for our tests, lets not. <filter class="solr.ASCIIFoldingFilterFactory"/>-->
+        <filter class="solr.LowerCaseFilterFactory"/>
+
+        <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+      </analyzer>
+      <analyzer type="query">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <!--<filter class="solr.ASCIIFoldingFilterFactory"/>-->
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- adds a stop filter -->
+    <fieldType name="tagStop" class="solr.TextField" positionIncrementGap="100"
+               postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.StopFilterFactory" /><!-- by default english stopwords -->
+
+        <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+      </analyzer>
+      <analyzer type="query">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.StopFilterFactory" /><!-- by default english stopwords -->
+      </analyzer>
+    </fieldType>
+
+    <!-- adds a WordDelimiterFilter producing stacked/synonym tokens -->
+    <fieldType name="tagWDF" class="solr.TextField" positionIncrementGap="100"
+               postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+      <analyzer type="index">
+        <tokenizer class="solr.WhitespaceTokenizerFactory" />
+        <filter class="solr.WordDelimiterGraphFilterFactory"
+                generateWordParts="1" generateNumberParts="1"
+                catenateWords="1" catenateNumbers="1" catenateAll="0"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+
+        <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+      </analyzer>
+      <analyzer type="query">
+        <tokenizer class="solr.WhitespaceTokenizerFactory" />
+        <filter class="solr.WordDelimiterGraphFilterFactory"
+                generateWordParts="1" generateNumberParts="1"
+                catenateWords="0" catenateNumbers="0" catenateAll="0"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- Problem: in XML can't write \u001F
+    <fieldType name="tagPartial" class="solr.TextField" positionIncrementGap="100"
+               postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory"/>
+
+        <filter class="solr.ShingleFilterFactory" tokenSeparator="&#x1F;"
+                outputUnigramsIfNoShingles="true" maxShingleSize="10"  />
+      </analyzer>
+      <analyzer type="query">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
+    -->
+
+    <fieldType name="tagXml" class="solr.TextField" positionIncrementGap="100"
+               postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory"/>
+
+        <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+      </analyzer>
+      <analyzer type="query">
+        <charFilter class="solr.HTMLStripCharFilterFactory" /><!-- ADDED THIS! -->
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <fieldType name="tagAttribute" class="solr.TextField" positionIncrementGap="100" postingsFormat="FST50">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory" />
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+
+        <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+      </analyzer>
+      <analyzer type="query">
+        <!-- 32 just for tests, bumps posInc -->
+        <tokenizer class="solr.StandardTokenizerFactory"
+                   maxTokenLength="32"/>
+        <!--
+         NOTE: This used the WordLengthTaggingFilterFactory to test the
+         TaggingAttribute. The WordLengthTaggingFilter set the TaggingAttribute
+         for words based on their length. The attribute is ignored at indexing
+         time, but the Tagger will use it to only start tags for words that are
+         equals or longer as the configured minLength.
+         -->
+        <filter class="org.apache.solr.handler.tagger.WordLengthTaggingFilterFactory" minLength="4"/>
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <!-- <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" /> -->
+        <!-- in this example, we will only use synonyms at query time
+        <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+        -->
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+      <analyzer type="query">
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <!-- <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" /> -->
+        <!-- <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> -->
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+  </types>
+
+
+</schema>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
new file mode 100644
index 0000000..e0d3677
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" ?>
+<!--
+  This software was produced for the U. S. Government
+  under Contract No. W15P7T-11-C-F600, and is
+  subject to the Rights in Noncommercial Computer Software
+  and Noncommercial Computer Software Documentation
+  Clause 252.227-7014 (JUN 1995)
+
+  Copyright 2013 The MITRE Corporation. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!-- a basic solrconfig that tests can use when they want simple minimal solrconfig/schema
+     DO NOT ADD THINGS TO THIS CONFIG! -->
+<config>
+  <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <dataDir>${solr.data.dir:}</dataDir>
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+
+  <!-- for postingsFormat="..." -->
+  <codecFactory name="CodecFactory" class="solr.SchemaCodecFactory" />
+
+  <!-- since Solr 4.8: -->
+  <schemaFactory name="SchemaFactory" class="solr.ClassicIndexSchemaFactory" />
+
+  <query>
+    <!-- illustrate putting in memory for warm-up -->
+    <listener event="firstSearcher" class="solr.QuerySenderListener">
+      <arr name="queries">
+        <lst><str name="q">name_tag:[* TO *]</str></lst>
+      </arr>
+    </listener>
+    <listener event="newSearcher" class="solr.QuerySenderListener">
+      <arr name="queries">
+        <lst><str name="q">name_tag:[* TO *]</str></lst>
+      </arr>
+    </listener>
+  </query>
+
+  <requestHandler name="/select" class="solr.SearchHandler"></requestHandler>
+
+  <requestHandler name="/tag" class="solr.TaggerRequestHandler">
+    <lst name="defaults">
+      <str name="field">name_tag</str>
+    </lst>
+  </requestHandler>
+
+</config>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java
new file mode 100644
index 0000000..8d31ad0
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java
@@ -0,0 +1,153 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+
+import org.apache.lucene.document.Field;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.StreamingResponseCallback;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Tests that we can skip serialization of the documents when embedding
+ * Solr.
+ */
+public class EmbeddedSolrNoSerializeTest extends SolrTestCaseJ4 {
+
+  static EmbeddedSolrServer solrServer;
+
+  @BeforeClass
+  public static void init() throws Exception {
+    initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+    solrServer = new EmbeddedSolrServer(h.getCoreContainer(), "collection1");
+    //we don't need to close the EmbeddedSolrServer because SolrTestCaseJ4 closes the core
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    super.setUp();
+    clearIndex();
+    assertU(adoc("id", "9999", "name", "Boston"));
+    assertU(commit());
+  }
+
+  @Test
+  public void testTag() throws SolrServerException, IOException {
+    ModifiableSolrParams params = params();
+    String input = "foo boston bar";//just one tag;
+    QueryRequest req = new SolrTaggerRequest(params, input);
+    req.setPath("/tag");
+
+    QueryResponse rsp = req.process(solrServer);
+    SolrDocumentList results= (SolrDocumentList) rsp.getResponse().get("response");
+    assertNotNull(rsp.getResponse().get("tags"));
+    assertNotNull(results.get(0));
+  }
+
+  @SuppressWarnings("serial")
+  public static class SolrTaggerRequest extends QueryRequest {
+
+    private final String input;
+
+    public SolrTaggerRequest(SolrParams p, String input) {
+      super(p, METHOD.POST);
+      this.input = input;
+    }
+
+    // Deprecated in 7.2 but should live on until 8.x
+    @SuppressWarnings("deprecation")
+    @Override
+    public Collection<ContentStream> getContentStreams() {
+      return Collections.singleton(new ContentStreamBase.StringStream(input));
+    }
+
+    // As of 7.2.  But won't work until: https://issues.apache.org/jira/browse/SOLR-12142
+//    @Override
+//    public RequestWriter.ContentWriter getContentWriter(String expectedType) {
+//      return new RequestWriter.StringPayloadContentWriter(input, "text/plain; charset=UTF8");
+//    }
+  }
+
+  @Test
+  public void testSearch() throws Exception {
+    QueryResponse rsp = solrServer.query(params("q", "name:Boston"));
+    assertNotNull(rsp.getResults().get(0));
+  }
+
+  @Test
+  public void testAssertTagStreamingWithSolrTaggerRequest() throws Exception {
+    doTestAssertTagStreaming(SolrTaggerRequest::new);
+  }
+
+  @Test
+  @Ignore("As of Solr 7, stream.body is disabled by default for security ") // DWS: dubious, IMO
+  // and it can't be enabled with EmbeddedSolrServer until SOLR-12126
+  public void testAssertTagStreamingWithStreamBodyParam() throws Exception {
+    doTestAssertTagStreaming((params, input) -> {
+      params.set("stream.body", input);
+      return new QueryRequest(params);
+    });
+  }
+
+  public void doTestAssertTagStreaming(BiFunction<ModifiableSolrParams,String,QueryRequest> newQueryRequest) throws IOException, SolrServerException {
+    ModifiableSolrParams params = params();
+    String input = "foo boston bar";//just one tag;
+    QueryRequest req = newQueryRequest.apply(params, input);
+    req.setPath("/tag");
+
+    final AtomicReference<SolrDocument> refDoc = new AtomicReference<>();
+    req.setStreamingResponseCallback(new StreamingResponseCallback() {
+      @Override
+      public void streamSolrDocument(SolrDocument doc) {
+        refDoc.set(doc);
+      }
+
+      @Override
+      public void streamDocListInfo(long numFound, long start, Float maxScore) {
+
+      }
+    });
+    QueryResponse rsp = req.process(solrServer);
+    assertNotNull(rsp.getResponse().get("tags"));
+    assertNotNull(refDoc.get());
+    assertEquals("Boston", ((Field)refDoc.get().getFieldValue("name")).stringValue());
+  }
+}


[18/18] lucene-solr:jira/solr-12438: SOLR-12438: Improvements to the call forwarding. RefGuide doc changes.

Posted by ab...@apache.org.
SOLR-12438: Improvements to the call forwarding. RefGuide doc changes.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/53d9d081
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/53d9d081
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/53d9d081

Branch: refs/heads/jira/solr-12438
Commit: 53d9d081b39afdb6bca02c5e454da2ab1f9a180e
Parents: 3e72481
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Jun 7 14:14:08 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Jun 7 14:14:08 2018 +0200

----------------------------------------------------------------------
 .../org/apache/solr/core/CoreContainer.java     |  20 +-
 .../handler/admin/MetricsHistoryHandler.java    |  81 ++--
 solr/solr-ref-guide/src/metrics-history.adoc    | 382 +++++++++++--------
 3 files changed, 287 insertions(+), 196 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53d9d081/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 5cbcab5..37a660f 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -739,6 +739,7 @@ public class CoreContainer {
     status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
   }
 
+  // MetricsHistoryHandler supports both cloud and standalone configs
   private void createMetricsHistoryHandler() {
     PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
     Map<String, Object> initArgs;
@@ -762,8 +763,13 @@ public class CoreContainer {
         name = "localhost";
       }
       cloudManager = null;
-      client = new EmbeddedSolrServer(this, CollectionAdminParams.SYSTEM_COLL);
-      // enable local metrics
+      client = new EmbeddedSolrServer(this, CollectionAdminParams.SYSTEM_COLL) {
+        @Override
+        public void close() throws IOException {
+          // do nothing - we close the container ourselves
+        }
+      };
+      // enable local metrics unless specifically set otherwise
       if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_NODES_PROP)) {
         initArgs.put(MetricsHistoryHandler.ENABLE_NODES_PROP, true);
       }
@@ -820,6 +826,12 @@ public class CoreContainer {
 
     ExecutorUtil.shutdownAndAwaitTermination(coreContainerWorkExecutor);
     replayUpdatesExecutor.shutdownAndAwaitTermination();
+
+    if (metricsHistoryHandler != null) {
+      IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());
+      metricsHistoryHandler.close();
+    }
+
     if (metricManager != null) {
       metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
       metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.jvm));
@@ -838,10 +850,6 @@ public class CoreContainer {
       } catch (Exception e) {
         log.warn("Error removing live node. Continuing to close CoreContainer", e);
       }
-      if (metricsHistoryHandler != null) {
-        IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());
-        metricsHistoryHandler.close();
-      }
       if (metricManager != null) {
         metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.cluster));
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53d9d081/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
index 89731d2..03b545f 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -24,6 +24,8 @@ import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -46,6 +48,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.DoubleAdder;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -112,30 +115,32 @@ import static org.apache.solr.common.params.CommonParams.ID;
 public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider, Closeable {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-  public static final List<String> DEFAULT_CORE_COUNTERS = new ArrayList<String>() {{
-    add("QUERY./select.requests");
-    add("UPDATE./update.requests");
-  }};
-  public static final List<String> DEFAULT_CORE_GAUGES = new ArrayList<String>() {{
-    add("INDEX.sizeInBytes");
-  }};
-  public static final List<String> DEFAULT_NODE_GAUGES = new ArrayList<String>() {{
-    add("CONTAINER.fs.coreRoot.usableSpace");
-  }};
-  public static final List<String> DEFAULT_JVM_GAUGES = new ArrayList<String>() {{
-    add("memory.heap.used");
-    add("os.processCpuLoad");
-    add("os.systemLoadAverage");
-  }};
+  public static final List<String> DEFAULT_CORE_COUNTERS = new ArrayList<>();
+  public static final List<String> DEFAULT_CORE_GAUGES = new ArrayList<>();
+  public static final List<String> DEFAULT_NODE_GAUGES = new ArrayList<>();
+  public static final List<String> DEFAULT_JVM_GAUGES = new ArrayList<>();
 
   public static final String NUM_SHARDS_KEY = "numShards";
   public static final String NUM_REPLICAS_KEY = "numReplicas";
   public static final String NUM_NODES_KEY = "numNodes";
 
-  public static final List<String> DEFAULT_COLLECTION_GAUGES = new ArrayList<String>() {{
-    add(NUM_SHARDS_KEY);
-    add(NUM_REPLICAS_KEY);
-  }};
+  public static final List<String> DEFAULT_COLLECTION_GAUGES = new ArrayList<>();
+
+  static {
+    DEFAULT_JVM_GAUGES.add("memory.heap.used");
+    DEFAULT_JVM_GAUGES.add("os.processCpuLoad");
+    DEFAULT_JVM_GAUGES.add("os.systemLoadAverage");
+
+    DEFAULT_NODE_GAUGES.add("CONTAINER.fs.coreRoot.usableSpace");
+
+    DEFAULT_CORE_GAUGES.add("INDEX.sizeInBytes");
+
+    DEFAULT_CORE_COUNTERS.add("QUERY./select.requests");
+    DEFAULT_CORE_COUNTERS.add("UPDATE./update.requests");
+
+    DEFAULT_COLLECTION_GAUGES.add(NUM_SHARDS_KEY);
+    DEFAULT_COLLECTION_GAUGES.add(NUM_REPLICAS_KEY);
+  }
 
   public static final String COLLECT_PERIOD_PROP = "collectPeriod";
   public static final String SYNC_PERIOD_PROP = "syncPeriod";
@@ -698,7 +703,8 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
         List<Pair<String, Long>> lst = factory.list(rows);
         lst.forEach(p -> {
           SimpleOrderedMap<Object> data = new SimpleOrderedMap<>();
-          data.add("lastModified", p.second());
+          // RrdDb always uses seconds - convert here for compatibility
+          data.add("lastModified", TimeUnit.SECONDS.convert(p.second(), TimeUnit.MILLISECONDS));
           data.add("node", nodeName);
           res.add(p.first(), data);
         });
@@ -753,7 +759,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
         rsp.add("success", "ok");
         break;
     }
-    // when using in-memory DBs non-overseer node has no access to overseer DBs
+    // when using in-memory DBs non-overseer node has no access to overseer DBs - in this case
     // forward the request to Overseer leader if available
     if (!factory.isPersistent()) {
       String leader = getOverseerLeader();
@@ -766,7 +772,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
     SimpleOrderedMap<Object> apiState = new SimpleOrderedMap<>();
     apiState.add("enableReplicas", enableReplicas);
     apiState.add("enableNodes", enableNodes);
-    apiState.add("mode", enable ? (factory.isPersistent() ? "index" : "memory") : "disabled");
+    apiState.add("mode", enable ? (factory.isPersistent() ? "index" : "memory") : "inactive");
     if (!factory.isPersistent()) {
       apiState.add("message", "WARNING: metrics history is not being persisted. Create .system collection to start persisting history.");
     }
@@ -775,10 +781,20 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
   }
 
   private NamedList<Object> handleRemoteRequest(String nodeName, SolrQueryRequest req) {
-    String url = Utils.getBaseUrlForNodeName(nodeName, overseerUrlScheme);
+    String baseUrl = Utils.getBaseUrlForNodeName(nodeName, overseerUrlScheme);
+    String url;
+    try {
+      URL u = new URL(baseUrl);
+      u = new URL(u.getProtocol(), u.getHost(), u.getPort(), "/api/cluster/metrics/history");
+      url = u.toString();
+    } catch (MalformedURLException e) {
+      log.warn("Invalid Overseer url '" + baseUrl + "', unable to fetch remote metrics history", e);
+      return null;
+    }
+    // always use javabin
     ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
     params.set(CommonParams.WT, "javabin");
-    url = url + req.getPath() + "?" + params.toString();
+    url = url + "?" + params.toString();
     try {
       byte[] data = cloudManager.httpRequest(url, SolrRequest.METHOD.GET, null, null, HttpClientUtil.DEFAULT_CONNECT_TIMEOUT, true);
       // response is always a NamedList
@@ -856,18 +872,17 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
       if (format != Format.GRAPH) {
         // add timestamps separately from values
         long[] timestamps = fd.getTimestamps();
-        str.setLength(0);
-        for (int i = 0; i < timestamps.length; i++) {
-          if (format == Format.LIST) {
-            map.add("timestamps", timestamps[i]);
-          } else {
+        if (format == Format.LIST) {
+          // Arrays.asList works only on arrays of Objects
+          map.add("timestamps", Arrays.stream(timestamps).boxed().collect(Collectors.toList()));
+        } else {
+          str.setLength(0);
+          for (int i = 0; i < timestamps.length; i++) {
             if (i > 0) {
               str.append('\n');
             }
             str.append(String.valueOf(timestamps[i]));
           }
-        }
-        if (format == Format.STRING) {
           map.add("timestamps", str.toString());
         }
       }
@@ -920,9 +935,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
             values.add(name, str.toString());
             break;
           case LIST:
-            for (int i = 0; i < vals.length; i++) {
-              values.add(name, vals[i]);
-            }
+            values.add(name, Arrays.stream(vals).boxed().collect(Collectors.toList()));
             break;
         }
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53d9d081/solr/solr-ref-guide/src/metrics-history.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/metrics-history.adoc b/solr/solr-ref-guide/src/metrics-history.adoc
index 5dc1c3c..e39f66e 100644
--- a/solr/solr-ref-guide/src/metrics-history.adoc
+++ b/solr/solr-ref-guide/src/metrics-history.adoc
@@ -18,15 +18,16 @@
 
 == Design
 === Round-robin databases
-When Solr runs in "cloud" mode it collects long-term history of certain key metrics. This information
-can be used for very simple monitoring and troubleshooting, but also some Solr Cloud components
-(eg. autoscaling) can use this data for making informed decisions based on long-term
-trends of selected metrics.
+Solr collects long-term history of certain key metrics both in SolrCloud and in standalone mode.
+This information can be used for very simple monitoring and troubleshooting, but also some
+Solr Cloud components (eg. autoscaling) can use this data for making informed decisions based on
+long-term trends of selected metrics.
 
 [IMPORTANT]
 ====
-Metrics history is available ONLY in SolrCloud mode, it's not supported in standalone Solr. Also,
-the `.system` collection must exist if metrics history should be persisted.
+The `.system` collection must exist if metrics history should be persisted. If this collection
+is absent then metrics history will still be collected and kept in memory but it will be lost
+on node restart.
 ====
 
 This data is maintained as multi-resolution time series, with a fixed total number of data points
@@ -61,14 +62,16 @@ update operations than storing each data point in a separate Solr document. Metr
 detailed data from each database, including retrieval of all individual datapoints.
 
 Databases are identified primarily by their corresponding metric registry name, so for databases that
-keep track of aggregated metrics this will be eg. `solr.jvm`, `solr.node`, `solr.collection.gettingstarted`,
-and for databases with non-aggregated metrics this will be eg. `solr.jvm.localhost:8983_solr`,
-`solr.node.localhost:7574_solr`, `solr.core.gettingstarted.shard1.replica_n1`.
+keep track of aggregated metrics this will be eg. `solr.jvm`, `solr.node`, `solr.collection.gettingstarted`.
+For databases with non-aggregated metrics the name consists of the registry name, optionally with a node name
+to identify databases with the same name coming from different nodes. For example, per-node databases are
+name like this: `solr.jvm.localhost:8983_solr`, `solr.node.localhost:7574_solr`, but per-replica names are
+already unique across the cluster so they are named like this: `solr.core.gettingstarted.shard1.replica_n1`.
 
 === Collected metrics
 Currently the following selected metrics are tracked:
 
-* `solr.core` and `solr.collection` metrics:
+* Non-aggregated `solr.core` and aggregated `solr.collection` metrics:
 ** `QUERY./select.requests`
 ** `UPDATE./update.requests`
 ** `INDEX.sizeInBytes`
@@ -78,6 +81,7 @@ Currently the following selected metrics are tracked:
 * `solr.node` metrics:
 ** `CONTAINER.fs.coreRoot.usableSpace`
 ** `numNodes` (aggregated, number of live nodes)
+
 * `solr.jvm` metrics:
 ** `memory.heap.used`
 ** `os.processCpuLoad`
@@ -86,6 +90,10 @@ Currently the following selected metrics are tracked:
 Separate databases are created for each of these groups, and each database keeps data for
 all metrics listed in that group.
 
+NOTE: Currently this list is not configurable. Also, if you change this list in the code then
+all existing databases must be first removed from the `.system` collection because RRD4j doesn't allow
+adding new datasources once the database is created.
+
 === SolrRrdBackendFactory
 This component is responsible for managing in-memory databases and periodically saving them
 to the `.system` collection. If the `.system` collection is not available the updates to the
@@ -101,7 +109,8 @@ collecting and periodically updating the in-memory databases.
 This handler also performs aggregation of metrics on per-collection level, and on a cluster level.
 By default only these aggregated metrics are tracked - historic data from each node and each replica
 in each collection is not collected separately. Aggregated databases are managed on the Overseer leader
-node.
+node but they are still accessible from other nodes even if they are not persisted - the handler redirects
+the call from originating node to the current Overseer leader.
 
 The handler assumes that a simple aggregation (sum of partial metric values from each resource) is
 sufficient. This happens to make sense for the default built-in sets of metrics. Future extensions will
@@ -135,6 +144,7 @@ databases.
 
 `collectPeriod`:: integer, in seconds, default is 60. Metrics values will be collected and respective
 databases updated every `collectPeriod` seconds.
+
 [IMPORTANT]
 ====
 Value of `collectPeriod` must be at least 1, and if it's changed then all previously existing databases
@@ -142,9 +152,9 @@ with their historic data must be manually removed (new databases will be created
 ====
 
 `syncPeriod`:: integer, in seconds, default is 60. Data from modified databases will be saved to Solr
-every `syncPeriod` seconds. When accessing the databases via REST API the visibility of most recent
-data depends on this period, because requests accessing the data from other nodes see only the
-version of the data that is stored in the `.system` collection.
+every `syncPeriod` seconds. When accessing the databases via REST API in `index` mode the visibility of
+most recent data depends on this period, because requests accessing the data from other nodes see only
+the version of the data that is stored in the `.system` collection.
 
 === Example configuration
 Example `/clusterprops.json` file with metrics history configuration that turns on the collection of
@@ -154,6 +164,7 @@ properties unrelated to metrics history API.
 [source,json]
 ----
 {
+...
   "metrics" : {
     "history" : {
       "enable" : true,
@@ -161,42 +172,86 @@ properties unrelated to metrics history API.
       "syncPeriod" : 300
     }
   }
+...
 }
 ----
 
 == Metrics History API
-Main entry point for accessing metrics history is `/admin/metrics/history` (or `/api/cluster/metrics/history` for
-v2 API).
+Main entry point for accessing metrics history is `/admin/metrics/history` (or `/api/cluster/metrics/history`
+for v2 API).
 
 The following sections describe actions available in this API. All calls have at least one
 required parameter `action`.
 
+All responses contain a section named `state`, which reports the current internal state of the API:
+
+`enableReplicas`:: boolean, corresponds to the `enableReplicas` configuration setting.
+`enableNodes`:: boolean, corresponds to the `enableNodes` configuration setting.
+`mode`:: one of the following values:
+* `inactive` - when metrics collection is disabled (but access to existing metrics history is still available).
+* `memory` - when metrics history is kept only in memory because `.system` collection doesn't exist. In this mode
+clients can access metrics history available on the node that received the reuqest and on the Overseer leader.
+* `index` - when metrics history is periodically stored in the `.system` collection. Data available in memory on
+the node that accepted the request is retrieved from memory, any other data is retrieved from the
+`.system` collection (so it's at least `syncPeriod` old).
+
+Also, the response header section (`responseHeader`) contains `zkConnected` boolean property that indicates
+whether the current node is a part of SolrCloud cluster.
+
 === List databases (`action=list`)
 This call produces a list of available databases. It supports the following parameters:
 
-`rows`:: optional integer, default is 500. Maximum number of results to return
+`rows`:: optional integer, default is 500. Maximum number of results to return.
 
 Example:
+In this SolrCloud example the API is in `memory` mode, and the request was made to a node that is
+not Overseer leader. The API transparently forwarded the request to Overseer leader.
 [source,bash]
 ----
-curl http://localhost:8983/solr/admin/metrics/history?action=list&rows=10
+curl http://localhost:7574/solr/admin/metrics/history?action=list&rows=10
 ----
 [source,json]
 ----
 {
-    "responseHeader": {
-        "status": 0,
-        "QTime": 16
+  "responseHeader": {
+    "zkConnected": true,
+    "status": 0,
+    "QTime": 9
+  },
+  "metrics": {
+    "solr.collection..system": {
+      "lastModified": 1528360138,
+      "node": "127.0.0.1:8983_solr"
+    },
+    "solr.collection.gettingstarted": {
+      "lastModified": 1528360138,
+      "node": "127.0.0.1:8983_solr"
+    },
+    "solr.jvm": {
+      "lastModified": 1528360138,
+      "node": "127.0.0.1:8983_solr"
     },
-    "metrics": [
-        "solr.collection..system",
-        "solr.collection.gettingstarted",
-        "solr.jvm",
-        "solr.node"
-    ]
+    "solr.node": {
+      "lastModified": 1528360138,
+      "node": "127.0.0.1:8983_solr"
+    }
+  },
+  "state": {
+    "enableReplicas": false,
+    "enableNodes": false,
+    "mode": "memory"
+  }
 }
 ----
 
+Note the presence of the `node` element in each section, which shows where the information is coming
+from - when API is in `memory` mode this indicates which results are local and which ones are retrieved
+from the Overseer leader node. When the API is in `index` mode this element always shows the node name that
+received the request (because the data is retrieved from the `.system` collection anyway).
+
+Each section also contains a `lastModified` element, which contains the last modification time when the
+database was update. All timestamps returned from this API correspond to Unix epoch time in seconds.
+
 === Database status (`action=status`)
 This call provides detailed status of the selected database.
 
@@ -207,66 +262,71 @@ The following parameters are supported:
 Example:
 [source,bash]
 ----
-curl http://localhost:8983/solr/admin/metrics/history?action=status&name=solr.collection.gettingstarted
+curl http://localhost:7574/solr/admin/metrics/history?action=status&name=solr.collection.gettingstarted
 ----
 [source,json]
 ----
 {
-    "responseHeader": {
-        "status": 0,
-        "QTime": 38
-    },
-    "metrics": [
-        "solr.collection.gettingstarted",
-        [
-            "status",
-            {
-                "lastModified": 1527268438,
-                "step": 60,
-                "datasourceCount": 5,
-                "archiveCount": 5,
-                "datasourceNames": [
-                    "numShards",
-                    "numReplicas",
-                    "QUERY./select.requests",
-                    "UPDATE./update.requests",
-                    "INDEX.sizeInBytes"
-                ],
-                "datasources": [
-                    {
-                        "datasource": "DS:numShards:GAUGE:120:U:U",
-                        "lastValue": 2
-                    },
-                    {
-                        "datasource": "DS:QUERY./select.requests:COUNTER:120:U:U",
-                        "lastValue": 8786
-                    },
-                    ...
-                ],
-                "archives": [
-                    {
-                        "archive": "RRA:AVERAGE:0.5:1:240",
-                        "steps": 1,
-                        "consolFun": "AVERAGE",
-                        "xff": 0.5,
-                        "startTime": 1527254040,
-                        "endTime": 1527268380,
-                        "rows": 240
-                    },
-                    {
-                        "archive": "RRA:AVERAGE:0.5:10:288",
-                        "steps": 10,
-                        "consolFun": "AVERAGE",
-                        "xff": 0.5,
-                        "startTime": 1527096000,
-                        "endTime": 1527268200,
-                        "rows": 288
-                    },
-                    ...
-                ]
-            }
+  "responseHeader": {
+    "zkConnected": true,
+    "status": 0,
+    "QTime": 46
+  },
+  "metrics": {
+    "solr.collection.gettingstarted": {
+      "status": {
+        "lastModified": 1528318361,
+        "step": 60,
+        "datasourceCount": 5,
+        "archiveCount": 5,
+        "datasourceNames": [
+          "numShards",
+          "numReplicas",
+          "QUERY./select.requests",
+          "UPDATE./update.requests",
+          "INDEX.sizeInBytes"
+        ],
+        "datasources": [
+          {
+            "datasource": "DS:numShards:GAUGE:120:U:U",
+            "lastValue": 2
+          },
+          {
+            "datasource": "DS:numReplicas:GAUGE:120:U:U",
+            "lastValue": 4
+          },
+          ...
+        ],
+        "archives": [
+          {
+            "archive": "RRA:AVERAGE:0.5:1:240",
+            "steps": 1,
+            "consolFun": "AVERAGE",
+            "xff": 0.5,
+            "startTime": 1528303980,
+            "endTime": 1528318320,
+            "rows": 240
+          },
+          {
+            "archive": "RRA:AVERAGE:0.5:10:288",
+            "steps": 10,
+            "consolFun": "AVERAGE",
+            "xff": 0.5,
+            "startTime": 1528146000,
+            "endTime": 1528318200,
+            "rows": 288
+          },
+          ...
         ]
-    ]
+      },
+      "node": "127.0.0.1:7574_solr"
+    }
+  },
+  "state": {
+    "enableReplicas": false,
+    "enableNodes": false,
+    "mode": "index"
+  }
 }
 ----
 
@@ -286,7 +346,7 @@ values (because points from all datasources in a given time series share the sam
 * `graph` - data is returned as PNG images, Base64-encoded, containing graphs of each time series values over time.
 
 In each case the response is structured in a similar way: archive identifiers are keys in a JSON map,
-and timestamps / datapoints / graphs are values.
+all data is placed in a `data` element, with timestamps / datapoints / graphs as values in lists or maps.
 
 ==== Examples
 This is the output using the default `list` format:
@@ -297,37 +357,49 @@ curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.colle
 [source,json]
 ----
 {
-    "responseHeader": {
-        "status": 0,
-        "QTime": 36
-    },
-    "metrics": [
-        "solr.collection.gettingstarted",
-        [
-            "data",
-            {
-                "RRA:AVERAGE:0.5:1:240": {
-                    "timestamps":1527254460,
-                    "timestamps":1527254520,
-                    "timestamps":1527254580,
-                    ...
-                    "values": {
-                        "numShards": "NaN",
-                        "numShards": 2.0,
-                        "numShards": 2.0,
-                        ...
-                        "numReplicas": "NaN",
-                        "numReplicas": 4.0,
-                        "numReplicas": 4.0,
-                        ...
-                        "QUERY./select.requests": "NaN",
-                        "QUERY./select.requests": 123,
-                        "QUERY./select.requests": 456,
-                        ...
-                    }
-                },
-                "RRA:AVERAGE:0.5:10:288": {
-...
+  "responseHeader": {
+    "zkConnected": true,
+    "status": 0,
+    "QTime": 4
+  },
+  "metrics": {
+    "solr.collection.gettingstarted": {
+      "data": {
+        "RRA:AVERAGE:0.5:1:240": {
+          "timestamps": [
+            1528304160,
+            1528304220,
+            ...
+          ],
+          "values": {
+            "numShards": [
+              "NaN",
+              2.0,
+              ...
+            ],
+            "numReplicas": [
+              "NaN",
+              4.0,
+              ...
+            ],
+            ...
+          }
+        },
+        "RRA:AVERAGE:0.5:10:288": {
+          "timestamps": [
+            1528145400,
+            1528146000,
+          ...
+      "lastModified": 1528318606,
+      "node": "127.0.0.1:8983_solr"
+    }
+  },
+  "state": {
+    "enableReplicas": false,
+    "enableNodes": false,
+    "mode": "index"
+  }
+}
 ----
 
 This is the output when using the `string` format:
@@ -338,25 +410,24 @@ curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.colle
 [source,json]
 ----
 {
-    "responseHeader": {
-        "status": 0,
-        "QTime": 11
-    },
-    "metrics": [
-        "solr.collection.gettingstarted",
-        [
-            "data",
-            {
-                "RRA:AVERAGE:0.5:1:240": {
-                    "timestamps": "1527254820\n1527254880\n1527254940\n...",
-                    "values": {
-                        "numShards": "NaN\n2.0\n2.0\n2.0\n2.0\n2.0\n2.0\n...",
-                        "numReplicas": "NaN\n4.0\n4.0\n4.0\n4.0\n4.0\n4.0\n...",
-                        "QUERY./select.requests": "NaN\n123\n456\n789\n...",
-                        ...
-                    }
-                },
-                "RRA:AVERAGE:0.5:10:288": {
+  "responseHeader": {
+    "zkConnected": true,
+    "status": 0,
+    "QTime": 2
+  },
+  "metrics": {
+    "solr.collection.gettingstarted": {
+      "data": {
+        "RRA:AVERAGE:0.5:1:240": {
+          "timestamps": "1527254820\n1527254880\n1527254940\n...",
+          "values": {
+            "numShards": "NaN\n2.0\n2.0\n2.0\n2.0\n2.0\n2.0\n...",
+            "numReplicas": "NaN\n4.0\n4.0\n4.0\n4.0\n4.0\n4.0\n...",
+            "QUERY./select.requests": "NaN\n123\n456\n789\n...",
+            ...
+          }
+        },
+        "RRA:AVERAGE:0.5:10:288": {
                 ...
 ----
 
@@ -368,29 +439,28 @@ curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.colle
 [source,json]
 ----
 {
-    "responseHeader": {
-        "status": 0,
-        "QTime": 2275
-    },
-    "metrics": [
-        "solr.collection.gettingstarted",
-        [
-            "data",
-            {
-                "RRA:AVERAGE:0.5:1:240": {
-                    "values": {
-                        "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
-                        "numReplicas": "iVBORw0KGgoAAAANSUhEUgAAAkQA...",
-                        "QUERY./select.requests": "iVBORw0KGgoAAAANS...",
-                        ...
-                    }
-                },
-                "RRA:AVERAGE:0.5:10:288": {
-                    "values": {
-                        "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
-                        ...
-                },
-                ...
+  "responseHeader": {
+    "zkConnected": true,
+    "status": 0,
+    "QTime": 2
+  },
+  "metrics": {
+    "solr.collection.gettingstarted": {
+      "data": {
+        "RRA:AVERAGE:0.5:1:240": {
+          "values": {
+            "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
+            "numReplicas": "iVBORw0KGgoAAAANSUhEUgAAAkQA...",
+            "QUERY./select.requests": "iVBORw0KGgoAAAANS...",
+            ...
+          }
+        },
+        "RRA:AVERAGE:0.5:10:288": {
+          "values": {
+            "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
+            ...
+          },
+        ...
 ----
 
 .Example 60 sec resolution history graph for `QUERY./select.requests` metric


[04/18] lucene-solr:jira/solr-12438: LUCENE-7960: fix Solr test to include mandatory args

Posted by ab...@apache.org.
LUCENE-7960: fix Solr test to include mandatory args


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c5875980
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c5875980
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c5875980

Branch: refs/heads/jira/solr-12438
Commit: c587598096cde769c299594fb26d0a23b7bd5930
Parents: 7d0b64f
Author: David Smiley <ds...@apache.org>
Authored: Tue Jun 5 12:30:34 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Jun 5 12:30:34 2018 -0400

----------------------------------------------------------------------
 .../core/src/test/org/apache/solr/core/ResourceLoaderTest.java | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c5875980/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java b/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
index 8ab9606..785195a 100644
--- a/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
+++ b/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
@@ -72,7 +72,8 @@ public class ResourceLoaderTest extends SolrTestCaseJ4 {
     
     Class<?> clazz = ResourceLoaderAware.class;
     // Check ResourceLoaderAware valid objects
-    assertAwareCompatibility(clazz, new NGramFilterFactory(new HashMap<>()));
+    //noinspection unchecked
+    assertAwareCompatibility(clazz, new NGramFilterFactory(map("minGramSize", "1", "maxGramSize", "2")));
     assertAwareCompatibility(clazz, new KeywordTokenizerFactory(new HashMap<>()));
     
     // Make sure it throws an error for invalid objects
@@ -98,8 +99,9 @@ public class ResourceLoaderTest extends SolrTestCaseJ4 {
     assertAwareCompatibility(clazz, new JSONResponseWriter());
     
     // Make sure it throws an error for invalid objects
+    //noinspection unchecked
     invalid = new Object[] {
-        new NGramFilterFactory(new HashMap<>()),
+        new NGramFilterFactory(map("minGramSize", "1", "maxGramSize", "2")),
         "hello",  new Float( 12.3f ),
         new KeywordTokenizerFactory(new HashMap<>())
     };