You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/06/07 12:18:19 UTC
[01/18] lucene-solr:jira/solr-12438: SOLR-12444: Updating a cluster
policy fails
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-12438 2fa260c14 -> 53d9d081b
SOLR-12444: Updating a cluster policy fails
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/add77d27
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/add77d27
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/add77d27
Branch: refs/heads/jira/solr-12438
Commit: add77d272582e909e9a7cf008d35ea72c3914230
Parents: 6624773
Author: Noble Paul <no...@apache.org>
Authored: Tue Jun 5 22:29:06 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jun 5 22:29:06 2018 +1000
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../autoscaling/AutoScalingHandlerTest.java | 21 ++++++++++++++++++++
.../client/solrj/cloud/autoscaling/Clause.java | 2 +-
3 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/add77d27/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c88de10..1b454a0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -1312,6 +1312,8 @@ Bug Fixes
* SOLR-11477: Disallow resolving of external entities in the XML query parser (defType=xmlparser).
(Michael Stepankin, Olga Barinova, Uwe Schindler, Christine Poerschke)
+* SOLR-12444: Updating a cluster policy fails (noble)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/add77d27/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 483b60c..cf11953 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud.autoscaling;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
@@ -26,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -1011,6 +1013,25 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertEquals(5L, properties.get(AutoScalingParams.ACTION_THROTTLE_PERIOD_SECONDS));
}
+ public void testUpdatePolicy() throws IOException, SolrServerException {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setPropertiesCommand = "{'set-cluster-policy': [" +
+ "{'cores': '<4','node': '#ANY'}]}";
+ solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals("<4", Utils.getObjectByPath(response,false,"cluster-policy[0]/cores"));
+ assertEquals("#ANY", Utils.getObjectByPath(response,false,"cluster-policy[0]/node"));
+ setPropertiesCommand = "{'set-cluster-policy': [" +
+ "{'cores': '<3','node': '#ANY'}]}";
+ solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
+ req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
+ response = solrClient.request(req);
+ System.out.println(response);
+
+
+ }
+
static class AutoScalingRequest extends SolrRequest {
protected final String message;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/add77d27/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index c739588..8f198bd 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -140,7 +140,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Clause that = (Clause)o;
- return compareTo(that) == 0;
+ return Objects.equals(this.original, that.original);
}
void addTags(Collection<String> params) {
[13/18] lucene-solr:jira/solr-12438: LUCENE-8299: Geo3D wrapper uses
new polygon method factory that gives better support for polygons with many
points (>100)
Posted by ab...@apache.org.
LUCENE-8299: Geo3D wrapper uses new polygon method factory that gives better support for polygons with many points (>100)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/af7dfb18
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/af7dfb18
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/af7dfb18
Branch: refs/heads/jira/solr-12438
Commit: af7dfb182ebc175df44b2d0a846a7d11f58bf84b
Parents: 0358fcb
Author: Ignacio Vera <iv...@apache.org>
Authored: Wed Jun 6 11:39:39 2018 +0200
Committer: Ignacio Vera <iv...@apache.org>
Committed: Wed Jun 6 11:39:39 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../spatial/spatial4j/Geo3dShapeFactory.java | 10 ++++-----
.../lucene/spatial/spatial4j/Geo3dRptTest.java | 23 ++++++++++++++++++--
3 files changed, 28 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af7dfb18/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9eecc42..8f307fe 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -267,6 +267,9 @@ Other
* LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)
+* LUCENE-8299: Geo3D wrapper uses new polygon method factory that gives better
+ support for polygons with many points (>100). (Ignacio vera)
+
* LUCENE-8261: InterpolatedProperties.interpolate and recursive property
references. (Steve Rowe, Dawid Weiss)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af7dfb18/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
index a6147df..ccbd6df 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShapeFactory.java
@@ -311,7 +311,7 @@ public class Geo3dShapeFactory implements S2ShapeFactory {
*/
private class Geo3dPolygonBuilder extends Geo3dPointBuilder<PolygonBuilder> implements PolygonBuilder {
- List<GeoPolygon> polyHoles;
+ List<GeoPolygonFactory.PolygonDescription> polyHoles = new ArrayList<>();
@Override
public HoleBuilder hole() {
@@ -321,10 +321,7 @@ public class Geo3dShapeFactory implements S2ShapeFactory {
class Geo3dHoleBuilder extends Geo3dPointBuilder<PolygonBuilder.HoleBuilder> implements PolygonBuilder.HoleBuilder {
@Override
public PolygonBuilder endHole() {
- if (polyHoles == null) {
- polyHoles = new ArrayList<>();
- }
- polyHoles.add(GeoPolygonFactory.makeGeoPolygon(planetModel, points));
+ polyHoles.add(new GeoPolygonFactory.PolygonDescription(points));
return Geo3dPolygonBuilder.this;
}
}
@@ -332,7 +329,8 @@ public class Geo3dShapeFactory implements S2ShapeFactory {
@SuppressWarnings("unchecked")
@Override
public Shape build() {
- GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, points, polyHoles);
+ GeoPolygonFactory.PolygonDescription description = new GeoPolygonFactory.PolygonDescription(points, polyHoles);
+ GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, description);
return new Geo3dShape<>(polygon, context);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af7dfb18/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
index d3b144f..eb6ed5b 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
@@ -18,9 +18,11 @@ package org.apache.lucene.spatial.spatial4j;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import org.apache.lucene.spatial.SpatialTestData;
import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
@@ -96,7 +98,7 @@ public class Geo3dRptTest extends RandomSpatialOpStrategyTestCase {
points.add(new GeoPoint(planetModel, 14 * DEGREES_TO_RADIANS, -180 * DEGREES_TO_RADIANS));
points.add(new GeoPoint(planetModel, -15 * DEGREES_TO_RADIANS, 153 * DEGREES_TO_RADIANS));
- final Shape triangle = new Geo3dShape(GeoPolygonFactory.makeGeoPolygon(planetModel, points),ctx);
+ final Shape triangle = new Geo3dShape<>(GeoPolygonFactory.makeGeoPolygon(planetModel, points),ctx);
final Rectangle rect = ctx.makeRectangle(-49, -45, 73, 86);
testOperation(rect,SpatialOperation.Intersects,triangle, false);
}
@@ -116,7 +118,7 @@ public class Geo3dRptTest extends RandomSpatialOpStrategyTestCase {
new GeoPoint(planetModel, 54.0 * DEGREES_TO_RADIANS, 165.0 * DEGREES_TO_RADIANS),
new GeoPoint(planetModel, -90.0 * DEGREES_TO_RADIANS, 0.0)};
final GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 29 * DEGREES_TO_RADIANS, pathPoints);
- final Shape shape = new Geo3dShape(path,ctx);
+ final Shape shape = new Geo3dShape<>(path,ctx);
final Rectangle rect = ctx.makeRectangle(131, 143, 39, 54);
testOperation(rect,SpatialOperation.Intersects,shape,true);
}
@@ -146,6 +148,23 @@ public class Geo3dRptTest extends RandomSpatialOpStrategyTestCase {
return new Geo3dShape<>(areaShape, ctx);
}
+ @Test
+ public void testOperationsFromFile() throws IOException {
+ setupStrategy();
+ final Iterator<SpatialTestData> indexedSpatialData = getSampleData( "states-poly.txt");
+ final List<Shape> indexedShapes = new ArrayList<>();
+ while(indexedSpatialData.hasNext()) {
+ indexedShapes.add(indexedSpatialData.next().shape);
+ }
+ final Iterator<SpatialTestData> querySpatialData = getSampleData( "states-bbox.txt");
+ final List<Shape> queryShapes = new ArrayList<>();
+ while(querySpatialData.hasNext()) {
+ queryShapes.add(querySpatialData.next().shape);
+ queryShapes.add(randomQueryShape());
+ }
+ testOperation(SpatialOperation.Intersects, indexedShapes, queryShapes, random().nextBoolean());
+ }
+
//TODO move to a new test class?
@Test
public void testWKT() throws Exception {
[09/18] lucene-solr:jira/solr-12438: SOLR-12444: added more
assertions to the test
Posted by ab...@apache.org.
SOLR-12444: added more assertions to the test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9ff3f5a1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9ff3f5a1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9ff3f5a1
Branch: refs/heads/jira/solr-12438
Commit: 9ff3f5a13614c9aefe50c84b9abc653490f536c4
Parents: 124b5e0
Author: Noble Paul <no...@apache.org>
Authored: Wed Jun 6 13:13:20 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Wed Jun 6 13:13:20 2018 +1000
----------------------------------------------------------------------
.../apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ff3f5a1/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index cf11953..cf57c48 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -1027,8 +1027,8 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
response = solrClient.request(req);
- System.out.println(response);
-
+ assertEquals("<3", Utils.getObjectByPath(response,false,"cluster-policy[0]/cores"));
+ assertEquals("#ANY", Utils.getObjectByPath(response,false,"cluster-policy[0]/node"));
}
[15/18] lucene-solr:jira/solr-12438: SOLR-12445: Upgrade Dropwizard
Metrics to version 3.2.6.
Posted by ab...@apache.org.
SOLR-12445: Upgrade Dropwizard Metrics to version 3.2.6.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3022bbce
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3022bbce
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3022bbce
Branch: refs/heads/jira/solr-12438
Commit: 3022bbce2ec493c759eceb4fac7eeab0fb908b59
Parents: c1f4290
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 15:14:22 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 15:14:22 2018 +0200
----------------------------------------------------------------------
lucene/ivy-versions.properties | 2 +-
solr/licenses/metrics-core-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-core-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-ganglia-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-ganglia-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-graphite-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-graphite-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-jetty9-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-jetty9-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-jvm-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-jvm-3.2.6.jar.sha1 | 1 +
11 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 981bbe6..49a1190 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -60,7 +60,7 @@ com.sun.jersey.version = 1.9
/dom4j/dom4j = 1.6.1
/info.ganglia.gmetric4j/gmetric4j = 1.0.7
-io.dropwizard.metrics.version = 3.2.2
+io.dropwizard.metrics.version = 3.2.6
/io.dropwizard.metrics/metrics-core = ${io.dropwizard.metrics.version}
/io.dropwizard.metrics/metrics-ganglia = ${io.dropwizard.metrics.version}
/io.dropwizard.metrics/metrics-graphite = ${io.dropwizard.metrics.version}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-core-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-core-3.2.2.jar.sha1 b/solr/licenses/metrics-core-3.2.2.jar.sha1
deleted file mode 100644
index d14a04e..0000000
--- a/solr/licenses/metrics-core-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cd9886f498ee2ab2d994f0c779e5553b2c450416
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-core-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-core-3.2.6.jar.sha1 b/solr/licenses/metrics-core-3.2.6.jar.sha1
new file mode 100644
index 0000000..13fae6d
--- /dev/null
+++ b/solr/licenses/metrics-core-3.2.6.jar.sha1
@@ -0,0 +1 @@
+62fe170cffeded1cef60e9e3402a93b45ce14327
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-ganglia-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-ganglia-3.2.2.jar.sha1 b/solr/licenses/metrics-ganglia-3.2.2.jar.sha1
deleted file mode 100644
index e5d8496..0000000
--- a/solr/licenses/metrics-ganglia-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d5bb1883e9b0daf0e4187e558746f5058f4585c1
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-ganglia-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-ganglia-3.2.6.jar.sha1 b/solr/licenses/metrics-ganglia-3.2.6.jar.sha1
new file mode 100644
index 0000000..32c9d30
--- /dev/null
+++ b/solr/licenses/metrics-ganglia-3.2.6.jar.sha1
@@ -0,0 +1 @@
+a44039835eafd2dad8842a9ed16a60c088c5b7ef
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-graphite-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-graphite-3.2.2.jar.sha1 b/solr/licenses/metrics-graphite-3.2.2.jar.sha1
deleted file mode 100644
index 5d11db4..0000000
--- a/solr/licenses/metrics-graphite-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-908e8cbec1bbdb2f4023334e424c7de2832a95af
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-graphite-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-graphite-3.2.6.jar.sha1 b/solr/licenses/metrics-graphite-3.2.6.jar.sha1
new file mode 100644
index 0000000..26a1bbc
--- /dev/null
+++ b/solr/licenses/metrics-graphite-3.2.6.jar.sha1
@@ -0,0 +1 @@
+ecbc470e9097bb3d7ff0232cca47f3badde2e20b
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jetty9-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jetty9-3.2.2.jar.sha1 b/solr/licenses/metrics-jetty9-3.2.2.jar.sha1
deleted file mode 100644
index 92d3508..0000000
--- a/solr/licenses/metrics-jetty9-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3fc94d99f41dc3f5be5483c81828138104df4449
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jetty9-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jetty9-3.2.6.jar.sha1 b/solr/licenses/metrics-jetty9-3.2.6.jar.sha1
new file mode 100644
index 0000000..5d7bfa4
--- /dev/null
+++ b/solr/licenses/metrics-jetty9-3.2.6.jar.sha1
@@ -0,0 +1 @@
+5dae1c13d8607663fbc7b22cf8c05aacd22f802e
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jvm-3.2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jvm-3.2.2.jar.sha1 b/solr/licenses/metrics-jvm-3.2.2.jar.sha1
deleted file mode 100644
index 0c02f93..0000000
--- a/solr/licenses/metrics-jvm-3.2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9cbf2030242f7ffb97fae23f8a81421eb8d4ad45
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3022bbce/solr/licenses/metrics-jvm-3.2.6.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/metrics-jvm-3.2.6.jar.sha1 b/solr/licenses/metrics-jvm-3.2.6.jar.sha1
new file mode 100644
index 0000000..219d02b
--- /dev/null
+++ b/solr/licenses/metrics-jvm-3.2.6.jar.sha1
@@ -0,0 +1 @@
+a7a475393fe47dfee2042415430da3f01d4fe94e
[11/18] lucene-solr:jira/solr-12438: SOLR-12250: Create the temporary
tlog file properly
Posted by ab...@apache.org.
SOLR-12250: Create the temporary tlog file properly
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2f19ae19
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2f19ae19
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2f19ae19
Branch: refs/heads/jira/solr-12438
Commit: 2f19ae1907e63864e19eda5b6325083f61f9cf66
Parents: 2b406a5
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed Jun 6 15:50:19 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed Jun 6 15:50:19 2018 +0700
----------------------------------------------------------------------
.../test/org/apache/solr/update/TransactionLogTest.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f19ae19/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
index d2b4b26..100d4d9 100644
--- a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
+++ b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
@@ -17,8 +17,10 @@
package org.apache.solr.update;
+import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Locale;
@@ -29,9 +31,11 @@ import org.junit.Test;
public class TransactionLogTest extends LuceneTestCase {
@Test
- public void testBigLastAddSize() throws IOException {
- String tlogFileName = String.format(Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, 0);
- try (TransactionLog transactionLog = new TransactionLog(Files.createTempFile(tlogFileName, "").toFile(), new ArrayList<>())) {
+ public void testBigLastAddSize() {
+ String tlogFileName = String.format(Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, Long.MAX_VALUE);
+ Path path = createTempDir();
+ File logFile = new File(path.toFile(), tlogFileName);
+ try (TransactionLog transactionLog = new TransactionLog(logFile, new ArrayList<>())) {
transactionLog.lastAddSize = 2000000000;
AddUpdateCommand updateCommand = new AddUpdateCommand(null);
updateCommand.solrDoc = new SolrInputDocument();
[08/18] lucene-solr:jira/solr-12438: SOLR-11453: fix typos in the
CHANGES entry
Posted by ab...@apache.org.
SOLR-11453: fix typos in the CHANGES entry
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/124b5e04
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/124b5e04
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/124b5e04
Branch: refs/heads/jira/solr-12438
Commit: 124b5e047df697bfd61f1d14f87a0ee5d24f68d8
Parents: 0c6f38a
Author: Varun Thacker <va...@apache.org>
Authored: Tue Jun 5 15:16:12 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Tue Jun 5 15:16:12 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/124b5e04/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5adc677..6dd4889 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -70,8 +70,8 @@ Upgrade Notes
To return the previous behavior pass false to skipCommitOnMasterVersionZero in slave section of replication
handler configuration, or pass it to the fetchindex command.
-* SOLR-11453: Configuring slowQueryThresholdMillis now logs slow requests to a separate file - solr_slow_requests.log .
- Previously they would get logged in the solr.xml file
+* SOLR-11453: Configuring slowQueryThresholdMillis now logs slow requests to a separate file - solr_slow_requests.log.
+ Previously they would get logged in the solr.log file.
New Features
----------------------
@@ -140,7 +140,7 @@ New Features
* SOLR-12328: JSON Facet API: Domain change with graph query.
(Daniel Meehl, Kevin Watters, yonik)
-* SOLR-11453: Configuring slowQueryThresholdMillis logs slow requests to a separate file - solr_slow_requests.log .
+* SOLR-11453: Configuring slowQueryThresholdMillis logs slow requests to a separate file - solr_slow_requests.log.
(Shawn Heisey, Remko Popma, Varun Thacker)
* SOLR-12401: Add getValue() and setValue() Stream Evaluators (Joel Bernstein, janhoy)
[02/18] lucene-solr:jira/solr-12438: SOLR-12387: fixing a test failure
Posted by ab...@apache.org.
SOLR-12387: fixing a test failure
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f9d807af
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f9d807af
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f9d807af
Branch: refs/heads/jira/solr-12438
Commit: f9d807af755f33dea1384cb8b9c591875076dd1a
Parents: add77d2
Author: Noble Paul <no...@apache.org>
Authored: Tue Jun 5 23:14:11 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jun 5 23:14:11 2018 +1000
----------------------------------------------------------------------
.../solr/handler/admin/CollectionsHandler.java | 5 +++--
.../solr/common/cloud/ClusterProperties.java | 18 ++++++++++++++++++
.../apache/solr/common/cloud/ZkStateReader.java | 6 ++++++
3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f9d807af/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 01d2fe8..269bb50 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -206,9 +206,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
return this.coreContainer;
}
- protected void copyFromClusterProp(Map<String, Object> props, String prop) {
+ protected void copyFromClusterProp(Map<String, Object> props, String prop) throws IOException {
if (props.get(prop) != null) return;//if it's already specified , return
- Object defVal = coreContainer.getZkController().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null);
+ Object defVal = new ClusterProperties(coreContainer.getZkController().getZkStateReader().getZkClient())
+ .getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null);
if (defVal != null) props.put(prop, String.valueOf(defVal));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f9d807af/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
index 87896da..446923b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import org.apache.solr.common.SolrException;
@@ -68,6 +69,23 @@ public class ClusterProperties {
}
/**
+ * Read the value of a cluster property, returning a default if it is not set
+ *
+ * @param key the property name or the full path to the property as a list of parts.
+ * @param defaultValue the default value
+ * @param <T> the type of the property
+ * @return the property value
+ * @throws IOException if there is an error reading the value from the cluster
+ */
+ @SuppressWarnings("unchecked")
+ public <T> T getClusterProperty(List<String> key, T defaultValue) throws IOException {
+ T value = (T) Utils.getObjectByPath(getClusterProperties(), false, key);
+ if (value == null)
+ return defaultValue;
+ return value;
+ }
+
+ /**
* Return the cluster properties
* @throws IOException if there is an error reading properties from the cluster
*/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f9d807af/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 6b65c34..a86c5e2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -961,6 +961,12 @@ public class ZkStateReader implements Closeable {
return value;
}
+ /**Same as the above but allows a full json path as a list of parts
+ *
+ * @param keyPath path to the property example ["collectionDefauls", "numShards"]
+ * @param defaultValue a default value to use if no such property exists
+ * @return the cluster property, or a default if the property is not set
+ */
public <T> T getClusterProperty(List<String> keyPath, T defaultValue) {
T value = (T) Utils.getObjectByPath( clusterProperties, false, keyPath);
if (value == null)
[10/18] lucene-solr:jira/solr-12438: LUCENE-8346: Remove final
modifier on QueryBuilder#createSpanQuery to allow override
Posted by ab...@apache.org.
LUCENE-8346: Remove final modifier on QueryBuilder#createSpanQuery to allow override
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2b406a57
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2b406a57
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2b406a57
Branch: refs/heads/jira/solr-12438
Commit: 2b406a57c41c28a95db1deefc17527e600fa244d
Parents: 9ff3f5a
Author: Jim Ferenczi <ji...@apache.org>
Authored: Wed Jun 6 09:18:25 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed Jun 6 09:18:25 2018 +0200
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2b406a57/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java b/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
index 2cb066b..697e3bb 100644
--- a/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
@@ -346,9 +346,9 @@ public class QueryBuilder {
/**
* Creates a span query from the tokenstream. In the case of a single token, a simple <code>SpanTermQuery</code> is
- * returned. When multiple tokens, an ordered <code>SpanNearQuery</code> with slop of 0 is returned.
+ * returned. When multiple tokens, an ordered <code>SpanNearQuery</code> with slop 0 is returned.
*/
- protected final SpanQuery createSpanQuery(TokenStream in, String field) throws IOException {
+ protected SpanQuery createSpanQuery(TokenStream in, String field) throws IOException {
TermToBytesRefAttribute termAtt = in.getAttribute(TermToBytesRefAttribute.class);
if (termAtt == null) {
return null;
[17/18] lucene-solr:jira/solr-12438: Merge branch 'master' into
jira/solr-12438
Posted by ab...@apache.org.
Merge branch 'master' into jira/solr-12438
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3e724818
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3e724818
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3e724818
Branch: refs/heads/jira/solr-12438
Commit: 3e724818c39623d40024025c7dfe5c1be694c547
Parents: 2fa260c d163159
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 16:48:27 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 16:48:27 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/util/QueryBuilder.java | 4 +-
lucene/ivy-versions.properties | 2 +-
.../spatial/spatial4j/Geo3dShapeFactory.java | 10 +-
.../lucene/spatial/spatial4j/Geo3dRptTest.java | 23 +-
solr/CHANGES.txt | 19 +-
solr/NOTICE.txt | 14 +
.../src/java/org/apache/solr/core/SolrCore.java | 2 +-
.../apache/solr/core/SolrResourceLoader.java | 12 +-
.../solr/handler/admin/CollectionsHandler.java | 5 +-
.../solr/handler/tagger/OffsetCorrector.java | 178 +++++++++
.../solr/handler/tagger/TagClusterReducer.java | 103 +++++
.../org/apache/solr/handler/tagger/TagLL.java | 176 ++++++++
.../org/apache/solr/handler/tagger/Tagger.java | 230 +++++++++++
.../handler/tagger/TaggerRequestHandler.java | 397 +++++++++++++++++++
.../solr/handler/tagger/TaggingAttribute.java | 65 +++
.../handler/tagger/TaggingAttributeImpl.java | 79 ++++
.../solr/handler/tagger/TermPrefixCursor.java | 189 +++++++++
.../solr/handler/tagger/XmlOffsetCorrector.java | 113 ++++++
.../solr/handler/tagger/package-info.java | 27 ++
.../org/apache/solr/search/QParserPlugin.java | 80 ++--
.../solr/collection1/conf/schema-tagger.xml | 187 +++++++++
.../solr/collection1/conf/solrconfig-tagger.xml | 59 +++
.../autoscaling/AutoScalingHandlerTest.java | 21 +
.../autoscaling/sim/SimSolrCloudTestCase.java | 56 ++-
.../apache/solr/core/ResourceLoaderTest.java | 6 +-
.../tagger/EmbeddedSolrNoSerializeTest.java | 153 +++++++
.../handler/tagger/RandomizedTaggerTest.java | 150 +++++++
.../apache/solr/handler/tagger/Tagger2Test.java | 175 ++++++++
.../apache/solr/handler/tagger/TaggerTest.java | 296 ++++++++++++++
.../solr/handler/tagger/TaggerTestCase.java | 251 ++++++++++++
.../handler/tagger/TaggingAttributeTest.java | 73 ++++
.../handler/tagger/WordLengthTaggingFilter.java | 110 +++++
.../tagger/WordLengthTaggingFilterFactory.java | 67 ++++
.../handler/tagger/XmlInterpolationTest.java | 224 +++++++++++
.../solr/search/TestStandardQParsers.java | 10 +-
.../apache/solr/update/TransactionLogTest.java | 12 +-
solr/licenses/metrics-core-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-core-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-ganglia-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-ganglia-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-graphite-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-graphite-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-jetty9-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-jetty9-3.2.6.jar.sha1 | 1 +
solr/licenses/metrics-jvm-3.2.2.jar.sha1 | 1 -
solr/licenses/metrics-jvm-3.2.6.jar.sha1 | 1 +
solr/solr-ref-guide/src/searching.adoc | 33 +-
solr/solr-ref-guide/src/the-tagger-handler.adoc | 265 +++++++++++++
.../client/solrj/cloud/autoscaling/Clause.java | 2 +-
.../solr/common/cloud/ClusterProperties.java | 18 +
.../apache/solr/common/cloud/ZkStateReader.java | 6 +
52 files changed, 3803 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
[07/18] lucene-solr:jira/solr-12438: SOLR-12233: QParserPlugin's
static registry of builtins can be optimized to avoid needless ClassLoader
activity on SolrCore load.
Posted by ab...@apache.org.
SOLR-12233: QParserPlugin's static registry of builtins can be optimized
to avoid needless ClassLoader activity on SolrCore load.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0c6f38a3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0c6f38a3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0c6f38a3
Branch: refs/heads/jira/solr-12438
Commit: 0c6f38a315d0df5abd01e7d4efe481bc53444a49
Parents: cf63392
Author: Jeff <je...@salesforce.com>
Authored: Tue Jun 5 15:11:32 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Jun 5 15:11:32 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 6 +-
.../src/java/org/apache/solr/core/SolrCore.java | 2 +-
.../org/apache/solr/search/QParserPlugin.java | 80 ++++++++++----------
.../solr/search/TestStandardQParsers.java | 10 +--
4 files changed, 51 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 479406f..5adc677 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -155,7 +155,7 @@ New Features
* SOLR-12389: support deeply nested json objects in clusterprops.json (noble)
* SOLR-12376: Added the TaggerRequestHandler (AKA SolrTextTagger) for tagging text. It's used as a component of
- NER/ERD systems including query-understanding. See the ref guide for more info. (David Smiley
+ NER/ERD systems including query-understanding. See the ref guide for more info. (David Smiley)
Bug Fixes
----------------------
@@ -332,6 +332,10 @@ Optimizations
* SOLR-9922: Write buffering updates to another tlog. (Cao Manh Dat)
+* SOLR-12233: QParserPlugin's built-in static registry now holds actual QParserPlugin instances instead of class
+ references. This is consistent with other plugin registries and allows a SolrCore to load faster.
+ (Jeff Miller, David Smiley)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 99c0cca..feab22d 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -958,7 +958,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
initIndex(prev != null, reload);
initWriters();
- qParserPlugins.init(createInstances(QParserPlugin.standardPlugins), this);
+ qParserPlugins.init(QParserPlugin.standardPlugins, this);
valueSourceParsers.init(ValueSourceParser.standardValueSourceParsers, this);
transformerFactories.init(TransformerFactory.defaultFactories, this);
loadSearchComponents();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
index f80bc9c..b20c3c8 100644
--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
@@ -37,53 +37,53 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
public static final String DEFAULT_QTYPE = LuceneQParserPlugin.NAME;
/**
- * Internal use - name to class mappings of builtin parsers.
+ * Internal use - name to parser for the builtin parsers.
* Each query parser plugin extending {@link QParserPlugin} has own instance of standardPlugins.
* This leads to cyclic dependencies of static fields and to case when NAME field is not yet initialized.
* This result to NPE during initialization.
* For every plugin, listed here, NAME field has to be final and static.
*/
- public static final Map<String, Class<? extends QParserPlugin>> standardPlugins;
+ public static final Map<String, QParserPlugin> standardPlugins;
static {
- HashMap<String, Class<? extends QParserPlugin>> map = new HashMap<>(30, 1);
- map.put(LuceneQParserPlugin.NAME, LuceneQParserPlugin.class);
- map.put(FunctionQParserPlugin.NAME, FunctionQParserPlugin.class);
- map.put(PrefixQParserPlugin.NAME, PrefixQParserPlugin.class);
- map.put(BoostQParserPlugin.NAME, BoostQParserPlugin.class);
- map.put(DisMaxQParserPlugin.NAME, DisMaxQParserPlugin.class);
- map.put(ExtendedDismaxQParserPlugin.NAME, ExtendedDismaxQParserPlugin.class);
- map.put(FieldQParserPlugin.NAME, FieldQParserPlugin.class);
- map.put(RawQParserPlugin.NAME, RawQParserPlugin.class);
- map.put(TermQParserPlugin.NAME, TermQParserPlugin.class);
- map.put(TermsQParserPlugin.NAME, TermsQParserPlugin.class);
- map.put(NestedQParserPlugin.NAME, NestedQParserPlugin.class);
- map.put(FunctionRangeQParserPlugin.NAME, FunctionRangeQParserPlugin.class);
- map.put(SpatialFilterQParserPlugin.NAME, SpatialFilterQParserPlugin.class);
- map.put(SpatialBoxQParserPlugin.NAME, SpatialBoxQParserPlugin.class);
- map.put(JoinQParserPlugin.NAME, JoinQParserPlugin.class);
- map.put(SurroundQParserPlugin.NAME, SurroundQParserPlugin.class);
- map.put(SwitchQParserPlugin.NAME, SwitchQParserPlugin.class);
- map.put(MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class);
- map.put(BlockJoinParentQParserPlugin.NAME, BlockJoinParentQParserPlugin.class);
- map.put(BlockJoinChildQParserPlugin.NAME, BlockJoinChildQParserPlugin.class);
- map.put(FiltersQParserPlugin.NAME, FiltersQParserPlugin.class);
- map.put(CollapsingQParserPlugin.NAME, CollapsingQParserPlugin.class);
- map.put(SimpleQParserPlugin.NAME, SimpleQParserPlugin.class);
- map.put(ComplexPhraseQParserPlugin.NAME, ComplexPhraseQParserPlugin.class);
- map.put(ReRankQParserPlugin.NAME, ReRankQParserPlugin.class);
- map.put(ExportQParserPlugin.NAME, ExportQParserPlugin.class);
- map.put(MLTQParserPlugin.NAME, MLTQParserPlugin.class);
- map.put(HashQParserPlugin.NAME, HashQParserPlugin.class);
- map.put(GraphQParserPlugin.NAME, GraphQParserPlugin.class);
- map.put(XmlQParserPlugin.NAME, XmlQParserPlugin.class);
- map.put(GraphTermsQParserPlugin.NAME, GraphTermsQParserPlugin.class);
- map.put(IGainTermsQParserPlugin.NAME, IGainTermsQParserPlugin.class);
- map.put(TextLogisticRegressionQParserPlugin.NAME, TextLogisticRegressionQParserPlugin.class);
- map.put(SignificantTermsQParserPlugin.NAME, SignificantTermsQParserPlugin.class);
- map.put(PayloadScoreQParserPlugin.NAME, PayloadScoreQParserPlugin.class);
- map.put(PayloadCheckQParserPlugin.NAME, PayloadCheckQParserPlugin.class);
- map.put(BoolQParserPlugin.NAME, BoolQParserPlugin.class);
+ HashMap<String, QParserPlugin> map = new HashMap<>(30, 1);
+ map.put(LuceneQParserPlugin.NAME, new LuceneQParserPlugin());
+ map.put(FunctionQParserPlugin.NAME, new FunctionQParserPlugin());
+ map.put(PrefixQParserPlugin.NAME, new PrefixQParserPlugin());
+ map.put(BoostQParserPlugin.NAME, new BoostQParserPlugin());
+ map.put(DisMaxQParserPlugin.NAME, new DisMaxQParserPlugin());
+ map.put(ExtendedDismaxQParserPlugin.NAME, new ExtendedDismaxQParserPlugin());
+ map.put(FieldQParserPlugin.NAME, new FieldQParserPlugin());
+ map.put(RawQParserPlugin.NAME, new RawQParserPlugin());
+ map.put(TermQParserPlugin.NAME, new TermQParserPlugin());
+ map.put(TermsQParserPlugin.NAME, new TermsQParserPlugin());
+ map.put(NestedQParserPlugin.NAME, new NestedQParserPlugin());
+ map.put(FunctionRangeQParserPlugin.NAME, new FunctionRangeQParserPlugin());
+ map.put(SpatialFilterQParserPlugin.NAME, new SpatialFilterQParserPlugin());
+ map.put(SpatialBoxQParserPlugin.NAME, new SpatialBoxQParserPlugin());
+ map.put(JoinQParserPlugin.NAME, new JoinQParserPlugin());
+ map.put(SurroundQParserPlugin.NAME, new SurroundQParserPlugin());
+ map.put(SwitchQParserPlugin.NAME, new SwitchQParserPlugin());
+ map.put(MaxScoreQParserPlugin.NAME, new MaxScoreQParserPlugin());
+ map.put(BlockJoinParentQParserPlugin.NAME, new BlockJoinParentQParserPlugin());
+ map.put(BlockJoinChildQParserPlugin.NAME, new BlockJoinChildQParserPlugin());
+ map.put(FiltersQParserPlugin.NAME, new FiltersQParserPlugin());
+ map.put(CollapsingQParserPlugin.NAME, new CollapsingQParserPlugin());
+ map.put(SimpleQParserPlugin.NAME, new SimpleQParserPlugin());
+ map.put(ComplexPhraseQParserPlugin.NAME, new ComplexPhraseQParserPlugin());
+ map.put(ReRankQParserPlugin.NAME, new ReRankQParserPlugin());
+ map.put(ExportQParserPlugin.NAME, new ExportQParserPlugin());
+ map.put(MLTQParserPlugin.NAME, new MLTQParserPlugin());
+ map.put(HashQParserPlugin.NAME, new HashQParserPlugin());
+ map.put(GraphQParserPlugin.NAME, new GraphQParserPlugin());
+ map.put(XmlQParserPlugin.NAME, new XmlQParserPlugin());
+ map.put(GraphTermsQParserPlugin.NAME, new GraphTermsQParserPlugin());
+ map.put(IGainTermsQParserPlugin.NAME, new IGainTermsQParserPlugin());
+ map.put(TextLogisticRegressionQParserPlugin.NAME, new TextLogisticRegressionQParserPlugin());
+ map.put(SignificantTermsQParserPlugin.NAME, new SignificantTermsQParserPlugin());
+ map.put(PayloadScoreQParserPlugin.NAME, new PayloadScoreQParserPlugin());
+ map.put(PayloadCheckQParserPlugin.NAME, new PayloadCheckQParserPlugin());
+ map.put(BoolQParserPlugin.NAME, new BoolQParserPlugin());
standardPlugins = Collections.unmodifiableMap(map);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c6f38a3/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java b/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
index ff9ffff..cab9026 100644
--- a/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
+++ b/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
@@ -16,15 +16,15 @@
*/
package org.apache.solr.search;
-import org.apache.lucene.util.LuceneTestCase;
-import org.junit.Test;
-
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
/**
* Check standard query parsers for class loading problems during initialization (NAME field is final and static).
* Because every query plugin extend {@link org.apache.solr.search.QParserPlugin} and contains own instance of {@link org.apache.solr.search.QParserPlugin#standardPlugins},
@@ -50,9 +50,9 @@ public class TestStandardQParsers extends LuceneTestCase {
List<String> notFinal = new ArrayList<>(QParserPlugin.standardPlugins.size());
List<String> mismatch = new ArrayList<>(QParserPlugin.standardPlugins.size());
- for (Map.Entry<String, Class<? extends QParserPlugin>> pair : QParserPlugin.standardPlugins.entrySet()) {
+ for (Map.Entry<String, QParserPlugin> pair : QParserPlugin.standardPlugins.entrySet()) {
String regName = pair.getKey();
- Class<? extends QParserPlugin> clazz = pair.getValue();
+ Class<? extends QParserPlugin> clazz = pair.getValue().getClass();;
Field nameField = clazz.getField(FIELD_NAME);
int modifiers = nameField.getModifiers();
[14/18] lucene-solr:jira/solr-12438: SOLR-11911: Move simulator reset
to @Before method.
Posted by ab...@apache.org.
SOLR-11911: Move simulator reset to @Before method.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c1f42906
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c1f42906
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c1f42906
Branch: refs/heads/jira/solr-12438
Commit: c1f42906ef1e0929f3887736dd9bb5bd0f9975c8
Parents: af7dfb1
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 11:06:50 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 11:48:42 2018 +0200
----------------------------------------------------------------------
.../autoscaling/sim/SimSolrCloudTestCase.java | 56 ++++++++++----------
1 file changed, 27 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1f42906/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
index e83f72f..270e7e7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
@@ -80,40 +80,38 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 {
@Override
public void setUp() throws Exception {
super.setUp();
- if (cluster != null) {
- // clear any persisted configuration
- cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
- cluster.getDistribStateManager().setData(ZkStateReader.ROLES, Utils.toJSON(new HashMap<>()), -1);
- cluster.getSimClusterStateProvider().simDeleteAllCollections();
- cluster.simClearSystemCollection();
- cluster.getSimNodeStateProvider().simRemoveDeadNodes();
- cluster.getSimClusterStateProvider().simRemoveDeadNodes();
- // restore the expected number of nodes
- int currentSize = cluster.getLiveNodesSet().size();
- if (currentSize < clusterNodeCount) {
- int addCnt = clusterNodeCount - currentSize;
- while (addCnt-- > 0) {
- cluster.simAddNode();
- }
- } else if (currentSize > clusterNodeCount) {
- cluster.simRemoveRandomNodes(currentSize - clusterNodeCount, true, random());
- }
- // clean any persisted trigger state or events
- removeChildren(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
- removeChildren(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
- removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
- removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
- cluster.getSimClusterStateProvider().simResetLeaderThrottles();
- cluster.simRestartOverseer(null);
- cluster.getTimeSource().sleep(5000);
- cluster.simResetOpCounts();
- }
}
@Before
- public void checkClusterConfiguration() {
+ public void checkClusterConfiguration() throws Exception {
if (cluster == null)
throw new RuntimeException("SimCloudManager not configured - have you called configureCluster()?");
+ // clear any persisted configuration
+ cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
+ cluster.getDistribStateManager().setData(ZkStateReader.ROLES, Utils.toJSON(new HashMap<>()), -1);
+ cluster.getSimClusterStateProvider().simDeleteAllCollections();
+ cluster.simClearSystemCollection();
+ cluster.getSimNodeStateProvider().simRemoveDeadNodes();
+ cluster.getSimClusterStateProvider().simRemoveDeadNodes();
+ // restore the expected number of nodes
+ int currentSize = cluster.getLiveNodesSet().size();
+ if (currentSize < clusterNodeCount) {
+ int addCnt = clusterNodeCount - currentSize;
+ while (addCnt-- > 0) {
+ cluster.simAddNode();
+ }
+ } else if (currentSize > clusterNodeCount) {
+ cluster.simRemoveRandomNodes(currentSize - clusterNodeCount, true, random());
+ }
+ // clean any persisted trigger state or events
+ removeChildren(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
+ removeChildren(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
+ removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
+ removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
+ cluster.getSimClusterStateProvider().simResetLeaderThrottles();
+ cluster.simRestartOverseer(null);
+ cluster.getTimeSource().sleep(5000);
+ cluster.simResetOpCounts();
}
protected void removeChildren(String path) throws Exception {
[16/18] lucene-solr:jira/solr-12438: SOLR-12445: Update CHANGES.
Posted by ab...@apache.org.
SOLR-12445: Update CHANGES.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d1631593
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d1631593
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d1631593
Branch: refs/heads/jira/solr-12438
Commit: d1631593f3a0ff9f58468ff0f32d7c1e8a404567
Parents: 3022bbc
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jun 6 15:30:32 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jun 6 15:30:32 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d1631593/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6dd4889..c93ccd8 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -295,6 +295,8 @@ Bug Fixes
* SOLR-12354: Register the /admin/info/key end-point at the startup time to avoid 404 (noble)
+* SOLR-12445: Upgrade Dropwizard Metrics to version 3.2.6. (ab)
+
Optimizations
----------------------
[03/18] lucene-solr:jira/solr-12438: SOLR-12417: doc: fix CHANGES
credit
Posted by ab...@apache.org.
SOLR-12417: doc: fix CHANGES credit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7d0b64f9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7d0b64f9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7d0b64f9
Branch: refs/heads/jira/solr-12438
Commit: 7d0b64f9d5484c1f03ccf422bdee50c31443344c
Parents: f9d807a
Author: yonik <yo...@apache.org>
Authored: Tue Jun 5 10:55:53 2018 -0400
Committer: yonik <yo...@apache.org>
Committed: Tue Jun 5 10:55:53 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d0b64f9/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 1b454a0..66d2026 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -283,7 +283,7 @@ Bug Fixes
* SOLR-12374: SnapShooter.getIndexCommit can forget to decref the searcher; though it's not clear in practice when.
(David Smiley)
-* SOLR-12417: velocity response writer should enforce valid function name for v.json parameter (yonik)
+* SOLR-12417: velocity response writer should enforce valid function name for v.json parameter (Mano Kovacs, yonik)
* SOLR-12271: Fixed bug in how Analytics component reads negative values from float and double fields. (Houston Putman)
[12/18] lucene-solr:jira/solr-12438: SOLR-12250: Fix precommit
Posted by ab...@apache.org.
SOLR-12250: Fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0358fcb1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0358fcb1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0358fcb1
Branch: refs/heads/jira/solr-12438
Commit: 0358fcb1759038f76d91c124703c5d3244125f75
Parents: 2f19ae1
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed Jun 6 15:52:15 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed Jun 6 15:52:15 2018 +0700
----------------------------------------------------------------------
solr/core/src/test/org/apache/solr/update/TransactionLogTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0358fcb1/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
index 100d4d9..66ecbc6 100644
--- a/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
+++ b/solr/core/src/test/org/apache/solr/update/TransactionLogTest.java
@@ -18,8 +18,6 @@
package org.apache.solr.update;
import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Locale;
[05/18] lucene-solr:jira/solr-12438: SOLR-12376: New
TaggerRequestHandler (SolrTextTagger).
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java
new file mode 100644
index 0000000..cb742a8
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/RandomizedTaggerTest.java
@@ -0,0 +1,150 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Random;
+import java.util.Set;
+
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Randomly generate taggable text and verify via simple tag algorithm.
+ */
+@Repeat(iterations = 10)
+public class RandomizedTaggerTest extends TaggerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+ }
+
+ @Test
+ public void test() throws Exception {
+ final Random R = random();
+
+ Set<String> names = new HashSet<>();
+ //random list of single-word names
+ final int NUM_SINGLES = 4;//RandomInts.randomIntBetween(R, 1, 5);
+ for (int i = 0; i < NUM_SINGLES; i++) {
+ if (i == 0)//first is a big string (perhaps triggers bugs related to growing buffers)
+ names.add(randomStringOfLength(16, 32));
+ else
+ names.add(randomString());
+ }
+
+ //add random list of multi-word names, partially including existing names
+ final int NUM_MULTI = 10;
+ for (int i = 0; i < NUM_MULTI; i++) {
+ final int numWords = RandomNumbers.randomIntBetween(R, 2, 4);
+ StringBuilder buf = new StringBuilder();
+ for (int j = 0; j < numWords; j++) {
+ if (j != 0)
+ buf.append(' ');
+ if (R.nextBoolean()) {//new likely non-existent word
+ buf.append(randomString());
+ } else {//existing word (possible multi-word from prev iteration)
+ buf.append(RandomPicks.randomFrom(R, names));
+ }
+ }
+ names.add(buf.toString());
+ }
+
+ // BUILD NAMES
+ buildNames(names.toArray(new String[names.size()]));
+
+ // QUERY LOOP
+ for (int tTries = 0; tTries < 10 * RANDOM_MULTIPLIER; tTries++) {
+ // Build up random input, similar to multi-word random names above
+ StringBuilder input = new StringBuilder();
+ final int INPUT_WORD_LEN = 20;
+ input.append(' ');//must start with space based on assertBruteForce logic
+ for (int i = 0; i < INPUT_WORD_LEN; i++) {
+ if (R.nextBoolean()) {//new likely non-existent word
+ input.append(randomString());
+ } else {//existing word (possible multi-word from prev iteration)
+ input.append(RandomPicks.randomFrom(R, NAMES));
+ }
+ input.append(' ');//must end with a space
+ }
+
+ boolean madeIt = false;
+ try {
+ assertBruteForce(input.toString());
+ madeIt = true;
+ } finally {
+ if (!madeIt) {
+ System.out.println("Reproduce with:");
+ System.out.print(" buildNames(");
+ for (int i = 0; i < NAMES.size(); i++) {
+ if (i != 0)
+ System.out.print(',');
+ System.out.print('"');
+ System.out.print(NAMES.get(i));
+ System.out.print('"');
+ }
+ System.out.println(");");
+ System.out.println(" assertBruteForce(\"" + input+"\");");
+ }
+ }
+ }
+
+ }
+
+ private void assertBruteForce(String input) throws Exception {
+ assert input.matches(" .* ");
+ baseParams.set("overlaps", "ALL");
+
+ //loop through NAMES and find all tag offsets
+ List<TestTag> testTags = new ArrayList<>();
+ for (String name : NAMES) {
+ String spaceName = " "+name+" ";
+ int off = 0;
+ while (true) {
+ int idx = input.indexOf(spaceName, off);
+ if (idx < 0)
+ break;
+ testTags.add(new TestTag(idx + 1, idx + 1 + name.length(), name, name));
+ off = idx + 1;
+ }
+ }
+
+ //assert
+ assertTags(reqDoc(input), testTags.toArray(new TestTag[testTags.size()]));
+ }
+
+ private String randomString() { return randomStringOfLength(1, 1); }
+
+ private String randomStringOfLength(int min, int max) {
+ return RandomStrings.randomAsciiLettersOfLengthBetween(random(), min, max).toLowerCase(Locale.ROOT);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java b/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java
new file mode 100644
index 0000000..c7580e1
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/Tagger2Test.java
@@ -0,0 +1,175 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.nio.charset.StandardCharsets;
+
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Test the {@link TaggerRequestHandler}.
+ */
+public class Tagger2Test extends TaggerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ baseParams.set("overlaps", "LONGEST_DOMINANT_RIGHT");
+ }
+
+ /** whole matching, no sub-tags */
+ @Test
+ public void testLongestDominantRight() throws Exception {
+ buildNames("in", "San", "in San", "Francisco", "San Francisco",
+ "San Francisco State College", "College of California",
+ "Clayton", "Clayton North", "North Carolina");
+
+ assertTags("He lived in San Francisco.",
+ "in", "San Francisco");
+
+ assertTags("He enrolled in San Francisco State College of California",
+ "in", "San Francisco State College");
+
+ assertTags("He lived in Clayton North Carolina",
+ "in", "Clayton", "North Carolina");
+
+ }
+
+ // As of Lucene/Solr 4.9, StandardTokenizer never does this anymore (reported to Lucene dev-list,
+ // Jan 26th 2015. Honestly it's not particularly important to us but it renders this test
+ // pointless.
+ /** Orig issue https://github.com/OpenSextant/SolrTextTagger/issues/2 related: #13 */
+ @Test
+ @Ignore
+ public void testVeryLongWord() throws Exception {
+ String SANFRAN = "San Francisco";
+ buildNames(SANFRAN);
+
+ // exceeds default 255 max token length which means it in-effect becomes a stop-word
+ StringBuilder STOP = new StringBuilder(260);//>255
+ for (int i = 0; i < STOP.capacity(); i++) {
+ STOP.append((char) ('0' + (i % 10)));
+ }
+
+ String doc = "San " + STOP + " Francisco";
+ assertTags(doc);//no match due to default stop word handling
+ //and we find it when we ignore stop words
+ assertTags(reqDoc(doc, "ignoreStopwords", "true"), new TestTag(0, doc.length(), doc, lookupByName(SANFRAN)));
+ }
+
+ /** Support for stopwords (posInc > 1);
+ * discussion: https://github.com/OpenSextant/SolrTextTagger/issues/13 */
+ @Test
+ public void testStopWords() throws Exception {
+ baseParams.set("field", "name_tagStop");//stop filter (pos inc enabled) index & query
+
+ String SOUTHOFWALES = "South of Wales";//'of' is stop word index time & query
+ String ACITYA = "A City A";
+
+ buildNames(SOUTHOFWALES, ACITYA);
+
+ //round-trip works
+ assertTags(reqDoc(SOUTHOFWALES), new TestTag(0, SOUTHOFWALES.length(), SOUTHOFWALES,
+ lookupByName(SOUTHOFWALES)));
+ // but offsets doesn't include stopword when leading or trailing...
+ assertTags(reqDoc(ACITYA), new TestTag(2, 6, "City",
+ lookupByName(ACITYA)));
+ //break on stop words
+ assertTags(reqDoc(SOUTHOFWALES, "ignoreStopwords", "false"));//match nothing
+ }
+
+ /** Tests WordDelimiterGraphFilter, stacked/synonymous tokens at index time (catenate options) */
+ @Test
+ public void testWDF() throws Exception {
+ baseParams.set("field", "name_tagWDF");
+
+ final String WINSTONSALEM = "City of Winston-Salem";//hyphen
+ final String BOSTONHARBOR = "Boston Harbor";//space
+ buildNames(WINSTONSALEM, BOSTONHARBOR);
+
+ //round-trip works
+ assertTags(reqDoc(WINSTONSALEM), new TestTag(0, WINSTONSALEM.length(), WINSTONSALEM,
+ lookupByName(WINSTONSALEM)));
+
+ // space separated works
+ final String WS_SPACE = WINSTONSALEM.replace('-', ' ');
+ assertTags(reqDoc(WS_SPACE),
+ new TestTag(0, WS_SPACE.length(), WS_SPACE,
+ lookupByName(WINSTONSALEM)));
+
+ //must be full match
+ assertTags(reqDoc("Winston"));//match nothing
+ assertTags(reqDoc("Salem"));//match nothing
+
+ // round-trip works
+ assertTags(reqDoc(BOSTONHARBOR), new TestTag(0, BOSTONHARBOR.length(), BOSTONHARBOR,
+ lookupByName(BOSTONHARBOR)));
+
+ // hyphen separated works
+ final String BH_HYPHEN = BOSTONHARBOR.replace(' ', '-');
+ assertTags(reqDoc(BH_HYPHEN),
+ new TestTag(0, BH_HYPHEN.length(), BH_HYPHEN,
+ lookupByName(BOSTONHARBOR)));
+ //must be full match
+ assertTags(reqDoc("Boston"));//match nothing
+ assertTags(reqDoc("Harbor"));//match nothing
+ }
+
+ /** Ensure character offsets work for multi-byte characters */
+ @Test
+ public void testMultibyteChar() throws Exception {
+ // https://unicode-table.com/en/2019/
+ // 0 1 2 3 4
+ // 01234567890123456789012345678901234567890
+ String TEXT = "He mentionned ’Obama’ in the White House";
+ assertEquals(40, TEXT.length()); // char length (in Java, UTF16)
+
+ String QUOTE = TEXT.substring(14, 15);
+ assertEquals(8217, QUOTE.codePointAt(0));
+
+ //UTF8
+ assertEquals(3, QUOTE.getBytes(StandardCharsets.UTF_8).length);
+ assertEquals(1, "a".getBytes(StandardCharsets.UTF_8).length);
+ assertEquals(40 + 2*2, TEXT.getBytes(StandardCharsets.UTF_8).length);
+
+ //UTF16 big endian (by specifying big/little endian, there is no "byte order mark")
+ assertEquals(2, QUOTE.getBytes(StandardCharsets.UTF_16BE).length);
+ assertEquals(2, "a".getBytes(StandardCharsets.UTF_16BE).length);
+ assertEquals(40 * 2, TEXT.getBytes(StandardCharsets.UTF_16BE).length);
+
+
+ buildNames("Obama");
+
+ assertTags(TEXT, "Obama");
+
+ // TODO test surrogate pairs (i.e. code points not in the BMP)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java
new file mode 100644
index 0000000..93b11b5
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTest.java
@@ -0,0 +1,296 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+/**
+ * The original test for {@link TaggerRequestHandler}.
+ */
+public class TaggerTest extends TaggerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+ }
+
+ private void indexAndBuild() throws Exception {
+ N[] names = N.values();
+ String[] namesStrs = new String[names.length];
+ for (int i = 0; i < names.length; i++) {
+ namesStrs[i] = names[i].getName();
+ }
+ buildNames(namesStrs);
+ }
+
+ /** Name corpus */
+ enum N {
+ //keep order to retain ord()
+ London, London_Business_School, Boston, City_of_London,
+ of, the//filtered out of the corpus by a custom query
+ ;
+
+ String getName() { return name().replace('_',' '); }
+ static N lookupByName(String name) { return N.valueOf(name.replace(' ', '_')); }
+ int getId() { return ordinal(); }
+ }
+
+ public void testFormat() throws Exception {
+ baseParams.set("overlaps", "NO_SUB");
+ indexAndBuild();
+
+ String rspStr = _testFormatRequest(false);
+ String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<response>\n" +
+ "\n" +
+ "<int name=\"tagsCount\">1</int>\n" +
+ "<arr name=\"tags\">\n" +
+ " <lst>\n" +
+ " <int name=\"startOffset\">0</int>\n" +
+ " <int name=\"endOffset\">22</int>\n" +
+ " <arr name=\"ids\">\n" +
+ " <str>1</str>\n" +
+ " </arr>\n" +
+ " </lst>\n" +
+ "</arr>\n" +
+ "<result name=\"response\" numFound=\"1\" start=\"0\">\n" +
+ " <doc>\n" +
+ " <str name=\"id\">1</str>\n" +
+ " <str name=\"name\">London Business School</str></doc>\n" +
+ "</result>\n" +
+ "</response>\n";
+ assertEquals(expected, rspStr);
+ }
+
+ public void testFormatMatchText() throws Exception {
+ baseParams.set("overlaps", "NO_SUB");
+ indexAndBuild();
+
+ String rspStr = _testFormatRequest(true);
+ String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<response>\n" +
+ "\n" +
+ "<int name=\"tagsCount\">1</int>\n" +
+ "<arr name=\"tags\">\n" +
+ " <lst>\n" +
+ " <int name=\"startOffset\">0</int>\n" +
+ " <int name=\"endOffset\">22</int>\n" +
+ " <str name=\"matchText\">london business school</str>\n" +
+ " <arr name=\"ids\">\n" +
+ " <str>1</str>\n" +
+ " </arr>\n" +
+ " </lst>\n" +
+ "</arr>\n" +
+ "<result name=\"response\" numFound=\"1\" start=\"0\">\n" +
+ " <doc>\n" +
+ " <str name=\"id\">1</str>\n" +
+ " <str name=\"name\">London Business School</str></doc>\n" +
+ "</result>\n" +
+ "</response>\n";
+ assertEquals(expected, rspStr);
+ }
+
+ private String _testFormatRequest(boolean matchText) throws Exception {
+ String doc = "london business school";//just one tag
+ SolrQueryRequest req = reqDoc(doc, "indent", "on", "omitHeader", "on", "matchText", ""+matchText);
+ String rspStr = h.query(req);
+ req.close();
+ return rspStr;
+ }
+
+ /** Partial matching, no sub-tags */
+ @Ignore //TODO ConcatenateGraphFilter uses a special separator char that we can't put into XML (invalid char)
+ public void testPartialMatching() throws Exception {
+ baseParams.set("field", "name_tagPartial");
+ baseParams.set("overlaps", "NO_SUB");
+ baseParams.set("fq", "NOT name:(of the)");//test filtering
+ indexAndBuild();
+
+ //these match nothing
+ assertTags(reqDoc("") );
+ assertTags(reqDoc(" ") );
+ assertTags(reqDoc("the") );
+
+ String doc;
+
+ //just London Business School via "school" substring
+ doc = "school";
+ assertTags(reqDoc(doc), tt(doc,"school", 0, N.London_Business_School));
+
+ doc = "a school";
+ assertTags(reqDoc(doc), tt(doc,"school", 0, N.London_Business_School));
+
+ doc = "school a";
+ assertTags(reqDoc(doc), tt(doc,"school", 0, N.London_Business_School));
+
+ //More interesting
+
+ doc = "school City";
+ assertTags(reqDoc(doc),
+ tt(doc, "school", 0, N.London_Business_School),
+ tt(doc, "City", 0, N.City_of_London) );
+
+ doc = "City of London Business School";
+ assertTags(reqDoc(doc), //no plain London (sub-tag)
+ tt(doc, "City of London", 0, N.City_of_London),
+ tt(doc, "London Business School", 0, N.London_Business_School));
+ }
+
+ /** whole matching, no sub-tags */
+ public void testWholeMatching() throws Exception {
+ baseParams.set("overlaps", "NO_SUB");
+ baseParams.set("fq", "NOT name:(of the)");//test filtering
+ indexAndBuild();
+
+ //these match nothing
+ assertTags(reqDoc(""));
+ assertTags(reqDoc(" ") );
+ assertTags(reqDoc("the") );
+
+ //partial on N.London_Business_School matches nothing
+ assertTags(reqDoc("school") );
+ assertTags(reqDoc("a school") );
+ assertTags(reqDoc("school a") );
+ assertTags(reqDoc("school City") );
+
+ String doc;
+
+ doc = "school business london";//backwards
+ assertTags(reqDoc(doc), tt(doc,"london", 0, N.London));
+
+ doc = "of London Business School";
+ assertTags(reqDoc(doc), //no plain London (sub-tag)
+ tt(doc, "London Business School", 0, N.London_Business_School));
+
+ //More interesting
+ doc = "City of London Business School";
+ assertTags(reqDoc(doc), //no plain London (sub-tag)
+ tt(doc, "City of London", 0, N.City_of_London),
+ tt(doc, "London Business School", 0, N.London_Business_School));
+
+ doc = "City of London Business";
+ assertTags(reqDoc(doc), //no plain London (sub-tag) no Business (partial-match)
+ tt(doc, "City of London", 0, N.City_of_London));
+
+ doc = "London Business magazine";
+ assertTags(reqDoc(doc), //Just London; L.B.S. fails
+ tt(doc, "London", 0, N.London));
+ }
+
+ /** whole matching, with sub-tags */
+ public void testSubTags() throws Exception {
+ baseParams.set("overlaps", "ALL");
+ baseParams.set("fq", "NOT name:(of the)");//test filtering
+ indexAndBuild();
+
+ //these match nothing
+ assertTags(reqDoc(""));
+ assertTags(reqDoc(" ") );
+ assertTags(reqDoc("the") );
+
+ //partial on N.London_Business_School matches nothing
+ assertTags(reqDoc("school") );
+ assertTags(reqDoc("a school") );
+ assertTags(reqDoc("school a") );
+ assertTags(reqDoc("school City") );
+
+ String doc;
+
+ doc = "school business london";//backwards
+ assertTags(reqDoc(doc), tt(doc,"london", 0, N.London));
+
+ //More interesting
+ doc = "City of London Business School";
+ assertTags(reqDoc(doc),
+ tt(doc, "City of London", 0, N.City_of_London),
+ tt(doc, "London", 0, N.London),
+ tt(doc, "London Business School", 0, N.London_Business_School));
+
+ doc = "City of London Business";
+ assertTags(reqDoc(doc),
+ tt(doc, "City of London", 0, N.City_of_London),
+ tt(doc, "London", 0, N.London));
+ }
+
+ public void testMultipleFilterQueries() throws Exception {
+ baseParams.set("overlaps", "ALL");
+
+ // build up the corpus with some additional fields for filtering purposes
+ deleteByQueryAndGetVersion("*:*", null);
+
+ int i = 0;
+ assertU(adoc("id", ""+i++, "name", N.London.getName(), "type", "city", "country", "UK"));
+ assertU(adoc("id", ""+i++, "name", N.London_Business_School.getName(), "type", "school", "country", "UK"));
+ assertU(adoc("id", ""+i++, "name", N.Boston.getName(), "type", "city", "country", "US"));
+ assertU(adoc("id", ""+i++, "name", N.City_of_London.getName(), "type", "org", "country", "UK"));
+ assertU(commit());
+
+ // not calling buildNames so that we can bring along extra attributes for filtering
+ NAMES = Arrays.stream(N.values()).map(N::getName).collect(Collectors.toList());
+
+ // phrase that matches everything
+ String doc = "City of London Business School in Boston";
+
+ // first do no filtering
+ ModifiableSolrParams p = new ModifiableSolrParams();
+ p.add(CommonParams.Q, "*:*");
+ assertTags(reqDoc(doc, p),
+ tt(doc, "City of London", 0, N.City_of_London),
+ tt(doc, "London", 0, N.London),
+ tt(doc, "London Business School", 0, N.London_Business_School),
+ tt(doc, "Boston", 0, N.Boston));
+
+ // add a single fq
+ p.add(CommonParams.FQ, "type:city");
+ assertTags(reqDoc(doc, p),
+ tt(doc, "London", 0, N.London),
+ tt(doc, "Boston", 0, N.Boston));
+
+ // add another fq
+ p.add(CommonParams.FQ, "country:US");
+ assertTags(reqDoc(doc, p),
+ tt(doc, "Boston", 0, N.Boston));
+ }
+
+ private TestTag tt(String doc, String substring, int substringIndex, N name) {
+ assert substringIndex == 0;
+
+ //little bit of copy-paste code from super.tt()
+ int startOffset = -1, endOffset;
+ int substringIndex1 = 0;
+ for(int i = 0; i <= substringIndex1; i++) {
+ startOffset = doc.indexOf(substring, ++startOffset);
+ assert startOffset >= 0 : "The test itself is broken";
+ }
+ endOffset = startOffset+ substring.length();//1 greater (exclusive)
+ return new TestTag(startOffset, endOffset, substring, lookupByName(name.getName()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java
new file mode 100644
index 0000000..e525ce9
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java
@@ -0,0 +1,251 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+import org.apache.commons.lang.builder.CompareToBuilder;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.lucene.document.Document;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.junit.Rule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class TaggerTestCase extends SolrTestCaseJ4 {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Rule
+ public TestWatcher watchman = new TestWatcher() {
+ @Override
+ protected void starting(Description description) {
+ log.info("{} being run...", description.getDisplayName());
+ }
+ };
+
+ protected final ModifiableSolrParams baseParams = new ModifiableSolrParams();
+
+ //populated in buildNames; tested in assertTags
+ protected static List<String> NAMES;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ baseParams.clear();
+ baseParams.set(CommonParams.QT, "/tag");
+ baseParams.set(CommonParams.WT, "xml");
+ }
+
+ protected void assertTags(String doc, String... tags) throws Exception {
+ TestTag[] tts = new TestTag[tags.length];
+ for (int i = 0; i < tags.length; i++) {
+ tts[i] = tt(doc, tags[i]);
+ }
+ assertTags(reqDoc(doc), tts);
+ }
+
+ protected static void buildNames(String... names) throws Exception {
+ deleteByQueryAndGetVersion("*:*", null);
+ NAMES = Arrays.asList(names);
+ //Collections.sort(NAMES);
+ int i = 0;
+ for (String n : NAMES) {
+ assertU(adoc("id", ""+(i++), "name", n));
+ }
+ assertU(commit());
+ }
+
+ protected String lookupByName(String name) {
+ for (String n : NAMES) {
+ if (n.equalsIgnoreCase(name))
+ return n;
+ }
+ return null;
+ }
+
+ protected TestTag tt(String doc, String substring) {
+ int startOffset = -1, endOffset;
+ int substringIndex = 0;
+ for(int i = 0; i <= substringIndex; i++) {
+ startOffset = doc.indexOf(substring,++startOffset);
+ assert startOffset >= 0 : "The test itself is broken";
+ }
+ endOffset = startOffset+substring.length();//1 greater (exclusive)
+ return new TestTag(startOffset, endOffset, substring, lookupByName(substring));
+ }
+
+ /** Asserts the tags. Will call req.close(). */
+ protected void assertTags(SolrQueryRequest req, TestTag... eTags) throws Exception {
+ try {
+ SolrQueryResponse rsp = h.queryAndResponse(req.getParams().get(CommonParams.QT), req);
+ TestTag[] aTags = pullTagsFromResponse(req, rsp);
+
+ String message;
+ if (aTags.length > 10)
+ message = null;
+ else
+ message = Arrays.asList(aTags).toString();
+ Arrays.sort(eTags);
+ assertSortedArrayEquals(message, eTags, aTags);
+
+ } finally {
+ req.close();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected TestTag[] pullTagsFromResponse(SolrQueryRequest req, SolrQueryResponse rsp ) throws IOException {
+ NamedList rspValues = rsp.getValues();
+ Map<String, String> matchingNames = new HashMap<>();
+ SolrIndexSearcher searcher = req.getSearcher();
+ DocList docList = (DocList) rspValues.get("response");
+ DocIterator iter = docList.iterator();
+ while (iter.hasNext()) {
+ int docId = iter.next();
+ Document doc = searcher.doc(docId);
+ String id = doc.getField("id").stringValue();
+ String name = lookupByName(doc.get("name"));
+ assertEquals("looking for "+name, NAMES.indexOf(name)+"", id);
+ matchingNames.put(id, name);
+ }
+
+ //build TestTag[] aTags from response ('a' is actual)
+ List<NamedList> mTagsList = (List<NamedList>) rspValues.get("tags");
+ List<TestTag> aTags = new ArrayList<>();
+ for (NamedList map : mTagsList) {
+ List<String> foundIds = (List<String>) map.get("ids");
+ for (String id : foundIds) {
+ aTags.add(new TestTag(
+ ((Number)map.get("startOffset")).intValue(),
+ ((Number)map.get("endOffset")).intValue(),
+ null,
+ matchingNames.get(id)));
+ }
+ }
+ return aTags.toArray(new TestTag[0]);
+ }
+
+ /** REMEMBER to close() the result req object. */
+ protected SolrQueryRequest reqDoc(String doc, String... moreParams) {
+ return reqDoc(doc, params(moreParams));
+ }
+
+ /** REMEMBER to close() the result req object. */
+ protected SolrQueryRequest reqDoc(String doc, SolrParams moreParams) {
+ log.debug("Test doc: "+doc);
+ SolrParams params = SolrParams.wrapDefaults(moreParams, baseParams);
+ SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(), params) {};
+ Iterable<ContentStream> stream = Collections.singleton((ContentStream)new ContentStreamBase.StringStream(doc));
+ req.setContentStreams(stream);
+ return req;
+ }
+
+ /** Asserts the sorted arrays are equals, with a helpful error message when not.*/
+ public void assertSortedArrayEquals(String message, Object[] expecteds, Object[] actuals) {
+ AssertionError error = null;
+ try {
+ assertArrayEquals(null, expecteds, actuals);
+ } catch (AssertionError e) {
+ error = e;
+ }
+ if (error == null)
+ return;
+ TreeSet<Object> expectedRemaining = new TreeSet<>(Arrays.asList(expecteds));
+ expectedRemaining.removeAll(Arrays.asList(actuals));
+ if (!expectedRemaining.isEmpty())
+ fail(message+": didn't find expected "+expectedRemaining.first()+" (of "+expectedRemaining.size()+"); "+ error);
+ TreeSet<Object> actualsRemaining = new TreeSet<>(Arrays.asList(actuals));
+ actualsRemaining.removeAll(Arrays.asList(expecteds));
+ fail(message+": didn't expect "+actualsRemaining.first()+" (of "+actualsRemaining.size()+"); "+ error);
+ }
+
+ class TestTag implements Comparable {
+ final int startOffset, endOffset;
+ final String substring;
+ final String docName;
+
+ TestTag(int startOffset, int endOffset, String substring, String docName) {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ this.substring = substring;
+ this.docName = docName;
+ }
+
+ @Override
+ public String toString() {
+ return "TestTag{" +
+ "[" + startOffset + "-" + endOffset + "]" +
+ " doc=" + NAMES.indexOf(docName) + ":'" + docName + "'" +
+ (docName.equals(substring) || substring == null ? "" : " substr="+substring)+
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ TestTag that = (TestTag) obj;
+ return new EqualsBuilder()
+ .append(this.startOffset, that.startOffset)
+ .append(this.endOffset, that.endOffset)
+ .append(this.docName, that.docName)
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return startOffset;//cheesy but acceptable
+ }
+
+ @Override
+ public int compareTo(Object o) {
+ TestTag that = (TestTag) o;
+ return new CompareToBuilder()
+ .append(this.startOffset, that.startOffset)
+ .append(this.endOffset, that.endOffset)
+ .append(this.docName,that.docName)
+ .toComparison();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java
new file mode 100644
index 0000000..39c7828
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/TaggingAttributeTest.java
@@ -0,0 +1,73 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the {@link TaggerRequestHandler} with
+ * a Analyzer chain that does use the {@link TaggingAttribute}. See the test
+ * configuration under 'taggingattribute'.
+ */
+public class TaggingAttributeTest extends TaggerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+ }
+
+ /**
+ * Whole matching, no sub-tags. Links only words with > 3 letters.
+ * Because of that "San" is not used to start tags
+ *
+ */
+ @Test
+ public void testTaggingAttribute() throws Exception {
+ baseParams.set("field", "name_tagAttribute"); // has WordLengthTaggingFilter using the TaggingAttribute
+ // this test is based on the longest dominant right test, so we use the
+ // the same TagClusterReducer setting
+ baseParams.set("overlaps", "LONGEST_DOMINANT_RIGHT");
+
+ buildNames("in", "San", "in San", "Francisco", "San Francisco",
+ "San Francisco State College", "College of California",
+ "Clayton", "Clayton North", "North Carolina");
+
+ assertTags("He lived in San Francisco.",
+ //"in", "San Francisco"); //whis would be expected without taggable
+ "Francisco");// this are the expected results with taggable
+
+ assertTags("He enrolled in San Francisco State College of California",
+ //"in", "San Francisco State College"); //without taggable enabled
+ "Francisco", "College of California");// With taggable
+ //NOTE this also tests that started tags are advanced for non-taggable
+ // tokens, as otherwise 'College of California' would not be
+ // suggested.
+
+ assertTags("He lived in Clayton North Carolina",
+ //"in", "Clayton", "North Carolina");
+ "Clayton", "North Carolina");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java
new file mode 100644
index 0000000..237a8b8
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilter.java
@@ -0,0 +1,110 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+
+/**
+ * Simple TokenFilter that lookup only Tokens with more as the parsed number
+ * of chars.<p>
+ * <b>NOTE:</b>This implementation is only intended to be used as an example
+ * and for unit testing the {@link TaggingAttribute} feature. Typically
+ * implementations will be based on NLP results (e.g. using POS tags or
+ * detected Named Entities).
+ * <p>
+ * <b>Example Usage:</b><p>
+ * Currently the usage requires to modify the Analyzer as defined by the
+ * <code>indexedField</code>. An alternative would be to allow the configuration
+ * of a special FieldType in the schema.xml and use this Analyzer for processing
+ * the text sent to the request.<p>
+ * While the current solution is fine for direct API usage, defining the
+ * Analyzer in the schema.xml would be better suitable for using this feature
+ * with the {@link TaggerRequestHandler}.
+ *
+ * <pre class="prettyprint">
+ * Analyzer analyzer = req.getSchema().getField(indexedField).getType().getAnalyzer();
+ * //get the TokenStream from the Analyzer
+ * TokenStream baseStream = analyzer.tokenStream("", reader);
+ * //add a FilterStream that sets the LookupAttribute to the end
+ * TokenStream filterStream = new WordLengthLookupFilter(baseStream);
+ * //create the Tagger using the modified analyzer chain.
+ * new Tagger(corpus, filterStream, tagClusterReducer) {
+ *
+ * protected void tagCallback(int startOffset, int endOffset, long docIdsKey) {
+ * //implement the callback
+ * }
+ *
+ * }.process();
+ * </pre>
+ */
+public class WordLengthTaggingFilter extends TokenFilter {
+
+ /**
+ * The default minimum length is <code>3</code>
+ */
+ public static final int DEFAULT_MIN_LENGTH = 3;
+ private final TaggingAttribute lookupAtt = addAttribute(TaggingAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private int minLength;
+
+ /**
+ * TokenFilter only marks tokens to be looked up with equals or more as
+ * {@link #DEFAULT_MIN_LENGTH} characters
+ */
+ public WordLengthTaggingFilter(TokenStream input) {
+ this(input, null);
+ }
+
+ /**
+ * TokenFilter only marks tokens to be looked up with equals or more characters
+ * as the parsed minimum.
+ *
+ * @param input the TokenStream to consume tokens from
+ * @param minLength The minimum length to lookup a Token. <code>null</code>
+ * or <= 0 to use the #DEFAULT_MIN_LENGTH
+ */
+ public WordLengthTaggingFilter(TokenStream input, Integer minLength) {
+ super(input);
+ if (minLength == null || minLength <= 0) {
+ this.minLength = DEFAULT_MIN_LENGTH;
+ } else {
+ this.minLength = minLength;
+ }
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ int size = offsetAtt.endOffset() - offsetAtt.startOffset();
+ lookupAtt.setTaggable(size >= minLength);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java
new file mode 100644
index 0000000..dbfc538
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/WordLengthTaggingFilterFactory.java
@@ -0,0 +1,67 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class WordLengthTaggingFilterFactory extends TokenFilterFactory {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String MIN_LENGTH = "minLength";
+
+ private final Integer minLength;
+
+ public WordLengthTaggingFilterFactory(Map<String, String> args) {
+ super(args);
+ int minLength = -1;
+ Object value = args.get(MIN_LENGTH);
+ if (value != null) {
+ try {
+ minLength = Integer.parseInt(value.toString());
+ } catch (NumberFormatException e) {
+ log.warn("Unable to parse minLength from value 'minLength=\"{}\"'", value);
+
+ }
+ }
+ if (minLength <= 0) {
+ log.info("use default minLength={}", WordLengthTaggingFilter.DEFAULT_MIN_LENGTH);
+ this.minLength = null;
+ } else {
+ log.info("set minLength={}", minLength);
+ this.minLength = minLength;
+ }
+ }
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new WordLengthTaggingFilter(input, minLength);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java
new file mode 100644
index 0000000..d7dd5df
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/XmlInterpolationTest.java
@@ -0,0 +1,224 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.xml.sax.InputSource;
+
+public class XmlInterpolationTest extends TaggerTestCase {
+
+ private static DocumentBuilder xmlDocBuilder;
+
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ DocumentBuilderFactory xmlDocBuilderFactory = DocumentBuilderFactory.newInstance();
+ xmlDocBuilderFactory.setValidating(true);
+ xmlDocBuilderFactory.setNamespaceAware(true);
+ xmlDocBuilder = xmlDocBuilderFactory.newDocumentBuilder();
+
+ initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ baseParams.set("field", "name_tagXml");
+ baseParams.set("overlaps", "LONGEST_DOMINANT_RIGHT");
+ baseParams.set("xmlOffsetAdjust", "true");
+ }
+
+ @Test
+ public void test() throws Exception {
+ buildNames("start end");
+
+ assertXmlTag("<doc>before start <!-- c --> end after</doc>", true);
+ assertXmlTag("<doc>before start <br/> end after</doc>", true);
+ assertXmlTag("<doc>before <em>start</em> <b>end</b> after</doc>", true);
+ assertXmlTag("<doc>before <em>start</em> end after</doc>", true);
+ assertXmlTag("<doc>before start end<em> after</em></doc>", true);
+ assertXmlTag("<doc><em>before </em>start end after</doc>", true);//adjacent tags
+ assertXmlTag("<doc>before <b> <em>start</em> </b> end after</doc>", true);
+ assertXmlTag("<doc>before <b> <em>start</em> </b> <em> end </em> after</doc>", true);
+
+ assertXmlTag("<doc><p>before start</p> end after</doc>", false);
+ assertXmlTag("<doc>before start <p>end after</p> </doc>", false);
+
+ assertXmlTag("<doc>before <em a='A' b='B'>start</em> <b a='A' b='B'>end</b> after</doc>", true);
+ }
+
+ @Test(expected = SolrException.class)
+ public void testInvalidXml() throws Exception {
+ assertXmlTag("notXml", false);
+ }
+
+ @Test(expected = Exception.class)
+ public void testValidatingXml() throws Exception {
+ validateXml("foo");
+ }
+
+ protected void assertXmlTag(String docText, boolean expected) throws Exception {
+ final SolrQueryRequest req = reqDoc(docText);
+ try { // 5.4 and beyond we can use try-with-resources
+ final SolrQueryResponse rsp = h.queryAndResponse(req.getParams().get("qt"), req);
+ final TestTag[] testTags = pullTagsFromResponse(req, rsp);
+ if (!expected) {
+ assertEquals(0, testTags.length);
+ } else {
+ assertEquals(1, testTags.length);
+ final TestTag tag = testTags[0];
+ validateXml(insertAnchorAtOffsets(docText, tag.startOffset, tag.endOffset, tag.docName));
+ }
+ } finally {
+ req.close();
+ }
+ }
+
+ protected void validateXml(String xml) throws Exception {
+ // the "parse" method also validates XML, will throw an exception if mis-formatted
+ xmlDocBuilder.parse(new InputSource(new StringReader(xml)));
+ }
+
+
+ @Test
+ public void testLuceneHtmlFilterBehavior() {
+ String docText;
+
+ //Close tag adjacent to start & end results in end offset including the close tag. LUCENE-5734
+ docText = "<doc><a><b>start</b> end</a></doc>";
+ assertArrayEquals(tagExpect(docText, "start", "end</a>"), analyzeTagOne(docText, "start", "end"));
+
+ //Space after "end" means offset doesn't include </a>
+ docText = "<doc><a><b>start</b> end </a></doc>";
+ assertArrayEquals(tagExpect(docText, "start", "end"), analyzeTagOne(docText, "start", "end"));
+
+ //Matches entity at end
+ final String endStr = String.format(Locale.ROOT, "en&#x%02x;", (int) 'd');
+ docText = "<doc>start " + endStr + "</doc>";
+ assertArrayEquals(tagExpect(docText, "start", endStr), analyzeTagOne(docText, "start", "end"));
+ //... and at start
+ final String startStr = String.format(Locale.ROOT, "&#x%02x;tart", (int) 's');
+ docText = "<doc>" + startStr + " end</doc>";
+ assertArrayEquals(tagExpect(docText, startStr, "end"), analyzeTagOne(docText, "start", "end"));
+
+ //Test ignoring proc instructions & comments. Note: doesn't expand the entity to "start".
+ docText = "<!DOCTYPE start [ "
+ + "<!ENTITY start \"start\">"
+ + "]><start><?start start ?><!-- start --><start/>&start;</start>";
+ assertArrayEquals(new int[]{-1, -1}, analyzeTagOne(docText, "start", "start"));
+
+ //Test entity behavior
+ docText = " — – & &foo;   a b";
+ assertArrayEquals(new String[]{"—", "–", "&", "&foo;", "\u00A0", "a", "b"},
+ analyzeReturnTokens(docText));
+
+ //Observe offset adjustment of trailing entity to end tag
+ docText = "foo bar";
+ assertArrayEquals(tagExpect(docText, "foo", "foo"), analyzeTagOne(docText, "foo", "foo"));
+ }
+
+ private String insertAnchorAtOffsets(String docText, int startOffset, int endOffset, String id) {
+ String insertStart = "<A id='"+ id +"'>";// (normally we'd escape id)
+ String insertEnd = "</A>";
+ return docText.substring(0, startOffset)
+ + insertStart
+ + docText.substring(startOffset, endOffset)
+ + insertEnd
+ + docText.substring(endOffset);
+ }
+
+ private int[] tagExpect(String docText, String start, String end) {
+ return new int[]{docText.indexOf(start), docText.indexOf(end) + end.length()};
+ }
+
+ private int[] analyzeTagOne(String docText, String start, String end) {
+ int[] result = {-1, -1};
+
+ Reader filter = new HTMLStripCharFilter(new StringReader(docText));
+
+ WhitespaceTokenizer ts = new WhitespaceTokenizer();
+ final CharTermAttribute termAttribute = ts.addAttribute(CharTermAttribute.class);
+ final OffsetAttribute offsetAttribute = ts.addAttribute(OffsetAttribute.class);
+ try {
+ ts.setReader(filter);
+ ts.reset();
+ while (ts.incrementToken()) {
+ final String termString = termAttribute.toString();
+ if (termString.equals(start))
+ result[0] = offsetAttribute.startOffset();
+ if (termString.equals(end)) {
+ result[1] = offsetAttribute.endOffset();
+ return result;
+ }
+ }
+ ts.end();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ IOUtils.closeQuietly(ts);
+ }
+ return result;
+ }
+
+ private String[] analyzeReturnTokens(String docText) {
+ List<String> result = new ArrayList<>();
+
+ Reader filter = new HTMLStripCharFilter(new StringReader(docText),
+ Collections.singleton("unescaped"));
+ WhitespaceTokenizer ts = new WhitespaceTokenizer();
+ final CharTermAttribute termAttribute = ts.addAttribute(CharTermAttribute.class);
+ try {
+ ts.setReader(filter);
+ ts.reset();
+ while (ts.incrementToken()) {
+ result.add(termAttribute.toString());
+ }
+ ts.end();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ IOUtils.closeQuietly(ts);
+ }
+ return result.toArray(new String[result.size()]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/solr-ref-guide/src/searching.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/searching.adoc b/solr/solr-ref-guide/src/searching.adoc
index 145c1a4..753c2d8 100644
--- a/solr/solr-ref-guide/src/searching.adoc
+++ b/solr/solr-ref-guide/src/searching.adoc
@@ -1,5 +1,35 @@
= Searching
-:page-children: overview-of-searching-in-solr, velocity-search-ui, relevance, query-syntax-and-parsing, json-request-api, json-facet-api, faceting, highlighting, spell-checking, query-re-ranking, transforming-result-documents, suggester, morelikethis, pagination-of-results, collapse-and-expand-results, result-grouping, result-clustering, spatial-search, the-terms-component, the-term-vector-component, the-stats-component, the-query-elevation-component, response-writers, near-real-time-searching, realtime-get, exporting-result-sets, streaming-expressions, parallel-sql-interface, analytics
+:page-children: overview-of-searching-in-solr, +
+ velocity-search-ui, +
+ relevance, +
+ query-syntax-and-parsing, +
+ json-request-api, +
+ json-facet-api, +
+ faceting, +
+ highlighting, +
+ spell-checking, +
+ query-re-ranking, +
+ transforming-result-documents, +
+ suggester, +
+ morelikethis, +
+ pagination-of-results, +
+ collapse-and-expand-results, +
+ result-grouping, +
+ result-clustering, +
+ spatial-search, +
+ the-terms-component, +
+ the-term-vector-component, +
+ the-stats-component, +
+ the-query-elevation-component, +
+ the-tagger-handler, +
+ response-writers, +
+ near-real-time-searching, +
+ realtime-get, +
+ exporting-result-sets, +
+ streaming-expressions, +
+ parallel-sql-interface, +
+ analytics
+
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -50,6 +80,7 @@ This section describes how Solr works with search requests. It covers the follow
* <<the-term-vector-component.adoc#the-term-vector-component,The Term Vector Component>>: How to get term information about specific documents.
* <<the-stats-component.adoc#the-stats-component,The Stats Component>>: How to return information from numeric fields within a document set.
* <<the-query-elevation-component.adoc#the-query-elevation-component,The Query Elevation Component>>: How to force documents to the top of the results for certain queries.
+* <<the-tagger-handler.adoc#the-tagger-handler,The Tagger Handler>>: The SolrTextTagger, for basic named entity tagging in text.
* <<response-writers.adoc#response-writers,Response Writers>>: Detailed information about configuring and using Solr's response writers.
* <<near-real-time-searching.adoc#near-real-time-searching,Near Real Time Searching>>: How to include documents in search results nearly immediately after they are indexed.
* <<realtime-get.adoc#realtime-get,RealTime Get>>: How to get the latest version of a document without opening a searcher.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/solr-ref-guide/src/the-tagger-handler.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/the-tagger-handler.adoc b/solr/solr-ref-guide/src/the-tagger-handler.adoc
new file mode 100644
index 0000000..14ba8ed
--- /dev/null
+++ b/solr/solr-ref-guide/src/the-tagger-handler.adoc
@@ -0,0 +1,265 @@
+[[the-tagger-handler]]
+= The Tagger Handler
+
+The "Tagger" Request Handler, AKA the "SolrTextTagger" is a "text tagger".
+Given a dictionary (a Solr index) with a name-like field,
+ you post text to this request handler and it will return every occurrence of one of those names with offsets and other document metadata desired.
+It's used for named entity recognition (NER).
+It doesn't do any NLP (outside of Lucene text analysis) so it's said to be a "naive tagger",
+ but it's definitely useful as-is and a more complete NER or ERD (entity recognition and disambiguation)
+ system can be built with this as a key component.
+The SolrTextTagger might be used on queries for query-understanding or large documents as well.
+
+To get a sense of how to use it, jump to the tutorial below.
+
+The tagger does not yet support a sharded index.
+Tens, perhaps hundreds of millions of names (documents) are supported, mostly limited by memory.
+
+[[tagger-configuration]]
+== Configuration
+
+The Solr schema needs 2 things:
+
+* A unique key field (see `<uniqueKey>`).
+ Recommended field settings: set `docValues=true`
+* A tag field, a TextField, with `ConcatenateGraphFilterFactory` at the end of the index chain (not the query chain):
+ Set `preservePositionIncrements=false` on that filter.
+ Recommended field settings: `omitNorms=true`, `omitTermFreqAndPositions=true` and `postingsFormat=FST50`
+
+The text field's _index analysis chain_, aside from needing ConcatenateGraphFilterFactory at the end,
+ can otherwise have whatever tokenizer and filters suit your matching preferences.
+It can have multi-word synonyms and use WordDelimiterGraphFilterFactory for example.
+However, do _not_ use FlattenGraphFilterFactory as it will interfere with ConcatenateGraphFilterFactory.
+Position gaps (e.g. stop words) get ignored; it's not (yet) supported for the gap to be significant.
+
+The text field's _query analysis chain_, on the other hand, is more limited.
+There should not be tokens at the same position, thus no synonym expansion -- do that at index time instead.
+Stop words (or any other filter introducing a position gap) are supported.
+At runtime the tagger can be configured to either treat it as a tag break or to ignore it.
+
+The Solr config needs the `solr.TagRequestHandler` defined, which supports `defaults`, `invariants`, and `appends`
+sections just like the search handler.
+
+[[tagger-parameters]]
+== Tagger Parameters
+
+The tagger's execution is completely configurable with request parameters. Only `field` is required.
+
+`field`::
+ The tag field that serves as the dictionary.
+ This is required; you'll probably specify it in the request handler.
+
+`fq`::
+ You can specify some number of _filter queries_ to limit the dictionary used for tagging.
+ This parameter is the same as is used by the `solr.SearchHandler`.
+
+`rows`::
+ The maximum number of documents to return, but defaulting to 10000 for a tag request.
+ This parameter is the same as is used by the `solr.SearchHandler`.
+
+`fl`::
+ Solr's standard param for listing the fields to return.
+ This parameter is the same as is used by the `solr.SearchHandler`.
+
+`overlaps`::
+ Choose the algorithm to determine which tags in an overlapping set should be retained, versus being pruned away.
+ Options are:
+
+ * `ALL`: Emit all tags.
+ * `NO_SUB`: Don't emit a tag that is completely within another tag (i.e. no subtag).
+ * `LONGEST_DOMINANT_RIGHT`: Given a cluster of overlapping tags, emit the longest one (by character length).
+ If there is a tie, pick the right-most.
+ Remove any tags overlapping with this tag then repeat the algorithm to potentially find other tags
+ that can be emitted in the cluster.
+
+`matchText`::
+ A boolean indicating whether to return the matched text in the tag response.
+ This will trigger the tagger to fully buffer the input before tagging.
+
+`tagsLimit`::
+ The maximum number of tags to return in the response.
+ Tagging effectively stops after this point.
+ By default this is 1000.
+
+`skipAltTokens`::
+ A boolean flag used to suppress errors that can occur if, for example,
+ you enable synonym expansion at query time in the analyzer, which you normally shouldn't do.
+ Let this default to false unless you know that such tokens can't be avoided.
+
+`ignoreStopwords`::
+ A boolean flag that causes stopwords (or any condition causing positions to skip like >255 char words)
+ to be ignored as if it wasn't there.
+ Otherwise, the behavior is to treat them as breaks in tagging on the presumption your indexed text-analysis
+ configuration doesn't have a StopWordFilter.
+ By default the indexed analysis chain is checked for the presence of a StopWordFilter and if found
+ then ignoreStopWords is true if unspecified.
+ You probably shouldn't have a StopWordFilter configured and probably won't need to set this param either.
+
+`xmlOffsetAdjust`::
+ A boolean indicating that the input is XML and furthermore that the offsets of returned tags should be adjusted as
+ necessary to allow for the client to insert an openening and closing element at the tag offset pair.
+ If it isn't possible to do so then the tag will be omitted.
+ You are expected to configure `HTMLStripCharFilterFactory` in the schema when using this option.
+ This will trigger the tagger to fully buffer the input before tagging.
+
+Solr's parameters for controlling the response format are supported, like:
+ `echoParams`, `wt`, `indent`, etc.
+
+[[tagger-tutorial-with-geonames]]
+== Tutorial with Geonames
+
+This is a tutorial that demonstrates how to configure and use the text
+tagger with the popular Geonames data set. It's more than a tutorial;
+it's a how-to with information that wasn't described above.
+
+[[tagger-create-and-configure-a-solr-collection]]
+=== Create and Configure a Solr Collection
+
+Create a Solr collection named "geonames". For the tutorial, we'll
+assume the default "data-driven" configuration. It's good for
+experimentation and getting going fast but not for production or being
+optimal.
+
+....
+bin/solr create -c geonames
+....
+
+[[tagger-configuring]]
+==== Configuring
+
+We need to configure the schema first. The "data driven" mode we're
+using allows us to keep this step fairly minimal -- we just need to
+declare a field type, 2 fields, and a copy-field. The critical part
+up-front is to define the "tag" field type. There are many many ways to
+configure text analysis; and we're not going to get into those choices
+here. But an important bit is the `ConcatenateGraphFilterFactory` at the
+end of the index analyzer chain. Another important bit for performance
+is postingsFormat=FST50 resulting in a compact FST based in-memory data
+structure that is especially beneficial for the text tagger.
+
+Schema configuration:
+
+....
+curl -X POST -H 'Content-type:application/json' http://localhost:8983/solr/geonames/schema -d '{
+ "add-field-type":{
+ "name":"tag",
+ "class":"solr.TextField",
+ "postingsFormat":"FST50",
+ "omitNorms":true,
+ "omitTermFreqAndPositions":true,
+ "indexAnalyzer":{
+ "tokenizer":{
+ "class":"solr.StandardTokenizerFactory" },
+ "filters":[
+ {"class":"solr.EnglishPossessiveFilterFactory"},
+ {"class":"solr.ASCIIFoldingFilterFactory"},
+ {"class":"solr.LowerCaseFilterFactory"},
+ {"class":"solr.ConcatenateGraphFilterFactory", "preservePositionIncrements":false }
+ ]},
+ "queryAnalyzer":{
+ "tokenizer":{
+ "class":"solr.StandardTokenizerFactory" },
+ "filters":[
+ {"class":"solr.EnglishPossessiveFilterFactory"},
+ {"class":"solr.ASCIIFoldingFilterFactory"},
+ {"class":"solr.LowerCaseFilterFactory"}
+ ]}
+ },
+
+ "add-field":{ "name":"name", "type":"text_general"},
+
+ "add-field":{ "name":"name_tag", "type":"tag", "stored":false },
+
+ "add-copy-field":{ "source":"name", "dest":[ "name_tag" ]}
+}'
+....
+
+Configure a custom Solr Request Handler:
+
+....
+curl -X POST -H 'Content-type:application/json' http://localhost:8983/solr/geonames/config -d '{
+ "add-requesthandler" : {
+ "name": "/tag",
+ "class":"solr.TaggerRequestHandler",
+ "defaults":{ "field":"name_tag" }
+ }
+}'
+....
+
+[[tagger-load-some-sample-data]]
+=== Load Some Sample Data
+
+We'll go with some Geonames.org data in CSV format. Solr is quite
+flexible in loading data in a variety of formats. This
+http://download.geonames.org/export/dump/cities1000.zip[cities1000.zip]
+should be almost 7MB file expanding to a cities1000.txt file around
+22.2MB containing 145k lines, each a city in the world of at least 1000
+population.
+
+Using bin/post:
+....
+bin/post -c geonames -type text/csv \
+ -params 'optimize=true&separator=%09&encapsulator=%00&fieldnames=id,name,,alternative_names,latitude,longitude,,,countrycode,,,,,,population,elevation,,timezone,lastupdate' \
+ /tmp/cities1000.txt
+....
+or using curl:
+....
+curl -X POST --data-binary @/path/to/cities1000.txt -H 'Content-type:application/csv' \
+ 'http://localhost:8983/solr/geonames/update?commit=true&optimize=true&separator=%09&encapsulator=%00&fieldnames=id,name,,alternative_names,latitude,longitude,,,countrycode,,,,,,population,elevation,,timezone,lastupdate'
+....
+
+That might take around 35 seconds; it depends. It can be a lot faster if
+the schema were tuned to only have what we truly need (no text search if
+not needed).
+
+In that command we said optimize=true to put the index in a state that
+will make tagging faster. The encapsulator=%00 is a bit of a hack to
+disable the default double-quote.
+
+[[tagger-tag-time]]
+=== Tag Time!
+
+This is a trivial example tagging a small piece of text. For more
+options, see the earlier documentation.
+
+....
+curl -X POST \
+ 'http://localhost:8983/solr/geonames/tag?overlaps=NO_SUB&tagsLimit=5000&fl=id,name,countrycode&wt=json&indent=on' \
+ -H 'Content-Type:text/plain' -d 'Hello New York City'
+....
+
+The response should be this (the QTime may vary):
+
+....
+{
+ "responseHeader":{
+ "status":0,
+ "QTime":1},
+ "tagsCount":1,
+ "tags":[[
+ "startOffset",6,
+ "endOffset",19,
+ "ids",["5128581"]]],
+ "response":{"numFound":1,"start":0,"docs":[
+ {
+ "id":"5128581",
+ "name":["New York City"],
+ "countrycode":["US"]}]
+ }}
+....
+
+[[tagger-tips]]
+== Tips
+
+Performance Tips:
+
+* Follow the recommended configuration field settings, especially `postingsFormat=FST50`.
+* "optimize" after loading your dictionary down to 1 Lucene segment, or at least to as few as possible.
+* For bulk tagging lots of documents, there are some strategies, not mutually exclusive:
+** Batch them.
+ The tagger doesn't directly support batching but as a hack you can send a bunch of documents concatenated with
+ a nonsense word that is not in the dictionary like "ZZYYXXAABBCC" between them.
+ You'll need to keep track of the character offsets of these so you can subtract them from the results.
+** For reducing tagging latency even further, consider embedding Solr with `EmbeddedSolrServer`.
+ See `EmbeddedSolrNoSerializeTest`.
+** Use more than one thread -- perhaps as many as there are CPU cores available to Solr.
\ No newline at end of file
[06/18] lucene-solr:jira/solr-12438: SOLR-12376: New
TaggerRequestHandler (SolrTextTagger).
Posted by ab...@apache.org.
SOLR-12376: New TaggerRequestHandler (SolrTextTagger).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cf633921
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cf633921
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cf633921
Branch: refs/heads/jira/solr-12438
Commit: cf63392183ffc96428fc4c52f546fec2cdf766d5
Parents: c587598
Author: David Smiley <ds...@apache.org>
Authored: Tue Jun 5 14:04:55 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Jun 5 14:04:55 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
solr/NOTICE.txt | 14 +
.../apache/solr/core/SolrResourceLoader.java | 12 +-
.../solr/handler/tagger/OffsetCorrector.java | 178 +++++++++
.../solr/handler/tagger/TagClusterReducer.java | 103 +++++
.../org/apache/solr/handler/tagger/TagLL.java | 176 ++++++++
.../org/apache/solr/handler/tagger/Tagger.java | 230 +++++++++++
.../handler/tagger/TaggerRequestHandler.java | 397 +++++++++++++++++++
.../solr/handler/tagger/TaggingAttribute.java | 65 +++
.../handler/tagger/TaggingAttributeImpl.java | 79 ++++
.../solr/handler/tagger/TermPrefixCursor.java | 189 +++++++++
.../solr/handler/tagger/XmlOffsetCorrector.java | 113 ++++++
.../solr/handler/tagger/package-info.java | 27 ++
.../solr/collection1/conf/schema-tagger.xml | 187 +++++++++
.../solr/collection1/conf/solrconfig-tagger.xml | 59 +++
.../tagger/EmbeddedSolrNoSerializeTest.java | 153 +++++++
.../handler/tagger/RandomizedTaggerTest.java | 150 +++++++
.../apache/solr/handler/tagger/Tagger2Test.java | 175 ++++++++
.../apache/solr/handler/tagger/TaggerTest.java | 296 ++++++++++++++
.../solr/handler/tagger/TaggerTestCase.java | 251 ++++++++++++
.../handler/tagger/TaggingAttributeTest.java | 73 ++++
.../handler/tagger/WordLengthTaggingFilter.java | 110 +++++
.../tagger/WordLengthTaggingFilterFactory.java | 67 ++++
.../handler/tagger/XmlInterpolationTest.java | 224 +++++++++++
solr/solr-ref-guide/src/searching.adoc | 33 +-
solr/solr-ref-guide/src/the-tagger-handler.adoc | 265 +++++++++++++
26 files changed, 3622 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 66d2026..479406f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -154,6 +154,9 @@ New Features
* SOLR-12389: support deeply nested json objects in clusterprops.json (noble)
+* SOLR-12376: Added the TaggerRequestHandler (AKA SolrTextTagger) for tagging text. It's used as a component of
+ NER/ERD systems including query-understanding. See the ref guide for more info. (David Smiley
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/NOTICE.txt
----------------------------------------------------------------------
diff --git a/solr/NOTICE.txt b/solr/NOTICE.txt
index fd954f4..a5b2070 100644
--- a/solr/NOTICE.txt
+++ b/solr/NOTICE.txt
@@ -537,3 +537,17 @@ See http://www.restlet.org/
Protocol Buffers - Google's data interchange format
Copyright 2008 Google Inc.
http://code.google.com/apis/protocolbuffers/
+
+=========================================================================
+== SolrTextTagger Notice ==
+=========================================================================
+
+The TaggerRequestHandler and related classes in its package came from the
+OpenSextant Solr Text Tagger,
+Copyright 2013 The MITRE Corporation. All Rights Reserved.
+
+ This software was produced for the U. S. Government
+ under Contract No. W15P7T-11-C-F600, and is
+ subject to the Rights in Noncommercial Computer Software
+ and Noncommercial Computer Software Documentation
+ Clause 252.227-7014 (JUN 1995)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 22753dd..0ff5c7b 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -16,6 +16,10 @@
*/
package org.apache.solr.core;
+import javax.naming.Context;
+import javax.naming.InitialContext;
+import javax.naming.NamingException;
+import javax.naming.NoInitialContextException;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
@@ -47,10 +51,6 @@ import java.util.concurrent.ConcurrentSkipListSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import javax.naming.Context;
-import javax.naming.InitialContext;
-import javax.naming.NamingException;
-import javax.naming.NoInitialContextException;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -88,9 +88,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
static final String project = "solr";
static final String base = "org.apache" + "." + project;
static final String[] packages = {
- "", "analysis.", "schema.", "handler.", "search.", "update.", "core.", "response.", "request.",
+ "", "analysis.", "schema.", "handler.", "handler.tagger.", "search.", "update.", "core.", "response.", "request.",
"update.processor.", "util.", "spelling.", "handler.component.", "handler.dataimport.",
- "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.","handler.admin.",
+ "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.", "handler.admin.",
"cloud.autoscaling."
};
private static final java.lang.String SOLR_CORE_NAME = "solr.core.name";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java b/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java
new file mode 100644
index 0000000..1fb4911
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/OffsetCorrector.java
@@ -0,0 +1,178 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.util.Arrays;
+
+import com.carrotsearch.hppc.IntArrayList;
+
+public abstract class OffsetCorrector {
+
+ //TODO support a streaming style of consuming input text so that we need not take a
+ // String. Trickier because we need to keep more information as we parse to know when tags
+ // are adjacent with/without whitespace
+
+ //Data structure requirements:
+ // Given a character offset:
+ // * determine what tagId is it's parent.
+ // * determine if it is adjacent to the parent open tag, ignoring whitespace
+ // * determine if it is adjacent to the parent close tag, ignoring whitespace
+ // Given a tagId:
+ // * What is it's parent tagId
+ // * What's the char offset of the start and end of the open tag
+ // * What's the char offset of the start and end of the close tag
+
+ /** Document text. */
+ protected final String docText;
+
+ /** Array of tag info comprised of 5 int fields:
+ * [int parentTag, int openStartOff, int openEndOff, int closeStartOff, int closeEndOff].
+ * It's size indicates how many tags there are. Tag's are ID'ed sequentially from 0. */
+ protected final IntArrayList tagInfo;
+
+ /** offsets of parent tag id change (ascending order) */
+ protected final IntArrayList parentChangeOffsets;
+ /** tag id; parallel array to parentChangeOffsets */
+ protected final IntArrayList parentChangeIds;
+
+ protected final int[] offsetPair = new int[] { -1, -1};//non-thread-safe state
+
+ /** Disjoint start and end span offsets (inclusive) of non-taggable sections. Null if none. */
+ protected final IntArrayList nonTaggableOffsets;
+
+ /**
+ * Initialize based on the document text.
+ * @param docText non-null structured content.
+ * @param hasNonTaggable if there may be "non-taggable" tags to track
+ */
+ protected OffsetCorrector(String docText, boolean hasNonTaggable) {
+ this.docText = docText;
+ final int guessNumElements = Math.max(docText.length() / 20, 4);
+
+ tagInfo = new IntArrayList(guessNumElements * 5);
+ parentChangeOffsets = new IntArrayList(guessNumElements * 2);
+ parentChangeIds = new IntArrayList(guessNumElements * 2);
+ nonTaggableOffsets = hasNonTaggable ? new IntArrayList(guessNumElements / 5) : null;
+ }
+
+ /** Corrects the start and end offset pair. It will return null if it can't
+ * due to a failure to keep the offsets balance-able, or if it spans "non-taggable" tags.
+ * The start (left) offset is pulled left as needed over whitespace and opening tags. The end
+ * (right) offset is pulled right as needed over whitespace and closing tags. It's returned as
+ * a 2-element array.
+ * <p>Note that the returned array is internally reused; just use it to examine the response.
+ */
+ public int[] correctPair(int leftOffset, int rightOffset) {
+ rightOffset = correctEndOffsetForCloseElement(rightOffset);
+ if (spansNonTaggable(leftOffset, rightOffset))
+ return null;
+
+ int startTag = lookupTag(leftOffset);
+ //offsetPair[0] = Math.max(offsetPair[0], getOpenStartOff(startTag));
+ int endTag = lookupTag(rightOffset-1);
+ //offsetPair[1] = Math.min(offsetPair[1], getCloseStartOff(endTag));
+
+ // Find the ancestor tag enclosing offsetPair. And bump out left offset along the way.
+ int iTag = startTag;
+ for (; !tagEnclosesOffset(iTag, rightOffset); iTag = getParentTag(iTag)) {
+ //Ensure there is nothing except whitespace thru OpenEndOff
+ int tagOpenEndOff = getOpenEndOff(iTag);
+ if (hasNonWhitespace(tagOpenEndOff, leftOffset))
+ return null;
+ leftOffset = getOpenStartOff(iTag);
+ }
+ final int ancestorTag = iTag;
+ // Bump out rightOffset until we get to ancestorTag.
+ for (iTag = endTag; iTag != ancestorTag; iTag = getParentTag(iTag)) {
+ //Ensure there is nothing except whitespace thru CloseStartOff
+ int tagCloseStartOff = getCloseStartOff(iTag);
+ if (hasNonWhitespace(rightOffset, tagCloseStartOff))
+ return null;
+ rightOffset = getCloseEndOff(iTag);
+ }
+
+ offsetPair[0] = leftOffset;
+ offsetPair[1] = rightOffset;
+ return offsetPair;
+ }
+
+ /** Correct endOffset for adjacent element at the right side. E.g. offsetPair might point to:
+ * <pre>
+ * foo</tag>
+ * </pre>
+ * and this method pulls the end offset left to the '<'. This is necessary for use with
+ * {@link org.apache.lucene.analysis.charfilter.HTMLStripCharFilter}.
+ *
+ * See https://issues.apache.org/jira/browse/LUCENE-5734 */
+ protected int correctEndOffsetForCloseElement(int endOffset) {
+ if (docText.charAt(endOffset-1) == '>') {
+ final int newEndOffset = docText.lastIndexOf('<', endOffset - 2);
+ if (newEndOffset > offsetPair[0])//just to be sure
+ return newEndOffset;
+ }
+ return endOffset;
+ }
+
+ protected boolean hasNonWhitespace(int start, int end) {
+ for (int i = start; i < end; i++) {
+ if (!Character.isWhitespace(docText.charAt(i)))
+ return true;
+ }
+ return false;
+ }
+
+ protected boolean tagEnclosesOffset(int tag, int off) {
+ return off >= getOpenStartOff(tag) && off < getCloseEndOff(tag);
+ }
+
+ protected int getParentTag(int tag) { return tagInfo.get(tag * 5 + 0); }
+ protected int getOpenStartOff(int tag) { return tagInfo.get(tag * 5 + 1); }
+ protected int getOpenEndOff(int tag) { return tagInfo.get(tag * 5 + 2); }
+ protected int getCloseStartOff(int tag) { return tagInfo.get(tag * 5 + 3); }
+ protected int getCloseEndOff(int tag) { return tagInfo.get(tag * 5 + 4); }
+
+ protected int lookupTag(int off) {
+ int idx = Arrays.binarySearch(parentChangeOffsets.buffer, 0, parentChangeOffsets.size(), off);
+ if (idx < 0)
+ idx = (-idx - 1) - 1;//round down
+ return parentChangeIds.get(idx);
+ }
+
+ protected boolean spansNonTaggable(int startOff, int endOff) {
+ if (nonTaggableOffsets == null)
+ return false;
+ int idx = Arrays.binarySearch(nonTaggableOffsets.buffer, 0, nonTaggableOffsets.size(), startOff);
+ //if tag start coincides with first or last char of non-taggable span then result is true.
+ // (probably never happens since those characters are actual element markup)
+ if (idx >= 0)
+ return true;
+ idx = -idx - 1;//modify for where we would insert
+ //if idx is odd then our span intersects a non-taggable span; return true
+ if ((idx & 1) == 1)
+ return true;
+ //it's non-taggable if the next non-taggable start span is before our endOff
+ if (idx == nonTaggableOffsets.size())
+ return false;
+ return nonTaggableOffsets.get(idx) < endOff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java b/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java
new file mode 100644
index 0000000..9310a04
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TagClusterReducer.java
@@ -0,0 +1,103 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+public interface TagClusterReducer {
+ /**
+ * Reduces the linked-list to only those tags that should be emitted
+ * @param head not null; 1-element array to head which isn't null either
+ */
+ void reduce(TagLL[] head);
+
+ static final TagClusterReducer ALL = new TagClusterReducer() {
+ @Override
+ public void reduce(TagLL[] head) {
+ }
+ };
+
+ static final TagClusterReducer NO_SUB = new TagClusterReducer() {
+ @Override
+ public void reduce(TagLL[] head) {
+ //loop forward over all tags
+ for (TagLL tag = head[0].nextTag; tag != null; tag = tag.nextTag) {
+ //loop backwards over prev tags from this tag
+ for (TagLL tPrev = tag.prevTag; tPrev != null; tPrev = tPrev.prevTag) {
+ assert tPrev.startOffset <= tag.startOffset;
+ //if a previous tag's endOffset is <= this one's, tForward can be removed
+ if (tPrev.endOffset >= tag.endOffset) {
+ tag.removeLL();
+ break;
+ } else if (tPrev.startOffset == tag.startOffset) {
+ tPrev.removeLL();
+ //continue; 'tag' is still valid
+ }
+ }
+ }
+ }
+ };
+
+ static final TagClusterReducer LONGEST_DOMINANT_RIGHT = new TagClusterReducer() {
+ @Override
+ public void reduce(TagLL[] head) {
+
+ //--Optimize for common single-tag case
+ if (head[0].nextTag == null)
+ return;
+
+ while (true) {
+ //--Find longest not already marked
+ TagLL longest = null;
+ for (TagLL t = head[0]; t != null; t = t.nextTag) {
+ if (!t.mark && (longest == null || t.charLen() >= longest.charLen()))
+ longest = t;
+ }
+ if (longest == null)
+ break;
+ //--Mark longest (so we return it eventually)
+ longest.mark = true;
+ //--Remove tags overlapping this longest
+ for (TagLL t = head[0]; t != null; t = t.nextTag) {
+ if (t.mark)
+ continue;
+
+ if (t.overlaps(longest)) {
+ t.removeLL();
+ } else if (t.startOffset >= longest.endOffset) {
+ break;//no subsequent can possibly overlap
+ }
+ }
+ }//loop
+
+ //all-remaining should be marked
+// for (TagLL t = head; t != null; t = t.nextTag) {
+// assert t.mark;
+//// if (!t.mark) {
+//// t.removeLL();
+//// if (head == t)
+//// head = t.nextTag;
+//// }
+// }
+ assert head[0].mark;
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java b/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java
new file mode 100644
index 0000000..e8bb0a3
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TagLL.java
@@ -0,0 +1,176 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * This is a Tag -- a startOffset, endOffset and value.
+ * <p>
+ * A Tag starts without a value in an
+ * "advancing" state. {@link #advance(org.apache.lucene.util.BytesRef, int)}
+ * is called with subsequent words and then eventually it won't advance any
+ * more, and value is set (could be null).
+ * <p>
+ * A Tag is also a doubly-linked-list (hence the LL in the name). All tags share
+ * a reference to the head via a 1-element array, which is potentially modified
+ * if any of the linked-list methods are called. Tags in the list should have
+ * equal or increasing start offsets.
+ */
+public class TagLL{
+
+ private final TagLL[] head;//a shared pointer to the head; 1 element
+ TagLL prevTag, nextTag; // linked list
+
+ private TermPrefixCursor cursor;
+
+ final int startOffset;//inclusive
+ int endOffset;//exclusive
+ Object value;//null means unset
+
+ /** optional boolean used by some TagClusterReducer's */
+ boolean mark = false;
+
+ TagLL(TagLL[] head, TermPrefixCursor cursor, int startOffset, int endOffset, Object value) {
+ this.head = head;
+ this.cursor = cursor;
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ this.value = value;
+ }
+
+ /**
+ * Advances this tag with "word" at offset "offset". If this tag is not in
+ * an advancing state then it does nothing. If it is advancing and prior to
+ * advancing further it sees a value, then a non-advancing tag may be inserted
+ * into the LL as side-effect. If this returns false (it didn't advance) and
+ * if there is no value, then it will also be removed.
+ *
+ *
+ * @param word The next word or null if at an end
+ * @param offset The last character in word's offset in the underlying
+ * stream. If word is null then it's meaningless.
+ *
+ * @return Whether it advanced or not.
+ */
+ boolean advance(BytesRef word, int offset) throws IOException {
+ if (!isAdvancing())
+ return false;
+
+ Object iVal = cursor.getDocIds();
+
+ if (word != null && cursor.advance(word)) {
+
+ if (iVal != null) {
+ addBeforeLL(new TagLL(head, null, startOffset, endOffset, iVal));
+ }
+
+ assert offset >= endOffset;
+ endOffset = offset;
+ return true;
+ } else {
+ this.value = iVal;
+ this.cursor = null;
+ if (iVal == null)
+ removeLL();
+ return false;
+ }
+ }
+
+ /** Removes this tag from the chain, connecting prevTag and nextTag. Does not
+ * modify "this" object's pointers, so the caller can refer to nextTag after
+ * removing it. */
+ public void removeLL() {
+ if (head[0] == this)
+ head[0] = nextTag;
+ if (prevTag != null) {
+ prevTag.nextTag = nextTag;
+ }
+ if (nextTag != null) {
+ nextTag.prevTag = prevTag;
+ }
+ }
+
+ void addBeforeLL(TagLL tag) {
+ assert tag.startOffset <= startOffset;
+ if (prevTag != null) {
+ assert prevTag.startOffset <= tag.startOffset;
+ prevTag.nextTag = tag;
+ tag.prevTag = prevTag;
+ } else {
+ assert head[0] == this;
+ head[0] = tag;
+ }
+ prevTag = tag;
+ tag.nextTag = this;
+ }
+
+ void addAfterLL(TagLL tag) {
+ assert tag.startOffset >= startOffset;
+ if (nextTag != null) {
+ assert nextTag.startOffset >= tag.startOffset;
+ nextTag.prevTag = tag;
+ tag.nextTag = nextTag;
+ }
+ nextTag = tag;
+ tag.prevTag = this;
+ }
+
+ public int charLen() {
+ return endOffset - startOffset;
+ }
+
+ public TagLL getNextTag() {
+ return nextTag;
+ }
+
+ public TagLL getPrevTag() {
+ return prevTag;
+ }
+
+ public int getStartOffset() {
+ return startOffset;
+ }
+ public int getEndOffset() {
+ return endOffset;
+ }
+ public boolean overlaps(TagLL other) {
+ //don't use >= or <= because startOffset is inclusive while endOffset is exclusive
+ if (startOffset < other.startOffset)
+ return endOffset > other.startOffset;
+ else
+ return startOffset < other.endOffset;
+ }
+
+ boolean isAdvancing() {
+ return cursor != null;
+ }
+
+ @Override
+ public String toString() {
+ return (prevTag != null ? '*' : '-') + "|" + (nextTag != null ? '*' : '-') +
+ " " + startOffset + " to " + endOffset + (isAdvancing() ? '+' : " #" + value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java b/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java
new file mode 100644
index 0000000..12a4cf0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/Tagger.java
@@ -0,0 +1,230 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IntsRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tags maximum string of words in a corpus. This is a callback-style API
+ * in which you implement {@link #tagCallback(int, int, Object)}.
+ *
+ * This class should be independently usable outside Solr.
+ */
+public abstract class Tagger {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final TokenStream tokenStream;
+ private final TermToBytesRefAttribute byteRefAtt;
+ private final PositionIncrementAttribute posIncAtt;
+ private final OffsetAttribute offsetAtt;
+ private final TaggingAttribute taggingAtt;
+
+ private final TagClusterReducer tagClusterReducer;
+ private final Terms terms;
+ private final Bits liveDocs;
+ private final boolean skipAltTokens;
+ private final boolean ignoreStopWords;
+
+ private Map<BytesRef, IntsRef> docIdsCache;
+
+ /** Whether the WARNING about skipped tokens was already logged. */
+ private boolean loggedSkippedAltTokenWarning = false;
+
+ public Tagger(Terms terms, Bits liveDocs, TokenStream tokenStream,
+ TagClusterReducer tagClusterReducer, boolean skipAltTokens,
+ boolean ignoreStopWords) throws IOException {
+ this.terms = terms;
+ this.liveDocs = liveDocs;
+ this.tokenStream = tokenStream;
+ this.skipAltTokens = skipAltTokens;
+ this.ignoreStopWords = ignoreStopWords;
+ byteRefAtt = tokenStream.addAttribute(TermToBytesRefAttribute.class);
+ posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
+ offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
+ taggingAtt = tokenStream.addAttribute(TaggingAttribute.class);
+ tokenStream.reset();
+
+ this.tagClusterReducer = tagClusterReducer;
+ }
+
+ public void enableDocIdsCache(int initSize) {
+ if (initSize > 0)
+ docIdsCache = new HashMap<>(initSize);
+ }
+
+ public void process() throws IOException {
+ if (terms == null)
+ return;
+
+ //a shared pointer to the head used by this method and each Tag instance.
+ final TagLL[] head = new TagLL[1];
+
+ TermPrefixCursor cursor = null;//re-used
+
+ //boolean switch used to log warnings in case tokens where skipped during tagging.
+ boolean skippedTokens = false;
+
+ while (tokenStream.incrementToken()) {
+ if (log.isTraceEnabled()) {
+ log.trace("Token: {}, posInc: {}, offset: [{},{}]",
+ byteRefAtt, posIncAtt.getPositionIncrement(),
+ offsetAtt.startOffset(), offsetAtt.endOffset());
+ }
+ //check for posInc < 1 (alternate Tokens, such as expanded Synonyms)
+ if (posIncAtt.getPositionIncrement() < 1) {
+ //(a) Deal with this as a configuration issue and throw an exception
+ if (!skipAltTokens) {
+ //TODO throw UnsupportedTokenException when PhraseBuilder is ported
+ throw new IllegalStateException("Query Analyzer generates alternate "
+ + "Tokens (posInc == 0). Please adapt your Analyzer configuration or "
+ + "enable '" + TaggerRequestHandler.SKIP_ALT_TOKENS + "' to skip such "
+ + "tokens. NOTE: enabling '" + TaggerRequestHandler.SKIP_ALT_TOKENS
+ + "' might result in wrong tagging results if the index time analyzer "
+ + "is not configured accordingly. For detailed information see "
+ + "https://github.com/OpenSextant/SolrTextTagger/pull/11#issuecomment-24936225");
+ } else {
+ //(b) In case the index time analyser had indexed all variants (users
+ // need to ensure that) processing of alternate tokens can be skipped
+ // as anyways all alternatives will be contained in the FST.
+ skippedTokens = true;
+ log.trace(" ... ignored token");
+ continue;
+ }
+ }
+ //-- If PositionIncrement > 1 (stopwords)
+ if (!ignoreStopWords && posIncAtt.getPositionIncrement() > 1) {
+ log.trace(" - posInc > 1 ... mark cluster as done");
+ advanceTagsAndProcessClusterIfDone(head, null);
+ }
+
+ final BytesRef term;
+ //NOTE: we need to lookup tokens if
+ // * the LookupAtt is true OR
+ // * there are still advancing tags (to find the longest possible match)
+ if(taggingAtt.isTaggable() || head[0] != null){
+ //-- Lookup the term id from the next token
+ term = byteRefAtt.getBytesRef();
+ if (term.length == 0) {
+ throw new IllegalArgumentException("term: " + term.utf8ToString() + " analyzed to a zero-length token");
+ }
+ } else { //no current cluster AND lookup == false ...
+ term = null; //skip this token
+ }
+
+ //-- Process tag
+ advanceTagsAndProcessClusterIfDone(head, term);
+
+ //-- only create new Tags for Tokens we need to lookup
+ if (taggingAtt.isTaggable() && term != null) {
+
+ //determine if the terms index has a term starting with the provided term
+ // TODO create a pool of these cursors to reuse them more? could be trivial impl
+ if (cursor == null)// (else the existing cursor will be re-used)
+ cursor = new TermPrefixCursor(terms.iterator(), liveDocs, docIdsCache);
+ if (cursor.advance(term)) {
+ TagLL newTail = new TagLL(head, cursor, offsetAtt.startOffset(), offsetAtt.endOffset(), null);
+ cursor = null;//because the new tag now "owns" this instance
+ //and add it to the end
+ if (head[0] == null) {
+ head[0] = newTail;
+ } else {
+ for (TagLL t = head[0]; true; t = t.nextTag) {
+ if (t.nextTag == null) {
+ t.addAfterLL(newTail);
+ break;
+ }
+ }
+ }
+ }
+ }//if termId >= 0
+ }//end while(incrementToken())
+
+ //-- Finish all tags
+ advanceTagsAndProcessClusterIfDone(head, null);
+ assert head[0] == null;
+
+ if(!loggedSkippedAltTokenWarning && skippedTokens){
+ loggedSkippedAltTokenWarning = true; //only log once
+ log.warn("The Tagger skipped some alternate tokens (tokens with posInc == 0) "
+ + "while processing text. This may cause problems with some Analyzer "
+ + "configurations (e.g. query time synonym expansion). For details see "
+ + "https://github.com/OpenSextant/SolrTextTagger/pull/11#issuecomment-24936225");
+ }
+
+ tokenStream.end();
+ //tokenStream.close(); caller closes because caller acquired it
+ }
+
+ private void advanceTagsAndProcessClusterIfDone(TagLL[] head, BytesRef term) throws IOException {
+ //-- Advance tags
+ final int endOffset = term != null ? offsetAtt.endOffset() : -1;
+ boolean anyAdvance = false;
+ for (TagLL t = head[0]; t != null; t = t.nextTag) {
+ anyAdvance |= t.advance(term, endOffset);
+ }
+
+ //-- Process cluster if done
+ if (!anyAdvance && head[0] != null) {
+ tagClusterReducer.reduce(head);
+ for (TagLL t = head[0]; t != null; t = t.nextTag) {
+ assert t.value != null;
+ tagCallback(t.startOffset, t.endOffset, t.value);
+ }
+ head[0] = null;
+ }
+ }
+
+ /**
+ * Invoked by {@link #process()} for each tag found. endOffset is always >= the endOffset
+ * given in the previous call.
+ *
+ * @param startOffset The character offset of the original stream where the tag starts.
+ * @param endOffset One more than the character offset of the original stream where the tag ends.
+ * @param docIdsKey A reference to the matching docIds that can be resolved via {@link #lookupDocIds(Object)}.
+ */
+ protected abstract void tagCallback(int startOffset, int endOffset, Object docIdsKey);
+
+ /**
+ * Returns a sorted array of integer docIds given the corresponding key.
+ * @param docIdsKey The lookup key.
+ * @return Not null
+ */
+ protected IntsRef lookupDocIds(Object docIdsKey) {
+ return (IntsRef) docIdsKey;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java
new file mode 100644
index 0000000..a972e47
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java
@@ -0,0 +1,397 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import javax.xml.stream.XMLStreamException;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import com.google.common.io.CharStreams;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.StopFilterFactory;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IntsRef;
+import org.apache.solr.analysis.TokenizerChain;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.BitDocSet;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.DocSlice;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SolrReturnFields;
+import org.apache.solr.search.SyntaxError;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Scans posted text, looking for matching strings in the Solr index.
+ * The public static final String members are request parameters.
+ * This handler is also called the "SolrTextTagger".
+ *
+ * @since 7.4.0
+ */
+public class TaggerRequestHandler extends RequestHandlerBase {
+
+ /** Request parameter. */
+ public static final String OVERLAPS = "overlaps";
+ /** Request parameter. */
+ public static final String TAGS_LIMIT = "tagsLimit";
+ /** Request parameter. */
+ public static final String MATCH_TEXT = "matchText";
+ /** Request parameter. */
+ public static final String SKIP_ALT_TOKENS = "skipAltTokens";
+ /** Request parameter. */
+ public static final String IGNORE_STOPWORDS = "ignoreStopwords";
+ /** Request parameter. */
+ public static final String XML_OFFSET_ADJUST = "xmlOffsetAdjust";
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Override
+ public String getDescription() {
+ return "Processes input text to find matching tokens stored in the index.";
+ }
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+
+ //--Read params
+ final String indexedField = req.getParams().get("field");
+ if (indexedField == null)
+ throw new RuntimeException("required param 'field'");
+
+ final TagClusterReducer tagClusterReducer =
+ chooseTagClusterReducer(req.getParams().get(OVERLAPS));
+ final int rows = req.getParams().getInt(CommonParams.ROWS, 10000);
+ final int tagsLimit = req.getParams().getInt(TAGS_LIMIT, 1000);
+ final boolean addMatchText = req.getParams().getBool(MATCH_TEXT, false);
+ final SchemaField idSchemaField = req.getSchema().getUniqueKeyField();
+ if (idSchemaField == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The tagger requires a" +
+ "uniqueKey in the schema.");//TODO this could be relaxed
+ }
+ final boolean skipAltTokens = req.getParams().getBool(SKIP_ALT_TOKENS, false);
+ final boolean ignoreStopWords = req.getParams().getBool(IGNORE_STOPWORDS,
+ fieldHasIndexedStopFilter(indexedField, req));
+
+ //--Get posted data
+ Reader inputReader = null;
+ Iterable<ContentStream> streams = req.getContentStreams();
+ if (streams != null) {
+ Iterator<ContentStream> iter = streams.iterator();
+ if (iter.hasNext()) {
+ inputReader = iter.next().getReader();
+ }
+ if (iter.hasNext()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ getClass().getSimpleName()+" does not support multiple ContentStreams"); //TODO support bulk tagging?
+ }
+ }
+ if (inputReader == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ getClass().getSimpleName()+" requires text to be POSTed to it");
+ }
+
+ // We may or may not need to read the input into a string
+ final InputStringLazy inputStringFuture = new InputStringLazy(inputReader);
+
+ final OffsetCorrector offsetCorrector = getOffsetCorrector(req.getParams(), inputStringFuture);
+
+ final String inputString;//only populated if needed
+ if (addMatchText || inputStringFuture.inputString != null) {
+ //Read the input fully into a String buffer that we'll need later,
+ // then replace the input with a reader wrapping the buffer.
+ inputString = inputStringFuture.call();
+ inputReader.close();
+ inputReader = new StringReader(inputString);
+ } else {
+ inputString = null;//not used
+ }
+
+ final SolrIndexSearcher searcher = req.getSearcher();
+ final FixedBitSet matchDocIdsBS = new FixedBitSet(searcher.maxDoc());
+ final List tags = new ArrayList(2000);
+
+ try {
+ Analyzer analyzer = req.getSchema().getField(indexedField).getType().getQueryAnalyzer();
+ try (TokenStream tokenStream = analyzer.tokenStream("", inputReader)) {
+ Terms terms = searcher.getSlowAtomicReader().terms(indexedField);
+ if (terms == null)
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "field " + indexedField + " has no indexed data");
+ Tagger tagger = new Tagger(terms, computeDocCorpus(req), tokenStream, tagClusterReducer,
+ skipAltTokens, ignoreStopWords) {
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void tagCallback(int startOffset, int endOffset, Object docIdsKey) {
+ if (tags.size() >= tagsLimit)
+ return;
+ if (offsetCorrector != null) {
+ int[] offsetPair = offsetCorrector.correctPair(startOffset, endOffset);
+ if (offsetPair == null) {
+ log.debug("Discarded offsets [{}, {}] because couldn't balance XML.",
+ startOffset, endOffset);
+ return;
+ }
+ startOffset = offsetPair[0];
+ endOffset = offsetPair[1];
+ }
+
+ NamedList tag = new NamedList();
+ tag.add("startOffset", startOffset);
+ tag.add("endOffset", endOffset);
+ if (addMatchText)
+ tag.add("matchText", inputString.substring(startOffset, endOffset));
+ //below caches, and also flags matchDocIdsBS
+ tag.add("ids", lookupSchemaDocIds(docIdsKey));
+ tags.add(tag);
+ }
+
+ Map<Object, List> docIdsListCache = new HashMap<>(2000);
+
+ ValueSourceAccessor uniqueKeyCache = new ValueSourceAccessor(searcher,
+ idSchemaField.getType().getValueSource(idSchemaField, null));
+
+ @SuppressWarnings("unchecked")
+ private List lookupSchemaDocIds(Object docIdsKey) {
+ List schemaDocIds = docIdsListCache.get(docIdsKey);
+ if (schemaDocIds != null)
+ return schemaDocIds;
+ IntsRef docIds = lookupDocIds(docIdsKey);
+ //translate lucene docIds to schema ids
+ schemaDocIds = new ArrayList(docIds.length);
+ for (int i = docIds.offset; i < docIds.offset + docIds.length; i++) {
+ int docId = docIds.ints[i];
+ assert i == docIds.offset || docIds.ints[i - 1] < docId : "not sorted?";
+ matchDocIdsBS.set(docId);//also, flip docid in bitset
+ try {
+ schemaDocIds.add(uniqueKeyCache.objectVal(docId));//translates here
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+ assert !schemaDocIds.isEmpty();
+
+ docIdsListCache.put(docIds, schemaDocIds);
+ return schemaDocIds;
+ }
+
+ };
+ tagger.enableDocIdsCache(2000);//TODO configurable
+ tagger.process();
+ }
+ } finally {
+ inputReader.close();
+ }
+ rsp.add("tagsCount",tags.size());
+ rsp.add("tags", tags);
+
+ rsp.setReturnFields(new SolrReturnFields( req ));
+
+ //Solr's standard name for matching docs in response
+ rsp.add("response", getDocList(rows, matchDocIdsBS));
+ }
+
+ private static class InputStringLazy implements Callable<String> {
+ final Reader inputReader;
+ String inputString;
+
+ InputStringLazy(Reader inputReader) {
+ this.inputReader = inputReader;
+ }
+
+ @Override
+ public String call() throws IOException {
+ if (inputString == null) {
+ inputString = CharStreams.toString(inputReader);
+ }
+ return inputString;
+ }
+ }
+
+ protected OffsetCorrector getOffsetCorrector(SolrParams params, Callable<String> inputStringProvider) throws Exception {
+ final boolean xmlOffsetAdjust = params.getBool(XML_OFFSET_ADJUST, false);
+ if (!xmlOffsetAdjust) {
+ return null;
+ }
+ try {
+ return new XmlOffsetCorrector(inputStringProvider.call());
+ } catch (XMLStreamException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Expecting XML but wasn't: " + e, e);
+ }
+ }
+
+ private DocList getDocList(int rows, FixedBitSet matchDocIdsBS) throws IOException {
+ //Now we must supply a Solr DocList and add it to the response.
+ // Typically this is gotten via a SolrIndexSearcher.search(), but in this case we
+ // know exactly what documents to return, the order doesn't matter nor does
+ // scoring.
+ // Ideally an implementation of DocList could be directly implemented off
+ // of a BitSet, but there are way too many methods to implement for a minor
+ // payoff.
+ int matchDocs = matchDocIdsBS.cardinality();
+ int[] docIds = new int[ Math.min(rows, matchDocs) ];
+ DocIdSetIterator docIdIter = new BitSetIterator(matchDocIdsBS, 1);
+ for (int i = 0; i < docIds.length; i++) {
+ docIds[i] = docIdIter.nextDoc();
+ }
+ return new DocSlice(0, docIds.length, docIds, null, matchDocs, 1f);
+ }
+
+ private TagClusterReducer chooseTagClusterReducer(String overlaps) {
+ TagClusterReducer tagClusterReducer;
+ if (overlaps == null || overlaps.equals("NO_SUB")) {
+ tagClusterReducer = TagClusterReducer.NO_SUB;
+ } else if (overlaps.equals("ALL")) {
+ tagClusterReducer = TagClusterReducer.ALL;
+ } else if (overlaps.equals("LONGEST_DOMINANT_RIGHT")) {
+ tagClusterReducer = TagClusterReducer.LONGEST_DOMINANT_RIGHT;
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "unknown tag overlap mode: "+overlaps);
+ }
+ return tagClusterReducer;
+ }
+
+ /**
+ * The set of documents matching the provided 'fq' (filter query). Don't include deleted docs
+ * either. If null is returned, then all docs are available.
+ */
+ private Bits computeDocCorpus(SolrQueryRequest req) throws SyntaxError, IOException {
+ final String[] corpusFilterQueries = req.getParams().getParams("fq");
+ final SolrIndexSearcher searcher = req.getSearcher();
+ final Bits docBits;
+ if (corpusFilterQueries != null && corpusFilterQueries.length > 0) {
+ List<Query> filterQueries = new ArrayList<Query>(corpusFilterQueries.length);
+ for (String corpusFilterQuery : corpusFilterQueries) {
+ QParser qParser = QParser.getParser(corpusFilterQuery, null, req);
+ try {
+ filterQueries.add(qParser.parse());
+ } catch (SyntaxError e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+ }
+
+ final DocSet docSet = searcher.getDocSet(filterQueries);//hopefully in the cache
+ //note: before Solr 4.7 we could call docSet.getBits() but no longer.
+ if (docSet instanceof BitDocSet) {
+ docBits = ((BitDocSet)docSet).getBits();
+ } else {
+ docBits = new Bits() {
+
+ @Override
+ public boolean get(int index) {
+ return docSet.exists(index);
+ }
+
+ @Override
+ public int length() {
+ return searcher.maxDoc();
+ }
+ };
+ }
+ } else {
+ docBits = searcher.getSlowAtomicReader().getLiveDocs();
+ }
+ return docBits;
+ }
+
+ private boolean fieldHasIndexedStopFilter(String field, SolrQueryRequest req) {
+ FieldType fieldType = req.getSchema().getFieldType(field);
+ Analyzer analyzer = fieldType.getIndexAnalyzer();//index analyzer
+ if (analyzer instanceof TokenizerChain) {
+ TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
+ TokenFilterFactory[] tokenFilterFactories = tokenizerChain.getTokenFilterFactories();
+ for (TokenFilterFactory tokenFilterFactory : tokenFilterFactories) {
+ if (tokenFilterFactory instanceof StopFilterFactory)
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** See LUCENE-4541 or {@link org.apache.solr.response.transform.ValueSourceAugmenter}. */
+ static class ValueSourceAccessor {
+ private final List<LeafReaderContext> readerContexts;
+ private final ValueSource valueSource;
+ private final Map fContext;
+ private final FunctionValues[] functionValuesPerSeg;
+ private final int[] functionValuesDocIdPerSeg;
+
+ ValueSourceAccessor(IndexSearcher searcher, ValueSource valueSource) {
+ readerContexts = searcher.getIndexReader().leaves();
+ this.valueSource = valueSource;
+ fContext = ValueSource.newContext(searcher);
+ functionValuesPerSeg = new FunctionValues[readerContexts.size()];
+ functionValuesDocIdPerSeg = new int[readerContexts.size()];
+ }
+
+ Object objectVal(int topDocId) throws IOException {
+ // lookup segment level stuff:
+ int segIdx = ReaderUtil.subIndex(topDocId, readerContexts);
+ LeafReaderContext rcontext = readerContexts.get(segIdx);
+ int segDocId = topDocId - rcontext.docBase;
+ // unfortunately Lucene 7.0 requires forward only traversal (with no reset method).
+ // So we need to track our last docId (per segment) and re-fetch the FunctionValues. :-(
+ FunctionValues functionValues = functionValuesPerSeg[segIdx];
+ if (functionValues == null || segDocId < functionValuesDocIdPerSeg[segIdx]) {
+ functionValues = functionValuesPerSeg[segIdx] = valueSource.getValues(fContext, rcontext);
+ }
+ functionValuesDocIdPerSeg[segIdx] = segDocId;
+
+ // get value:
+ return functionValues.objectVal(segDocId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java
new file mode 100644
index 0000000..b7803e4
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttribute.java
@@ -0,0 +1,65 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.Attribute;
+
+/**
+ * Attribute used by the {@link Tagger} to decide if a token can start a
+ * new {@link TagLL tag}.
+ * <p>
+ * By default this Attribute will return <code>true</code>, but it might be
+ * reset by some {@link TokenFilter} added to the {@link TokenStream} used
+ * to analyze the parsed text. Typically this will be done based on NLP
+ * processing results (e.g. to only lookup Named Entities).
+ * <p>
+ * NOTE: that all Tokens are used to advance existing {@link TagLL tags}.
+ */
+public interface TaggingAttribute extends Attribute {
+
+ /**
+ * By default this Attribute will be initialised with <code>true</code>.
+ * This ensures that all tokens are taggable by default (especially if
+ * the {@link TaggingAttribute} is not set by any component in the configured
+ * {@link TokenStream}
+ */
+ public static final boolean DEFAULT_TAGGABLE = true;
+
+ /**
+ * Getter for the taggable state of the current Token
+ *
+ * @return the state
+ */
+ public boolean isTaggable();
+
+ /**
+ * Setter for the taggable state. Typically called by code within
+ * {@link TokenFilter#incrementToken()}.
+ *
+ * @param lookup the state
+ */
+ public void setTaggable(boolean lookup);
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java
new file mode 100644
index 0000000..55ecfbc6
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TaggingAttributeImpl.java
@@ -0,0 +1,79 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+
+/**
+ * Implementation of the {@link TaggingAttribute}
+ */
+public class TaggingAttributeImpl extends AttributeImpl implements TaggingAttribute {
+
+ /**
+ * the private field initialised with {@link TaggingAttribute#DEFAULT_TAGGABLE}
+ */
+ private boolean taggable = TaggingAttribute.DEFAULT_TAGGABLE;
+
+ /*
+ * (non-Javadoc)
+ * @see org.opensextant.solrtexttagger.LookupAttribute#isLookup()
+ */
+ @Override
+ public boolean isTaggable() {
+ return taggable;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.opensextant.solrtexttagger.LookupAttribute#setLookup(boolean)
+ */
+ @Override
+ public void setTaggable(boolean lookup) {
+ this.taggable = lookup;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.lucene.util.AttributeImpl#clear()
+ */
+ @Override
+ public void clear() {
+ taggable = DEFAULT_TAGGABLE;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.lucene.util.AttributeImpl#copyTo(org.apache.lucene.util.AttributeImpl)
+ */
+ @Override
+ public void copyTo(AttributeImpl target) {
+ ((TaggingAttribute) target).setTaggable(taggable);
+ }
+
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ reflector.reflect(TaggingAttribute.class, "taggable", isTaggable());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java b/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java
new file mode 100644
index 0000000..1e82dbe
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/TermPrefixCursor.java
@@ -0,0 +1,189 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.analysis.miscellaneous.ConcatenateGraphFilter;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.IntsRef;
+
+/**
+ * Cursor into the terms that advances by prefix.
+ */
+class TermPrefixCursor {
+
+ //Note: this could be a lot more efficient if MemoryPostingsFormat supported ordinal lookup.
+ // Maybe that could be added to Lucene.
+
+ // TODO add bloom filter of hashcode of first ~ 6 bytes to avoid lookup into terms dict?
+
+ private static final byte SEPARATOR_CHAR = ConcatenateGraphFilter.SEP_LABEL; // used to be ' '; TODO configurable?
+ private static final IntsRef EMPTY_INTSREF = new IntsRef();
+
+ private final TermsEnum termsEnum;
+ private final Bits liveDocs;
+ private final Map<BytesRef, IntsRef> docIdsCache;
+
+ private BytesRef prefixBuf;//we append to this
+ private BytesRefBuilder prefixBufBuilder = new BytesRefBuilder();
+ private boolean prefixBufOnLoan;//if true, PB is loaned; needs to be copied
+ private PostingsEnum postingsEnum;
+ private IntsRef docIds;
+
+ TermPrefixCursor(TermsEnum termsEnum, Bits liveDocs, Map<BytesRef, IntsRef> docIdsCache) {
+ this.termsEnum = termsEnum;
+ this.liveDocs = liveDocs;
+ this.docIdsCache = docIdsCache;
+ }
+
+ /** Appends the separator char (if not the first) plus the given word to the prefix buffer,
+ * then seeks to it. If the seek fails, false is returned and this cursor
+ * can be re-used as if in a new state. The {@code word} BytesRef is considered temporary,
+ * and is not saved within this class. */
+ boolean advance(BytesRef word) throws IOException {
+ if (prefixBuf == null) { // first advance
+ //set prefixBuf to word temporary. When advance() completes, we either null out or copy.
+ prefixBuf = word;
+ prefixBufOnLoan = true;
+ if (seekPrefix()) {//... and we have to
+ ensureBufIsACopy();
+ return true;
+ } else {
+ prefixBuf = null;//just to be darned sure 'word' isn't referenced here
+ return false;
+ }
+
+ } else { // subsequent advance
+ //append to existing
+ assert !prefixBufOnLoan;
+
+ prefixBufBuilder.append(SEPARATOR_CHAR);
+ prefixBufBuilder.append(word);
+ prefixBuf = prefixBufBuilder.get();
+ if (seekPrefix()) {
+ return true;
+ } else {
+ prefixBuf = null;
+ return false;
+ }
+ }
+ }
+
+ private void ensureBufIsACopy() {
+ if (!prefixBufOnLoan)
+ return;
+
+ prefixBufBuilder.clear();
+ prefixBufBuilder.copyBytes(prefixBuf);
+ prefixBuf = prefixBufBuilder.get();
+ prefixBufOnLoan = false;
+ }
+
+ /** Seeks to prefixBuf or the next term that is prefixed by prefixBuf plus the separator char.
+ * Sets docIds. **/
+ private boolean seekPrefix() throws IOException {
+ TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(prefixBuf);
+
+ docIds = null;//invalidate
+ switch (seekStatus) {
+ case END:
+ return false;
+
+ case FOUND:
+ postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
+ docIds = postingsEnumToIntsRef(postingsEnum, liveDocs);
+ if (docIds.length > 0) {
+ return true;
+ }
+
+ //Pretend we didn't find it; go to next term
+ docIds = null;
+ if (termsEnum.next() == null) { // case END
+ return false;
+ }
+ //fall through to NOT_FOUND
+
+ case NOT_FOUND:
+ //termsEnum must start with prefixBuf to continue
+ BytesRef teTerm = termsEnum.term();
+
+ if (teTerm.length > prefixBuf.length) {
+ for (int i = 0; i < prefixBuf.length; i++) {
+ if (prefixBuf.bytes[prefixBuf.offset + i] != teTerm.bytes[teTerm.offset + i])
+ return false;
+ }
+ if (teTerm.bytes[teTerm.offset + prefixBuf.length] != SEPARATOR_CHAR)
+ return false;
+ return true;
+ }
+ return false;
+ }
+ throw new IllegalStateException(seekStatus.toString());
+ }
+
+ /** Returns an IntsRef either cached or reading postingsEnum. Not null. */
+ private IntsRef postingsEnumToIntsRef(PostingsEnum postingsEnum, Bits liveDocs) throws IOException {
+ // (The cache can have empty IntsRefs)
+
+ //lookup prefixBuf in a cache
+ if (docIdsCache != null) {
+ docIds = docIdsCache.get(prefixBuf);
+ if (docIds != null) {
+ return docIds;
+ }
+ }
+
+ //read postingsEnum
+ docIds = new IntsRef(termsEnum.docFreq());
+ int docId;
+ while ((docId = postingsEnum.nextDoc()) != PostingsEnum.NO_MORE_DOCS) {
+ if (liveDocs != null && !liveDocs.get(postingsEnum.docID())) {
+ continue;
+ }
+ docIds.ints[docIds.length++] = docId;
+ }
+ if (docIds.length == 0)
+ docIds = EMPTY_INTSREF;
+
+ //cache
+ if (docIdsCache != null) {
+ ensureBufIsACopy();
+ //clone is shallow; that's okay as the prefix isn't overwritten; it's just appended to
+ docIdsCache.put(prefixBuf.clone(), docIds);
+ }
+ return docIds;
+ }
+
+ /** The docIds of the last call to advance, if it returned true. It might be null, but
+ * its length won't be 0. Treat as immutable. */
+ IntsRef getDocIds() {
+ assert docIds == null || docIds.length != 0;
+ return docIds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java b/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java
new file mode 100644
index 0000000..576328f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/XmlOffsetCorrector.java
@@ -0,0 +1,113 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import javax.xml.stream.XMLResolver;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.XMLEvent;
+import java.io.InputStream;
+import java.io.StringReader;
+
+import com.ctc.wstx.stax.WstxInputFactory;
+import org.apache.commons.io.input.ClosedInputStream;
+import org.codehaus.stax2.LocationInfo;
+import org.codehaus.stax2.XMLInputFactory2;
+import org.codehaus.stax2.XMLStreamReader2;
+
+/**
+ * Corrects offsets to adjust for XML formatted data. The goal is such that the caller should be
+ * able to insert a start XML tag at the start offset and a corresponding end XML tag at the end
+ * offset of the tagger, and have it be valid XML. See {@link #correctPair(int, int)}.
+ *
+ * This will not work on invalid XML.
+ *
+ * Not thread-safe.
+ */
+public class XmlOffsetCorrector extends OffsetCorrector {
+
+ //TODO use StAX without hard requirement on woodstox. xmlStreamReader.getLocation().getCharacterOffset()
+
+ private static final XMLInputFactory2 XML_INPUT_FACTORY;
+ static {
+ // note: similar code in Solr's EmptyEntityResolver
+ XML_INPUT_FACTORY = new WstxInputFactory();
+ XML_INPUT_FACTORY.setXMLResolver(new XMLResolver() {
+ @Override
+ public InputStream resolveEntity(String publicId, String systemId, String baseURI, String namespace) {
+ return ClosedInputStream.CLOSED_INPUT_STREAM;
+ }
+ });
+ // TODO disable DTD?
+ // XML_INPUT_FACTORY.setProperty(XMLInputFactory.IS_VALIDATING, Boolean.FALSE)
+ XML_INPUT_FACTORY.configureForSpeed();
+ }
+
+ /**
+ * Initialize based on the document text.
+ * @param docText non-null XML content.
+ * @throws XMLStreamException If there's a problem parsing the XML.
+ */
+ public XmlOffsetCorrector(String docText) throws XMLStreamException {
+ super(docText, false);
+
+ int tagCounter = 0;
+ int thisTag = -1;
+
+ //note: we *could* add a virtual outer tag to guarantee all text is in the context of a tag,
+ // but we shouldn't need to because there is no findable text outside the top element.
+
+ final XMLStreamReader2 xmlStreamReader =
+ (XMLStreamReader2) XML_INPUT_FACTORY.createXMLStreamReader(new StringReader(docText));
+
+ while (xmlStreamReader.hasNext()) {
+ int eventType = xmlStreamReader.next();
+ switch (eventType) {
+ case XMLEvent.START_ELEMENT: {
+ tagInfo.ensureCapacity(tagInfo.size() + 5);
+ final int parentTag = thisTag;
+ final LocationInfo info = xmlStreamReader.getLocationInfo();
+ tagInfo.add(parentTag);
+ tagInfo.add((int) info.getStartingCharOffset(), (int) info.getEndingCharOffset());
+ tagInfo.add(-1, -1);//these 2 will be populated when we get to the close tag
+ thisTag = tagCounter++;
+
+ parentChangeOffsets.add((int) info.getStartingCharOffset());
+ parentChangeIds.add(thisTag);
+ break;
+ }
+ case XMLEvent.END_ELEMENT: {
+ final LocationInfo info = xmlStreamReader.getLocationInfo();
+ tagInfo.set(5 * thisTag + 3, (int) info.getStartingCharOffset());
+ tagInfo.set(5 * thisTag + 4, (int) info.getEndingCharOffset());
+ thisTag = getParentTag(thisTag);
+
+ parentChangeOffsets.add((int) info.getEndingCharOffset());
+ parentChangeIds.add(thisTag);
+ break;
+ }
+ default: //do nothing
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java b/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java
new file mode 100644
index 0000000..c2055b3
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/tagger/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The {@link org.apache.solr.handler.tagger.TaggerRequestHandler} and supporting classes.
+ * This was formerly known as OpenSextant's SolrTextTagger.
+ */
+package org.apache.solr.handler.tagger;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml
new file mode 100644
index 0000000..051cd10
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-tagger.xml
@@ -0,0 +1,187 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ This software was produced for the U. S. Government
+ under Contract No. W15P7T-11-C-F600, and is
+ subject to the Rights in Noncommercial Computer Software
+ and Noncommercial Computer Software Documentation
+ Clause 252.227-7014 (JUN 1995)
+
+ Copyright 2013 The MITRE Corporation. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<schema name="minimal" version="1.6">
+
+ <fields>
+ <field name="id" type="string" docValues="true" required="true"/>
+ <field name="name" type="string"/>
+ <!-- freq, positions, and norms are not needed by the tagger. However if you
+ intend to have this field be used for general search, you should not exclude
+ these stats. -->
+ <field name="name_tag" type="tag" stored="false"/>
+ <field name="name_tagStop" type="tagStop" stored="false"/>
+ <field name="name_tagWDF" type="tagWDF" stored="false"/>
+ <!--<field name="name_tagPartial" type="tagPartial" stored="false"/>-->
+ <field name="name_tagXml" type="tagXml" stored="false"/>
+ <field name="name_tagAttribute" type="tagAttribute" stored="false"/>
+
+ <copyField source="name" dest="name_tag"/>
+ <copyField source="name" dest="name_tagStop"/>
+ <copyField source="name" dest="name_tagWDF"/>
+ <!--<copyField source="name" dest="name_tagPartial"/>-->
+ <copyField source="name" dest="name_tagXml"/>
+ <copyField source="name" dest="name_tagAttribute"/>
+
+ <dynamicField name="*" type="string" indexed="true" stored="true"/>
+ </fields>
+
+ <uniqueKey>id</uniqueKey>
+
+ <types>
+ <fieldType name="string" class="solr.StrField"/>
+
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+
+ <fieldType name="tag" class="solr.TextField" positionIncrementGap="100"
+ postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <!--Usually good but for our tests, lets not. <filter class="solr.ASCIIFoldingFilterFactory"/>-->
+ <filter class="solr.LowerCaseFilterFactory"/>
+
+ <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <!--<filter class="solr.ASCIIFoldingFilterFactory"/>-->
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- adds a stop filter -->
+ <fieldType name="tagStop" class="solr.TextField" positionIncrementGap="100"
+ postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.StopFilterFactory" /><!-- by default english stopwords -->
+
+ <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.StopFilterFactory" /><!-- by default english stopwords -->
+ </analyzer>
+ </fieldType>
+
+ <!-- adds a WordDelimiterFilter producing stacked/synonym tokens -->
+ <fieldType name="tagWDF" class="solr.TextField" positionIncrementGap="100"
+ postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory" />
+ <filter class="solr.WordDelimiterGraphFilterFactory"
+ generateWordParts="1" generateNumberParts="1"
+ catenateWords="1" catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+
+ <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory" />
+ <filter class="solr.WordDelimiterGraphFilterFactory"
+ generateWordParts="1" generateNumberParts="1"
+ catenateWords="0" catenateNumbers="0" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Problem: in XML can't write \u001F
+ <fieldType name="tagPartial" class="solr.TextField" positionIncrementGap="100"
+ postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+
+ <filter class="solr.ShingleFilterFactory" tokenSeparator=""
+ outputUnigramsIfNoShingles="true" maxShingleSize="10" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ -->
+
+ <fieldType name="tagXml" class="solr.TextField" positionIncrementGap="100"
+ postingsFormat="FST50" omitTermFreqAndPositions="true" omitNorms="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+
+ <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+ </analyzer>
+ <analyzer type="query">
+ <charFilter class="solr.HTMLStripCharFilterFactory" /><!-- ADDED THIS! -->
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="tagAttribute" class="solr.TextField" positionIncrementGap="100" postingsFormat="FST50">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+
+ <filter class="solr.ConcatenateGraphFilterFactory" preservePositionIncrements="false" />
+ </analyzer>
+ <analyzer type="query">
+ <!-- 32 just for tests, bumps posInc -->
+ <tokenizer class="solr.StandardTokenizerFactory"
+ maxTokenLength="32"/>
+ <!--
+ NOTE: This used the WordLengthTaggingFilterFactory to test the
+ TaggingAttribute. The WordLengthTaggingFilter set the TaggingAttribute
+ for words based on their length. The attribute is ignored at indexing
+ time, but the Tagger will use it to only start tags for words that are
+ equals or longer as the configured minLength.
+ -->
+ <filter class="org.apache.solr.handler.tagger.WordLengthTaggingFilterFactory" minLength="4"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <!-- <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" /> -->
+ <!-- in this example, we will only use synonyms at query time
+ <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+ -->
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <!-- <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" /> -->
+ <!-- <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> -->
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ </types>
+
+
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
new file mode 100644
index 0000000..e0d3677
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" ?>
+<!--
+ This software was produced for the U. S. Government
+ under Contract No. W15P7T-11-C-F600, and is
+ subject to the Rights in Noncommercial Computer Software
+ and Noncommercial Computer Software Documentation
+ Clause 252.227-7014 (JUN 1995)
+
+ Copyright 2013 The MITRE Corporation. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!-- a basic solrconfig that tests can use when they want simple minimal solrconfig/schema
+ DO NOT ADD THINGS TO THIS CONFIG! -->
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <dataDir>${solr.data.dir:}</dataDir>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+
+ <!-- for postingsFormat="..." -->
+ <codecFactory name="CodecFactory" class="solr.SchemaCodecFactory" />
+
+ <!-- since Solr 4.8: -->
+ <schemaFactory name="SchemaFactory" class="solr.ClassicIndexSchemaFactory" />
+
+ <query>
+ <!-- illustrate putting in memory for warm-up -->
+ <listener event="firstSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ <lst><str name="q">name_tag:[* TO *]</str></lst>
+ </arr>
+ </listener>
+ <listener event="newSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ <lst><str name="q">name_tag:[* TO *]</str></lst>
+ </arr>
+ </listener>
+ </query>
+
+ <requestHandler name="/select" class="solr.SearchHandler"></requestHandler>
+
+ <requestHandler name="/tag" class="solr.TaggerRequestHandler">
+ <lst name="defaults">
+ <str name="field">name_tag</str>
+ </lst>
+ </requestHandler>
+
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf633921/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java
new file mode 100644
index 0000000..8d31ad0
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java
@@ -0,0 +1,153 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.tagger;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+
+import org.apache.lucene.document.Field;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.StreamingResponseCallback;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Tests that we can skip serialization of the documents when embedding
+ * Solr.
+ */
+public class EmbeddedSolrNoSerializeTest extends SolrTestCaseJ4 {
+
+ static EmbeddedSolrServer solrServer;
+
+ @BeforeClass
+ public static void init() throws Exception {
+ initCore("solrconfig-tagger.xml", "schema-tagger.xml");
+ solrServer = new EmbeddedSolrServer(h.getCoreContainer(), "collection1");
+ //we don't need to close the EmbeddedSolrServer because SolrTestCaseJ4 closes the core
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ clearIndex();
+ assertU(adoc("id", "9999", "name", "Boston"));
+ assertU(commit());
+ }
+
+ @Test
+ public void testTag() throws SolrServerException, IOException {
+ ModifiableSolrParams params = params();
+ String input = "foo boston bar";//just one tag;
+ QueryRequest req = new SolrTaggerRequest(params, input);
+ req.setPath("/tag");
+
+ QueryResponse rsp = req.process(solrServer);
+ SolrDocumentList results= (SolrDocumentList) rsp.getResponse().get("response");
+ assertNotNull(rsp.getResponse().get("tags"));
+ assertNotNull(results.get(0));
+ }
+
+ @SuppressWarnings("serial")
+ public static class SolrTaggerRequest extends QueryRequest {
+
+ private final String input;
+
+ public SolrTaggerRequest(SolrParams p, String input) {
+ super(p, METHOD.POST);
+ this.input = input;
+ }
+
+ // Deprecated in 7.2 but should live on until 8.x
+ @SuppressWarnings("deprecation")
+ @Override
+ public Collection<ContentStream> getContentStreams() {
+ return Collections.singleton(new ContentStreamBase.StringStream(input));
+ }
+
+ // As of 7.2. But won't work until: https://issues.apache.org/jira/browse/SOLR-12142
+// @Override
+// public RequestWriter.ContentWriter getContentWriter(String expectedType) {
+// return new RequestWriter.StringPayloadContentWriter(input, "text/plain; charset=UTF8");
+// }
+ }
+
+ @Test
+ public void testSearch() throws Exception {
+ QueryResponse rsp = solrServer.query(params("q", "name:Boston"));
+ assertNotNull(rsp.getResults().get(0));
+ }
+
+ @Test
+ public void testAssertTagStreamingWithSolrTaggerRequest() throws Exception {
+ doTestAssertTagStreaming(SolrTaggerRequest::new);
+ }
+
+ @Test
+ @Ignore("As of Solr 7, stream.body is disabled by default for security ") // DWS: dubious, IMO
+ // and it can't be enabled with EmbeddedSolrServer until SOLR-12126
+ public void testAssertTagStreamingWithStreamBodyParam() throws Exception {
+ doTestAssertTagStreaming((params, input) -> {
+ params.set("stream.body", input);
+ return new QueryRequest(params);
+ });
+ }
+
+ public void doTestAssertTagStreaming(BiFunction<ModifiableSolrParams,String,QueryRequest> newQueryRequest) throws IOException, SolrServerException {
+ ModifiableSolrParams params = params();
+ String input = "foo boston bar";//just one tag;
+ QueryRequest req = newQueryRequest.apply(params, input);
+ req.setPath("/tag");
+
+ final AtomicReference<SolrDocument> refDoc = new AtomicReference<>();
+ req.setStreamingResponseCallback(new StreamingResponseCallback() {
+ @Override
+ public void streamSolrDocument(SolrDocument doc) {
+ refDoc.set(doc);
+ }
+
+ @Override
+ public void streamDocListInfo(long numFound, long start, Float maxScore) {
+
+ }
+ });
+ QueryResponse rsp = req.process(solrServer);
+ assertNotNull(rsp.getResponse().get("tags"));
+ assertNotNull(refDoc.get());
+ assertEquals("Boston", ((Field)refDoc.get().getFieldValue("name")).stringValue());
+ }
+}
[18/18] lucene-solr:jira/solr-12438: SOLR-12438: Improvements to the
call forwarding. RefGuide doc changes.
Posted by ab...@apache.org.
SOLR-12438: Improvements to the call forwarding. RefGuide doc changes.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/53d9d081
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/53d9d081
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/53d9d081
Branch: refs/heads/jira/solr-12438
Commit: 53d9d081b39afdb6bca02c5e454da2ab1f9a180e
Parents: 3e72481
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Jun 7 14:14:08 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Jun 7 14:14:08 2018 +0200
----------------------------------------------------------------------
.../org/apache/solr/core/CoreContainer.java | 20 +-
.../handler/admin/MetricsHistoryHandler.java | 81 ++--
solr/solr-ref-guide/src/metrics-history.adoc | 382 +++++++++++--------
3 files changed, 287 insertions(+), 196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53d9d081/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 5cbcab5..37a660f 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -739,6 +739,7 @@ public class CoreContainer {
status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
}
+ // MetricsHistoryHandler supports both cloud and standalone configs
private void createMetricsHistoryHandler() {
PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
Map<String, Object> initArgs;
@@ -762,8 +763,13 @@ public class CoreContainer {
name = "localhost";
}
cloudManager = null;
- client = new EmbeddedSolrServer(this, CollectionAdminParams.SYSTEM_COLL);
- // enable local metrics
+ client = new EmbeddedSolrServer(this, CollectionAdminParams.SYSTEM_COLL) {
+ @Override
+ public void close() throws IOException {
+ // do nothing - we close the container ourselves
+ }
+ };
+ // enable local metrics unless specifically set otherwise
if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_NODES_PROP)) {
initArgs.put(MetricsHistoryHandler.ENABLE_NODES_PROP, true);
}
@@ -820,6 +826,12 @@ public class CoreContainer {
ExecutorUtil.shutdownAndAwaitTermination(coreContainerWorkExecutor);
replayUpdatesExecutor.shutdownAndAwaitTermination();
+
+ if (metricsHistoryHandler != null) {
+ IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());
+ metricsHistoryHandler.close();
+ }
+
if (metricManager != null) {
metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.jvm));
@@ -838,10 +850,6 @@ public class CoreContainer {
} catch (Exception e) {
log.warn("Error removing live node. Continuing to close CoreContainer", e);
}
- if (metricsHistoryHandler != null) {
- IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());
- metricsHistoryHandler.close();
- }
if (metricManager != null) {
metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.cluster));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53d9d081/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
index 89731d2..03b545f 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -24,6 +24,8 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -46,6 +48,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.DoubleAdder;
import java.util.function.Function;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.annotations.VisibleForTesting;
@@ -112,30 +115,32 @@ import static org.apache.solr.common.params.CommonParams.ID;
public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider, Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final List<String> DEFAULT_CORE_COUNTERS = new ArrayList<String>() {{
- add("QUERY./select.requests");
- add("UPDATE./update.requests");
- }};
- public static final List<String> DEFAULT_CORE_GAUGES = new ArrayList<String>() {{
- add("INDEX.sizeInBytes");
- }};
- public static final List<String> DEFAULT_NODE_GAUGES = new ArrayList<String>() {{
- add("CONTAINER.fs.coreRoot.usableSpace");
- }};
- public static final List<String> DEFAULT_JVM_GAUGES = new ArrayList<String>() {{
- add("memory.heap.used");
- add("os.processCpuLoad");
- add("os.systemLoadAverage");
- }};
+ public static final List<String> DEFAULT_CORE_COUNTERS = new ArrayList<>();
+ public static final List<String> DEFAULT_CORE_GAUGES = new ArrayList<>();
+ public static final List<String> DEFAULT_NODE_GAUGES = new ArrayList<>();
+ public static final List<String> DEFAULT_JVM_GAUGES = new ArrayList<>();
public static final String NUM_SHARDS_KEY = "numShards";
public static final String NUM_REPLICAS_KEY = "numReplicas";
public static final String NUM_NODES_KEY = "numNodes";
- public static final List<String> DEFAULT_COLLECTION_GAUGES = new ArrayList<String>() {{
- add(NUM_SHARDS_KEY);
- add(NUM_REPLICAS_KEY);
- }};
+ public static final List<String> DEFAULT_COLLECTION_GAUGES = new ArrayList<>();
+
+ static {
+ DEFAULT_JVM_GAUGES.add("memory.heap.used");
+ DEFAULT_JVM_GAUGES.add("os.processCpuLoad");
+ DEFAULT_JVM_GAUGES.add("os.systemLoadAverage");
+
+ DEFAULT_NODE_GAUGES.add("CONTAINER.fs.coreRoot.usableSpace");
+
+ DEFAULT_CORE_GAUGES.add("INDEX.sizeInBytes");
+
+ DEFAULT_CORE_COUNTERS.add("QUERY./select.requests");
+ DEFAULT_CORE_COUNTERS.add("UPDATE./update.requests");
+
+ DEFAULT_COLLECTION_GAUGES.add(NUM_SHARDS_KEY);
+ DEFAULT_COLLECTION_GAUGES.add(NUM_REPLICAS_KEY);
+ }
public static final String COLLECT_PERIOD_PROP = "collectPeriod";
public static final String SYNC_PERIOD_PROP = "syncPeriod";
@@ -698,7 +703,8 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
List<Pair<String, Long>> lst = factory.list(rows);
lst.forEach(p -> {
SimpleOrderedMap<Object> data = new SimpleOrderedMap<>();
- data.add("lastModified", p.second());
+ // RrdDb always uses seconds - convert here for compatibility
+ data.add("lastModified", TimeUnit.SECONDS.convert(p.second(), TimeUnit.MILLISECONDS));
data.add("node", nodeName);
res.add(p.first(), data);
});
@@ -753,7 +759,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
rsp.add("success", "ok");
break;
}
- // when using in-memory DBs non-overseer node has no access to overseer DBs
+ // when using in-memory DBs non-overseer node has no access to overseer DBs - in this case
// forward the request to Overseer leader if available
if (!factory.isPersistent()) {
String leader = getOverseerLeader();
@@ -766,7 +772,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
SimpleOrderedMap<Object> apiState = new SimpleOrderedMap<>();
apiState.add("enableReplicas", enableReplicas);
apiState.add("enableNodes", enableNodes);
- apiState.add("mode", enable ? (factory.isPersistent() ? "index" : "memory") : "disabled");
+ apiState.add("mode", enable ? (factory.isPersistent() ? "index" : "memory") : "inactive");
if (!factory.isPersistent()) {
apiState.add("message", "WARNING: metrics history is not being persisted. Create .system collection to start persisting history.");
}
@@ -775,10 +781,20 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
}
private NamedList<Object> handleRemoteRequest(String nodeName, SolrQueryRequest req) {
- String url = Utils.getBaseUrlForNodeName(nodeName, overseerUrlScheme);
+ String baseUrl = Utils.getBaseUrlForNodeName(nodeName, overseerUrlScheme);
+ String url;
+ try {
+ URL u = new URL(baseUrl);
+ u = new URL(u.getProtocol(), u.getHost(), u.getPort(), "/api/cluster/metrics/history");
+ url = u.toString();
+ } catch (MalformedURLException e) {
+ log.warn("Invalid Overseer url '" + baseUrl + "', unable to fetch remote metrics history", e);
+ return null;
+ }
+ // always use javabin
ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
params.set(CommonParams.WT, "javabin");
- url = url + req.getPath() + "?" + params.toString();
+ url = url + "?" + params.toString();
try {
byte[] data = cloudManager.httpRequest(url, SolrRequest.METHOD.GET, null, null, HttpClientUtil.DEFAULT_CONNECT_TIMEOUT, true);
// response is always a NamedList
@@ -856,18 +872,17 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
if (format != Format.GRAPH) {
// add timestamps separately from values
long[] timestamps = fd.getTimestamps();
- str.setLength(0);
- for (int i = 0; i < timestamps.length; i++) {
- if (format == Format.LIST) {
- map.add("timestamps", timestamps[i]);
- } else {
+ if (format == Format.LIST) {
+ // Arrays.asList works only on arrays of Objects
+ map.add("timestamps", Arrays.stream(timestamps).boxed().collect(Collectors.toList()));
+ } else {
+ str.setLength(0);
+ for (int i = 0; i < timestamps.length; i++) {
if (i > 0) {
str.append('\n');
}
str.append(String.valueOf(timestamps[i]));
}
- }
- if (format == Format.STRING) {
map.add("timestamps", str.toString());
}
}
@@ -920,9 +935,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
values.add(name, str.toString());
break;
case LIST:
- for (int i = 0; i < vals.length; i++) {
- values.add(name, vals[i]);
- }
+ values.add(name, Arrays.stream(vals).boxed().collect(Collectors.toList()));
break;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53d9d081/solr/solr-ref-guide/src/metrics-history.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/metrics-history.adoc b/solr/solr-ref-guide/src/metrics-history.adoc
index 5dc1c3c..e39f66e 100644
--- a/solr/solr-ref-guide/src/metrics-history.adoc
+++ b/solr/solr-ref-guide/src/metrics-history.adoc
@@ -18,15 +18,16 @@
== Design
=== Round-robin databases
-When Solr runs in "cloud" mode it collects long-term history of certain key metrics. This information
-can be used for very simple monitoring and troubleshooting, but also some Solr Cloud components
-(eg. autoscaling) can use this data for making informed decisions based on long-term
-trends of selected metrics.
+Solr collects long-term history of certain key metrics both in SolrCloud and in standalone mode.
+This information can be used for very simple monitoring and troubleshooting, but also some
+Solr Cloud components (eg. autoscaling) can use this data for making informed decisions based on
+long-term trends of selected metrics.
[IMPORTANT]
====
-Metrics history is available ONLY in SolrCloud mode, it's not supported in standalone Solr. Also,
-the `.system` collection must exist if metrics history should be persisted.
+The `.system` collection must exist if metrics history should be persisted. If this collection
+is absent then metrics history will still be collected and kept in memory but it will be lost
+on node restart.
====
This data is maintained as multi-resolution time series, with a fixed total number of data points
@@ -61,14 +62,16 @@ update operations than storing each data point in a separate Solr document. Metr
detailed data from each database, including retrieval of all individual datapoints.
Databases are identified primarily by their corresponding metric registry name, so for databases that
-keep track of aggregated metrics this will be eg. `solr.jvm`, `solr.node`, `solr.collection.gettingstarted`,
-and for databases with non-aggregated metrics this will be eg. `solr.jvm.localhost:8983_solr`,
-`solr.node.localhost:7574_solr`, `solr.core.gettingstarted.shard1.replica_n1`.
+keep track of aggregated metrics this will be eg. `solr.jvm`, `solr.node`, `solr.collection.gettingstarted`.
+For databases with non-aggregated metrics the name consists of the registry name, optionally with a node name
+to identify databases with the same name coming from different nodes. For example, per-node databases are
+name like this: `solr.jvm.localhost:8983_solr`, `solr.node.localhost:7574_solr`, but per-replica names are
+already unique across the cluster so they are named like this: `solr.core.gettingstarted.shard1.replica_n1`.
=== Collected metrics
Currently the following selected metrics are tracked:
-* `solr.core` and `solr.collection` metrics:
+* Non-aggregated `solr.core` and aggregated `solr.collection` metrics:
** `QUERY./select.requests`
** `UPDATE./update.requests`
** `INDEX.sizeInBytes`
@@ -78,6 +81,7 @@ Currently the following selected metrics are tracked:
* `solr.node` metrics:
** `CONTAINER.fs.coreRoot.usableSpace`
** `numNodes` (aggregated, number of live nodes)
+
* `solr.jvm` metrics:
** `memory.heap.used`
** `os.processCpuLoad`
@@ -86,6 +90,10 @@ Currently the following selected metrics are tracked:
Separate databases are created for each of these groups, and each database keeps data for
all metrics listed in that group.
+NOTE: Currently this list is not configurable. Also, if you change this list in the code then
+all existing databases must be first removed from the `.system` collection because RRD4j doesn't allow
+adding new datasources once the database is created.
+
=== SolrRrdBackendFactory
This component is responsible for managing in-memory databases and periodically saving them
to the `.system` collection. If the `.system` collection is not available the updates to the
@@ -101,7 +109,8 @@ collecting and periodically updating the in-memory databases.
This handler also performs aggregation of metrics on per-collection level, and on a cluster level.
By default only these aggregated metrics are tracked - historic data from each node and each replica
in each collection is not collected separately. Aggregated databases are managed on the Overseer leader
-node.
+node but they are still accessible from other nodes even if they are not persisted - the handler redirects
+the call from originating node to the current Overseer leader.
The handler assumes that a simple aggregation (sum of partial metric values from each resource) is
sufficient. This happens to make sense for the default built-in sets of metrics. Future extensions will
@@ -135,6 +144,7 @@ databases.
`collectPeriod`:: integer, in seconds, default is 60. Metrics values will be collected and respective
databases updated every `collectPeriod` seconds.
+
[IMPORTANT]
====
Value of `collectPeriod` must be at least 1, and if it's changed then all previously existing databases
@@ -142,9 +152,9 @@ with their historic data must be manually removed (new databases will be created
====
`syncPeriod`:: integer, in seconds, default is 60. Data from modified databases will be saved to Solr
-every `syncPeriod` seconds. When accessing the databases via REST API the visibility of most recent
-data depends on this period, because requests accessing the data from other nodes see only the
-version of the data that is stored in the `.system` collection.
+every `syncPeriod` seconds. When accessing the databases via REST API in `index` mode the visibility of
+most recent data depends on this period, because requests accessing the data from other nodes see only
+the version of the data that is stored in the `.system` collection.
=== Example configuration
Example `/clusterprops.json` file with metrics history configuration that turns on the collection of
@@ -154,6 +164,7 @@ properties unrelated to metrics history API.
[source,json]
----
{
+...
"metrics" : {
"history" : {
"enable" : true,
@@ -161,42 +172,86 @@ properties unrelated to metrics history API.
"syncPeriod" : 300
}
}
+...
}
----
== Metrics History API
-Main entry point for accessing metrics history is `/admin/metrics/history` (or `/api/cluster/metrics/history` for
-v2 API).
+Main entry point for accessing metrics history is `/admin/metrics/history` (or `/api/cluster/metrics/history`
+for v2 API).
The following sections describe actions available in this API. All calls have at least one
required parameter `action`.
+All responses contain a section named `state`, which reports the current internal state of the API:
+
+`enableReplicas`:: boolean, corresponds to the `enableReplicas` configuration setting.
+`enableNodes`:: boolean, corresponds to the `enableNodes` configuration setting.
+`mode`:: one of the following values:
+* `inactive` - when metrics collection is disabled (but access to existing metrics history is still available).
+* `memory` - when metrics history is kept only in memory because `.system` collection doesn't exist. In this mode
+clients can access metrics history available on the node that received the reuqest and on the Overseer leader.
+* `index` - when metrics history is periodically stored in the `.system` collection. Data available in memory on
+the node that accepted the request is retrieved from memory, any other data is retrieved from the
+`.system` collection (so it's at least `syncPeriod` old).
+
+Also, the response header section (`responseHeader`) contains `zkConnected` boolean property that indicates
+whether the current node is a part of SolrCloud cluster.
+
=== List databases (`action=list`)
This call produces a list of available databases. It supports the following parameters:
-`rows`:: optional integer, default is 500. Maximum number of results to return
+`rows`:: optional integer, default is 500. Maximum number of results to return.
Example:
+In this SolrCloud example the API is in `memory` mode, and the request was made to a node that is
+not Overseer leader. The API transparently forwarded the request to Overseer leader.
[source,bash]
----
-curl http://localhost:8983/solr/admin/metrics/history?action=list&rows=10
+curl http://localhost:7574/solr/admin/metrics/history?action=list&rows=10
----
[source,json]
----
{
- "responseHeader": {
- "status": 0,
- "QTime": 16
+ "responseHeader": {
+ "zkConnected": true,
+ "status": 0,
+ "QTime": 9
+ },
+ "metrics": {
+ "solr.collection..system": {
+ "lastModified": 1528360138,
+ "node": "127.0.0.1:8983_solr"
+ },
+ "solr.collection.gettingstarted": {
+ "lastModified": 1528360138,
+ "node": "127.0.0.1:8983_solr"
+ },
+ "solr.jvm": {
+ "lastModified": 1528360138,
+ "node": "127.0.0.1:8983_solr"
},
- "metrics": [
- "solr.collection..system",
- "solr.collection.gettingstarted",
- "solr.jvm",
- "solr.node"
- ]
+ "solr.node": {
+ "lastModified": 1528360138,
+ "node": "127.0.0.1:8983_solr"
+ }
+ },
+ "state": {
+ "enableReplicas": false,
+ "enableNodes": false,
+ "mode": "memory"
+ }
}
----
+Note the presence of the `node` element in each section, which shows where the information is coming
+from - when API is in `memory` mode this indicates which results are local and which ones are retrieved
+from the Overseer leader node. When the API is in `index` mode this element always shows the node name that
+received the request (because the data is retrieved from the `.system` collection anyway).
+
+Each section also contains a `lastModified` element, which contains the last modification time when the
+database was update. All timestamps returned from this API correspond to Unix epoch time in seconds.
+
=== Database status (`action=status`)
This call provides detailed status of the selected database.
@@ -207,66 +262,71 @@ The following parameters are supported:
Example:
[source,bash]
----
-curl http://localhost:8983/solr/admin/metrics/history?action=status&name=solr.collection.gettingstarted
+curl http://localhost:7574/solr/admin/metrics/history?action=status&name=solr.collection.gettingstarted
----
[source,json]
----
{
- "responseHeader": {
- "status": 0,
- "QTime": 38
- },
- "metrics": [
- "solr.collection.gettingstarted",
- [
- "status",
- {
- "lastModified": 1527268438,
- "step": 60,
- "datasourceCount": 5,
- "archiveCount": 5,
- "datasourceNames": [
- "numShards",
- "numReplicas",
- "QUERY./select.requests",
- "UPDATE./update.requests",
- "INDEX.sizeInBytes"
- ],
- "datasources": [
- {
- "datasource": "DS:numShards:GAUGE:120:U:U",
- "lastValue": 2
- },
- {
- "datasource": "DS:QUERY./select.requests:COUNTER:120:U:U",
- "lastValue": 8786
- },
- ...
- ],
- "archives": [
- {
- "archive": "RRA:AVERAGE:0.5:1:240",
- "steps": 1,
- "consolFun": "AVERAGE",
- "xff": 0.5,
- "startTime": 1527254040,
- "endTime": 1527268380,
- "rows": 240
- },
- {
- "archive": "RRA:AVERAGE:0.5:10:288",
- "steps": 10,
- "consolFun": "AVERAGE",
- "xff": 0.5,
- "startTime": 1527096000,
- "endTime": 1527268200,
- "rows": 288
- },
- ...
- ]
- }
+ "responseHeader": {
+ "zkConnected": true,
+ "status": 0,
+ "QTime": 46
+ },
+ "metrics": {
+ "solr.collection.gettingstarted": {
+ "status": {
+ "lastModified": 1528318361,
+ "step": 60,
+ "datasourceCount": 5,
+ "archiveCount": 5,
+ "datasourceNames": [
+ "numShards",
+ "numReplicas",
+ "QUERY./select.requests",
+ "UPDATE./update.requests",
+ "INDEX.sizeInBytes"
+ ],
+ "datasources": [
+ {
+ "datasource": "DS:numShards:GAUGE:120:U:U",
+ "lastValue": 2
+ },
+ {
+ "datasource": "DS:numReplicas:GAUGE:120:U:U",
+ "lastValue": 4
+ },
+ ...
+ ],
+ "archives": [
+ {
+ "archive": "RRA:AVERAGE:0.5:1:240",
+ "steps": 1,
+ "consolFun": "AVERAGE",
+ "xff": 0.5,
+ "startTime": 1528303980,
+ "endTime": 1528318320,
+ "rows": 240
+ },
+ {
+ "archive": "RRA:AVERAGE:0.5:10:288",
+ "steps": 10,
+ "consolFun": "AVERAGE",
+ "xff": 0.5,
+ "startTime": 1528146000,
+ "endTime": 1528318200,
+ "rows": 288
+ },
+ ...
]
- ]
+ },
+ "node": "127.0.0.1:7574_solr"
+ }
+ },
+ "state": {
+ "enableReplicas": false,
+ "enableNodes": false,
+ "mode": "index"
+ }
}
----
@@ -286,7 +346,7 @@ values (because points from all datasources in a given time series share the sam
* `graph` - data is returned as PNG images, Base64-encoded, containing graphs of each time series values over time.
In each case the response is structured in a similar way: archive identifiers are keys in a JSON map,
-and timestamps / datapoints / graphs are values.
+all data is placed in a `data` element, with timestamps / datapoints / graphs as values in lists or maps.
==== Examples
This is the output using the default `list` format:
@@ -297,37 +357,49 @@ curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.colle
[source,json]
----
{
- "responseHeader": {
- "status": 0,
- "QTime": 36
- },
- "metrics": [
- "solr.collection.gettingstarted",
- [
- "data",
- {
- "RRA:AVERAGE:0.5:1:240": {
- "timestamps":1527254460,
- "timestamps":1527254520,
- "timestamps":1527254580,
- ...
- "values": {
- "numShards": "NaN",
- "numShards": 2.0,
- "numShards": 2.0,
- ...
- "numReplicas": "NaN",
- "numReplicas": 4.0,
- "numReplicas": 4.0,
- ...
- "QUERY./select.requests": "NaN",
- "QUERY./select.requests": 123,
- "QUERY./select.requests": 456,
- ...
- }
- },
- "RRA:AVERAGE:0.5:10:288": {
-...
+ "responseHeader": {
+ "zkConnected": true,
+ "status": 0,
+ "QTime": 4
+ },
+ "metrics": {
+ "solr.collection.gettingstarted": {
+ "data": {
+ "RRA:AVERAGE:0.5:1:240": {
+ "timestamps": [
+ 1528304160,
+ 1528304220,
+ ...
+ ],
+ "values": {
+ "numShards": [
+ "NaN",
+ 2.0,
+ ...
+ ],
+ "numReplicas": [
+ "NaN",
+ 4.0,
+ ...
+ ],
+ ...
+ }
+ },
+ "RRA:AVERAGE:0.5:10:288": {
+ "timestamps": [
+ 1528145400,
+ 1528146000,
+ ...
+ "lastModified": 1528318606,
+ "node": "127.0.0.1:8983_solr"
+ }
+ },
+ "state": {
+ "enableReplicas": false,
+ "enableNodes": false,
+ "mode": "index"
+ }
+}
----
This is the output when using the `string` format:
@@ -338,25 +410,24 @@ curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.colle
[source,json]
----
{
- "responseHeader": {
- "status": 0,
- "QTime": 11
- },
- "metrics": [
- "solr.collection.gettingstarted",
- [
- "data",
- {
- "RRA:AVERAGE:0.5:1:240": {
- "timestamps": "1527254820\n1527254880\n1527254940\n...",
- "values": {
- "numShards": "NaN\n2.0\n2.0\n2.0\n2.0\n2.0\n2.0\n...",
- "numReplicas": "NaN\n4.0\n4.0\n4.0\n4.0\n4.0\n4.0\n...",
- "QUERY./select.requests": "NaN\n123\n456\n789\n...",
- ...
- }
- },
- "RRA:AVERAGE:0.5:10:288": {
+ "responseHeader": {
+ "zkConnected": true,
+ "status": 0,
+ "QTime": 2
+ },
+ "metrics": {
+ "solr.collection.gettingstarted": {
+ "data": {
+ "RRA:AVERAGE:0.5:1:240": {
+ "timestamps": "1527254820\n1527254880\n1527254940\n...",
+ "values": {
+ "numShards": "NaN\n2.0\n2.0\n2.0\n2.0\n2.0\n2.0\n...",
+ "numReplicas": "NaN\n4.0\n4.0\n4.0\n4.0\n4.0\n4.0\n...",
+ "QUERY./select.requests": "NaN\n123\n456\n789\n...",
+ ...
+ }
+ },
+ "RRA:AVERAGE:0.5:10:288": {
...
----
@@ -368,29 +439,28 @@ curl http://localhost:8983/solr/admin/metrics/history?action=get&name=solr.colle
[source,json]
----
{
- "responseHeader": {
- "status": 0,
- "QTime": 2275
- },
- "metrics": [
- "solr.collection.gettingstarted",
- [
- "data",
- {
- "RRA:AVERAGE:0.5:1:240": {
- "values": {
- "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
- "numReplicas": "iVBORw0KGgoAAAANSUhEUgAAAkQA...",
- "QUERY./select.requests": "iVBORw0KGgoAAAANS...",
- ...
- }
- },
- "RRA:AVERAGE:0.5:10:288": {
- "values": {
- "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
- ...
- },
- ...
+ "responseHeader": {
+ "zkConnected": true,
+ "status": 0,
+ "QTime": 2
+ },
+ "metrics": {
+ "solr.collection.gettingstarted": {
+ "data": {
+ "RRA:AVERAGE:0.5:1:240": {
+ "values": {
+ "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
+ "numReplicas": "iVBORw0KGgoAAAANSUhEUgAAAkQA...",
+ "QUERY./select.requests": "iVBORw0KGgoAAAANS...",
+ ...
+ }
+ },
+ "RRA:AVERAGE:0.5:10:288": {
+ "values": {
+ "numShards": "iVBORw0KGgoAAAANSUhEUgAAAkQAAA...",
+ ...
+ },
+ ...
----
.Example 60 sec resolution history graph for `QUERY./select.requests` metric
[04/18] lucene-solr:jira/solr-12438: LUCENE-7960: fix Solr test to
include mandatory args
Posted by ab...@apache.org.
LUCENE-7960: fix Solr test to include mandatory args
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c5875980
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c5875980
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c5875980
Branch: refs/heads/jira/solr-12438
Commit: c587598096cde769c299594fb26d0a23b7bd5930
Parents: 7d0b64f
Author: David Smiley <ds...@apache.org>
Authored: Tue Jun 5 12:30:34 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Jun 5 12:30:34 2018 -0400
----------------------------------------------------------------------
.../core/src/test/org/apache/solr/core/ResourceLoaderTest.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c5875980/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java b/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
index 8ab9606..785195a 100644
--- a/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
+++ b/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
@@ -72,7 +72,8 @@ public class ResourceLoaderTest extends SolrTestCaseJ4 {
Class<?> clazz = ResourceLoaderAware.class;
// Check ResourceLoaderAware valid objects
- assertAwareCompatibility(clazz, new NGramFilterFactory(new HashMap<>()));
+ //noinspection unchecked
+ assertAwareCompatibility(clazz, new NGramFilterFactory(map("minGramSize", "1", "maxGramSize", "2")));
assertAwareCompatibility(clazz, new KeywordTokenizerFactory(new HashMap<>()));
// Make sure it throws an error for invalid objects
@@ -98,8 +99,9 @@ public class ResourceLoaderTest extends SolrTestCaseJ4 {
assertAwareCompatibility(clazz, new JSONResponseWriter());
// Make sure it throws an error for invalid objects
+ //noinspection unchecked
invalid = new Object[] {
- new NGramFilterFactory(new HashMap<>()),
+ new NGramFilterFactory(map("minGramSize", "1", "maxGramSize", "2")),
"hello", new Float( 12.3f ),
new KeywordTokenizerFactory(new HashMap<>())
};